[IPoIB] Synchronized port down event handling with SA queries.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 \r
45 /* Amount of physical memory to register. */\r
46 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
47 \r
48 /* Number of work completions to chain for send and receive polling. */\r
49 #define MAX_SEND_WC             8\r
50 #define MAX_RECV_WC             16\r
51 \r
52 \r
53 ib_gid_t        bcast_mgid_template = {\r
54         0xff,                                                           /* multicast field */\r
55         0x12,                                                           /* scope (to be filled in) */\r
56         0x40, 0x1b,                                                     /* IPv4 signature */\r
57         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
58         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
59         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
60 };\r
61 \r
62 \r
63 #ifdef _DEBUG_\r
64 /* Handy pointer for debug use. */\r
65 ipoib_port_t    *gp_ipoib_port;\r
66 #endif\r
67 \r
68 \r
69 /******************************************************************************\r
70 *\r
71 * Declarations\r
72 *\r
73 ******************************************************************************/\r
74 static void\r
75 __port_construct(\r
76         IN                              ipoib_port_t* const                     p_port );\r
77 \r
78 static ib_api_status_t\r
79 __port_init(\r
80         IN                              ipoib_port_t* const                     p_port,\r
81         IN                              ipoib_adapter_t* const          p_adapter,\r
82         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
83 \r
84 static void\r
85 __port_destroying(\r
86         IN                              cl_obj_t* const                         p_obj );\r
87 \r
88 static void\r
89 __port_cleanup(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_free(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 \r
97 /******************************************************************************\r
98 *\r
99 * IB resource manager operations\r
100 *\r
101 ******************************************************************************/\r
102 static void\r
103 __ib_mgr_construct(\r
104         IN                              ipoib_port_t* const                     p_port );\r
105 \r
106 static ib_api_status_t\r
107 __ib_mgr_init(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static void\r
111 __ib_mgr_destroy(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __qp_event(\r
116         IN                              ib_async_event_rec_t            *p_event_rec );\r
117 \r
118 static void\r
119 __cq_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static ib_api_status_t\r
123 __ib_mgr_activate(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 /******************************************************************************\r
127 *\r
128 * Buffer manager operations.\r
129 *\r
130 ******************************************************************************/\r
131 static void\r
132 __buf_mgr_construct(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 static ib_api_status_t\r
136 __buf_mgr_init(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static void\r
140 __buf_mgr_destroy(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static cl_status_t\r
144 __recv_ctor(\r
145         IN                              void* const                                     p_object,\r
146         IN                              void*                                           context,\r
147                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
148 \r
149 #if !IPOIB_INLINE_RECV\r
150 static void\r
151 __recv_dtor(\r
152         IN              const   cl_pool_item_t* const           p_pool_item,\r
153         IN                              void                                            *context );\r
154 #endif  /* IPOIB_INLINE_RECV */\r
155 \r
156 static inline ipoib_send_desc_t*\r
157 __buf_mgr_get_send(\r
158         IN                              ipoib_port_t* const                     p_port );\r
159 \r
160 static inline void\r
161 __buf_mgr_put_send(\r
162         IN                              ipoib_port_t* const                     p_port,\r
163         IN                              ipoib_send_desc_t* const        p_desc );\r
164 \r
165 static inline ipoib_recv_desc_t*\r
166 __buf_mgr_get_recv(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_recv(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_recv_desc_t* const        p_desc,\r
173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
174 \r
175 static inline void\r
176 __buf_mgr_put_recv_list(\r
177         IN                              ipoib_port_t* const                     p_port,\r
178         IN                              cl_qlist_t* const                       p_list );\r
179 \r
180 static inline NDIS_PACKET*\r
181 __buf_mgr_get_ndis_pkt(\r
182         IN                              ipoib_port_t* const                     p_port,\r
183         IN                              ipoib_recv_desc_t* const        p_desc );\r
184 \r
185 \r
186 /******************************************************************************\r
187 *\r
188 * Receive manager operations.\r
189 *\r
190 ******************************************************************************/\r
191 static void\r
192 __recv_mgr_construct(\r
193         IN                              ipoib_port_t* const                     p_port );\r
194 \r
195 static ib_api_status_t\r
196 __recv_mgr_init(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static void\r
200 __recv_mgr_destroy(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 /* Posts receive buffers to the receive queue. */\r
204 static ib_api_status_t\r
205 __recv_mgr_repost(\r
206         IN                              ipoib_port_t* const                     p_port,\r
207         IN                              uint32_t                                        num_wrs );\r
208 \r
209 static void\r
210 __recv_cb(\r
211         IN              const   ib_cq_handle_t                          h_cq,\r
212         IN                              void                                            *cq_context );\r
213 \r
214 static void\r
215 __recv_get_endpts(\r
216         IN                              ipoib_port_t* const                     p_port,\r
217         IN                              ipoib_recv_desc_t* const        p_desc,\r
218         IN                              ib_wc_t* const                          p_wc,\r
219                 OUT                     ipoib_endpt_t** const           pp_src,\r
220                 OUT                     ipoib_endpt_t** const           pp_dst );\r
221 \r
222 static uint32_t\r
223 __recv_mgr_filter(\r
224         IN                              ipoib_port_t* const                     p_port,\r
225         IN                              ib_wc_t* const                          p_done_wc_list,\r
226                 OUT                     cl_qlist_t* const                       p_done_list,\r
227                 OUT                     cl_qlist_t* const                       p_bad_list );\r
228 \r
229 static ib_api_status_t\r
230 __recv_gen(\r
231         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
232                 OUT                     eth_pkt_t* const                        p_eth,\r
233         IN                              ipoib_endpt_t* const            p_src,\r
234         IN                              ipoib_endpt_t* const            p_dst );\r
235 \r
236 static ib_api_status_t\r
237 __recv_dhcp(\r
238         IN                              ipoib_port_t* const                     p_port,\r
239         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
240                 OUT                     eth_pkt_t* const                        p_eth,\r
241         IN                              ipoib_endpt_t* const            p_src,\r
242         IN                              ipoib_endpt_t* const            p_dst );\r
243 \r
244 static ib_api_status_t\r
245 __recv_arp(\r
246         IN                              ipoib_port_t* const                     p_port,\r
247         IN                              ib_wc_t* const                          p_wc,\r
248         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
249                 OUT                     eth_pkt_t* const                        p_eth,\r
250         IN                              ipoib_endpt_t** const           p_src,\r
251         IN                              ipoib_endpt_t* const            p_dst );\r
252 \r
253 static ib_api_status_t\r
254 __recv_mgr_prepare_pkt(\r
255         IN                              ipoib_port_t* const                     p_port,\r
256         IN                              ipoib_recv_desc_t* const        p_desc,\r
257                 OUT                     NDIS_PACKET** const                     pp_packet );\r
258 \r
259 static uint32_t\r
260 __recv_mgr_build_pkt_array(\r
261         IN                              ipoib_port_t* const                     p_port,\r
262                 OUT                     cl_qlist_t* const                       p_done_list );\r
263 \r
264 /******************************************************************************\r
265 *\r
266 * Send manager operations.\r
267 *\r
268 ******************************************************************************/\r
269 static void\r
270 __send_mgr_construct(\r
271         IN                              ipoib_port_t* const                     p_port );\r
272 \r
273 static void\r
274 __send_mgr_destroy(\r
275         IN                              ipoib_port_t* const                     p_port );\r
276 \r
277 static NDIS_STATUS\r
278 __send_gen(\r
279         IN                              ipoib_port_t* const                     p_port,\r
280         IN                              ipoib_send_desc_t* const        p_desc );\r
281 \r
282 static NDIS_STATUS\r
283 __send_mgr_filter_ip(\r
284         IN                              ipoib_port_t* const                     p_port,\r
285         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
286         IN                              NDIS_BUFFER*                            p_buf,\r
287         IN                              size_t                                          buf_len,\r
288         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
289 \r
290 static NDIS_STATUS\r
291 __send_mgr_filter_udp(\r
292         IN                              ipoib_port_t* const                     p_port,\r
293         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
294         IN                              NDIS_BUFFER*                            p_buf,\r
295         IN                              size_t                                          buf_len,\r
296         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
297 \r
298 static NDIS_STATUS\r
299 __send_mgr_filter_dhcp(\r
300         IN                              ipoib_port_t* const                     p_port,\r
301         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
302         IN                              NDIS_BUFFER*                            p_buf,\r
303         IN                              size_t                                          buf_len,\r
304         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
305 \r
306 static NDIS_STATUS\r
307 __send_mgr_filter_arp(\r
308         IN                              ipoib_port_t* const                     p_port,\r
309         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
310         IN                              NDIS_BUFFER*                            p_buf,\r
311         IN                              size_t                                          buf_len,\r
312         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
313 \r
314 static void\r
315 __process_failed_send(\r
316         IN                              ipoib_port_t* const                     p_port,\r
317         IN                              ipoib_send_desc_t* const        p_desc,\r
318         IN              const   NDIS_STATUS                                     status );\r
319 \r
320 static void\r
321 __send_cb(\r
322         IN              const   ib_cq_handle_t                          h_cq,\r
323         IN                              void                                            *cq_context );\r
324 \r
325 \r
326 /******************************************************************************\r
327 *\r
328 * Endpoint manager operations\r
329 *\r
330 ******************************************************************************/\r
331 static void\r
332 __endpt_mgr_construct(\r
333         IN                              ipoib_port_t* const                     p_port );\r
334 \r
335 static ib_api_status_t\r
336 __endpt_mgr_init(\r
337         IN                              ipoib_port_t* const                     p_port );\r
338 \r
339 static void\r
340 __endpt_mgr_destroy(\r
341         IN                              ipoib_port_t* const                     p_port );\r
342 \r
343 /****f* IPoIB/__endpt_mgr_remove_all\r
344 * NAME\r
345 *       __endpt_mgr_remove_all\r
346 *\r
347 * DESCRIPTION\r
348 *       Removes all enpoints from the port, dereferencing them to initiate\r
349 *       destruction.\r
350 *\r
351 * SYNOPSIS\r
352 */\r
353 static void\r
354 __endpt_mgr_remove_all(\r
355         IN                              ipoib_port_t* const                     p_port );\r
356 /*\r
357 ********/\r
358 \r
359 static void\r
360 __endpt_mgr_remove(\r
361         IN                              ipoib_port_t* const                     p_port,\r
362         IN                              ipoib_endpt_t* const            p_endpt );\r
363 \r
364 static void\r
365 __endpt_mgr_reset_all(\r
366         IN                              ipoib_port_t* const                     p_port );\r
367 \r
368 static inline NDIS_STATUS\r
369 __endpt_mgr_ref(\r
370         IN                              ipoib_port_t* const                     p_port,\r
371         IN              const   mac_addr_t                                      mac,\r
372                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
373 \r
374 static inline NDIS_STATUS\r
375 __endpt_mgr_get_gid_qpn(\r
376         IN                              ipoib_port_t* const                     p_port,\r
377         IN              const   mac_addr_t                                      mac,\r
378                 OUT                     ib_gid_t* const                         p_gid,\r
379                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
380 \r
381 static inline ipoib_endpt_t*\r
382 __endpt_mgr_get_by_gid(\r
383         IN                              ipoib_port_t* const                     p_port,\r
384         IN              const   ib_gid_t* const                         p_gid );\r
385 \r
386 static inline ipoib_endpt_t*\r
387 __endpt_mgr_get_by_lid(\r
388         IN                              ipoib_port_t* const                     p_port,\r
389         IN              const   net16_t                                         lid );\r
390 \r
391 static inline void\r
392 __endpt_mgr_insert_locked(\r
393         IN                              ipoib_port_t* const                     p_port,\r
394         IN              const   mac_addr_t                                      mac,\r
395         IN                              ipoib_endpt_t* const            p_endpt );\r
396 \r
397 static inline void\r
398 __endpt_mgr_insert(\r
399         IN                              ipoib_port_t* const                     p_port,\r
400         IN              const   mac_addr_t                                      mac,\r
401         IN                              ipoib_endpt_t* const            p_endpt );\r
402 \r
403 static ib_api_status_t\r
404 __endpt_mgr_add_local(\r
405         IN                              ipoib_port_t* const                     p_port,\r
406         IN                              ib_port_info_t* const           p_port_info );\r
407 \r
408 static ib_api_status_t\r
409 __endpt_mgr_add_bcast(\r
410         IN                              ipoib_port_t* const                     p_port,\r
411         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
412 \r
413 /******************************************************************************\r
414 *\r
415 * MCast operations.\r
416 *\r
417 ******************************************************************************/\r
418 static ib_api_status_t\r
419 __port_get_bcast(\r
420         IN                              ipoib_port_t* const                     p_port );\r
421 \r
422 static ib_api_status_t\r
423 __port_join_bcast(\r
424         IN                              ipoib_port_t* const                     p_port,\r
425         IN                              ib_member_rec_t* const          p_member_rec );\r
426 \r
427 static ib_api_status_t\r
428 __port_create_bcast(\r
429         IN                              ipoib_port_t* const                     p_port );\r
430 \r
431 static void\r
432 __port_info_cb(\r
433         IN                              ib_query_rec_t                          *p_query_rec );\r
434 \r
435 \r
436 static void\r
437 __bcast_get_cb(\r
438         IN                              ib_query_rec_t                          *p_query_rec );\r
439 \r
440 \r
441 static void\r
442 __bcast_cb(\r
443         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
444 \r
445 \r
446 static void\r
447 __mcast_cb(\r
448         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
449 \r
450 \r
451 static intn_t\r
452 __gid_cmp(\r
453         IN              const   void* const                                     p_key1,\r
454         IN              const   void* const                                     p_key2 )\r
455 {\r
456         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
457 }\r
458 \r
459 \r
460 /******************************************************************************\r
461 *\r
462 * Implementation\r
463 *\r
464 ******************************************************************************/\r
465 ib_api_status_t\r
466 ipoib_create_port(\r
467         IN                              ipoib_adapter_t* const          p_adapter,\r
468         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
469                 OUT                     ipoib_port_t** const            pp_port )\r
470 {\r
471         ib_api_status_t         status;\r
472         ipoib_port_t            *p_port;\r
473 \r
474         IPOIB_ENTER( IPOIB_DBG_INIT );\r
475 \r
476         CL_ASSERT( !p_adapter->p_port );\r
477 \r
478         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
479                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
480         if( !p_port )\r
481         {\r
482                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
483                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
484                         sizeof(ipoib_port_t)) );\r
485                 return IB_INSUFFICIENT_MEMORY;\r
486         }\r
487 \r
488 #ifdef _DEBUG_\r
489         gp_ipoib_port = p_port;\r
490 #endif\r
491 \r
492         __port_construct( p_port );\r
493 \r
494         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
495         if( status != IB_SUCCESS )\r
496         {\r
497                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
498                         ("ipoib_port_init returned %s.\n", \r
499                         p_adapter->p_ifc->get_err_str( status )) );\r
500                 __port_cleanup( &p_port->obj );\r
501                 __port_free( &p_port->obj );\r
502                 return status;\r
503         }\r
504 \r
505         *pp_port = p_port;\r
506         IPOIB_EXIT( IPOIB_DBG_INIT );\r
507         return IB_SUCCESS;\r
508 }\r
509 \r
510 \r
511 void\r
512 ipoib_port_destroy(\r
513         IN                              ipoib_port_t* const                     p_port )\r
514 {\r
515         IPOIB_ENTER( IPOIB_DBG_INIT );\r
516 \r
517         CL_ASSERT( p_port );\r
518         CL_ASSERT( p_port->p_adapter );\r
519         CL_ASSERT( !p_port->p_adapter->p_port );\r
520 \r
521         cl_obj_destroy( &p_port->obj );\r
522 \r
523         IPOIB_EXIT( IPOIB_DBG_INIT );\r
524 }\r
525 \r
526 \r
527 static void\r
528 __port_construct(\r
529         IN                              ipoib_port_t* const                     p_port )\r
530 {\r
531         IPOIB_ENTER( IPOIB_DBG_INIT );\r
532 \r
533         p_port->state = IB_QPS_RESET;\r
534 \r
535         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
536         cl_spinlock_construct( &p_port->send_lock );\r
537         cl_spinlock_construct( &p_port->recv_lock );\r
538         __ib_mgr_construct( p_port );\r
539         __buf_mgr_construct( p_port );\r
540 \r
541         __recv_mgr_construct( p_port );\r
542         __send_mgr_construct( p_port );\r
543 \r
544         __endpt_mgr_construct( p_port );\r
545 \r
546         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
547 \r
548         IPOIB_EXIT( IPOIB_DBG_INIT );\r
549 }\r
550 \r
551 \r
552 static ib_api_status_t\r
553 __port_init(\r
554         IN                              ipoib_port_t* const                     p_port,\r
555         IN                              ipoib_adapter_t* const          p_adapter,\r
556         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
557 {\r
558         cl_status_t                     cl_status;\r
559         ib_api_status_t         status;\r
560 \r
561         IPOIB_ENTER( IPOIB_DBG_INIT );\r
562 \r
563         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
564         p_port->p_adapter = p_adapter;\r
565 \r
566         cl_status = cl_spinlock_init( &p_port->send_lock );\r
567         if( cl_status != CL_SUCCESS )\r
568         {\r
569                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
570                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
571                 return IB_ERROR;\r
572         }\r
573 \r
574         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
575         if( cl_status != CL_SUCCESS )\r
576         {\r
577                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
578                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
579                 return IB_ERROR;\r
580         }\r
581 \r
582         /* Initialize the IB resource manager. */\r
583         status = __ib_mgr_init( p_port );\r
584         if( status != IB_SUCCESS )\r
585         {\r
586                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
587                         ("__ib_mgr_init returned %s\n", \r
588                         p_adapter->p_ifc->get_err_str( status )) );\r
589                 return status;\r
590         }\r
591 \r
592         /* Initialize the buffer manager. */\r
593         status = __buf_mgr_init( p_port );\r
594         if( status != IB_SUCCESS )\r
595         {\r
596                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
597                         ("__buf_mgr_init returned %s\n", \r
598                         p_adapter->p_ifc->get_err_str( status )) );\r
599                 return status;\r
600         }\r
601 \r
602         /* Initialize the receive manager. */\r
603         status = __recv_mgr_init( p_port );\r
604         if( status != IB_SUCCESS )\r
605         {\r
606                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
607                         ("__recv_mgr_init returned %s\n", \r
608                         p_adapter->p_ifc->get_err_str( status )) );\r
609                 return status;\r
610         }\r
611 \r
612         /* Initialize the endpoint manager. */\r
613         status = __endpt_mgr_init( p_port );\r
614         if( status != IB_SUCCESS )\r
615         {\r
616                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
617                         ("__endpt_mgr_init returned %s\n", \r
618                         p_adapter->p_ifc->get_err_str( status )) );\r
619                 return status;\r
620         }\r
621 \r
622         /* We only ever destroy from the PnP callback thread. */\r
623         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
624                 __port_destroying, __port_cleanup, __port_free );\r
625         if( cl_status != CL_SUCCESS )\r
626         {\r
627                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
628                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
629                 return IB_ERROR;\r
630         }\r
631 \r
632         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
633 \r
634         IPOIB_EXIT( IPOIB_DBG_INIT );\r
635         return IB_SUCCESS;\r
636 }\r
637 \r
638 \r
639 static void\r
640 __port_destroying(\r
641         IN                              cl_obj_t* const                         p_obj )\r
642 {\r
643         ipoib_port_t    *p_port;\r
644 \r
645         IPOIB_ENTER( IPOIB_DBG_INIT );\r
646 \r
647         CL_ASSERT( p_obj );\r
648 \r
649         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
650 \r
651         ipoib_port_down( p_port );\r
652 \r
653         __endpt_mgr_remove_all( p_port );\r
654 \r
655         IPOIB_EXIT( IPOIB_DBG_INIT );\r
656 }\r
657 \r
658 \r
659 static void\r
660 __port_cleanup(\r
661         IN                              cl_obj_t* const                         p_obj )\r
662 {\r
663         ipoib_port_t    *p_port;\r
664 \r
665         IPOIB_ENTER( IPOIB_DBG_INIT );\r
666 \r
667         CL_ASSERT( p_obj );\r
668 \r
669         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
670 \r
671         /* Wait for all sends and receives to get flushed. */\r
672         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
673                 cl_thread_suspend( 0 );\r
674 \r
675         /* Destroy the send and receive managers before closing the CA. */\r
676         __ib_mgr_destroy( p_port );\r
677 \r
678         IPOIB_EXIT( IPOIB_DBG_INIT );\r
679 }\r
680 \r
681 \r
682 static void\r
683 __port_free(\r
684         IN                              cl_obj_t* const                         p_obj )\r
685 {\r
686         ipoib_port_t    *p_port;\r
687 \r
688         IPOIB_ENTER( IPOIB_DBG_INIT );\r
689 \r
690         CL_ASSERT( p_obj );\r
691 \r
692         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
693 \r
694         __endpt_mgr_destroy( p_port );\r
695         __recv_mgr_destroy( p_port );\r
696         __send_mgr_destroy( p_port );\r
697         __buf_mgr_destroy( p_port );\r
698 \r
699         cl_spinlock_destroy( &p_port->send_lock );\r
700         cl_spinlock_destroy( &p_port->recv_lock );\r
701 \r
702         cl_obj_deinit( p_obj );\r
703 \r
704         cl_free( p_port );\r
705 \r
706         IPOIB_EXIT( IPOIB_DBG_INIT );\r
707 }\r
708 \r
709 \r
710 \r
711 /******************************************************************************\r
712 *\r
713 * IB resource manager implementation.\r
714 *\r
715 ******************************************************************************/\r
716 static void\r
717 __ib_mgr_construct(\r
718         IN                              ipoib_port_t* const                     p_port )\r
719 {\r
720         IPOIB_ENTER( IPOIB_DBG_INIT );\r
721 \r
722         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
723 \r
724         IPOIB_EXIT( IPOIB_DBG_INIT );\r
725 }\r
726 \r
727 \r
728 static ib_api_status_t\r
729 __ib_mgr_init(\r
730         IN                              ipoib_port_t* const                     p_port )\r
731 {\r
732         ib_api_status_t         status;\r
733         ib_cq_create_t          cq_create;\r
734         ib_qp_create_t          qp_create;\r
735         ib_phys_create_t        phys_create;\r
736         ib_phys_range_t         phys_range;\r
737         uint64_t                        vaddr;\r
738         net32_t                         rkey;\r
739         ib_qp_attr_t            qp_attr;\r
740 \r
741         IPOIB_ENTER( IPOIB_DBG_INIT );\r
742 \r
743         /* Open the CA. */\r
744         status = p_port->p_adapter->p_ifc->open_ca(\r
745                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
746                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
747         if( status != IB_SUCCESS )\r
748         {\r
749                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
750                         EVENT_IPOIB_OPEN_CA, 1, status );\r
751                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
752                         ("ib_open_ca returned %s\n", \r
753                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
754                 return status;\r
755         }\r
756 \r
757         /* Allocate the PD. */\r
758         status = p_port->p_adapter->p_ifc->alloc_pd(\r
759                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
760         if( status != IB_SUCCESS )\r
761         {\r
762                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
763                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
764                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
765                         ("ib_alloc_pd returned %s\n", \r
766                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
767                 return status;\r
768         }\r
769 \r
770         /* Allocate receive CQ. */\r
771         cq_create.size = p_port->p_adapter->params.rq_depth;\r
772         cq_create.pfn_comp_cb = __recv_cb;\r
773         cq_create.h_wait_obj = NULL;\r
774 \r
775         status = p_port->p_adapter->p_ifc->create_cq(\r
776                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
777                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
778         if( status != IB_SUCCESS )\r
779         {\r
780                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
781                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
782                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
783                         ("ib_create_cq returned %s.\n", \r
784                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
785                 return status;\r
786         }\r
787 \r
788         /* Allocate send CQ. */\r
789         cq_create.size = p_port->p_adapter->params.sq_depth;\r
790         cq_create.pfn_comp_cb = __send_cb;\r
791 \r
792         status = p_port->p_adapter->p_ifc->create_cq(\r
793                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
794                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
795         if( status != IB_SUCCESS )\r
796         {\r
797                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
798                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
799                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
800                         ("ib_create_cq returned %s.\n", \r
801                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
802                 return status;\r
803         }\r
804 \r
805         /* Allocate the QP. */\r
806         cl_memclr( &qp_create, sizeof(qp_create) );\r
807         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
808         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
809         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
810         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
811         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
812         //TODO: Figure out the right number of SGE entries for sends.\r
813         qp_create.sq_sge = MAX_SEND_SGE;\r
814         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
815         qp_create.sq_signaled = TRUE;\r
816         status = p_port->p_adapter->p_ifc->create_qp(\r
817                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
818                 __qp_event, &p_port->ib_mgr.h_qp );\r
819         if( status != IB_SUCCESS )\r
820         {\r
821                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
822                         EVENT_IPOIB_CREATE_QP, 1, status );\r
823                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
824                         ("ib_create_qp returned %s\n", \r
825                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
826                 return status;\r
827         }\r
828         /* Query the QP so we can get our QPN. */\r
829         status = p_port->p_adapter->p_ifc->query_qp(\r
830                 p_port->ib_mgr.h_qp, &qp_attr );\r
831         if( status != IB_SUCCESS )\r
832         {\r
833                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
834                         EVENT_IPOIB_QUERY_QP, 1, status );\r
835                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
836                         ("ib_query_qp returned %s\n", \r
837                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
838                 return status;\r
839         }\r
840         p_port->ib_mgr.qpn = qp_attr.num;\r
841 \r
842         /* Register all of physical memory */\r
843         phys_create.length = MEM_REG_SIZE;\r
844         phys_create.num_ranges = 1;\r
845         phys_create.range_array = &phys_range;\r
846         phys_create.buf_offset = 0;\r
847         phys_create.hca_page_size = PAGE_SIZE;\r
848         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
849         phys_range.base_addr = 0;\r
850         phys_range.size = MEM_REG_SIZE;\r
851         vaddr = 0;\r
852         status = p_port->p_adapter->p_ifc->reg_phys(\r
853                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
854                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
855         if( status != IB_SUCCESS )\r
856         {\r
857                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
858                         EVENT_IPOIB_REG_PHYS, 1, status );\r
859                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
860                         ("ib_reg_phys returned %s\n", \r
861                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
862                 return status;\r
863         }\r
864 \r
865         IPOIB_EXIT( IPOIB_DBG_INIT );\r
866         return IB_SUCCESS;\r
867 }\r
868 \r
869 \r
870 static void\r
871 __ib_mgr_destroy(\r
872         IN                              ipoib_port_t* const                     p_port )\r
873 {\r
874         ib_api_status_t status;\r
875 \r
876         IPOIB_ENTER( IPOIB_DBG_INIT );\r
877 \r
878         if( p_port->ib_mgr.h_ca )\r
879         {\r
880                 status =\r
881                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
882                 CL_ASSERT( status == IB_SUCCESS );\r
883                 p_port->ib_mgr.h_ca = NULL;\r
884         }\r
885 \r
886         IPOIB_EXIT( IPOIB_DBG_INIT );\r
887 }\r
888 \r
889 \r
890 \r
891 /******************************************************************************\r
892 *\r
893 * Buffer manager implementation.\r
894 *\r
895 ******************************************************************************/\r
896 static void\r
897 __buf_mgr_construct(\r
898         IN                              ipoib_port_t* const                     p_port )\r
899 {\r
900         IPOIB_ENTER( IPOIB_DBG_INIT );\r
901 \r
902         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
903 \r
904         p_port->buf_mgr.h_packet_pool = NULL;\r
905         p_port->buf_mgr.h_buffer_pool = NULL;\r
906 \r
907         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
908                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
909 \r
910         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
911         p_port->buf_mgr.h_send_buf_pool = NULL;\r
912 \r
913         IPOIB_EXIT( IPOIB_DBG_INIT );\r
914 }\r
915 \r
916 \r
917 static ib_api_status_t\r
918 __buf_mgr_init(\r
919         IN                              ipoib_port_t* const                     p_port )\r
920 {\r
921         cl_status_t             cl_status;\r
922         NDIS_STATUS             ndis_status;\r
923         ipoib_params_t  *p_params;\r
924 \r
925         IPOIB_ENTER(IPOIB_DBG_INIT );\r
926 \r
927         CL_ASSERT( p_port );\r
928         CL_ASSERT( p_port->p_adapter );\r
929 \r
930         p_params = &p_port->p_adapter->params;\r
931 \r
932         /* Allocate the receive descriptor pool */\r
933         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
934                 p_params->rq_depth * p_params->recv_pool_ratio,\r
935 #if IPOIB_INLINE_RECV\r
936                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
937 #else   /* IPOIB_INLINE_RECV */\r
938                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
939 #endif  /* IPOIB_INLINE_RECV */\r
940         if( cl_status != CL_SUCCESS )\r
941         {\r
942                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
943                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
944                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
945                         ("cl_qpool_init for recvs returned %s\n",\r
946                         cl_status_text[cl_status]) );\r
947                 return IB_INSUFFICIENT_MEMORY;\r
948         }\r
949 \r
950         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
951         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
952                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
953         if( ndis_status != NDIS_STATUS_SUCCESS )\r
954         {\r
955                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
956                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
957                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
958                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
959                 return IB_INSUFFICIENT_RESOURCES;\r
960         }\r
961 \r
962         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
963                 p_params->rq_depth );\r
964         if( ndis_status != NDIS_STATUS_SUCCESS )\r
965         {\r
966                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
967                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
968                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
969                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
970                 return IB_INSUFFICIENT_RESOURCES;\r
971         }\r
972 \r
973         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
974         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
975                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
976         if( ndis_status != NDIS_STATUS_SUCCESS )\r
977         {\r
978                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
979                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
980                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
981                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
982                 return IB_INSUFFICIENT_RESOURCES;\r
983         }\r
984 \r
985         NdisAllocateBufferPool( &ndis_status,\r
986                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
987         if( ndis_status != NDIS_STATUS_SUCCESS )\r
988         {\r
989                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
990                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
991                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
992                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
993                 return IB_INSUFFICIENT_RESOURCES;\r
994         }\r
995 \r
996         IPOIB_EXIT( IPOIB_DBG_INIT );\r
997         return IB_SUCCESS;\r
998 }\r
999 \r
1000 \r
1001 static void\r
1002 __buf_mgr_destroy(\r
1003         IN                              ipoib_port_t* const                     p_port )\r
1004 {\r
1005         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1006 \r
1007         CL_ASSERT( p_port );\r
1008 \r
1009         /* Destroy the send packet and buffer pools. */\r
1010         if( p_port->buf_mgr.h_send_buf_pool )\r
1011                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1012         if( p_port->buf_mgr.h_send_pkt_pool )\r
1013                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1014 \r
1015         /* Destroy the receive packet and buffer pools. */\r
1016         if( p_port->buf_mgr.h_buffer_pool )\r
1017                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1018         if( p_port->buf_mgr.h_packet_pool )\r
1019                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1020 \r
1021         /* Free the receive and send descriptors. */\r
1022         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1023 \r
1024         /* Free the lookaside list of scratch buffers. */\r
1025         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1026 \r
1027         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1028 }\r
1029 \r
1030 \r
1031 static cl_status_t\r
1032 __recv_ctor(\r
1033         IN                              void* const                                     p_object,\r
1034         IN                              void*                                           context,\r
1035                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1036 {\r
1037         ipoib_recv_desc_t       *p_desc;\r
1038         ipoib_port_t            *p_port;\r
1039         uint32_t                        ds0_len;\r
1040 \r
1041         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1042 \r
1043         CL_ASSERT( p_object );\r
1044         CL_ASSERT( context );\r
1045 \r
1046         p_desc = (ipoib_recv_desc_t*)p_object;\r
1047         p_port = (ipoib_port_t*)context;\r
1048 \r
1049         /* Setup the work request. */\r
1050         p_desc->wr.ds_array = p_desc->local_ds;\r
1051         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1052 \r
1053 #if IPOIB_INLINE_RECV\r
1054         /* Sanity check on the receive buffer layout */\r
1055         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1056                 (void*)&p_desc->buf.ib.pkt.type );\r
1057         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1058 \r
1059         /* Setup the local data segment. */\r
1060         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1061         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1062         ds0_len =\r
1063                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1064         if( ds0_len >= sizeof(recv_buf_t) )\r
1065         {\r
1066                 /* The whole buffer is within a page. */\r
1067                 p_desc->local_ds[0].length = ds0_len;\r
1068                 p_desc->wr.num_ds = 1;\r
1069         }\r
1070         else\r
1071         {\r
1072                 /* The buffer crosses page boundaries. */\r
1073                 p_desc->local_ds[0].length = ds0_len;\r
1074                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1075                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1076                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1077                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1078                 p_desc->wr.num_ds = 2;\r
1079         }\r
1080 #else   /* IPOIB_INLINE_RECV */\r
1081         /* Allocate the receive buffer. */\r
1082         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1083         if( !p_desc->p_buf )\r
1084         {\r
1085                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1086                         ("Failed to allocate receive buffer.\n") );\r
1087                 return CL_INSUFFICIENT_MEMORY;\r
1088         }\r
1089 \r
1090         /* Sanity check on the receive buffer layout */\r
1091         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1092                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1093 \r
1094         /* Setup the local data segment. */\r
1095         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1096         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1097         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1098 #endif  /* IPOIB_INLINE_RECV */\r
1099 \r
1100         *pp_pool_item = &p_desc->item;\r
1101 \r
1102         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1103         return CL_SUCCESS;\r
1104 }\r
1105 \r
1106 \r
1107 #if !IPOIB_INLINE_RECV\r
1108 static void\r
1109 __recv_dtor(\r
1110         IN              const   cl_pool_item_t* const           p_pool_item,\r
1111         IN                              void                                            *context )\r
1112 {\r
1113         ipoib_recv_desc_t       *p_desc;\r
1114 \r
1115         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1116 \r
1117         UNUSED_PARAM( context );\r
1118 \r
1119         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1120 \r
1121         if( p_desc->p_buf )\r
1122                 cl_free( p_desc->p_buf );\r
1123 \r
1124         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1125 }\r
1126 #endif\r
1127 \r
1128 \r
1129 static inline ipoib_recv_desc_t*\r
1130 __buf_mgr_get_recv(\r
1131         IN                              ipoib_port_t* const                     p_port )\r
1132 {\r
1133         ipoib_recv_desc_t       *p_desc;\r
1134         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1135         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1136         /* Reference the port object for the send. */\r
1137         if( p_desc )\r
1138         {\r
1139                 cl_obj_ref( &p_port->obj );\r
1140                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1141 #if IPOIB_INLINE_RECV\r
1142                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1143                         cl_get_physaddr( &p_desc->buf ) );\r
1144 #else   /* IPOIB_INLINE_RECV */\r
1145                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1146                         cl_get_physaddr( p_desc->p_buf ) );\r
1147                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1148                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1149 #endif  /* IPOIB_INLINE_RECV */\r
1150                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1151         }\r
1152         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1153         return p_desc;\r
1154 }\r
1155 \r
1156 \r
1157 static inline void\r
1158 __buf_mgr_put_recv(\r
1159         IN                              ipoib_port_t* const                     p_port,\r
1160         IN                              ipoib_recv_desc_t* const        p_desc,\r
1161         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1162 {\r
1163         NDIS_BUFFER             *p_buf;\r
1164 \r
1165         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1166 \r
1167         if( p_packet )\r
1168         {\r
1169                 /* Unchain the NDIS buffer. */\r
1170                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1171                 CL_ASSERT( p_buf );\r
1172                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1173                 NdisDprFreePacketNonInterlocked( p_packet );\r
1174                 NdisFreeBuffer( p_buf );\r
1175         }\r
1176 \r
1177         /* Return the descriptor to its pools. */\r
1178         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1179 \r
1180         /*\r
1181          * Dereference the port object since the receive is no longer outstanding.\r
1182          */\r
1183         cl_obj_deref( &p_port->obj );\r
1184 \r
1185         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1186 }\r
1187 \r
1188 \r
1189 static inline void\r
1190 __buf_mgr_put_recv_list(\r
1191         IN                              ipoib_port_t* const                     p_port,\r
1192         IN                              cl_qlist_t* const                       p_list )\r
1193 {\r
1194         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1195         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1196         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1197 }\r
1198 \r
1199 \r
1200 static inline NDIS_PACKET*\r
1201 __buf_mgr_get_ndis_pkt(\r
1202         IN                              ipoib_port_t* const                     p_port,\r
1203         IN                              ipoib_recv_desc_t* const        p_desc )\r
1204 {\r
1205         NDIS_STATUS                             status;\r
1206         NDIS_PACKET                             *p_packet;\r
1207         NDIS_BUFFER                             *p_buffer;\r
1208 \r
1209         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1210 \r
1211         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1212                 p_port->buf_mgr.h_packet_pool );\r
1213         if( status != NDIS_STATUS_SUCCESS )\r
1214         {\r
1215                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1216                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1217                 return NULL;\r
1218         }\r
1219 \r
1220         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1221         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1222 \r
1223         NdisAllocateBuffer( &status, &p_buffer,\r
1224 #if IPOIB_INLINE_RECV\r
1225                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1226 #else   /* IPOIB_INLINE_RECV */\r
1227                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1228 #endif  /* IPOIB_INLINE_RECV */\r
1229         if( status != NDIS_STATUS_SUCCESS )\r
1230         {\r
1231                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1232                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1233                 NdisDprFreePacketNonInterlocked( p_packet );\r
1234                 return NULL;\r
1235         }\r
1236 \r
1237         NdisChainBufferAtFront( p_packet, p_buffer );\r
1238         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1239         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1240 \r
1241         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1242         return p_packet;\r
1243 }\r
1244 \r
1245 /******************************************************************************\r
1246 *\r
1247 * Receive manager implementation.\r
1248 *\r
1249 ******************************************************************************/\r
1250 static void\r
1251 __recv_mgr_construct(\r
1252         IN                              ipoib_port_t* const                     p_port )\r
1253 {\r
1254         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1255 \r
1256         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1257 \r
1258         p_port->recv_mgr.recv_pkt_array = NULL;\r
1259 \r
1260         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1261 }\r
1262 \r
1263 \r
1264 static ib_api_status_t\r
1265 __recv_mgr_init(\r
1266         IN                              ipoib_port_t* const                     p_port )\r
1267 {\r
1268         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1269 \r
1270         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1271         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1272                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1273         if( !p_port->recv_mgr.recv_pkt_array )\r
1274         {\r
1275                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1276                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1277                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1278                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1279                 return IB_INSUFFICIENT_MEMORY;\r
1280         }\r
1281 \r
1282         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1283         return IB_SUCCESS;\r
1284 }\r
1285 \r
1286 \r
1287 static void\r
1288 __recv_mgr_destroy(\r
1289         IN                              ipoib_port_t* const                     p_port )\r
1290 {\r
1291         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1292 \r
1293         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1294         CL_ASSERT( !p_port->recv_mgr.depth );\r
1295 \r
1296         if( p_port->recv_mgr.recv_pkt_array )\r
1297                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1298 \r
1299         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1300 }\r
1301 \r
1302 \r
1303 /* Posts receive buffers to the receive queue. */\r
1304 static ib_api_status_t\r
1305 __recv_mgr_repost(\r
1306         IN                              ipoib_port_t* const                     p_port,\r
1307         IN                              uint32_t                                        num_wrs )\r
1308 {\r
1309         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1310         ib_api_status_t         status;\r
1311         ib_recv_wr_t            *p_failed;\r
1312         PERF_DECLARE( GetRecv );\r
1313         PERF_DECLARE( PostRecv );\r
1314 \r
1315         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1316 \r
1317         CL_ASSERT( p_port );\r
1318         CL_ASSERT( num_wrs );\r
1319         cl_obj_lock( &p_port->obj );\r
1320         if( p_port->state == IB_QPS_ERROR )\r
1321         {\r
1322                 cl_obj_unlock( &p_port->obj );\r
1323                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1324                         ("Port in invalid state.  Not reposting.\n") );\r
1325                 return IB_SUCCESS;\r
1326         }\r
1327         cl_obj_ref( &p_port->obj );\r
1328         cl_obj_unlock( &p_port->obj );\r
1329 \r
1330         while( num_wrs-- )\r
1331         {\r
1332                 if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )\r
1333                 {\r
1334                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1335                                 ("Receive queue already full\n") );\r
1336                         break;\r
1337                 }\r
1338 \r
1339                 /* Pull receives out of the pool and chain them up. */\r
1340                 cl_perf_start( GetRecv );\r
1341                 p_next = __buf_mgr_get_recv( p_port );\r
1342                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1343                 if( !p_next )\r
1344                 {\r
1345                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1346                                 ("Out of receive descriptors!\n") );\r
1347                         break;\r
1348                 }\r
1349 \r
1350                 if( !p_tail )\r
1351                 {\r
1352                         p_tail = p_next;\r
1353                         p_next->wr.p_next = NULL;\r
1354                 }\r
1355                 else\r
1356                 {\r
1357                         p_next->wr.p_next = &p_head->wr;\r
1358                 }\r
1359 \r
1360                 p_head = p_next;\r
1361 \r
1362                 cl_atomic_inc( &p_port->recv_mgr.depth );\r
1363         }\r
1364 \r
1365         if( !p_head )\r
1366         {\r
1367                 cl_obj_deref( &p_port->obj );\r
1368                 IPOIB_EXIT( IPOIB_DBG_RECV );\r
1369                 return IB_SUCCESS;\r
1370         }\r
1371 \r
1372         cl_perf_start( PostRecv );\r
1373         status = p_port->p_adapter->p_ifc->post_recv(\r
1374                 p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1375         cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1376 \r
1377         /*\r
1378          * If we failed, fix up the work completion list and return those\r
1379          * buffers to the pool\r
1380          */\r
1381         if( status != IB_SUCCESS )\r
1382         {\r
1383                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1384                         ("ip_post_recv returned %s\n", \r
1385                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1386                 /* return the descriptors to the pool */\r
1387                 while( p_failed )\r
1388                 {\r
1389                         p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1390                         p_failed = p_failed->p_next;\r
1391 \r
1392                         __buf_mgr_put_recv( p_port, p_head, NULL );\r
1393                         cl_atomic_dec( &p_port->recv_mgr.depth );\r
1394                 }\r
1395         }\r
1396 \r
1397         cl_obj_deref( &p_port->obj );\r
1398 \r
1399         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1400         return status;\r
1401 }\r
1402 \r
1403 \r
1404 void\r
1405 ipoib_return_packet(\r
1406         IN                              NDIS_HANDLE                                     adapter_context,\r
1407         IN                              NDIS_PACKET                                     *p_packet )\r
1408 {\r
1409         cl_list_item_t          *p_item;\r
1410         ipoib_port_t            *p_port;\r
1411         ipoib_recv_desc_t       *p_desc;\r
1412         ib_api_status_t         status = IB_NOT_DONE;\r
1413         PERF_DECLARE( ReturnPacket );\r
1414         PERF_DECLARE( ReturnPutRecv );\r
1415         PERF_DECLARE( ReturnRepostRecv );\r
1416         PERF_DECLARE( ReturnPreparePkt );\r
1417         PERF_DECLARE( ReturnNdisIndicate );\r
1418 \r
1419         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1420 \r
1421         UNUSED_PARAM( adapter_context );\r
1422         CL_ASSERT( p_packet );\r
1423 \r
1424         cl_perf_start( ReturnPacket );\r
1425 \r
1426         /* Get the manager and descriptor from the packet. */\r
1427         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1428         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1429 \r
1430         cl_spinlock_acquire( &p_port->recv_lock );\r
1431         cl_perf_start( ReturnPutRecv );\r
1432         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1433         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1434 \r
1435         /* Repost buffers. */\r
1436         cl_perf_start( ReturnRepostRecv );\r
1437         __recv_mgr_repost( p_port, 0xFFFF );\r
1438         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1439 \r
1440         /* Complete any additional receives waiting for a packet. */\r
1441         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1442         do\r
1443         {\r
1444                 if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
1445                 {\r
1446                         cl_spinlock_release( &p_port->recv_lock );\r
1447                         break;\r
1448                 }\r
1449 \r
1450                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1451 \r
1452                 cl_perf_start( ReturnPreparePkt );\r
1453                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1454                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1455                 if( status == IB_SUCCESS )\r
1456                 {\r
1457                         cl_spinlock_release( &p_port->recv_lock );\r
1458                         cl_perf_start( ReturnNdisIndicate );\r
1459                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1460                                 &p_packet, 1 );\r
1461                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1462                 }\r
1463                 else if( status != IB_NOT_DONE )\r
1464                 {\r
1465                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1466                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1467                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1468                         /* Return the item to the head of the list. */\r
1469                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1470                         cl_spinlock_release( &p_port->recv_lock );\r
1471                 }\r
1472                 else\r
1473                 {\r
1474                         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1475                 }\r
1476 \r
1477         } while( status == IB_NOT_DONE );\r
1478         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1479 \r
1480         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1481 }\r
1482 \r
1483 \r
1484 static void\r
1485 __recv_cb(\r
1486         IN              const   ib_cq_handle_t                          h_cq,\r
1487         IN                              void                                            *cq_context )\r
1488 {\r
1489         ipoib_port_t            *p_port;\r
1490         ib_api_status_t         status;\r
1491         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1492         uint32_t                        pkt_cnt, recv_cnt = 0;\r
1493         cl_qlist_t                      done_list, bad_list;\r
1494         size_t                          i;\r
1495         PERF_DECLARE( RecvCompBundle );\r
1496         PERF_DECLARE( RecvCb );\r
1497         PERF_DECLARE( PollRecv );\r
1498         PERF_DECLARE( RepostRecv );\r
1499         PERF_DECLARE( FilterRecv );\r
1500         PERF_DECLARE( BuildPktArray );\r
1501         PERF_DECLARE( RecvNdisIndicate );\r
1502         PERF_DECLARE( RearmRecv );\r
1503         PERF_DECLARE( PutRecvList );\r
1504 \r
1505         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1506 \r
1507         cl_perf_clr( RecvCompBundle );\r
1508 \r
1509         cl_perf_start( RecvCb );\r
1510 \r
1511         UNUSED_PARAM( h_cq );\r
1512 \r
1513         p_port = (ipoib_port_t*)cq_context;\r
1514 \r
1515         cl_qlist_init( &done_list );\r
1516         cl_qlist_init( &bad_list );\r
1517 \r
1518         cl_obj_ref( &p_port->obj );\r
1519 \r
1520         for( i = 0; i < MAX_RECV_WC; i++ )\r
1521                 wc[i].p_next = &wc[i + 1];\r
1522         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1523 \r
1524         /*\r
1525          * We'll be accessing the endpoint map so take a reference\r
1526          * on it to prevent modifications.\r
1527          */\r
1528         cl_obj_lock( &p_port->obj );\r
1529         cl_atomic_inc( &p_port->endpt_rdr );\r
1530         cl_obj_unlock( &p_port->obj );\r
1531 \r
1532         do\r
1533         {\r
1534                 /* If we get here, then the list of WCs is intact. */\r
1535                 p_free = wc;\r
1536 \r
1537                 cl_perf_start( PollRecv );\r
1538                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1539                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1540                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1541                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1542 \r
1543                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1544                 cl_perf_start( FilterRecv );\r
1545                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1546                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1547 \r
1548         } while( !p_free );\r
1549 \r
1550         /* We're done looking at the endpoint map, release the reference. */\r
1551         cl_atomic_dec( &p_port->endpt_rdr );\r
1552 \r
1553         /* Update our posted depth. */\r
1554         cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
1555 \r
1556         cl_perf_start( BuildPktArray );\r
1557         /* Notify NDIS of any and all possible receive buffers. */\r
1558         pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
1559         cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1560 \r
1561         /* Only indicate receives if we actually had any. */\r
1562         if( pkt_cnt )\r
1563         {\r
1564                 cl_perf_start( RecvNdisIndicate );\r
1565                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1566                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1567                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1568         }\r
1569 \r
1570         cl_spinlock_acquire( &p_port->recv_lock );\r
1571 \r
1572         /* Return any discarded receives to the pool */\r
1573         cl_perf_start( PutRecvList );\r
1574         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1575         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1576 \r
1577         /* Repost receives. */\r
1578         cl_perf_start( RepostRecv );\r
1579         __recv_mgr_repost( p_port, recv_cnt );\r
1580         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1581 \r
1582         cl_spinlock_release( &p_port->recv_lock );\r
1583 \r
1584         /*\r
1585          * Rearm after filtering to prevent contention on the enpoint maps\r
1586          * and eliminate the possibility of having a call to\r
1587          * __endpt_mgr_insert find a duplicate.\r
1588          */\r
1589         cl_perf_start( RearmRecv );\r
1590         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1591                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1592         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1593         CL_ASSERT( status == IB_SUCCESS );\r
1594 \r
1595         cl_obj_deref( &p_port->obj );\r
1596 \r
1597         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1598         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1599 \r
1600         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1601 }\r
1602 \r
1603 \r
1604 static void\r
1605 __recv_get_endpts(\r
1606         IN                              ipoib_port_t* const                     p_port,\r
1607         IN                              ipoib_recv_desc_t* const        p_desc,\r
1608         IN                              ib_wc_t* const                          p_wc,\r
1609                 OUT                     ipoib_endpt_t** const           pp_src,\r
1610                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1611 {\r
1612         ib_api_status_t         status;\r
1613         mac_addr_t                      mac;\r
1614         PERF_DECLARE( GetEndptByGid );\r
1615         PERF_DECLARE( GetEndptByLid );\r
1616         PERF_DECLARE( EndptInsert );\r
1617 \r
1618         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1619 \r
1620         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1621         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1622         {\r
1623                 /* Lookup the source endpoints based on GID. */\r
1624                 cl_perf_start( GetEndptByGid );\r
1625                 *pp_src =\r
1626 #if IPOIB_INLINE_RECV\r
1627                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1628 #else   /* IPOIB_INLINE_RECV */\r
1629                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1630 #endif  /* IPOIB_INLINE_RECV */\r
1631                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1632 \r
1633                 /*\r
1634                  * Lookup the destination endpoint based on GID.\r
1635                  * This is used along with the packet filter to determine\r
1636                  * whether to report this to NDIS.\r
1637                  */\r
1638                 cl_perf_start( GetEndptByGid );\r
1639                 *pp_dst =\r
1640 #if IPOIB_INLINE_RECV\r
1641                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1642 #else   /* IPOIB_INLINE_RECV */\r
1643                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1644 #endif  /* IPOIB_INLINE_RECV */\r
1645                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1646 \r
1647                 /*\r
1648                  * Create the source endpoint if it does not exist.  Note that we\r
1649                  * can only do this for globally routed traffic since we need the\r
1650                  * information from the GRH to generate the MAC.\r
1651                  */\r
1652                 if( !*pp_src )\r
1653                 {\r
1654                         status = ipoib_mac_from_guid(\r
1655 #if IPOIB_INLINE_RECV\r
1656                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1657 #else   /* IPOIB_INLINE_RECV */\r
1658                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1659 #endif  /* IPOIB_INLINE_RECV */\r
1660                         if( status != IB_SUCCESS )\r
1661                         {\r
1662                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1663                                         ("ipoib_mac_from_guid returned %s\n",\r
1664                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1665                                 return;\r
1666                         }\r
1667 \r
1668                         /* Create the endpoint. */\r
1669 #if IPOIB_INLINE_RECV\r
1670                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1671 #else   /* IPOIB_INLINE_RECV */\r
1672                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1673 #endif  /* IPOIB_INLINE_RECV */\r
1674                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1675                         if( !*pp_src )\r
1676                         {\r
1677                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1678                                         ("ipoib_endpt_create returned %s\n",\r
1679                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1680                                 return;\r
1681                         }\r
1682                         cl_perf_start( EndptInsert );\r
1683                         cl_obj_lock( &p_port->obj );\r
1684                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1685                         cl_obj_unlock( &p_port->obj );\r
1686                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1687                 }\r
1688         }\r
1689         else\r
1690         {\r
1691                 /*\r
1692                  * Lookup the remote endpoint based on LID.  Note that only\r
1693                  * unicast traffic can be LID routed.\r
1694                  */\r
1695                 cl_perf_start( GetEndptByLid );\r
1696                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1697                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1698                 *pp_dst = p_port->p_local_endpt;\r
1699                 CL_ASSERT( *pp_dst );\r
1700         }\r
1701 \r
1702         if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1703         {\r
1704                 /* Update the QPN for the endpoint. */\r
1705                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1706                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1707                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1708                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1709                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1710                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1711         }\r
1712 \r
1713         if( *pp_src && *pp_dst )\r
1714         {\r
1715                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1716                         ("Recv:\n"\r
1717                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1718                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1719                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1720                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1721                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1722                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1723                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1724                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1725         }\r
1726 \r
1727         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1728 }\r
1729 \r
1730 \r
1731 static uint32_t\r
1732 __recv_mgr_filter(\r
1733         IN                              ipoib_port_t* const                     p_port,\r
1734         IN                              ib_wc_t* const                          p_done_wc_list,\r
1735                 OUT                     cl_qlist_t* const                       p_done_list,\r
1736                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1737 {\r
1738         ipoib_recv_desc_t               *p_desc;\r
1739         ib_wc_t                                 *p_wc;\r
1740         ipoib_pkt_t                             *p_ipoib;\r
1741         eth_pkt_t                               *p_eth;\r
1742         ipoib_endpt_t                   *p_src, *p_dst;\r
1743         ib_api_status_t                 status;\r
1744         uint32_t                                len, recv_cnt = 0;\r
1745         PERF_DECLARE( GetRecvEndpts );\r
1746         PERF_DECLARE( RecvGen );\r
1747         PERF_DECLARE( RecvTcp );\r
1748         PERF_DECLARE( RecvUdp );\r
1749         PERF_DECLARE( RecvDhcp );\r
1750         PERF_DECLARE( RecvArp );\r
1751 \r
1752         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1753 \r
1754         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1755         {\r
1756                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1757                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1758                 recv_cnt++;\r
1759 \r
1760                 if( p_wc->status != IB_WCS_SUCCESS )\r
1761                 {\r
1762                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1763                         {\r
1764                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1765                                         ("Failed completion %s\n",\r
1766                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1767                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1768                         }\r
1769                         else\r
1770                         {\r
1771                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1772                                         ("Flushed completion %s\n",\r
1773                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1774                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1775                         }\r
1776                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1777                         /* Dereference the port object on behalf of the failed receive. */\r
1778                         cl_obj_deref( &p_port->obj );\r
1779                         continue;\r
1780                 }\r
1781 \r
1782                 len = p_wc->length - sizeof(ib_grh_t);\r
1783 \r
1784                 if( len < sizeof(ipoib_hdr_t) )\r
1785                 {\r
1786                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1787                                 ("Received ETH packet < min size\n") );\r
1788                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1789                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1790                         cl_obj_deref( &p_port->obj );\r
1791                         continue;\r
1792                 }\r
1793 \r
1794                 /* Successful completion.  Get the receive information. */\r
1795                 cl_perf_start( GetRecvEndpts );\r
1796                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1797                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1798 \r
1799 #if IPOIB_INLINE_RECV\r
1800                 p_ipoib = &p_desc->buf.ib.pkt;\r
1801                 p_eth = &p_desc->buf.eth.pkt;\r
1802 #else   /* IPOIB_INLINE_RECV */\r
1803                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1804                 p_eth = &p_desc->p_buf->eth.pkt;\r
1805 #endif  /*IPOIB_INLINE_RECV */\r
1806 \r
1807                 if( p_src )\r
1808                 {\r
1809                         /* Don't report loopback traffic - we requested SW loopback. */\r
1810                         if( !cl_memcmp( &p_port->p_adapter->mac, &p_src->mac,\r
1811                                 sizeof(p_port->p_adapter->mac) ) )\r
1812                         {\r
1813                                 /*\r
1814                                  * "This is not the packet you're looking for" - don't update\r
1815                                  * receive statistics, the packet never happened.\r
1816                                  */\r
1817                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1818                                 /* Dereference the port object on behalf of the failed recv. */\r
1819                                 cl_obj_deref( &p_port->obj );\r
1820                                 continue;\r
1821                         }\r
1822                 }\r
1823 \r
1824                 switch( p_ipoib->hdr.type )\r
1825                 {\r
1826                 case ETH_PROT_TYPE_IP:\r
1827                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1828                         {\r
1829                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1830                                         ("Received IP packet < min size\n") );\r
1831                                 status = IB_INVALID_SETTING;\r
1832                                 break;\r
1833                         }\r
1834 \r
1835                         if( p_ipoib->type.ip.hdr.offset ||\r
1836                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1837                         {\r
1838                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1839                                 cl_perf_start( RecvTcp );\r
1840                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1841                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1842                                 break;\r
1843                         }\r
1844 \r
1845                         /* First packet of a UDP transfer. */\r
1846                         if( len <\r
1847                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1848                         {\r
1849                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1850                                         ("Received UDP packet < min size\n") );\r
1851                                 status = IB_INVALID_SETTING;\r
1852                                 break;\r
1853                         }\r
1854 \r
1855                         /* Check if DHCP conversion is required. */\r
1856                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1857                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1858                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1859                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1860                         {\r
1861                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1862                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1863                                 {\r
1864                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1865                                                 ("Received DHCP < min size\n") );\r
1866                                         status = IB_INVALID_SETTING;\r
1867                                         break;\r
1868                                 }\r
1869                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1870                                 cl_perf_start( RecvDhcp );\r
1871                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1872                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1873                         }\r
1874                         else\r
1875                         {\r
1876                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1877                                 cl_perf_start( RecvUdp );\r
1878                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1879                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1880                         }\r
1881                         break;\r
1882 \r
1883                 case ETH_PROT_TYPE_ARP:\r
1884                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1885                         {\r
1886                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1887                                         ("Received ARP < min size\n") );\r
1888                                 status = IB_INVALID_SETTING;\r
1889                                 break;\r
1890                         }\r
1891                         cl_perf_start( RecvArp );\r
1892                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1893                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1894                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1895                         break;\r
1896 \r
1897                 default:\r
1898                         /* Unfiltered.  Setup the ethernet header and report. */\r
1899                         cl_perf_start( RecvGen );\r
1900                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1901                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1902                 }\r
1903 \r
1904                 if( status != IB_SUCCESS )\r
1905                 {\r
1906                         /* Update stats. */\r
1907                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1908                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1909                         /* Dereference the port object on behalf of the failed receive. */\r
1910                         cl_obj_deref( &p_port->obj );\r
1911                 }\r
1912                 else\r
1913                 {\r
1914                         p_desc->len =\r
1915                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1916                         if( p_dst->h_mcast)\r
1917                         {\r
1918                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1919                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1920                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1921                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1922                                 {\r
1923                                         p_desc->type = PKT_TYPE_BCAST;\r
1924                                 }\r
1925                                 else\r
1926                                 {\r
1927                                         p_desc->type = PKT_TYPE_MCAST;\r
1928                                 }\r
1929                         }\r
1930                         else\r
1931                         {\r
1932                                 p_desc->type = PKT_TYPE_UCAST;\r
1933                         }\r
1934                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1935                 }\r
1936         }\r
1937 \r
1938         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1939         return recv_cnt;\r
1940 }\r
1941 \r
1942 \r
1943 static ib_api_status_t\r
1944 __recv_gen(\r
1945         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1946                 OUT                     eth_pkt_t* const                        p_eth,\r
1947         IN                              ipoib_endpt_t* const            p_src,\r
1948         IN                              ipoib_endpt_t* const            p_dst )\r
1949 {\r
1950         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1951 \r
1952         if( !p_src || !p_dst )\r
1953         {\r
1954                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1955                         ("Received packet with no matching endpoints.\n") );\r
1956                 return IB_NOT_DONE;\r
1957         }\r
1958 \r
1959         /*\r
1960          * Fill in the ethernet header.  Note that doing so will overwrite\r
1961          * the IPoIB header, so start by moving the information from the IPoIB\r
1962          * header.\r
1963          */\r
1964         p_eth->hdr.type = p_ipoib->hdr.type;\r
1965         p_eth->hdr.src = p_src->mac;\r
1966         p_eth->hdr.dst = p_dst->mac;\r
1967 \r
1968         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1969         return IB_SUCCESS;\r
1970 }\r
1971 \r
1972 \r
1973 static ib_api_status_t\r
1974 __recv_dhcp(\r
1975         IN                              ipoib_port_t* const                     p_port,\r
1976         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1977                 OUT                     eth_pkt_t* const                        p_eth,\r
1978         IN                              ipoib_endpt_t* const            p_src,\r
1979         IN                              ipoib_endpt_t* const            p_dst )\r
1980 {\r
1981         ib_api_status_t         status;\r
1982         dhcp_pkt_t                      *p_dhcp;\r
1983         uint8_t                         *p_option;\r
1984         uint8_t                         *p_cid = NULL;\r
1985         ib_gid_t                        gid;\r
1986         uint8_t                         msg = 0;\r
1987 \r
1988         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1989 \r
1990         UNUSED_PARAM( p_port );\r
1991 \r
1992         /* Create the ethernet header. */\r
1993         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1994         if( status != IB_SUCCESS )\r
1995         {\r
1996                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1997                         ("__recv_gen returned %s.\n", \r
1998                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1999                 return status;\r
2000         }\r
2001 \r
2002         /* Fixup the payload. */\r
2003         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2004         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2005         {\r
2006                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2007                         ("Invalid DHCP op code.\n") );\r
2008                 return IB_INVALID_SETTING;\r
2009         }\r
2010 \r
2011         /*\r
2012          * Find the client identifier option, making sure to skip\r
2013          * the "magic cookie".\r
2014          */\r
2015         p_option = &p_dhcp->options[0];\r
2016         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2017         {\r
2018                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2019                         ("DHCP cookie corrupted.\n") );\r
2020                 return IB_INVALID_PARAMETER;\r
2021         }\r
2022 \r
2023         p_option = &p_dhcp->options[4];\r
2024         while( *p_option != DHCP_OPT_END )\r
2025         {\r
2026                 switch( *p_option )\r
2027                 {\r
2028                 case DHCP_OPT_PAD:\r
2029                         p_option++;\r
2030                         break;\r
2031 \r
2032                 case DHCP_OPT_MSG:\r
2033                         msg = p_option[2];\r
2034                         p_option += 3;\r
2035                         break;\r
2036 \r
2037                 case DHCP_OPT_CLIENT_ID:\r
2038                         p_cid = p_option;\r
2039                         /* Fall through. */\r
2040 \r
2041                 default:\r
2042                         /*\r
2043                          * All other options have a length byte following the option code.\r
2044                          * Offset by the length to get to the next option.\r
2045                          */\r
2046                         p_option += (p_option[1] + 2);\r
2047                 }\r
2048         }\r
2049 \r
2050         switch( msg )\r
2051         {\r
2052         /* message from client */\r
2053         case DHCPDISCOVER:\r
2054         case DHCPREQUEST:\r
2055         case DHCPDECLINE:\r
2056         case DHCPRELEASE:\r
2057         case DHCPINFORM:\r
2058                 if( !p_cid )\r
2059                 {\r
2060                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2061                                 ("Failed to find required Client-identifier option.\n") );\r
2062                         return IB_INVALID_SETTING;\r
2063                 }\r
2064                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2065                 {\r
2066                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2067                                 ("Invalid hardware address type.\n") );\r
2068                         return IB_INVALID_SETTING;\r
2069                 }\r
2070                 break;\r
2071         /* message from DHCP server */\r
2072         case DHCPOFFER:\r
2073         case DHCPACK:\r
2074         case DHCPNAK:\r
2075                 break;\r
2076 \r
2077         default:\r
2078                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2079                         ("Invalide message type.\n") );\r
2080                 return IB_INVALID_PARAMETER;\r
2081         }\r
2082         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2083         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2084         p_dhcp->hlen = HW_ADDR_LEN;\r
2085 \r
2086         if( p_cid ) /* from client */\r
2087         {\r
2088                 /* Validate that the length and type of the option is as required. */\r
2089                 if( p_cid[1] != 21 )\r
2090                 {\r
2091                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2092                                 ("Client-identifier length not 21 as required.\n") );\r
2093                         return IB_INVALID_SETTING;\r
2094                 }\r
2095                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2096                 {\r
2097                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2098                                 ("Client-identifier type is wrong.\n") );\r
2099                         return IB_INVALID_SETTING;\r
2100                 }\r
2101                 /*\r
2102                  * Copy the GID value from the option so that we can make aligned\r
2103                  * accesses to the contents.\r
2104                  * Recover CID to standard type.\r
2105                  */\r
2106                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2107                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2108                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2109                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2110                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2111         }\r
2112         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2113         return status;\r
2114 }\r
2115 \r
2116 \r
2117 static ib_api_status_t\r
2118 __recv_arp(\r
2119         IN                              ipoib_port_t* const                     p_port,\r
2120         IN                              ib_wc_t* const                          p_wc,\r
2121         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2122                 OUT                     eth_pkt_t* const                        p_eth,\r
2123         IN                              ipoib_endpt_t** const           pp_src,\r
2124         IN                              ipoib_endpt_t* const            p_dst )\r
2125 {\r
2126         ib_api_status_t                 status;\r
2127         arp_pkt_t                               *p_arp;\r
2128         const ipoib_arp_pkt_t   *p_ib_arp;\r
2129         ib_gid_t                                gid;\r
2130         mac_addr_t                              mac;\r
2131         ipoib_hw_addr_t                 null_hw = {0};\r
2132 \r
2133         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2134 \r
2135         if( !p_dst )\r
2136         {\r
2137                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2138                         ("Unknown destination endpoint\n") );\r
2139                 return IB_INVALID_SETTING;\r
2140         }\r
2141 \r
2142         p_ib_arp = &p_ipoib->type.arp;\r
2143         p_arp = &p_eth->type.arp;\r
2144 \r
2145         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2146         {\r
2147                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2148                         ("ARP hardware type is not IB\n") );\r
2149                 return IB_INVALID_SETTING;\r
2150         }\r
2151 \r
2152         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2153         {\r
2154                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2155                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2156                 return IB_INVALID_SETTING;\r
2157         }\r
2158 \r
2159         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2160         {\r
2161                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2162                         ("ARP protocal type not IP\n") );\r
2163                 return IB_INVALID_SETTING;\r
2164         }\r
2165 \r
2166         /*\r
2167          * If we don't have a source, lookup the endpoint specified in the payload.\r
2168          */\r
2169         if( !*pp_src )\r
2170                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2171 \r
2172         /*\r
2173          * If the endpoint exists for the GID, make sure\r
2174          * the dlid and qpn match the arp.\r
2175          */\r
2176         if( *pp_src )\r
2177         {\r
2178                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2179                         sizeof(ib_gid_t) ) )\r
2180                 {\r
2181                         /*\r
2182                          * GIDs for the endpoint are different.  The ARP must\r
2183                          * have been proxied.  Dereference it.\r
2184                          */\r
2185                         *pp_src = NULL;\r
2186                 }\r
2187                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2188                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2189                 {\r
2190                         /* Out of date!  Destroy the endpoint and replace it. */\r
2191                         __endpt_mgr_remove( p_port, *pp_src );\r
2192                         *pp_src = NULL;\r
2193                 }\r
2194         }\r
2195 \r
2196         /* Do we need to create an endpoint for this GID? */\r
2197         if( !*pp_src )\r
2198         {\r
2199                 /* Copy the src GID to allow aligned access */\r
2200                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2201                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2202                 if( status != IB_SUCCESS )\r
2203                 {\r
2204                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2205                                 ("ipoib_mac_from_guid returned %s\n",\r
2206                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2207                         return status;\r
2208                 }\r
2209                 /*\r
2210                  * Create the endpoint.  Note that the LID is left blank and will be\r
2211                  * resolved by a path query as needed.  This is done because the\r
2212                  * remote LID/GID from the work completion may not be the original\r
2213                  * initiator.\r
2214                  */\r
2215                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2216                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2217                 if( !*pp_src )\r
2218                 {\r
2219                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2220                                 ("ipoib_endpt_create returned %s\n",\r
2221                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2222                         return status;\r
2223                 }\r
2224 \r
2225                 cl_obj_lock( &p_port->obj );\r
2226                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2227                 cl_obj_unlock( &p_port->obj );\r
2228         }\r
2229 \r
2230         CL_ASSERT( !cl_memcmp(\r
2231                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2232         CL_ASSERT( (*pp_src)->qpn ==\r
2233                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2234         /* Now swizzle the data. */\r
2235         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2236         p_arp->hw_size = sizeof(mac_addr_t);\r
2237         p_arp->src_hw = (*pp_src)->mac;\r
2238         p_arp->src_ip = p_ib_arp->src_ip;\r
2239 \r
2240         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2241         {\r
2242 \r
2243                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2244                 {\r
2245                         /*\r
2246                          * We received bcast ARP packet that means\r
2247                          * remote port lets everyone know it was changed IP/MAC\r
2248                          * or just activated\r
2249                          */\r
2250                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2251                         {\r
2252                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2253                                         ("ARP: is not ARP MCAST\n") );\r
2254                                 return IB_INVALID_SETTING;\r
2255                         }\r
2256 \r
2257                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2258                         p_dst->mac = p_port->p_local_endpt->mac;\r
2259                         /*\r
2260                          * we don't care what receiver ip addr is,\r
2261                          * as long as OS' ARP table is global  ???\r
2262                          */\r
2263                         p_arp->dst_ip = (net32_t)0;\r
2264 \r
2265                 }\r
2266                 else /* we've got reply to our ARP request */\r
2267                 {\r
2268                         p_arp->dst_hw = p_dst->mac;\r
2269                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2270                         CL_ASSERT( p_dst->qpn == \r
2271                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2272                 }\r
2273         }\r
2274         else /* we got ARP reqeust */\r
2275         {\r
2276                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2277                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2278         }\r
2279 \r
2280         /*\r
2281          * Create the ethernet header.  Note that this is done last so that\r
2282          * we have a chance to create a new endpoint.\r
2283          */\r
2284         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2285         if( status != IB_SUCCESS )\r
2286         {\r
2287                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2288                         ("__recv_gen returned %s.\n", \r
2289                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2290                 return status;\r
2291         }\r
2292 \r
2293         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2294         return IB_SUCCESS;\r
2295 }\r
2296 \r
2297 \r
2298 static ib_api_status_t\r
2299 __recv_mgr_prepare_pkt(\r
2300         IN                              ipoib_port_t* const                     p_port,\r
2301         IN                              ipoib_recv_desc_t* const        p_desc,\r
2302                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2303 {\r
2304         NDIS_STATUS                                                     status;\r
2305         uint32_t                                                        pkt_filter;\r
2306         ip_stat_sel_t                                           type;\r
2307         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2308         PERF_DECLARE( GetNdisPkt );\r
2309 \r
2310         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2311 \r
2312         pkt_filter = p_port->p_adapter->packet_filter;\r
2313         /* Check the packet filter. */\r
2314         switch( p_desc->type )\r
2315         {\r
2316         default:\r
2317         case PKT_TYPE_UCAST:\r
2318                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2319                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2320                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2321                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2322                 {\r
2323                         /* OK to report. */\r
2324                         type = IP_STAT_UCAST_BYTES;\r
2325                         status = NDIS_STATUS_SUCCESS;\r
2326                 }\r
2327                 else\r
2328                 {\r
2329                         type = IP_STAT_DROPPED;\r
2330                         status = NDIS_STATUS_FAILURE;\r
2331                 }\r
2332                 break;\r
2333         case PKT_TYPE_BCAST:\r
2334                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2335                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2336                 {\r
2337                         /* OK to report. */\r
2338                         type = IP_STAT_BCAST_BYTES;\r
2339                         status = NDIS_STATUS_SUCCESS;\r
2340                 }\r
2341                 else\r
2342                 {\r
2343                         type = IP_STAT_DROPPED;\r
2344                         status = NDIS_STATUS_FAILURE;\r
2345                 }\r
2346                 break;\r
2347         case PKT_TYPE_MCAST:\r
2348                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2349                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2350                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2351                 {\r
2352                         /* OK to report. */\r
2353                         type = IP_STAT_MCAST_BYTES;\r
2354                         status = NDIS_STATUS_SUCCESS;\r
2355                 }\r
2356                 else\r
2357                 {\r
2358                         type = IP_STAT_DROPPED;\r
2359                         status = NDIS_STATUS_FAILURE;\r
2360                 }\r
2361                 break;\r
2362         }\r
2363 \r
2364         if( status != NDIS_STATUS_SUCCESS )\r
2365         {\r
2366                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2367                 /* Return the receive descriptor to the pool. */\r
2368                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2369                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2370                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2371                 /*\r
2372                  * Return IB_NOT_DONE since the packet has been completed,\r
2373                  * but has not consumed an array entry.\r
2374                  */\r
2375                 return IB_NOT_DONE;\r
2376         }\r
2377 \r
2378         cl_perf_start( GetNdisPkt );\r
2379         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2380         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2381         if( !*pp_packet )\r
2382         {\r
2383                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2384                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2385                 return IB_INSUFFICIENT_RESOURCES;\r
2386         }\r
2387 \r
2388         /* Flag the checksums as having been calculated. */\r
2389         chksum.Value = 0;\r
2390         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2391         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2392         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2393         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2394                 (void*)(uintn_t)chksum.Value;\r
2395 \r
2396         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2397 \r
2398         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2399         return IB_SUCCESS;\r
2400 }\r
2401 \r
2402 \r
2403 static uint32_t\r
2404 __recv_mgr_build_pkt_array(\r
2405         IN                              ipoib_port_t* const                     p_port,\r
2406                 OUT                     cl_qlist_t* const                       p_done_list )\r
2407 {\r
2408         cl_list_item_t                  *p_item;\r
2409         ipoib_recv_desc_t               *p_desc;\r
2410         uint32_t                                i = 0;\r
2411         ib_api_status_t                 status;\r
2412         PERF_DECLARE( PreparePkt );\r
2413 \r
2414         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2415 \r
2416         cl_spinlock_acquire( &p_port->recv_lock );\r
2417 \r
2418         /* Move any existing receives to the head to preserve ordering. */\r
2419         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2420         p_item = cl_qlist_remove_head( p_done_list );\r
2421         while( p_item != cl_qlist_end( p_done_list ) )\r
2422         {\r
2423                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2424 \r
2425                 cl_perf_start( PreparePkt );\r
2426                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2427                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2428                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2429                 if( status == IB_SUCCESS )\r
2430                 {\r
2431                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2432                         i++;\r
2433                 }\r
2434                 else if( status != IB_NOT_DONE )\r
2435                 {\r
2436                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2437                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2438                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2439                         /* Put all completed receives on the port's done list. */\r
2440                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2441                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2442                         break;\r
2443                 }\r
2444 \r
2445                 p_item = cl_qlist_remove_head( p_done_list );\r
2446         }\r
2447 \r
2448         cl_spinlock_release( &p_port->recv_lock );\r
2449 \r
2450         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2451         return i;\r
2452 }\r
2453 \r
2454 \r
2455 \r
2456 \r
2457 /******************************************************************************\r
2458 *\r
2459 * Send manager implementation.\r
2460 *\r
2461 ******************************************************************************/\r
2462 static void\r
2463 __send_mgr_construct(\r
2464         IN                              ipoib_port_t* const                     p_port )\r
2465 {\r
2466         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2467         p_port->send_mgr.depth = 0;\r
2468         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2469         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2470 }\r
2471 \r
2472 \r
2473 static void\r
2474 __send_mgr_destroy(\r
2475         IN                              ipoib_port_t* const                     p_port )\r
2476 {\r
2477         cl_list_item_t  *p_item;\r
2478         NDIS_PACKET             *p_packet;\r
2479 \r
2480         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2481 \r
2482         /* Complete any pending packets. */\r
2483         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2484                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2485                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2486         {\r
2487                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2488                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2489                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2490         }\r
2491 \r
2492         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2493 }\r
2494 \r
2495 \r
2496 static NDIS_STATUS\r
2497 __send_mgr_filter(\r
2498         IN                              ipoib_port_t* const                     p_port,\r
2499         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2500         IN                              NDIS_BUFFER* const                      p_buf,\r
2501         IN                              size_t                                          buf_len,\r
2502         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2503 {\r
2504         NDIS_STATUS             status;\r
2505 \r
2506         PERF_DECLARE( FilterIp );\r
2507         PERF_DECLARE( FilterArp );\r
2508         PERF_DECLARE( SendGen );\r
2509 \r
2510         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2511 \r
2512         /*\r
2513          * We already checked the ethernet header length, so we know it's safe\r
2514          * to decrement the buf_len without underflowing.\r
2515          */\r
2516         buf_len -= sizeof(eth_hdr_t);\r
2517 \r
2518         switch( p_eth_hdr->type )\r
2519         {\r
2520         case ETH_PROT_TYPE_IP:\r
2521                 cl_perf_start( FilterIp );\r
2522                 status = __send_mgr_filter_ip(\r
2523                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2524                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2525                 break;\r
2526 \r
2527         case ETH_PROT_TYPE_ARP:\r
2528                 cl_perf_start( FilterArp );\r
2529                 status = __send_mgr_filter_arp(\r
2530                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2531                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2532                 break;\r
2533 \r
2534         default:\r
2535                 /*\r
2536                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2537                  * Just send the payload and hope for the best.\r
2538                  */\r
2539                 cl_perf_start( SendGen );\r
2540                 status = __send_gen( p_port, p_desc );\r
2541                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2542                 break;\r
2543         }\r
2544 \r
2545         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2546         return status;\r
2547 }\r
2548 \r
2549 \r
2550 static NDIS_STATUS\r
2551 __send_copy(\r
2552         IN                              ipoib_port_t* const                     p_port,\r
2553         IN                              ipoib_send_desc_t* const        p_desc )\r
2554 {\r
2555         NDIS_PACKET                             *p_packet;\r
2556         NDIS_BUFFER                             *p_buf;\r
2557         NDIS_STATUS                             status;\r
2558         UINT                                    tot_len, bytes_copied = 0;\r
2559 \r
2560         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2561 \r
2562         p_desc->p_buf = \r
2563                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2564         if( !p_desc->p_buf )\r
2565         {\r
2566                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2567                         ("Failed to allocate buffer for packet copy.\n") );\r
2568                 return NDIS_STATUS_RESOURCES;\r
2569         }\r
2570 \r
2571         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2572         if( status != NDIS_STATUS_SUCCESS )\r
2573         {\r
2574                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2575                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2576                 return status;\r
2577         }\r
2578 \r
2579         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2580                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2581         if( status != NDIS_STATUS_SUCCESS )\r
2582         {\r
2583                 NdisFreePacket( p_packet );\r
2584                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2585                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2586                 return status;\r
2587         }\r
2588 \r
2589         NdisChainBufferAtFront( p_packet, p_buf );\r
2590 \r
2591         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2592 \r
2593         /* Setup the work request. */\r
2594         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2595                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2596         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2597         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2598         p_desc->wr.num_ds = 2;\r
2599 \r
2600         /* Copy the packet. */\r
2601         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2602                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2603                 NormalPagePriority );\r
2604 \r
2605         /* Free our temp packet now that the data is copied. */\r
2606         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2607         NdisFreeBuffer( p_buf );\r
2608         NdisFreePacket( p_packet );\r
2609 \r
2610         if( bytes_copied != tot_len )\r
2611         {\r
2612                 /* Something went wrong.  Drop the packet. */\r
2613                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2614                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2615                         bytes_copied, tot_len) );\r
2616                 return NDIS_STATUS_RESOURCES;\r
2617         }\r
2618 \r
2619         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2620         return NDIS_STATUS_SUCCESS;\r
2621 }\r
2622 \r
2623 \r
2624 #if !IPOIB_USE_DMA\r
2625 /* Send using the MDL's page information rather than the SGL. */\r
2626 static ib_api_status_t\r
2627 __send_gen(\r
2628         IN                              ipoib_port_t* const                     p_port,\r
2629         IN                              ipoib_send_desc_t* const        p_desc )\r
2630 {\r
2631         uint32_t                                i, j = 1;\r
2632         ULONG                                   offset;\r
2633         MDL                                             *p_mdl;\r
2634         UINT                                    num_pages, tot_len;\r
2635         ULONG                                   buf_len;\r
2636         PPFN_NUMBER                             page_array;\r
2637         boolean_t                               hdr_done = FALSE;\r
2638         ib_api_status_t                 status;\r
2639 \r
2640         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2641 \r
2642         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2643                 &tot_len );\r
2644 \r
2645         if( !p_mdl )\r
2646         {\r
2647                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2648                         ("No buffers associated with packet.\n") );\r
2649                 return IB_ERROR;\r
2650         }\r
2651 \r
2652         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2653         if( num_pages >= MAX_SEND_SGE )\r
2654         {\r
2655                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2656                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2657                 status = __send_copy( p_port, p_desc );\r
2658                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2659                 return status;\r
2660         }\r
2661 \r
2662         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2663         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2664         /*\r
2665          * Assume that the ethernet header is always fully contained\r
2666          * in the first page of the first MDL.  This makes for much\r
2667          * simpler code.\r
2668          */\r
2669         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2670         CL_ASSERT( offset <= PAGE_SIZE );\r
2671 \r
2672         while( tot_len )\r
2673         {\r
2674                 buf_len = MmGetMdlByteCount( p_mdl );\r
2675                 page_array = MmGetMdlPfnArray( p_mdl );\r
2676                 CL_ASSERT( page_array );\r
2677                 i = 0;\r
2678                 if( !hdr_done )\r
2679                 {\r
2680                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2681                         /* Skip the ethernet header. */\r
2682                         buf_len -= sizeof(eth_hdr_t);\r
2683                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2684                         if( buf_len )\r
2685                         {\r
2686                                 /* The ethernet header is a subset of this MDL. */\r
2687                                 CL_ASSERT( i == 0 );\r
2688                                 if( offset < PAGE_SIZE )\r
2689                                 {\r
2690                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2691                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2692                                         /* Add the byte offset since we're on the 1st page. */\r
2693                                         p_desc->local_ds[j].vaddr += offset;\r
2694                                         if( offset + buf_len > PAGE_SIZE )\r
2695                                         {\r
2696                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2697                                                 buf_len -= p_desc->local_ds[j].length;\r
2698                                         }\r
2699                                         else\r
2700                                         {\r
2701                                                 p_desc->local_ds[j].length = buf_len;\r
2702                                                 buf_len = 0;\r
2703                                         }\r
2704                                         /* This data segment is done.  Move to the next. */\r
2705                                         j++;\r
2706                                 }\r
2707                                 /* This page is done.  Move to the next. */\r
2708                                 i++;\r
2709                         }\r
2710                         /* Done handling the ethernet header. */\r
2711                         hdr_done = TRUE;\r
2712                 }\r
2713 \r
2714                 /* Finish this MDL */\r
2715                 while( buf_len )\r
2716                 {\r
2717                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2718                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2719                         /* Add the first page's offset if we're on the first page. */\r
2720                         if( i == 0 )\r
2721                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2722 \r
2723                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2724                         {\r
2725                                 /* Buffers spans pages. */\r
2726                                 p_desc->local_ds[j].length =\r
2727                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2728                                 buf_len -= p_desc->local_ds[j].length;\r
2729                                 /* This page is done.  Move to the next. */\r
2730                                 i++;\r
2731                         }\r
2732                         else\r
2733                         {\r
2734                                 /* Last page of the buffer. */\r
2735                                 p_desc->local_ds[j].length = buf_len;\r
2736                                 buf_len = 0;\r
2737                         }\r
2738                         /* This data segment is done.  Move to the next. */\r
2739                         j++;\r
2740                 }\r
2741 \r
2742                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2743                 if( !tot_len )\r
2744                         break;\r
2745 \r
2746                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2747                 if( !p_mdl )\r
2748                 {\r
2749                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2750                                 ("Failed to get next buffer.\n") );\r
2751                         return IB_ERROR;\r
2752                 }\r
2753         }\r
2754 \r
2755         /* Set the number of data segments. */\r
2756         p_desc->wr.num_ds = j;\r
2757 \r
2758         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2759         return IB_SUCCESS;\r
2760 }\r
2761 \r
2762 #else\r
2763 \r
2764 static NDIS_STATUS\r
2765 __send_gen(\r
2766         IN                              ipoib_port_t* const                     p_port,\r
2767         IN                              ipoib_send_desc_t* const        p_desc )\r
2768 {\r
2769         ib_api_status_t                 status;\r
2770         SCATTER_GATHER_LIST             *p_sgl;\r
2771         uint32_t                                i, j = 1;\r
2772         uint32_t                                offset = sizeof(eth_hdr_t);\r
2773         PERF_DECLARE( SendCopy );\r
2774 \r
2775         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2776 \r
2777         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2778                 ScatterGatherListPacketInfo );\r
2779         if( !p_sgl )\r
2780         {\r
2781                 ASSERT( p_sgl );\r
2782                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2783                         ("Failed to get SGL from packet.\n") );\r
2784                 return NDIS_STATUS_FAILURE;\r
2785         }\r
2786 \r
2787         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2788         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2789                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2790                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2791                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2792         {\r
2793                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2794                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2795                 cl_perf_start( SendCopy );\r
2796                 status = __send_copy( p_port, p_desc );\r
2797                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2798                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2799                 return status;\r
2800         }\r
2801 \r
2802         /*\r
2803          * Skip the ethernet header.  It is either the first element,\r
2804          * or part of it.\r
2805          */\r
2806         i = 0;\r
2807         while( offset )\r
2808         {\r
2809                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2810                 {\r
2811                         offset -= p_sgl->Elements[i++].Length;\r
2812                 }\r
2813                 else\r
2814                 {\r
2815                         p_desc->local_ds[j].vaddr =\r
2816                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2817                         p_desc->local_ds[j].length =\r
2818                                 p_sgl->Elements[i].Length - offset;\r
2819                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2820                         i++;\r
2821                         j++;\r
2822                         break;\r
2823                 }\r
2824         }\r
2825         /* Now fill in the rest of the local data segments. */\r
2826         while( i < p_sgl->NumberOfElements )\r
2827         {\r
2828                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2829                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2830                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2831                 i++;\r
2832                 j++;\r
2833         }\r
2834 \r
2835         /* Set the number of data segments. */\r
2836         p_desc->wr.num_ds = j;\r
2837 \r
2838         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2839         return NDIS_STATUS_SUCCESS;\r
2840 }\r
2841 #endif\r
2842 \r
2843 \r
2844 static NDIS_STATUS\r
2845 __send_mgr_filter_ip(\r
2846         IN                              ipoib_port_t* const                     p_port,\r
2847         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2848         IN                              NDIS_BUFFER*                            p_buf,\r
2849         IN                              size_t                                          buf_len,\r
2850         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2851 {\r
2852         NDIS_STATUS             status;\r
2853         ip_hdr_t                *p_ip_hdr;\r
2854 \r
2855         PERF_DECLARE( QueryIp );\r
2856         PERF_DECLARE( SendTcp );\r
2857         PERF_DECLARE( FilterUdp );\r
2858 \r
2859         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2860 \r
2861         if( !buf_len )\r
2862         {\r
2863                 cl_perf_start( QueryIp );\r
2864                 NdisGetNextBuffer( p_buf, &p_buf );\r
2865                 if( !p_buf )\r
2866                 {\r
2867                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2868                                 ("Failed to get IP header buffer.\n") );\r
2869                         return NDIS_STATUS_FAILURE;\r
2870                 }\r
2871                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2872                 if( !p_ip_hdr )\r
2873                 {\r
2874                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2875                                 ("Failed to query IP header buffer.\n") );\r
2876                         return NDIS_STATUS_FAILURE;\r
2877                 }\r
2878                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2879         }\r
2880         else\r
2881         {\r
2882                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2883         }\r
2884         if( buf_len < sizeof(ip_hdr_t) )\r
2885         {\r
2886                 /* This buffer is done for.  Get the next buffer. */\r
2887                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2888                         ("Buffer too small for IP packet.\n") );\r
2889                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2890         }\r
2891 \r
2892         if( p_ip_hdr->offset ||\r
2893                 p_ip_hdr->prot != IP_PROT_UDP )\r
2894         {\r
2895                 /* Not a UDP packet. */\r
2896                 cl_perf_start( SendTcp );\r
2897                 status = __send_gen( p_port, p_desc );\r
2898                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2899                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2900                 return status;\r
2901         }\r
2902 \r
2903         buf_len -= sizeof(ip_hdr_t);\r
2904 \r
2905         cl_perf_start( FilterUdp );\r
2906         status = __send_mgr_filter_udp(\r
2907                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2908         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2909 \r
2910         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2911         return status;\r
2912 }\r
2913 \r
2914 \r
2915 static NDIS_STATUS\r
2916 __send_mgr_filter_udp(\r
2917         IN                              ipoib_port_t* const                     p_port,\r
2918         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2919         IN                              NDIS_BUFFER*                            p_buf,\r
2920         IN                              size_t                                          buf_len,\r
2921         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2922 {\r
2923         ib_api_status_t         status;\r
2924         udp_hdr_t                       *p_udp_hdr;\r
2925         PERF_DECLARE( QueryUdp );\r
2926         PERF_DECLARE( SendUdp );\r
2927         PERF_DECLARE( FilterDhcp );\r
2928 \r
2929         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2930 \r
2931         if( !buf_len )\r
2932         {\r
2933                 cl_perf_start( QueryUdp );\r
2934                 NdisGetNextBuffer( p_buf, &p_buf );\r
2935                 if( !p_buf )\r
2936                 {\r
2937                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2938                                 ("Failed to get UDP header buffer.\n") );\r
2939                         return NDIS_STATUS_FAILURE;\r
2940                 }\r
2941                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2942                 if( !p_udp_hdr )\r
2943                 {\r
2944                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2945                                 ("Failed to query UDP header buffer.\n") );\r
2946                         return NDIS_STATUS_FAILURE;\r
2947                 }\r
2948                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
2949         }\r
2950         else\r
2951         {\r
2952                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
2953         }\r
2954         /* Get the UDP header and check the destination port numbers. */\r
2955         if( buf_len < sizeof(udp_hdr_t) )\r
2956         {\r
2957                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2958                         ("Buffer not large enough for UDP packet.\n") );\r
2959                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2960         }\r
2961 \r
2962         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
2963                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r
2964                 (p_udp_hdr->src_port != DHCP_PORT_SERVER ||\r
2965                 p_udp_hdr->dst_port != DHCP_PORT_CLIENT) )\r
2966         {\r
2967                 /* Not a DHCP packet. */\r
2968                 cl_perf_start( SendUdp );\r
2969                 status = __send_gen( p_port, p_desc );\r
2970                 cl_perf_stop( &p_port->p_adapter->perf, SendUdp );\r
2971                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2972                 return status;\r
2973         }\r
2974 \r
2975         buf_len -= sizeof(udp_hdr_t);\r
2976 \r
2977         /* Allocate our scratch buffer. */\r
2978         p_desc->p_buf = (send_buf_t*)\r
2979                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2980         if( !p_desc->p_buf )\r
2981         {\r
2982                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2983                         ("Failed to query DHCP packet buffer.\n") );\r
2984                 return NDIS_STATUS_RESOURCES;\r
2985         }\r
2986         /* Copy the IP and UDP headers. */\r
2987         cl_memcpy( &p_desc->p_buf->ip.hdr, p_ip_hdr , sizeof(ip_hdr_t) );\r
2988         cl_memcpy(\r
2989                 &p_desc->p_buf->ip.prot.udp.hdr, p_udp_hdr, sizeof(udp_hdr_t) );\r
2990 \r
2991         cl_perf_start( FilterDhcp );\r
2992         status = __send_mgr_filter_dhcp(\r
2993                 p_port, p_udp_hdr, p_buf, buf_len, p_desc );\r
2994         cl_perf_stop( &p_port->p_adapter->perf, FilterDhcp );\r
2995 \r
2996         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2997         return status;\r
2998 }\r
2999 \r
3000 \r
3001 static NDIS_STATUS\r
3002 __send_mgr_filter_dhcp(\r
3003         IN                              ipoib_port_t* const                     p_port,\r
3004         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
3005         IN                              NDIS_BUFFER*                            p_buf,\r
3006         IN                              size_t                                          buf_len,\r
3007         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
3008 {\r
3009         dhcp_pkt_t                      *p_dhcp;\r
3010         dhcp_pkt_t                      *p_ib_dhcp;\r
3011         uint8_t                         *p_option, *p_cid = NULL;\r
3012         uint8_t                         msg = 0;\r
3013         size_t                          len;\r
3014         ib_gid_t                        gid;\r
3015 \r
3016         IPOIB_ENTER( IPOIB_DBG_SEND );\r
3017 \r
3018         if( !buf_len )\r
3019         {\r
3020                 NdisGetNextBuffer( p_buf, &p_buf );\r
3021                 if( !p_buf )\r
3022                 {\r
3023                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3024                                 ("Failed to get DHCP buffer.\n") );\r
3025                         return NDIS_STATUS_FAILURE;\r
3026                 }\r
3027                 NdisQueryBufferSafe( p_buf, &p_dhcp, &buf_len, NormalPagePriority );\r
3028                 if( !p_dhcp )\r
3029                 {\r
3030                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3031                                 ("Failed to query DHCP buffer.\n