[IPOIB] bugfix: endpt are not added to the port child list if port is marked as being...
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 \r
45 /* Amount of physical memory to register. */\r
46 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
47 \r
48 /* Number of work completions to chain for send and receive polling. */\r
49 #define MAX_SEND_WC             8\r
50 #define MAX_RECV_WC             16\r
51 \r
52 \r
53 ib_gid_t        bcast_mgid_template = {\r
54         0xff,                                                           /* multicast field */\r
55         0x12,                                                           /* scope (to be filled in) */\r
56         0x40, 0x1b,                                                     /* IPv4 signature */\r
57         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
58         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
59         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
60 };\r
61 \r
62 \r
63 #ifdef _DEBUG_\r
64 /* Handy pointer for debug use. */\r
65 ipoib_port_t    *gp_ipoib_port;\r
66 #endif\r
67 \r
68 \r
69 /******************************************************************************\r
70 *\r
71 * Declarations\r
72 *\r
73 ******************************************************************************/\r
74 static void\r
75 __port_construct(\r
76         IN                              ipoib_port_t* const                     p_port );\r
77 \r
78 static ib_api_status_t\r
79 __port_init(\r
80         IN                              ipoib_port_t* const                     p_port,\r
81         IN                              ipoib_adapter_t* const          p_adapter,\r
82         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
83 \r
84 static void\r
85 __port_destroying(\r
86         IN                              cl_obj_t* const                         p_obj );\r
87 \r
88 static void\r
89 __port_cleanup(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_free(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 \r
97 /******************************************************************************\r
98 *\r
99 * IB resource manager operations\r
100 *\r
101 ******************************************************************************/\r
102 static void\r
103 __ib_mgr_construct(\r
104         IN                              ipoib_port_t* const                     p_port );\r
105 \r
106 static ib_api_status_t\r
107 __ib_mgr_init(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static void\r
111 __ib_mgr_destroy(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __qp_event(\r
116         IN                              ib_async_event_rec_t            *p_event_rec );\r
117 \r
118 static void\r
119 __cq_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static ib_api_status_t\r
123 __ib_mgr_activate(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 /******************************************************************************\r
127 *\r
128 * Buffer manager operations.\r
129 *\r
130 ******************************************************************************/\r
131 static void\r
132 __buf_mgr_construct(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 static ib_api_status_t\r
136 __buf_mgr_init(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static void\r
140 __buf_mgr_destroy(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static cl_status_t\r
144 __recv_ctor(\r
145         IN                              void* const                                     p_object,\r
146         IN                              void*                                           context,\r
147                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
148 \r
149 #if !IPOIB_INLINE_RECV\r
150 static void\r
151 __recv_dtor(\r
152         IN              const   cl_pool_item_t* const           p_pool_item,\r
153         IN                              void                                            *context );\r
154 #endif  /* IPOIB_INLINE_RECV */\r
155 \r
156 static inline ipoib_send_desc_t*\r
157 __buf_mgr_get_send(\r
158         IN                              ipoib_port_t* const                     p_port );\r
159 \r
160 static inline void\r
161 __buf_mgr_put_send(\r
162         IN                              ipoib_port_t* const                     p_port,\r
163         IN                              ipoib_send_desc_t* const        p_desc );\r
164 \r
165 static inline ipoib_recv_desc_t*\r
166 __buf_mgr_get_recv(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_recv(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_recv_desc_t* const        p_desc,\r
173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
174 \r
175 static inline void\r
176 __buf_mgr_put_recv_list(\r
177         IN                              ipoib_port_t* const                     p_port,\r
178         IN                              cl_qlist_t* const                       p_list );\r
179 \r
180 static inline NDIS_PACKET*\r
181 __buf_mgr_get_ndis_pkt(\r
182         IN                              ipoib_port_t* const                     p_port,\r
183         IN                              ipoib_recv_desc_t* const        p_desc );\r
184 \r
185 \r
186 /******************************************************************************\r
187 *\r
188 * Receive manager operations.\r
189 *\r
190 ******************************************************************************/\r
191 static void\r
192 __recv_mgr_construct(\r
193         IN                              ipoib_port_t* const                     p_port );\r
194 \r
195 static ib_api_status_t\r
196 __recv_mgr_init(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static void\r
200 __recv_mgr_destroy(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 /* Posts receive buffers to the receive queue. */\r
204 static ib_api_status_t\r
205 __recv_mgr_repost(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_cb(\r
210         IN              const   ib_cq_handle_t                          h_cq,\r
211         IN                              void                                            *cq_context );\r
212 \r
213 static void\r
214 __recv_get_endpts(\r
215         IN                              ipoib_port_t* const                     p_port,\r
216         IN                              ipoib_recv_desc_t* const        p_desc,\r
217         IN                              ib_wc_t* const                          p_wc,\r
218                 OUT                     ipoib_endpt_t** const           pp_src,\r
219                 OUT                     ipoib_endpt_t** const           pp_dst );\r
220 \r
221 static int32_t\r
222 __recv_mgr_filter(\r
223         IN                              ipoib_port_t* const                     p_port,\r
224         IN                              ib_wc_t* const                          p_done_wc_list,\r
225                 OUT                     cl_qlist_t* const                       p_done_list,\r
226                 OUT                     cl_qlist_t* const                       p_bad_list );\r
227 \r
228 static ib_api_status_t\r
229 __recv_gen(\r
230         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
231                 OUT                     eth_pkt_t* const                        p_eth,\r
232         IN                              ipoib_endpt_t* const            p_src,\r
233         IN                              ipoib_endpt_t* const            p_dst );\r
234 \r
235 static ib_api_status_t\r
236 __recv_dhcp(\r
237         IN                              ipoib_port_t* const                     p_port,\r
238         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
239                 OUT                     eth_pkt_t* const                        p_eth,\r
240         IN                              ipoib_endpt_t* const            p_src,\r
241         IN                              ipoib_endpt_t* const            p_dst );\r
242 \r
243 static ib_api_status_t\r
244 __recv_arp(\r
245         IN                              ipoib_port_t* const                     p_port,\r
246         IN                              ib_wc_t* const                          p_wc,\r
247         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
248                 OUT                     eth_pkt_t* const                        p_eth,\r
249         IN                              ipoib_endpt_t** const           p_src,\r
250         IN                              ipoib_endpt_t* const            p_dst );\r
251 \r
252 static ib_api_status_t\r
253 __recv_mgr_prepare_pkt(\r
254         IN                              ipoib_port_t* const                     p_port,\r
255         IN                              ipoib_recv_desc_t* const        p_desc,\r
256                 OUT                     NDIS_PACKET** const                     pp_packet );\r
257 \r
258 static uint32_t\r
259 __recv_mgr_build_pkt_array(\r
260         IN                              ipoib_port_t* const                     p_port,\r
261         IN                              int32_t                                         shortage,\r
262                 OUT                     cl_qlist_t* const                       p_done_list,\r
263                 OUT                     int32_t* const                          p_discarded );\r
264 \r
265 /******************************************************************************\r
266 *\r
267 * Send manager operations.\r
268 *\r
269 ******************************************************************************/\r
270 static void\r
271 __send_mgr_construct(\r
272         IN                              ipoib_port_t* const                     p_port );\r
273 \r
274 static void\r
275 __send_mgr_destroy(\r
276         IN                              ipoib_port_t* const                     p_port );\r
277 \r
278 static NDIS_STATUS\r
279 __send_gen(\r
280         IN                              ipoib_port_t* const                     p_port,\r
281         IN                              ipoib_send_desc_t* const        p_desc );\r
282 \r
283 static NDIS_STATUS\r
284 __send_mgr_filter_ip(\r
285         IN                              ipoib_port_t* const                     p_port,\r
286         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
287         IN                              NDIS_BUFFER*                            p_buf,\r
288         IN                              size_t                                          buf_len,\r
289         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
290 \r
291 static NDIS_STATUS\r
292 __send_mgr_filter_udp(\r
293         IN                              ipoib_port_t* const                     p_port,\r
294         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
295         IN                              NDIS_BUFFER*                            p_buf,\r
296         IN                              size_t                                          buf_len,\r
297         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
298 \r
299 static NDIS_STATUS\r
300 __send_mgr_filter_dhcp(\r
301         IN                              ipoib_port_t* const                     p_port,\r
302         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
303         IN                              NDIS_BUFFER*                            p_buf,\r
304         IN                              size_t                                          buf_len,\r
305         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
306 \r
307 static NDIS_STATUS\r
308 __send_mgr_filter_arp(\r
309         IN                              ipoib_port_t* const                     p_port,\r
310         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
311         IN                              NDIS_BUFFER*                            p_buf,\r
312         IN                              size_t                                          buf_len,\r
313         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
314 \r
315 static void\r
316 __process_failed_send(\r
317         IN                              ipoib_port_t* const                     p_port,\r
318         IN                              ipoib_send_desc_t* const        p_desc,\r
319         IN              const   NDIS_STATUS                                     status );\r
320 \r
321 static void\r
322 __send_cb(\r
323         IN              const   ib_cq_handle_t                          h_cq,\r
324         IN                              void                                            *cq_context );\r
325 \r
326 \r
327 /******************************************************************************\r
328 *\r
329 * Endpoint manager operations\r
330 *\r
331 ******************************************************************************/\r
332 static void\r
333 __endpt_mgr_construct(\r
334         IN                              ipoib_port_t* const                     p_port );\r
335 \r
336 static ib_api_status_t\r
337 __endpt_mgr_init(\r
338         IN                              ipoib_port_t* const                     p_port );\r
339 \r
340 static void\r
341 __endpt_mgr_destroy(\r
342         IN                              ipoib_port_t* const                     p_port );\r
343 \r
344 /****f* IPoIB/__endpt_mgr_remove_all\r
345 * NAME\r
346 *       __endpt_mgr_remove_all\r
347 *\r
348 * DESCRIPTION\r
349 *       Removes all enpoints from the port, dereferencing them to initiate\r
350 *       destruction.\r
351 *\r
352 * SYNOPSIS\r
353 */\r
354 static void\r
355 __endpt_mgr_remove_all(\r
356         IN                              ipoib_port_t* const                     p_port );\r
357 /*\r
358 ********/\r
359 \r
360 static void\r
361 __endpt_mgr_remove(\r
362         IN                              ipoib_port_t* const                     p_port,\r
363         IN                              ipoib_endpt_t* const            p_endpt );\r
364 \r
365 static void\r
366 __endpt_mgr_reset_all(\r
367         IN                              ipoib_port_t* const                     p_port );\r
368 \r
369 static inline NDIS_STATUS\r
370 __endpt_mgr_ref(\r
371         IN                              ipoib_port_t* const                     p_port,\r
372         IN              const   mac_addr_t                                      mac,\r
373                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
374 \r
375 static inline NDIS_STATUS\r
376 __endpt_mgr_get_gid_qpn(\r
377         IN                              ipoib_port_t* const                     p_port,\r
378         IN              const   mac_addr_t                                      mac,\r
379                 OUT                     ib_gid_t* const                         p_gid,\r
380                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
381 \r
382 static inline ipoib_endpt_t*\r
383 __endpt_mgr_get_by_gid(\r
384         IN                              ipoib_port_t* const                     p_port,\r
385         IN              const   ib_gid_t* const                         p_gid );\r
386 \r
387 static inline ipoib_endpt_t*\r
388 __endpt_mgr_get_by_lid(\r
389         IN                              ipoib_port_t* const                     p_port,\r
390         IN              const   net16_t                                         lid );\r
391 \r
392 static inline ib_api_status_t\r
393 __endpt_mgr_insert_locked(\r
394         IN                              ipoib_port_t* const                     p_port,\r
395         IN              const   mac_addr_t                                      mac,\r
396         IN                              ipoib_endpt_t* const            p_endpt );\r
397 \r
398 static inline ib_api_status_t\r
399 __endpt_mgr_insert(\r
400         IN                              ipoib_port_t* const                     p_port,\r
401         IN              const   mac_addr_t                                      mac,\r
402         IN                              ipoib_endpt_t* const            p_endpt );\r
403 \r
404 static ib_api_status_t\r
405 __endpt_mgr_add_local(\r
406         IN                              ipoib_port_t* const                     p_port,\r
407         IN                              ib_port_info_t* const           p_port_info );\r
408 \r
409 static ib_api_status_t\r
410 __endpt_mgr_add_bcast(\r
411         IN                              ipoib_port_t* const                     p_port,\r
412         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
413 \r
414 /******************************************************************************\r
415 *\r
416 * MCast operations.\r
417 *\r
418 ******************************************************************************/\r
419 static ib_api_status_t\r
420 __port_get_bcast(\r
421         IN                              ipoib_port_t* const                     p_port );\r
422 \r
423 static ib_api_status_t\r
424 __port_join_bcast(\r
425         IN                              ipoib_port_t* const                     p_port,\r
426         IN                              ib_member_rec_t* const          p_member_rec );\r
427 \r
428 static ib_api_status_t\r
429 __port_create_bcast(\r
430         IN                              ipoib_port_t* const                     p_port );\r
431 \r
432 static void\r
433 __port_info_cb(\r
434         IN                              ib_query_rec_t                          *p_query_rec );\r
435 \r
436 \r
437 static void\r
438 __bcast_get_cb(\r
439         IN                              ib_query_rec_t                          *p_query_rec );\r
440 \r
441 \r
442 static void\r
443 __bcast_cb(\r
444         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
445 \r
446 \r
447 static void\r
448 __mcast_cb(\r
449         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
450 \r
451 \r
452 static intn_t\r
453 __gid_cmp(\r
454         IN              const   void* const                                     p_key1,\r
455         IN              const   void* const                                     p_key2 )\r
456 {\r
457         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
458 }\r
459 \r
460 \r
461 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
462 {\r
463         cl_obj_ref( &p_port->obj );\r
464 #if DBG\r
465         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
466         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
467                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
468 #endif\r
469 }\r
470 \r
471 \r
472 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
473 {\r
474         cl_obj_deref( &p_port->obj );\r
475 \r
476 #if DBG\r
477         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
478         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
479                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
480 #endif\r
481 }\r
482 \r
483 \r
484 /******************************************************************************\r
485 *\r
486 * Implementation\r
487 *\r
488 ******************************************************************************/\r
489 ib_api_status_t\r
490 ipoib_create_port(\r
491         IN                              ipoib_adapter_t* const          p_adapter,\r
492         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
493                 OUT                     ipoib_port_t** const            pp_port )\r
494 {\r
495         ib_api_status_t         status;\r
496         ipoib_port_t            *p_port;\r
497 \r
498         IPOIB_ENTER( IPOIB_DBG_INIT );\r
499 \r
500         CL_ASSERT( !p_adapter->p_port );\r
501 \r
502         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
503                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
504         if( !p_port )\r
505         {\r
506                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
507                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
508                         sizeof(ipoib_port_t)) );\r
509                 return IB_INSUFFICIENT_MEMORY;\r
510         }\r
511 \r
512 #ifdef _DEBUG_\r
513         gp_ipoib_port = p_port;\r
514 #endif\r
515 \r
516         __port_construct( p_port );\r
517 \r
518         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
519         if( status != IB_SUCCESS )\r
520         {\r
521                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
522                         ("ipoib_port_init returned %s.\n", \r
523                         p_adapter->p_ifc->get_err_str( status )) );\r
524                 __port_cleanup( &p_port->obj );\r
525                 __port_free( &p_port->obj );\r
526                 return status;\r
527         }\r
528 \r
529         *pp_port = p_port;\r
530         IPOIB_EXIT( IPOIB_DBG_INIT );\r
531         return IB_SUCCESS;\r
532 }\r
533 \r
534 \r
535 void\r
536 ipoib_port_destroy(\r
537         IN                              ipoib_port_t* const                     p_port )\r
538 {\r
539         IPOIB_ENTER( IPOIB_DBG_INIT );\r
540 \r
541         CL_ASSERT( p_port );\r
542         CL_ASSERT( p_port->p_adapter );\r
543         CL_ASSERT( !p_port->p_adapter->p_port );\r
544 \r
545         cl_obj_destroy( &p_port->obj );\r
546 \r
547         IPOIB_EXIT( IPOIB_DBG_INIT );\r
548 }\r
549 \r
550 \r
551 static void\r
552 __port_construct(\r
553         IN                              ipoib_port_t* const                     p_port )\r
554 {\r
555         IPOIB_ENTER( IPOIB_DBG_INIT );\r
556 \r
557         p_port->state = IB_QPS_RESET;\r
558 \r
559         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
560         cl_spinlock_construct( &p_port->send_lock );\r
561         cl_spinlock_construct( &p_port->recv_lock );\r
562         __ib_mgr_construct( p_port );\r
563         __buf_mgr_construct( p_port );\r
564 \r
565         __recv_mgr_construct( p_port );\r
566         __send_mgr_construct( p_port );\r
567 \r
568         __endpt_mgr_construct( p_port );\r
569 \r
570         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
571 \r
572         IPOIB_EXIT( IPOIB_DBG_INIT );\r
573 }\r
574 \r
575 \r
576 static ib_api_status_t\r
577 __port_init(\r
578         IN                              ipoib_port_t* const                     p_port,\r
579         IN                              ipoib_adapter_t* const          p_adapter,\r
580         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
581 {\r
582         cl_status_t                     cl_status;\r
583         ib_api_status_t         status;\r
584 \r
585         IPOIB_ENTER( IPOIB_DBG_INIT );\r
586 \r
587         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
588         p_port->p_adapter = p_adapter;\r
589 \r
590         cl_status = cl_spinlock_init( &p_port->send_lock );\r
591         if( cl_status != CL_SUCCESS )\r
592         {\r
593                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
594                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
595                 return IB_ERROR;\r
596         }\r
597 \r
598         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
599         if( cl_status != CL_SUCCESS )\r
600         {\r
601                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
602                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
603                 return IB_ERROR;\r
604         }\r
605 \r
606         /* Initialize the IB resource manager. */\r
607         status = __ib_mgr_init( p_port );\r
608         if( status != IB_SUCCESS )\r
609         {\r
610                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
611                         ("__ib_mgr_init returned %s\n", \r
612                         p_adapter->p_ifc->get_err_str( status )) );\r
613                 return status;\r
614         }\r
615 \r
616         /* Initialize the buffer manager. */\r
617         status = __buf_mgr_init( p_port );\r
618         if( status != IB_SUCCESS )\r
619         {\r
620                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
621                         ("__buf_mgr_init returned %s\n", \r
622                         p_adapter->p_ifc->get_err_str( status )) );\r
623                 return status;\r
624         }\r
625 \r
626         /* Initialize the receive manager. */\r
627         status = __recv_mgr_init( p_port );\r
628         if( status != IB_SUCCESS )\r
629         {\r
630                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
631                         ("__recv_mgr_init returned %s\n", \r
632                         p_adapter->p_ifc->get_err_str( status )) );\r
633                 return status;\r
634         }\r
635 \r
636         /* Initialize the endpoint manager. */\r
637         status = __endpt_mgr_init( p_port );\r
638         if( status != IB_SUCCESS )\r
639         {\r
640                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
641                         ("__endpt_mgr_init returned %s\n", \r
642                         p_adapter->p_ifc->get_err_str( status )) );\r
643                 return status;\r
644         }\r
645 \r
646         /* We only ever destroy from the PnP callback thread. */\r
647         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
648                 __port_destroying, __port_cleanup, __port_free );\r
649 \r
650 #if DBG\r
651         cl_atomic_inc( &p_port->ref[ref_init] );\r
652         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
653                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
654 #endif\r
655 \r
656         if( cl_status != CL_SUCCESS )\r
657         {\r
658                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
659                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
660                 return IB_ERROR;\r
661         }\r
662 \r
663         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
664         if( cl_status != CL_SUCCESS )\r
665         {\r
666                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
667                         ("cl_obj_insert_rel returned %s\n", cl_status_text[cl_status]) );\r
668                 cl_obj_destroy( &p_port->obj );\r
669                 return IB_ERROR;\r
670         }\r
671 \r
672 #if DBG\r
673         cl_atomic_inc( &p_port->ref[ref_init] );\r
674         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
675                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
676 #endif\r
677 \r
678         IPOIB_EXIT( IPOIB_DBG_INIT );\r
679         return IB_SUCCESS;\r
680 }\r
681 \r
682 \r
683 static void\r
684 __port_destroying(\r
685         IN                              cl_obj_t* const                         p_obj )\r
686 {\r
687         ipoib_port_t    *p_port;\r
688 \r
689         IPOIB_ENTER( IPOIB_DBG_INIT );\r
690 \r
691         CL_ASSERT( p_obj );\r
692 \r
693         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
694 \r
695         ipoib_port_down( p_port );\r
696 \r
697         __endpt_mgr_remove_all( p_port );\r
698 \r
699         ipoib_port_resume( p_port );\r
700 \r
701         IPOIB_EXIT( IPOIB_DBG_INIT );\r
702 }\r
703 \r
704 \r
705 static void\r
706 __port_cleanup(\r
707         IN                              cl_obj_t* const                         p_obj )\r
708 {\r
709         ipoib_port_t    *p_port;\r
710 \r
711         IPOIB_ENTER( IPOIB_DBG_INIT );\r
712 \r
713         CL_ASSERT( p_obj );\r
714 \r
715         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
716 \r
717         /* Wait for all sends and receives to get flushed. */\r
718         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
719                 cl_thread_suspend( 0 );\r
720 \r
721         /* Destroy the send and receive managers before closing the CA. */\r
722         __ib_mgr_destroy( p_port );\r
723 \r
724         IPOIB_EXIT( IPOIB_DBG_INIT );\r
725 }\r
726 \r
727 \r
728 static void\r
729 __port_free(\r
730         IN                              cl_obj_t* const                         p_obj )\r
731 {\r
732         ipoib_port_t    *p_port;\r
733 \r
734         IPOIB_ENTER( IPOIB_DBG_INIT );\r
735 \r
736         CL_ASSERT( p_obj );\r
737 \r
738         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
739 \r
740         __endpt_mgr_destroy( p_port );\r
741         __recv_mgr_destroy( p_port );\r
742         __send_mgr_destroy( p_port );\r
743         __buf_mgr_destroy( p_port );\r
744 \r
745         cl_spinlock_destroy( &p_port->send_lock );\r
746         cl_spinlock_destroy( &p_port->recv_lock );\r
747 \r
748         cl_obj_deinit( p_obj );\r
749 \r
750         cl_free( p_port );\r
751 \r
752         IPOIB_EXIT( IPOIB_DBG_INIT );\r
753 }\r
754 \r
755 \r
756 \r
757 /******************************************************************************\r
758 *\r
759 * IB resource manager implementation.\r
760 *\r
761 ******************************************************************************/\r
762 static void\r
763 __ib_mgr_construct(\r
764         IN                              ipoib_port_t* const                     p_port )\r
765 {\r
766         IPOIB_ENTER( IPOIB_DBG_INIT );\r
767 \r
768         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
769 \r
770         IPOIB_EXIT( IPOIB_DBG_INIT );\r
771 }\r
772 \r
773 \r
774 static ib_api_status_t\r
775 __ib_mgr_init(\r
776         IN                              ipoib_port_t* const                     p_port )\r
777 {\r
778         ib_api_status_t         status;\r
779         ib_cq_create_t          cq_create;\r
780         ib_qp_create_t          qp_create;\r
781         ib_phys_create_t        phys_create;\r
782         ib_phys_range_t         phys_range;\r
783         uint64_t                        vaddr;\r
784         net32_t                         rkey;\r
785         ib_qp_attr_t            qp_attr;\r
786 \r
787         IPOIB_ENTER( IPOIB_DBG_INIT );\r
788 \r
789         /* Open the CA. */\r
790         status = p_port->p_adapter->p_ifc->open_ca(\r
791                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
792                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
793         if( status != IB_SUCCESS )\r
794         {\r
795                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
796                         EVENT_IPOIB_OPEN_CA, 1, status );\r
797                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
798                         ("ib_open_ca returned %s\n", \r
799                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
800                 return status;\r
801         }\r
802 \r
803         /* Allocate the PD. */\r
804         status = p_port->p_adapter->p_ifc->alloc_pd(\r
805                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
806         if( status != IB_SUCCESS )\r
807         {\r
808                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
809                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
810                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
811                         ("ib_alloc_pd returned %s\n", \r
812                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
813                 return status;\r
814         }\r
815 \r
816         /* Allocate receive CQ. */\r
817         cq_create.size = p_port->p_adapter->params.rq_depth;\r
818         cq_create.pfn_comp_cb = __recv_cb;\r
819         cq_create.h_wait_obj = NULL;\r
820 \r
821         status = p_port->p_adapter->p_ifc->create_cq(\r
822                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
823                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
824         if( status != IB_SUCCESS )\r
825         {\r
826                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
827                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
828                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
829                         ("ib_create_cq returned %s.\n", \r
830                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
831                 return status;\r
832         }\r
833 \r
834         /* Allocate send CQ. */\r
835         cq_create.size = p_port->p_adapter->params.sq_depth;\r
836         cq_create.pfn_comp_cb = __send_cb;\r
837 \r
838         status = p_port->p_adapter->p_ifc->create_cq(\r
839                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
840                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
841         if( status != IB_SUCCESS )\r
842         {\r
843                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
844                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
845                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
846                         ("ib_create_cq returned %s.\n", \r
847                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
848                 return status;\r
849         }\r
850 \r
851         /* Allocate the QP. */\r
852         cl_memclr( &qp_create, sizeof(qp_create) );\r
853         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
854         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
855         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
856         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
857         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
858         //TODO: Figure out the right number of SGE entries for sends.\r
859         qp_create.sq_sge = MAX_SEND_SGE;\r
860         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
861         qp_create.sq_signaled = TRUE;\r
862         status = p_port->p_adapter->p_ifc->create_qp(\r
863                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
864                 __qp_event, &p_port->ib_mgr.h_qp );\r
865         if( status != IB_SUCCESS )\r
866         {\r
867                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
868                         EVENT_IPOIB_CREATE_QP, 1, status );\r
869                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
870                         ("ib_create_qp returned %s\n", \r
871                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
872                 return status;\r
873         }\r
874         /* Query the QP so we can get our QPN. */\r
875         status = p_port->p_adapter->p_ifc->query_qp(\r
876                 p_port->ib_mgr.h_qp, &qp_attr );\r
877         if( status != IB_SUCCESS )\r
878         {\r
879                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
880                         EVENT_IPOIB_QUERY_QP, 1, status );\r
881                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
882                         ("ib_query_qp returned %s\n", \r
883                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
884                 return status;\r
885         }\r
886         p_port->ib_mgr.qpn = qp_attr.num;\r
887 \r
888         /* Register all of physical memory */\r
889         phys_create.length = MEM_REG_SIZE;\r
890         phys_create.num_ranges = 1;\r
891         phys_create.range_array = &phys_range;\r
892         phys_create.buf_offset = 0;\r
893         phys_create.hca_page_size = PAGE_SIZE;\r
894         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
895         phys_range.base_addr = 0;\r
896         phys_range.size = MEM_REG_SIZE;\r
897         vaddr = 0;\r
898         status = p_port->p_adapter->p_ifc->reg_phys(\r
899                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
900                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
901         if( status != IB_SUCCESS )\r
902         {\r
903                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
904                         EVENT_IPOIB_REG_PHYS, 1, status );\r
905                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
906                         ("ib_reg_phys returned %s\n", \r
907                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
908                 return status;\r
909         }\r
910 \r
911         IPOIB_EXIT( IPOIB_DBG_INIT );\r
912         return IB_SUCCESS;\r
913 }\r
914 \r
915 \r
916 static void\r
917 __ib_mgr_destroy(\r
918         IN                              ipoib_port_t* const                     p_port )\r
919 {\r
920         ib_api_status_t status;\r
921 \r
922         IPOIB_ENTER( IPOIB_DBG_INIT );\r
923 \r
924         if( p_port->ib_mgr.h_ca )\r
925         {\r
926                 status =\r
927                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
928                 CL_ASSERT( status == IB_SUCCESS );\r
929                 p_port->ib_mgr.h_ca = NULL;\r
930         }\r
931 \r
932         IPOIB_EXIT( IPOIB_DBG_INIT );\r
933 }\r
934 \r
935 \r
936 \r
937 /******************************************************************************\r
938 *\r
939 * Buffer manager implementation.\r
940 *\r
941 ******************************************************************************/\r
942 static void\r
943 __buf_mgr_construct(\r
944         IN                              ipoib_port_t* const                     p_port )\r
945 {\r
946         IPOIB_ENTER( IPOIB_DBG_INIT );\r
947 \r
948         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
949 \r
950         p_port->buf_mgr.h_packet_pool = NULL;\r
951         p_port->buf_mgr.h_buffer_pool = NULL;\r
952 \r
953         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
954                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
955 \r
956         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
957         p_port->buf_mgr.h_send_buf_pool = NULL;\r
958 \r
959         IPOIB_EXIT( IPOIB_DBG_INIT );\r
960 }\r
961 \r
962 \r
963 static ib_api_status_t\r
964 __buf_mgr_init(\r
965         IN                              ipoib_port_t* const                     p_port )\r
966 {\r
967         cl_status_t             cl_status;\r
968         NDIS_STATUS             ndis_status;\r
969         ipoib_params_t  *p_params;\r
970 \r
971         IPOIB_ENTER(IPOIB_DBG_INIT );\r
972 \r
973         CL_ASSERT( p_port );\r
974         CL_ASSERT( p_port->p_adapter );\r
975 \r
976         p_params = &p_port->p_adapter->params;\r
977 \r
978         /* Allocate the receive descriptor pool */\r
979         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
980                 p_params->rq_depth * p_params->recv_pool_ratio,\r
981 #if IPOIB_INLINE_RECV\r
982                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
983 #else   /* IPOIB_INLINE_RECV */\r
984                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
985 #endif  /* IPOIB_INLINE_RECV */\r
986         if( cl_status != CL_SUCCESS )\r
987         {\r
988                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
989                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
990                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
991                         ("cl_qpool_init for recvs returned %s\n",\r
992                         cl_status_text[cl_status]) );\r
993                 return IB_INSUFFICIENT_MEMORY;\r
994         }\r
995 \r
996         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
997         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
998                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
999         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1000         {\r
1001                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1002                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1003                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1004                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1005                 return IB_INSUFFICIENT_RESOURCES;\r
1006         }\r
1007 \r
1008         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1009                 p_params->rq_depth );\r
1010         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1011         {\r
1012                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1013                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1014                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1015                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1016                 return IB_INSUFFICIENT_RESOURCES;\r
1017         }\r
1018 \r
1019         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1020         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1021                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1022         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1023         {\r
1024                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1025                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1026                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1027                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1028                 return IB_INSUFFICIENT_RESOURCES;\r
1029         }\r
1030 \r
1031         NdisAllocateBufferPool( &ndis_status,\r
1032                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1033         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1034         {\r
1035                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1036                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1037                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1038                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1039                 return IB_INSUFFICIENT_RESOURCES;\r
1040         }\r
1041 \r
1042         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1043         return IB_SUCCESS;\r
1044 }\r
1045 \r
1046 \r
1047 static void\r
1048 __buf_mgr_destroy(\r
1049         IN                              ipoib_port_t* const                     p_port )\r
1050 {\r
1051         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1052 \r
1053         CL_ASSERT( p_port );\r
1054 \r
1055         /* Destroy the send packet and buffer pools. */\r
1056         if( p_port->buf_mgr.h_send_buf_pool )\r
1057                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1058         if( p_port->buf_mgr.h_send_pkt_pool )\r
1059                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1060 \r
1061         /* Destroy the receive packet and buffer pools. */\r
1062         if( p_port->buf_mgr.h_buffer_pool )\r
1063                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1064         if( p_port->buf_mgr.h_packet_pool )\r
1065                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1066 \r
1067         /* Free the receive and send descriptors. */\r
1068         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1069 \r
1070         /* Free the lookaside list of scratch buffers. */\r
1071         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1072 \r
1073         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1074 }\r
1075 \r
1076 \r
1077 static cl_status_t\r
1078 __recv_ctor(\r
1079         IN                              void* const                                     p_object,\r
1080         IN                              void*                                           context,\r
1081                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1082 {\r
1083         ipoib_recv_desc_t       *p_desc;\r
1084         ipoib_port_t            *p_port;\r
1085         uint32_t                        ds0_len;\r
1086 \r
1087         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1088 \r
1089         CL_ASSERT( p_object );\r
1090         CL_ASSERT( context );\r
1091 \r
1092         p_desc = (ipoib_recv_desc_t*)p_object;\r
1093         p_port = (ipoib_port_t*)context;\r
1094 \r
1095         /* Setup the work request. */\r
1096         p_desc->wr.ds_array = p_desc->local_ds;\r
1097         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1098 \r
1099 #if IPOIB_INLINE_RECV\r
1100         /* Sanity check on the receive buffer layout */\r
1101         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1102                 (void*)&p_desc->buf.ib.pkt.type );\r
1103         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1104 \r
1105         /* Setup the local data segment. */\r
1106         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1107         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1108         ds0_len =\r
1109                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1110         if( ds0_len >= sizeof(recv_buf_t) )\r
1111         {\r
1112                 /* The whole buffer is within a page. */\r
1113                 p_desc->local_ds[0].length = ds0_len;\r
1114                 p_desc->wr.num_ds = 1;\r
1115         }\r
1116         else\r
1117         {\r
1118                 /* The buffer crosses page boundaries. */\r
1119                 p_desc->local_ds[0].length = ds0_len;\r
1120                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1121                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1122                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1123                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1124                 p_desc->wr.num_ds = 2;\r
1125         }\r
1126 #else   /* IPOIB_INLINE_RECV */\r
1127         /* Allocate the receive buffer. */\r
1128         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1129         if( !p_desc->p_buf )\r
1130         {\r
1131                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1132                         ("Failed to allocate receive buffer.\n") );\r
1133                 return CL_INSUFFICIENT_MEMORY;\r
1134         }\r
1135 \r
1136         /* Sanity check on the receive buffer layout */\r
1137         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1138                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1139 \r
1140         /* Setup the local data segment. */\r
1141         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1142         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1143         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1144 #endif  /* IPOIB_INLINE_RECV */\r
1145 \r
1146         *pp_pool_item = &p_desc->item;\r
1147 \r
1148         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1149         return CL_SUCCESS;\r
1150 }\r
1151 \r
1152 \r
1153 #if !IPOIB_INLINE_RECV\r
1154 static void\r
1155 __recv_dtor(\r
1156         IN              const   cl_pool_item_t* const           p_pool_item,\r
1157         IN                              void                                            *context )\r
1158 {\r
1159         ipoib_recv_desc_t       *p_desc;\r
1160 \r
1161         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1162 \r
1163         UNUSED_PARAM( context );\r
1164 \r
1165         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1166 \r
1167         if( p_desc->p_buf )\r
1168                 cl_free( p_desc->p_buf );\r
1169 \r
1170         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1171 }\r
1172 #endif\r
1173 \r
1174 \r
1175 static inline ipoib_recv_desc_t*\r
1176 __buf_mgr_get_recv(\r
1177         IN                              ipoib_port_t* const                     p_port )\r
1178 {\r
1179         ipoib_recv_desc_t       *p_desc;\r
1180         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1181         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1182         /* Reference the port object for the send. */\r
1183         if( p_desc )\r
1184         {\r
1185                 ipoib_port_ref( p_port, ref_get_recv );\r
1186                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1187 #if IPOIB_INLINE_RECV\r
1188                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1189                         cl_get_physaddr( &p_desc->buf ) );\r
1190 #else   /* IPOIB_INLINE_RECV */\r
1191                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1192                         cl_get_physaddr( p_desc->p_buf ) );\r
1193                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1194                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1195 #endif  /* IPOIB_INLINE_RECV */\r
1196                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1197         }\r
1198         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1199         return p_desc;\r
1200 }\r
1201 \r
1202 \r
1203 static inline void\r
1204 __buf_mgr_put_recv(\r
1205         IN                              ipoib_port_t* const                     p_port,\r
1206         IN                              ipoib_recv_desc_t* const        p_desc,\r
1207         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1208 {\r
1209         NDIS_BUFFER             *p_buf;\r
1210 \r
1211         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1212 \r
1213         if( p_packet )\r
1214         {\r
1215                 /* Unchain the NDIS buffer. */\r
1216                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1217                 CL_ASSERT( p_buf );\r
1218                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1219                 NdisDprFreePacketNonInterlocked( p_packet );\r
1220                 NdisFreeBuffer( p_buf );\r
1221         }\r
1222 \r
1223         /* Return the descriptor to its pools. */\r
1224         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1225 \r
1226         /*\r
1227          * Dereference the port object since the receive is no longer outstanding.\r
1228          */\r
1229         ipoib_port_deref( p_port, ref_get_recv );\r
1230         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1231 }\r
1232 \r
1233 \r
1234 static inline void\r
1235 __buf_mgr_put_recv_list(\r
1236         IN                              ipoib_port_t* const                     p_port,\r
1237         IN                              cl_qlist_t* const                       p_list )\r
1238 {\r
1239         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1240         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1241         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1242 }\r
1243 \r
1244 \r
1245 static inline NDIS_PACKET*\r
1246 __buf_mgr_get_ndis_pkt(\r
1247         IN                              ipoib_port_t* const                     p_port,\r
1248         IN                              ipoib_recv_desc_t* const        p_desc )\r
1249 {\r
1250         NDIS_STATUS                             status;\r
1251         NDIS_PACKET                             *p_packet;\r
1252         NDIS_BUFFER                             *p_buffer;\r
1253 \r
1254         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1255 \r
1256         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1257                 p_port->buf_mgr.h_packet_pool );\r
1258         if( status != NDIS_STATUS_SUCCESS )\r
1259         {\r
1260                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1261                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1262                 return NULL;\r
1263         }\r
1264 \r
1265         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1266         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1267 \r
1268         NdisAllocateBuffer( &status, &p_buffer,\r
1269 #if IPOIB_INLINE_RECV\r
1270                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1271 #else   /* IPOIB_INLINE_RECV */\r
1272                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1273 #endif  /* IPOIB_INLINE_RECV */\r
1274         if( status != NDIS_STATUS_SUCCESS )\r
1275         {\r
1276                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1277                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1278                 NdisDprFreePacketNonInterlocked( p_packet );\r
1279                 return NULL;\r
1280         }\r
1281 \r
1282         NdisChainBufferAtFront( p_packet, p_buffer );\r
1283         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1284 \r
1285         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1286         return p_packet;\r
1287 }\r
1288 \r
1289 /******************************************************************************\r
1290 *\r
1291 * Receive manager implementation.\r
1292 *\r
1293 ******************************************************************************/\r
1294 static void\r
1295 __recv_mgr_construct(\r
1296         IN                              ipoib_port_t* const                     p_port )\r
1297 {\r
1298         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1299 \r
1300         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1301 \r
1302         p_port->recv_mgr.recv_pkt_array = NULL;\r
1303 \r
1304         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1305 }\r
1306 \r
1307 \r
1308 static ib_api_status_t\r
1309 __recv_mgr_init(\r
1310         IN                              ipoib_port_t* const                     p_port )\r
1311 {\r
1312         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1313 \r
1314         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1315         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1316                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1317         if( !p_port->recv_mgr.recv_pkt_array )\r
1318         {\r
1319                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1320                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1321                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1322                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1323                 return IB_INSUFFICIENT_MEMORY;\r
1324         }\r
1325 \r
1326         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1327         return IB_SUCCESS;\r
1328 }\r
1329 \r
1330 \r
1331 static void\r
1332 __recv_mgr_destroy(\r
1333         IN                              ipoib_port_t* const                     p_port )\r
1334 {\r
1335         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1336 \r
1337         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1338         CL_ASSERT( !p_port->recv_mgr.depth );\r
1339 \r
1340         if( p_port->recv_mgr.recv_pkt_array )\r
1341                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1342 \r
1343         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1344 }\r
1345 \r
1346 \r
1347 /*\r
1348  * Posts receive buffers to the receive queue and returns the number\r
1349  * of receives needed to bring the RQ to its low water mark.  Note\r
1350  * that the value is signed, and can go negative.  All tests must\r
1351  * be for > 0.\r
1352  */\r
1353 static int32_t\r
1354 __recv_mgr_repost(\r
1355         IN                              ipoib_port_t* const                     p_port )\r
1356 {\r
1357         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1358         ib_api_status_t         status;\r
1359         ib_recv_wr_t            *p_failed;\r
1360         PERF_DECLARE( GetRecv );\r
1361         PERF_DECLARE( PostRecv );\r
1362 \r
1363         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1364 \r
1365         CL_ASSERT( p_port );\r
1366         cl_obj_lock( &p_port->obj );\r
1367         if( p_port->state != IB_QPS_RTS )\r
1368         {\r
1369                 cl_obj_unlock( &p_port->obj );\r
1370                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1371                         ("Port in invalid state.  Not reposting.\n") );\r
1372                 return 0;\r
1373         }\r
1374         ipoib_port_ref( p_port, ref_repost );\r
1375         cl_obj_unlock( &p_port->obj );\r
1376 \r
1377         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1378         {\r
1379                 /* Pull receives out of the pool and chain them up. */\r
1380                 cl_perf_start( GetRecv );\r
1381                 p_next = __buf_mgr_get_recv( p_port );\r
1382                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1383                 if( !p_next )\r
1384                 {\r
1385                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1386                                 ("Out of receive descriptors!\n") );\r
1387                         break;\r
1388                 }\r
1389 \r
1390                 if( !p_tail )\r
1391                 {\r
1392                         p_tail = p_next;\r
1393                         p_next->wr.p_next = NULL;\r
1394                 }\r
1395                 else\r
1396                 {\r
1397                         p_next->wr.p_next = &p_head->wr;\r
1398                 }\r
1399 \r
1400                 p_head = p_next;\r
1401 \r
1402                 p_port->recv_mgr.depth++;\r
1403         }\r
1404 \r
1405         if( p_head )\r
1406         {\r
1407                 cl_perf_start( PostRecv );\r
1408                 status = p_port->p_adapter->p_ifc->post_recv(\r
1409                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1410                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1411 \r
1412                 if( status != IB_SUCCESS )\r
1413                 {\r
1414                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1415                                 ("ip_post_recv returned %s\n", \r
1416                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1417                         /* return the descriptors to the pool */\r
1418                         while( p_failed )\r
1419                         {\r
1420                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1421                                 p_failed = p_failed->p_next;\r
1422 \r
1423                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1424                                 p_port->recv_mgr.depth--;\r
1425                         }\r
1426                 }\r
1427         }\r
1428 \r
1429         ipoib_port_deref( p_port, ref_repost );\r
1430         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1431         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1432 }\r
1433 \r
1434 \r
1435 void\r
1436 ipoib_return_packet(\r
1437         IN                              NDIS_HANDLE                                     adapter_context,\r
1438         IN                              NDIS_PACKET                                     *p_packet )\r
1439 {\r
1440         cl_list_item_t          *p_item;\r
1441         ipoib_port_t            *p_port;\r
1442         ipoib_recv_desc_t       *p_desc;\r
1443         ib_api_status_t         status = IB_NOT_DONE;\r
1444         int32_t                         shortage;\r
1445         PERF_DECLARE( ReturnPacket );\r
1446         PERF_DECLARE( ReturnPutRecv );\r
1447         PERF_DECLARE( ReturnRepostRecv );\r
1448         PERF_DECLARE( ReturnPreparePkt );\r
1449         PERF_DECLARE( ReturnNdisIndicate );\r
1450 \r
1451         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1452 \r
1453         UNUSED_PARAM( adapter_context );\r
1454         CL_ASSERT( p_packet );\r
1455 \r
1456         cl_perf_start( ReturnPacket );\r
1457 \r
1458         /* Get the port and descriptor from the packet. */\r
1459         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1460         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1461 \r
1462         cl_spinlock_acquire( &p_port->recv_lock );\r
1463 \r
1464         cl_perf_start( ReturnPutRecv );\r
1465         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1466         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1467 \r
1468         /* Repost buffers. */\r
1469         cl_perf_start( ReturnRepostRecv );\r
1470         shortage = __recv_mgr_repost( p_port );\r
1471         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1472 \r
1473         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1474                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1475                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1476         {\r
1477                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1478 \r
1479                 cl_perf_start( ReturnPreparePkt );\r
1480                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1481                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1482                 if( status == IB_SUCCESS )\r
1483                 {\r
1484                         if( shortage > 0 )\r
1485                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1486                         else\r
1487                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1488 \r
1489                         cl_spinlock_release( &p_port->recv_lock );\r
1490                         cl_perf_start( ReturnNdisIndicate );\r
1491                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1492                                 &p_packet, 1 );\r
1493                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1494                         cl_spinlock_acquire( &p_port->recv_lock );\r
1495 \r
1496                         if( shortage > 0 )\r
1497                         {\r
1498                                 cl_perf_start( ReturnPutRecv );\r
1499                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1500                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1501 \r
1502                                 /* Repost buffers. */\r
1503                                 cl_perf_start( ReturnRepostRecv );\r
1504                                 shortage = __recv_mgr_repost( p_port );\r
1505                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1506                         }\r
1507                 }\r
1508                 else if( status != IB_NOT_DONE )\r
1509                 {\r
1510                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1511                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1512                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1513                         /* Return the item to the head of the list. */\r
1514                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1515                         break;\r
1516                 }\r
1517         }\r
1518         cl_spinlock_release( &p_port->recv_lock );\r
1519         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1520 \r
1521         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1522 }\r
1523 \r
1524 \r
1525 static void\r
1526 __recv_cb(\r
1527         IN              const   ib_cq_handle_t                          h_cq,\r
1528         IN                              void                                            *cq_context )\r
1529 {\r
1530         ipoib_port_t            *p_port;\r
1531         ib_api_status_t         status;\r
1532         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1533         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1534         cl_qlist_t                      done_list, bad_list;\r
1535         size_t                          i;\r
1536         PERF_DECLARE( RecvCompBundle );\r
1537         PERF_DECLARE( RecvCb );\r
1538         PERF_DECLARE( PollRecv );\r
1539         PERF_DECLARE( RepostRecv );\r
1540         PERF_DECLARE( FilterRecv );\r
1541         PERF_DECLARE( BuildPktArray );\r
1542         PERF_DECLARE( RecvNdisIndicate );\r
1543         PERF_DECLARE( RearmRecv );\r
1544         PERF_DECLARE( PutRecvList );\r
1545 \r
1546         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1547 \r
1548         cl_perf_clr( RecvCompBundle );\r
1549 \r
1550         cl_perf_start( RecvCb );\r
1551 \r
1552         UNUSED_PARAM( h_cq );\r
1553 \r
1554         p_port = (ipoib_port_t*)cq_context;\r
1555 \r
1556         cl_qlist_init( &done_list );\r
1557         cl_qlist_init( &bad_list );\r
1558 \r
1559         ipoib_port_ref( p_port, ref_recv_cb );\r
1560         for( i = 0; i < MAX_RECV_WC; i++ )\r
1561                 wc[i].p_next = &wc[i + 1];\r
1562         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1563 \r
1564         /*\r
1565          * We'll be accessing the endpoint map so take a reference\r
1566          * on it to prevent modifications.\r
1567          */\r
1568         cl_obj_lock( &p_port->obj );\r
1569         cl_atomic_inc( &p_port->endpt_rdr );\r
1570         cl_obj_unlock( &p_port->obj );\r
1571 \r
1572         do\r
1573         {\r
1574                 /* If we get here, then the list of WCs is intact. */\r
1575                 p_free = wc;\r
1576 \r
1577                 cl_perf_start( PollRecv );\r
1578                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1579                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1580                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1581                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1582 \r
1583                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1584                 cl_perf_start( FilterRecv );\r
1585                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1586                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1587 \r
1588         } while( !p_free );\r
1589 \r
1590         /* We're done looking at the endpoint map, release the reference. */\r
1591         cl_atomic_dec( &p_port->endpt_rdr );\r
1592 \r
1593         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1594 \r
1595         cl_spinlock_acquire( &p_port->recv_lock );\r
1596 \r
1597         /* Update our posted depth. */\r
1598         p_port->recv_mgr.depth -= recv_cnt;\r
1599 \r
1600         /* Return any discarded receives to the pool */\r
1601         cl_perf_start( PutRecvList );\r
1602         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1603         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1604 \r
1605         do\r
1606         {\r
1607                 /* Repost ASAP so we don't starve the RQ. */\r
1608                 cl_perf_start( RepostRecv );\r
1609                 shortage = __recv_mgr_repost( p_port );\r
1610                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1611 \r
1612                 cl_perf_start( BuildPktArray );\r
1613                 /* Notify NDIS of any and all possible receive buffers. */\r
1614                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1615                         p_port, shortage, &done_list, &discarded );\r
1616                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1617 \r
1618                 /* Only indicate receives if we actually had any. */\r
1619                 if( discarded && shortage > 0 )\r
1620                 {\r
1621                         /* We may have thrown away packets, and have a shortage */\r
1622                         cl_perf_start( RepostRecv );\r
1623                         __recv_mgr_repost( p_port );\r
1624                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1625                 }\r
1626 \r
1627                 if( !pkt_cnt )\r
1628                         break;\r
1629 \r
1630                 cl_spinlock_release( &p_port->recv_lock );\r
1631 \r
1632                 cl_perf_start( RecvNdisIndicate );\r
1633                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1634                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1635                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1636 \r
1637                 /*\r
1638                  * Cap the number of receives to put back to what we just indicated\r
1639                  * with NDIS_STATUS_RESOURCES.\r
1640                  */\r
1641                 if( shortage > 0 )\r
1642                 {\r
1643                         if( pkt_cnt < shortage )\r
1644                                 shortage = pkt_cnt;\r
1645 \r
1646                         /* Return all but the last packet to the pool. */\r
1647                         cl_spinlock_acquire( &p_port->recv_lock );\r
1648                         while( shortage-- > 1 )\r
1649                         {\r
1650                                 __buf_mgr_put_recv( p_port,\r
1651                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1652                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1653                         }\r
1654                         cl_spinlock_release( &p_port->recv_lock );\r
1655 \r
1656                         /*\r
1657                          * Return the last packet as if NDIS returned it, so that we repost\r
1658                          * and report any other pending receives.\r
1659                          */\r
1660                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1661                 }\r
1662                 cl_spinlock_acquire( &p_port->recv_lock );\r
1663 \r
1664         } while( pkt_cnt );\r
1665         cl_spinlock_release( &p_port->recv_lock );\r
1666 \r
1667         /*\r
1668          * Rearm after filtering to prevent contention on the enpoint maps\r
1669          * and eliminate the possibility of having a call to\r
1670          * __endpt_mgr_insert find a duplicate.\r
1671          */\r
1672         cl_perf_start( RearmRecv );\r
1673         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1674                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1675         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1676         CL_ASSERT( status == IB_SUCCESS );\r
1677 \r
1678         ipoib_port_deref( p_port, ref_recv_cb );\r
1679 \r
1680         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1681 \r
1682         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1683 }\r
1684 \r
1685 \r
1686 static void\r
1687 __recv_get_endpts(\r
1688         IN                              ipoib_port_t* const                     p_port,\r
1689         IN                              ipoib_recv_desc_t* const        p_desc,\r
1690         IN                              ib_wc_t* const                          p_wc,\r
1691                 OUT                     ipoib_endpt_t** const           pp_src,\r
1692                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1693 {\r
1694         ib_api_status_t         status;\r
1695         mac_addr_t                      mac;\r
1696         PERF_DECLARE( GetEndptByGid );\r
1697         PERF_DECLARE( GetEndptByLid );\r
1698         PERF_DECLARE( EndptInsert );\r
1699 \r
1700         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1701 \r
1702         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1703         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1704         {\r
1705                 /* Lookup the source endpoints based on GID. */\r
1706                 cl_perf_start( GetEndptByGid );\r
1707                 *pp_src =\r
1708 #if IPOIB_INLINE_RECV\r
1709                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1710 #else   /* IPOIB_INLINE_RECV */\r
1711                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1712 #endif  /* IPOIB_INLINE_RECV */\r
1713                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1714 \r
1715                 /*\r
1716                  * Lookup the destination endpoint based on GID.\r
1717                  * This is used along with the packet filter to determine\r
1718                  * whether to report this to NDIS.\r
1719                  */\r
1720                 cl_perf_start( GetEndptByGid );\r
1721                 *pp_dst =\r
1722 #if IPOIB_INLINE_RECV\r
1723                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1724 #else   /* IPOIB_INLINE_RECV */\r
1725                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1726 #endif  /* IPOIB_INLINE_RECV */\r
1727                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1728 \r
1729                 /*\r
1730                  * Create the source endpoint if it does not exist.  Note that we\r
1731                  * can only do this for globally routed traffic since we need the\r
1732                  * information from the GRH to generate the MAC.\r
1733                  */\r
1734                 if( !*pp_src )\r
1735                 {\r
1736                         status = ipoib_mac_from_guid(\r
1737 #if IPOIB_INLINE_RECV\r
1738                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1739 #else   /* IPOIB_INLINE_RECV */\r
1740                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1741 #endif  /* IPOIB_INLINE_RECV */\r
1742                         if( status != IB_SUCCESS )\r
1743                         {\r
1744                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1745                                         ("ipoib_mac_from_guid returned %s\n",\r
1746                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1747                                 return;\r
1748                         }\r
1749 \r
1750                         /* Create the endpoint. */\r
1751 #if IPOIB_INLINE_RECV\r
1752                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1753 #else   /* IPOIB_INLINE_RECV */\r
1754                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1755 #endif  /* IPOIB_INLINE_RECV */\r
1756                                 0, p_wc->recv.ud.remote_qp );\r
1757                         if( !*pp_src )\r
1758                         {\r
1759                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1760                                         ("ipoib_endpt_create failed\n") );\r
1761                                 return;\r
1762                         }\r
1763                         cl_perf_start( EndptInsert );\r
1764                         cl_obj_lock( &p_port->obj );\r
1765                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1766                         if( status != IB_SUCCESS )\r
1767                         {\r
1768                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1769                                         ("__endpt_mgr_insert returned %s\n",\r
1770                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1771                                 return;\r
1772                         }\r
1773                         cl_obj_unlock( &p_port->obj );\r
1774                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1775                 }\r
1776         }\r
1777         else\r
1778         {\r
1779                 /*\r
1780                  * Lookup the remote endpoint based on LID.  Note that only\r
1781                  * unicast traffic can be LID routed.\r
1782                  */\r
1783                 cl_perf_start( GetEndptByLid );\r
1784                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1785                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1786                 *pp_dst = p_port->p_local_endpt;\r
1787                 CL_ASSERT( *pp_dst );\r
1788         }\r
1789 \r
1790         if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1791         {\r
1792                 /* Update the QPN for the endpoint. */\r
1793                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1794                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1795                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1796                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1797                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1798                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1799         }\r
1800 \r
1801         if( *pp_src && *pp_dst )\r
1802         {\r
1803                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1804                         ("Recv:\n"\r
1805                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1806                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1807                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1808                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1809                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1810                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1811                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1812                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1813         }\r
1814 \r
1815         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1816 }\r
1817 \r
1818 \r
1819 static int32_t\r
1820 __recv_mgr_filter(\r
1821         IN                              ipoib_port_t* const                     p_port,\r
1822         IN                              ib_wc_t* const                          p_done_wc_list,\r
1823                 OUT                     cl_qlist_t* const                       p_done_list,\r
1824                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1825 {\r
1826         ipoib_recv_desc_t               *p_desc;\r
1827         ib_wc_t                                 *p_wc;\r
1828         ipoib_pkt_t                             *p_ipoib;\r
1829         eth_pkt_t                               *p_eth;\r
1830         ipoib_endpt_t                   *p_src, *p_dst;\r
1831         ib_api_status_t                 status;\r
1832         uint32_t                                len;\r
1833         int32_t                                 recv_cnt = 0;\r
1834         PERF_DECLARE( GetRecvEndpts );\r
1835         PERF_DECLARE( RecvGen );\r
1836         PERF_DECLARE( RecvTcp );\r
1837         PERF_DECLARE( RecvUdp );\r
1838         PERF_DECLARE( RecvDhcp );\r
1839         PERF_DECLARE( RecvArp );\r
1840 \r
1841         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1842 \r
1843         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1844         {\r
1845                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1846                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1847                 recv_cnt++;\r
1848 \r
1849                 if( p_wc->status != IB_WCS_SUCCESS )\r
1850                 {\r
1851                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1852                         {\r
1853                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1854                                         ("Failed completion %s\n",\r
1855                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1856                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1857                         }\r
1858                         else\r
1859                         {\r
1860                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1861                                         ("Flushed completion %s\n",\r
1862                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1863                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1864                         }\r
1865                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1866                         /* Dereference the port object on behalf of the failed receive. */\r
1867                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1868                         continue;\r
1869                 }\r
1870 \r
1871                 len = p_wc->length - sizeof(ib_grh_t);\r
1872 \r
1873                 if( len < sizeof(ipoib_hdr_t) )\r
1874                 {\r
1875                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1876                                 ("Received ETH packet < min size\n") );\r
1877                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1878                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1879                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1880                         continue;\r
1881                 }\r
1882 \r
1883                 /* Successful completion.  Get the receive information. */\r
1884                 cl_perf_start( GetRecvEndpts );\r
1885                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1886                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1887 \r
1888 #if IPOIB_INLINE_RECV\r
1889                 p_ipoib = &p_desc->buf.ib.pkt;\r
1890                 p_eth = &p_desc->buf.eth.pkt;\r
1891 #else   /* IPOIB_INLINE_RECV */\r
1892                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1893                 p_eth = &p_desc->p_buf->eth.pkt;\r
1894 #endif  /*IPOIB_INLINE_RECV */\r
1895 \r
1896                 if( p_src )\r
1897                 {\r
1898                         /* Don't report loopback traffic - we requested SW loopback. */\r
1899                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1900                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1901                         {\r
1902                                 /*\r
1903                                  * "This is not the packet you're looking for" - don't update\r
1904                                  * receive statistics, the packet never happened.\r
1905                                  */\r
1906                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1907                                 /* Dereference the port object on behalf of the failed recv. */\r
1908                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
1909                                 continue;\r
1910                         }\r
1911                 }\r
1912 \r
1913                 switch( p_ipoib->hdr.type )\r
1914                 {\r
1915                 case ETH_PROT_TYPE_IP:\r
1916                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1917                         {\r
1918                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1919                                         ("Received IP packet < min size\n") );\r
1920                                 status = IB_INVALID_SETTING;\r
1921                                 break;\r
1922                         }\r
1923 \r
1924                         if( p_ipoib->type.ip.hdr.offset ||\r
1925                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1926                         {\r
1927                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1928                                 cl_perf_start( RecvTcp );\r
1929                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1930                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1931                                 break;\r
1932                         }\r
1933 \r
1934                         /* First packet of a UDP transfer. */\r
1935                         if( len <\r
1936                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1937                         {\r
1938                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1939                                         ("Received UDP packet < min size\n") );\r
1940                                 status = IB_INVALID_SETTING;\r
1941                                 break;\r
1942                         }\r
1943 \r
1944                         /* Check if DHCP conversion is required. */\r
1945                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1946                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1947                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1948                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1949                         {\r
1950                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1951                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1952                                 {\r
1953                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1954                                                 ("Received DHCP < min size\n") );\r
1955                                         status = IB_INVALID_SETTING;\r
1956                                         break;\r
1957                                 }\r
1958                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1959                                 cl_perf_start( RecvDhcp );\r
1960                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1961                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1962                         }\r
1963                         else\r
1964                         {\r
1965                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1966                                 cl_perf_start( RecvUdp );\r
1967                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1968                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1969                         }\r
1970                         break;\r
1971 \r
1972                 case ETH_PROT_TYPE_ARP:\r
1973                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1974                         {\r
1975                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1976                                         ("Received ARP < min size\n") );\r
1977                                 status = IB_INVALID_SETTING;\r
1978                                 break;\r
1979                         }\r
1980                         cl_perf_start( RecvArp );\r
1981                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1982                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1983                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1984                         break;\r
1985 \r
1986                 default:\r
1987                         /* Unfiltered.  Setup the ethernet header and report. */\r
1988                         cl_perf_start( RecvGen );\r
1989                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1990                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1991                 }\r
1992 \r
1993                 if( status != IB_SUCCESS )\r
1994                 {\r
1995                         /* Update stats. */\r
1996                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1997                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1998                         /* Dereference the port object on behalf of the failed receive. */\r
1999                         ipoib_port_deref( p_port, ref_recv_filter );\r
2000                 }\r
2001                 else\r
2002                 {\r
2003                         p_desc->len =\r
2004                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2005                         if( p_dst->h_mcast)\r
2006                         {\r
2007                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2008                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2009                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2010                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2011                                 {\r
2012                                         p_desc->type = PKT_TYPE_BCAST;\r
2013                                 }\r
2014                                 else\r
2015                                 {\r
2016                                         p_desc->type = PKT_TYPE_MCAST;\r
2017                                 }\r
2018                         }\r
2019                         else\r
2020                         {\r
2021                                 p_desc->type = PKT_TYPE_UCAST;\r
2022                         }\r
2023                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2024                 }\r
2025         }\r
2026 \r
2027         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2028         return recv_cnt;\r
2029 }\r
2030 \r
2031 \r
2032 static ib_api_status_t\r
2033 __recv_gen(\r
2034         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2035                 OUT                     eth_pkt_t* const                        p_eth,\r
2036         IN                              ipoib_endpt_t* const            p_src,\r
2037         IN                              ipoib_endpt_t* const            p_dst )\r
2038 {\r
2039         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2040 \r
2041         if( !p_src || !p_dst )\r
2042         {\r
2043                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2044                         ("Received packet with no matching endpoints.\n") );\r
2045                 return IB_NOT_DONE;\r
2046         }\r
2047 \r
2048         /*\r
2049          * Fill in the ethernet header.  Note that doing so will overwrite\r
2050          * the IPoIB header, so start by moving the information from the IPoIB\r
2051          * header.\r
2052          */\r
2053         p_eth->hdr.type = p_ipoib->hdr.type;\r
2054         p_eth->hdr.src = p_src->mac;\r
2055         p_eth->hdr.dst = p_dst->mac;\r
2056 \r
2057         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2058         return IB_SUCCESS;\r
2059 }\r
2060 \r
2061 \r
2062 static ib_api_status_t\r
2063 __recv_dhcp(\r
2064         IN                              ipoib_port_t* const                     p_port,\r
2065         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2066                 OUT                     eth_pkt_t* const                        p_eth,\r
2067         IN                              ipoib_endpt_t* const            p_src,\r
2068         IN                              ipoib_endpt_t* const            p_dst )\r
2069 {\r
2070         ib_api_status_t         status;\r
2071         dhcp_pkt_t                      *p_dhcp;\r
2072         uint8_t                         *p_option;\r
2073         uint8_t                         *p_cid = NULL;\r
2074         ib_gid_t                        gid;\r
2075         uint8_t                         msg = 0;\r
2076 \r
2077         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2078 \r
2079         UNUSED_PARAM( p_port );\r
2080 \r
2081         /* Create the ethernet header. */\r
2082         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2083         if( status != IB_SUCCESS )\r
2084         {\r
2085                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2086                         ("__recv_gen returned %s.\n", \r
2087                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2088                 return status;\r
2089         }\r
2090 \r
2091         /* Fixup the payload. */\r
2092         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2093         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2094         {\r
2095                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2096                         ("Invalid DHCP op code.\n") );\r
2097                 return IB_INVALID_SETTING;\r
2098         }\r
2099 \r
2100         /*\r
2101          * Find the client identifier option, making sure to skip\r
2102          * the "magic cookie".\r
2103          */\r
2104         p_option = &p_dhcp->options[0];\r
2105         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2106         {\r
2107                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2108                         ("DHCP cookie corrupted.\n") );\r
2109                 return IB_INVALID_PARAMETER;\r
2110         }\r
2111 \r
2112         p_option = &p_dhcp->options[4];\r
2113         while( *p_option != DHCP_OPT_END )\r
2114         {\r
2115                 switch( *p_option )\r
2116                 {\r
2117                 case DHCP_OPT_PAD:\r
2118                         p_option++;\r
2119                         break;\r
2120 \r
2121                 case DHCP_OPT_MSG:\r
2122                         msg = p_option[2];\r
2123                         p_option += 3;\r
2124                         break;\r
2125 \r
2126                 case DHCP_OPT_CLIENT_ID:\r
2127                         p_cid = p_option;\r
2128                         /* Fall through. */\r
2129 \r
2130                 default:\r
2131                         /*\r
2132                          * All other options have a length byte following the option code.\r
2133                          * Offset by the length to get to the next option.\r
2134                          */\r
2135                         p_option += (p_option[1] + 2);\r
2136                 }\r
2137         }\r
2138 \r
2139         switch( msg )\r
2140         {\r
2141         /* message from client */\r
2142         case DHCPDISCOVER:\r
2143         case DHCPREQUEST:\r
2144         case DHCPDECLINE:\r
2145         case DHCPRELEASE:\r
2146         case DHCPINFORM:\r
2147                 if( !p_cid )\r
2148                 {\r
2149                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2150                                 ("Failed to find required Client-identifier option.\n") );\r
2151                         return IB_INVALID_SETTING;\r
2152                 }\r
2153                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2154                 {\r
2155                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2156                                 ("Invalid hardware address type.\n") );\r
2157                         return IB_INVALID_SETTING;\r
2158                 }\r
2159                 break;\r
2160         /* message from DHCP server */\r
2161         case DHCPOFFER:\r
2162         case DHCPACK:\r
2163         case DHCPNAK:\r
2164                 break;\r
2165 \r
2166         default:\r
2167                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2168                         ("Invalide message type.\n") );\r
2169                 return IB_INVALID_PARAMETER;\r
2170         }\r
2171         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2172         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2173         p_dhcp->hlen = HW_ADDR_LEN;\r
2174 \r
2175         if( p_cid ) /* from client */\r
2176         {\r
2177                 /* Validate that the length and type of the option is as required. */\r
2178                 if( p_cid[1] != 21 )\r
2179                 {\r
2180                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2181                                 ("Client-identifier length not 21 as required.\n") );\r
2182                         return IB_INVALID_SETTING;\r
2183                 }\r
2184                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2185                 {\r
2186                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2187                                 ("Client-identifier type is wrong.\n") );\r
2188                         return IB_INVALID_SETTING;\r
2189                 }\r
2190                 /*\r
2191                  * Copy the GID value from the option so that we can make aligned\r
2192                  * accesses to the contents.\r
2193                  * Recover CID to standard type.\r
2194                  */\r
2195                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2196                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2197                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2198                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2199                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2200         }\r
2201         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2202         return status;\r
2203 }\r
2204 \r
2205 \r
2206 static ib_api_status_t\r
2207 __recv_arp(\r
2208         IN                              ipoib_port_t* const                     p_port,\r
2209         IN                              ib_wc_t* const                          p_wc,\r
2210         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2211                 OUT                     eth_pkt_t* const                        p_eth,\r
2212         IN                              ipoib_endpt_t** const           pp_src,\r
2213         IN                              ipoib_endpt_t* const            p_dst )\r
2214 {\r
2215         ib_api_status_t                 status;\r
2216         arp_pkt_t                               *p_arp;\r
2217         const ipoib_arp_pkt_t   *p_ib_arp;\r
2218         ib_gid_t                                gid;\r
2219         mac_addr_t                              mac;\r
2220         ipoib_hw_addr_t                 null_hw = {0};\r
2221 \r
2222         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2223 \r
2224         if( !p_dst )\r
2225         {\r
2226                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2227                         ("Unknown destination endpoint\n") );\r
2228                 return IB_INVALID_SETTING;\r
2229         }\r
2230 \r
2231         p_ib_arp = &p_ipoib->type.arp;\r
2232         p_arp = &p_eth->type.arp;\r
2233 \r
2234         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2235         {\r
2236                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2237                         ("ARP hardware type is not IB\n") );\r
2238                 return IB_INVALID_SETTING;\r
2239         }\r
2240 \r
2241         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2242         {\r
2243                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2244                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2245                 return IB_INVALID_SETTING;\r
2246         }\r
2247 \r
2248         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2249         {\r
2250                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2251                         ("ARP protocal type not IP\n") );\r
2252                 return IB_INVALID_SETTING;\r
2253         }\r
2254 \r
2255         /*\r
2256          * If we don't have a source, lookup the endpoint specified in the payload.\r
2257          */\r
2258         if( !*pp_src )\r
2259                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2260 \r
2261         /*\r
2262          * If the endpoint exists for the GID, make sure\r
2263          * the dlid and qpn match the arp.\r
2264          */\r
2265         if( *pp_src )\r
2266         {\r
2267                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2268                         sizeof(ib_gid_t) ) )\r
2269                 {\r
2270                         /*\r
2271                          * GIDs for the endpoint are different.  The ARP must\r
2272                          * have been proxied.  Dereference it.\r
2273                          */\r
2274                         *pp_src = NULL;\r
2275                 }\r
2276                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2277                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2278                 {\r
2279                         /* Out of date!  Destroy the endpoint and replace it. */\r
2280                         __endpt_mgr_remove( p_port, *pp_src );\r
2281                         *pp_src = NULL;\r
2282                 }\r
2283         }\r
2284 \r
2285         /* Do we need to create an endpoint for this GID? */\r
2286         if( !*pp_src )\r
2287         {\r
2288                 /* Copy the src GID to allow aligned access */\r
2289                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2290                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2291                 if( status != IB_SUCCESS )\r
2292                 {\r
2293                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2294                                 ("ipoib_mac_from_guid returned %s\n",\r
2295                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2296                         return status;\r
2297                 }\r
2298                 /*\r
2299                  * Create the endpoint.  Note that the LID is left blank and will be\r
2300                  * resolved by a path query as needed.  This is done because the\r
2301                  * remote LID/GID from the work completion may not be the original\r
2302                  * initiator.\r
2303                  */\r
2304                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2305                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2306 \r
2307                 if( !*pp_src )\r
2308                 {\r
2309                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2310                                 ("ipoib_endpt_create failed\n") );\r
2311                         return status;\r
2312                 }\r
2313 \r
2314                 cl_obj_lock( &p_port->obj );\r
2315                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2316                 if( status != IB_SUCCESS )\r
2317                 {\r
2318                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2319                                 ("__endpt_mgr_insert return %s \n",\r
2320                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2321                         return status;\r
2322                 }\r
2323 \r
2324                 cl_obj_unlock( &p_port->obj );\r
2325         }\r
2326 \r
2327         CL_ASSERT( !cl_memcmp(\r
2328                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2329         CL_ASSERT( (*pp_src)->qpn ==\r
2330                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2331         /* Now swizzle the data. */\r
2332         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2333         p_arp->hw_size = sizeof(mac_addr_t);\r
2334         p_arp->src_hw = (*pp_src)->mac;\r
2335         p_arp->src_ip = p_ib_arp->src_ip;\r
2336 \r
2337         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2338         {\r
2339 \r
2340                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2341                 {\r
2342                         /*\r
2343                          * We received bcast ARP packet that means\r
2344                          * remote port lets everyone know it was changed IP/MAC\r
2345                          * or just activated\r
2346                          */\r
2347                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2348                         {\r
2349                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2350                                         ("ARP: is not ARP MCAST\n") );\r
2351                                 return IB_INVALID_SETTING;\r
2352                         }\r
2353 \r
2354                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2355                         p_dst->mac = p_port->p_local_endpt->mac;\r
2356                         /*\r
2357                          * we don't care what receiver ip addr is,\r
2358                          * as long as OS' ARP table is global  ???\r
2359                          */\r
2360                         p_arp->dst_ip = (net32_t)0;\r
2361 \r
2362                 }\r
2363                 else /* we've got reply to our ARP request */\r
2364                 {\r
2365                         p_arp->dst_hw = p_dst->mac;\r
2366                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2367                         CL_ASSERT( p_dst->qpn == \r
2368                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2369                 }\r
2370         }\r
2371         else /* we got ARP reqeust */\r
2372         {\r
2373                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2374                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2375         }\r
2376 \r
2377         /*\r
2378          * Create the ethernet header.  Note that this is done last so that\r
2379          * we have a chance to create a new endpoint.\r
2380          */\r
2381         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2382         if( status != IB_SUCCESS )\r
2383         {\r
2384                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2385                         ("__recv_gen returned %s.\n", \r
2386                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2387                 return status;\r
2388         }\r
2389 \r
2390         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2391         return IB_SUCCESS;\r
2392 }\r
2393 \r
2394 \r
2395 static ib_api_status_t\r
2396 __recv_mgr_prepare_pkt(\r
2397         IN                              ipoib_port_t* const                     p_port,\r
2398         IN                              ipoib_recv_desc_t* const        p_desc,\r
2399                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2400 {\r
2401         NDIS_STATUS                                                     status;\r
2402         uint32_t                                                        pkt_filter;\r
2403         ip_stat_sel_t                                           type;\r
2404         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2405         PERF_DECLARE( GetNdisPkt );\r
2406 \r
2407         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2408 \r
2409         pkt_filter = p_port->p_adapter->packet_filter;\r
2410         /* Check the packet filter. */\r
2411         switch( p_desc->type )\r
2412         {\r
2413         default:\r
2414         case PKT_TYPE_UCAST:\r
2415                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2416                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2417                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2418                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2419                 {\r
2420                         /* OK to report. */\r
2421                         type = IP_STAT_UCAST_BYTES;\r
2422                         status = NDIS_STATUS_SUCCESS;\r
2423                 }\r
2424                 else\r
2425                 {\r
2426                         type = IP_STAT_DROPPED;\r
2427                         status = NDIS_STATUS_FAILURE;\r
2428                 }\r
2429                 break;\r
2430         case PKT_TYPE_BCAST:\r
2431                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2432                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2433                 {\r
2434                         /* OK to report. */\r
2435                         type = IP_STAT_BCAST_BYTES;\r
2436                         status = NDIS_STATUS_SUCCESS;\r
2437                 }\r
2438                 else\r
2439                 {\r
2440                         type = IP_STAT_DROPPED;\r
2441                         status = NDIS_STATUS_FAILURE;\r
2442                 }\r
2443                 break;\r
2444         case PKT_TYPE_MCAST:\r
2445                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2446                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2447                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2448                 {\r
2449                         /* OK to report. */\r
2450                         type = IP_STAT_MCAST_BYTES;\r
2451                         status = NDIS_STATUS_SUCCESS;\r
2452                 }\r
2453                 else\r
2454                 {\r
2455                         type = IP_STAT_DROPPED;\r
2456                         status = NDIS_STATUS_FAILURE;\r
2457                 }\r
2458                 break;\r
2459         }\r
2460 \r
2461         if( status != NDIS_STATUS_SUCCESS )\r
2462         {\r
2463                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2464                 /* Return the receive descriptor to the pool. */\r
2465                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2466                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2467                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2468                 /*\r
2469                  * Return IB_NOT_DONE since the packet has been completed,\r
2470                  * but has not consumed an array entry.\r
2471                  */\r
2472                 return IB_NOT_DONE;\r
2473         }\r
2474 \r
2475         cl_perf_start( GetNdisPkt );\r
2476         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2477         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2478         if( !*pp_packet )\r
2479         {\r
2480                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2481                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2482                 return IB_INSUFFICIENT_RESOURCES;\r
2483         }\r
2484 \r
2485         /* Flag the checksums as having been calculated. */\r
2486         chksum.Value = 0;\r
2487         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2488         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2489         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2490         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2491                 (void*)(uintn_t)chksum.Value;\r
2492 \r
2493         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2494 \r
2495         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2496         return IB_SUCCESS;\r
2497 }\r
2498 \r
2499 \r
2500 static uint32_t\r
2501 __recv_mgr_build_pkt_array(\r
2502         IN                              ipoib_port_t* const                     p_port,\r
2503         IN                              int32_t                                         shortage,\r
2504                 OUT                     cl_qlist_t* const                       p_done_list,\r
2505                 OUT                     int32_t* const                          p_discarded )\r
2506 {\r
2507         cl_list_item_t                  *p_item;\r
2508         ipoib_recv_desc_t               *p_desc;\r
2509         uint32_t                                i = 0;\r
2510         ib_api_status_t                 status;\r
2511         PERF_DECLARE( PreparePkt );\r
2512 \r
2513         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2514 \r
2515         *p_discarded = 0;\r
2516 \r
2517         /* Move any existing receives to the head to preserve ordering. */\r
2518         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2519         p_item = cl_qlist_remove_head( p_done_list );\r
2520         while( p_item != cl_qlist_end( p_done_list ) )\r
2521         {\r
2522                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2523 \r
2524                 cl_perf_start( PreparePkt );\r
2525                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2526                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2527                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2528                 if( status == IB_SUCCESS )\r
2529                 {\r
2530                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2531                         if( shortage-- > 0 )\r
2532                         {\r
2533                                 NDIS_SET_PACKET_STATUS(\r
2534                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2535                         }\r
2536                         else\r
2537                         {\r
2538                                 NDIS_SET_PACKET_STATUS(\r
2539                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2540                         }\r
2541                         i++;\r
2542                 }\r
2543                 else if( status == IB_NOT_DONE )\r
2544                 {\r
2545                         (*p_discarded)++;\r
2546                 }\r
2547                 else\r
2548                 {\r
2549                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2550                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2551                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2552                         /* Put all completed receives on the port's done list. */\r
2553                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2554                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2555                         break;\r
2556                 }\r
2557 \r
2558                 p_item = cl_qlist_remove_head( p_done_list );\r
2559         }\r
2560 \r
2561         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2562         return i;\r
2563 }\r
2564 \r
2565 \r
2566 \r
2567 \r
2568 /******************************************************************************\r
2569 *\r
2570 * Send manager implementation.\r
2571 *\r
2572 ******************************************************************************/\r
2573 static void\r
2574 __send_mgr_construct(\r
2575         IN                              ipoib_port_t* const                     p_port )\r
2576 {\r
2577         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2578         p_port->send_mgr.depth = 0;\r
2579         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2580         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2581 }\r
2582 \r
2583 \r
2584 static void\r
2585 __send_mgr_destroy(\r
2586         IN                              ipoib_port_t* const                     p_port )\r
2587 {\r
2588         cl_list_item_t  *p_item;\r
2589         NDIS_PACKET             *p_packet;\r
2590 \r
2591         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2592 \r
2593         /* Complete any pending packets. */\r
2594         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2595                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2596                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2597         {\r
2598                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2599                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2600                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2601         }\r
2602 \r
2603         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2604 }\r
2605 \r
2606 \r
2607 static NDIS_STATUS\r
2608 __send_mgr_filter(\r
2609         IN                              ipoib_port_t* const                     p_port,\r
2610         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2611         IN                              NDIS_BUFFER* const                      p_buf,\r
2612         IN                              size_t                                          buf_len,\r
2613         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2614 {\r
2615         NDIS_STATUS             status;\r
2616 \r
2617         PERF_DECLARE( FilterIp );\r
2618         PERF_DECLARE( FilterArp );\r
2619         PERF_DECLARE( SendGen );\r
2620 \r
2621         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2622 \r
2623         /*\r
2624          * We already checked the ethernet header length, so we know it's safe\r
2625          * to decrement the buf_len without underflowing.\r
2626          */\r
2627         buf_len -= sizeof(eth_hdr_t);\r
2628 \r
2629         switch( p_eth_hdr->type )\r
2630         {\r
2631         case ETH_PROT_TYPE_IP:\r
2632                 cl_perf_start( FilterIp );\r
2633                 status = __send_mgr_filter_ip(\r
2634                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2635                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2636                 break;\r
2637 \r
2638         case ETH_PROT_TYPE_ARP:\r
2639                 cl_perf_start( FilterArp );\r
2640                 status = __send_mgr_filter_arp(\r
2641                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2642                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2643                 break;\r
2644 \r
2645         default:\r
2646                 /*\r
2647                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2648                  * Just send the payload and hope for the best.\r
2649                  */\r
2650                 cl_perf_start( SendGen );\r
2651                 status = __send_gen( p_port, p_desc );\r
2652                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2653                 break;\r
2654         }\r
2655 \r
2656         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2657         return status;\r
2658 }\r
2659 \r
2660 \r
2661 static NDIS_STATUS\r
2662 __send_copy(\r
2663         IN                              ipoib_port_t* const                     p_port,\r
2664         IN                              ipoib_send_desc_t* const        p_desc )\r
2665 {\r
2666         NDIS_PACKET                             *p_packet;\r
2667         NDIS_BUFFER                             *p_buf;\r
2668         NDIS_STATUS                             status;\r
2669         UINT                                    tot_len, bytes_copied = 0;\r
2670 \r
2671         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2672 \r
2673         p_desc->p_buf = \r
2674                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2675         if( !p_desc->p_buf )\r
2676         {\r
2677                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2678                         ("Failed to allocate buffer for packet copy.\n") );\r
2679                 return NDIS_STATUS_RESOURCES;\r
2680         }\r
2681 \r
2682         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2683         if( status != NDIS_STATUS_SUCCESS )\r
2684         {\r
2685                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2686                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2687                 return status;\r
2688         }\r
2689 \r
2690         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2691                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2692         if( status != NDIS_STATUS_SUCCESS )\r
2693         {\r
2694                 NdisFreePacket( p_packet );\r
2695                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2696                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2697                 return status;\r
2698         }\r
2699 \r
2700         NdisChainBufferAtFront( p_packet, p_buf );\r
2701 \r
2702         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2703 \r
2704         /* Setup the work request. */\r
2705         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2706                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2707         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2708         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2709         p_desc->wr.num_ds = 2;\r
2710 \r
2711         /* Copy the packet. */\r
2712         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2713                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2714                 NormalPagePriority );\r
2715 \r
2716         /* Free our temp packet now that the data is copied. */\r
2717         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2718         NdisFreeBuffer( p_buf );\r
2719         NdisFreePacket( p_packet );\r
2720 \r
2721         if( bytes_copied != tot_len )\r
2722         {\r
2723                 /* Something went wrong.  Drop the packet. */\r
2724                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2725                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2726                         bytes_copied, tot_len) );\r
2727                 return NDIS_STATUS_RESOURCES;\r
2728         }\r
2729 \r
2730         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2731         return NDIS_STATUS_SUCCESS;\r
2732 }\r
2733 \r
2734 \r
2735 #if !IPOIB_USE_DMA\r
2736 /* Send using the MDL's page information rather than the SGL. */\r
2737 static ib_api_status_t\r
2738 __send_gen(\r
2739         IN                              ipoib_port_t* const                     p_port,\r
2740         IN                              ipoib_send_desc_t* const        p_desc )\r
2741 {\r
2742         uint32_t                                i, j = 1;\r
2743         ULONG                                   offset;\r
2744         MDL                                             *p_mdl;\r
2745         UINT                                    num_pages, tot_len;\r
2746         ULONG                                   buf_len;\r
2747         PPFN_NUMBER                             page_array;\r
2748         boolean_t                               hdr_done = FALSE;\r
2749         ib_api_status_t                 status;\r
2750 \r
2751         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2752 \r
2753         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2754                 &tot_len );\r
2755 \r
2756         if( !p_mdl )\r
2757         {\r
2758                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2759                         ("No buffers associated with packet.\n") );\r
2760                 return IB_ERROR;\r
2761         }\r
2762 \r
2763         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2764         if( num_pages >= MAX_SEND_SGE )\r
2765         {\r
2766                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2767                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2768                 status = __send_copy( p_port, p_desc );\r
2769                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2770                 return status;\r
2771         }\r
2772 \r
2773         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2774         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2775         /*\r
2776          * Assume that the ethernet header is always fully contained\r
2777          * in the first page of the first MDL.  This makes for much\r
2778          * simpler code.\r
2779          */\r
2780         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2781         CL_ASSERT( offset <= PAGE_SIZE );\r
2782 \r
2783         while( tot_len )\r
2784         {\r
2785                 buf_len = MmGetMdlByteCount( p_mdl );\r
2786                 page_array = MmGetMdlPfnArray( p_mdl );\r
2787                 CL_ASSERT( page_array );\r
2788                 i = 0;\r
2789                 if( !hdr_done )\r
2790                 {\r
2791                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2792                         /* Skip the ethernet header. */\r
2793                         buf_len -= sizeof(eth_hdr_t);\r
2794                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2795                         if( buf_len )\r
2796                         {\r
2797                                 /* The ethernet header is a subset of this MDL. */\r
2798                                 CL_ASSERT( i == 0 );\r
2799                                 if( offset < PAGE_SIZE )\r
2800                                 {\r
2801                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2802                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2803                                         /* Add the byte offset since we're on the 1st page. */\r
2804                                         p_desc->local_ds[j].vaddr += offset;\r
2805                                         if( offset + buf_len > PAGE_SIZE )\r
2806                                         {\r
2807                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2808                                                 buf_len -= p_desc->local_ds[j].length;\r
2809                                         }\r
2810                                         else\r
2811                                         {\r
2812                                                 p_desc->local_ds[j].length = buf_len;\r
2813                                                 buf_len = 0;\r
2814                                         }\r
2815                                         /* This data segment is done.  Move to the next. */\r
2816                                         j++;\r
2817                                 }\r
2818                                 /* This page is done.  Move to the next. */\r
2819                                 i++;\r
2820                         }\r
2821                         /* Done handling the ethernet header. */\r
2822                         hdr_done = TRUE;\r
2823                 }\r
2824 \r
2825                 /* Finish this MDL */\r
2826                 while( buf_len )\r
2827                 {\r
2828                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2829                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2830                         /* Add the first page's offset if we're on the first page. */\r
2831                         if( i == 0 )\r
2832                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2833 \r
2834                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2835                         {\r
2836                                 /* Buffers spans pages. */\r
2837                                 p_desc->local_ds[j].length =\r
2838                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2839                                 buf_len -= p_desc->local_ds[j].length;\r
2840                                 /* This page is done.  Move to the next. */\r
2841                                 i++;\r
2842                         }\r
2843                         else\r
2844                         {\r
2845                                 /* Last page of the buffer. */\r
2846                                 p_desc->local_ds[j].length = buf_len;\r
2847                                 buf_len = 0;\r
2848                         }\r
2849                         /* This data segment is done.  Move to the next. */\r
2850                         j++;\r
2851                 }\r
2852 \r
2853                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2854                 if( !tot_len )\r
2855                         break;\r
2856 \r
2857                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2858                 if( !p_mdl )\r
2859                 {\r
2860                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2861                                 ("Failed to get next buffer.\n") );\r
2862                         return IB_ERROR;\r
2863                 }\r
2864         }\r
2865 \r
2866         /* Set the number of data segments. */\r
2867         p_desc->wr.num_ds = j;\r
2868 \r
2869         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2870         return IB_SUCCESS;\r
2871 }\r
2872 \r
2873 #else\r
2874 \r
2875 static NDIS_STATUS\r
2876 __send_gen(\r
2877         IN                              ipoib_port_t* const                     p_port,\r
2878         IN                              ipoib_send_desc_t* const        p_desc )\r
2879 {\r
2880         ib_api_status_t                 status;\r
2881         SCATTER_GATHER_LIST             *p_sgl;\r
2882         uint32_t                                i, j = 1;\r
2883         uint32_t                                offset = sizeof(eth_hdr_t);\r
2884         PERF_DECLARE( SendCopy );\r
2885 \r
2886         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2887 \r
2888         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2889                 ScatterGatherListPacketInfo );\r
2890         if( !p_sgl )\r
2891         {\r
2892                 ASSERT( p_sgl );\r
2893                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2894                         ("Failed to get SGL from packet.\n") );\r
2895                 return NDIS_STATUS_FAILURE;\r
2896         }\r
2897 \r
2898         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2899         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2900                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2901                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2902                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2903         {\r
2904                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2905                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2906                 cl_perf_start( SendCopy );\r
2907                 status = __send_copy( p_port, p_desc );\r
2908                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2909                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2910                 return status;\r
2911         }\r
2912 \r
2913         /*\r
2914          * Skip the ethernet header.  It is either the first element,\r
2915          * or part of it.\r
2916          */\r
2917         i = 0;\r
2918         while( offset )\r
2919         {\r
2920                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2921                 {\r
2922                         offset -= p_sgl->Elements[i++].Length;\r
2923                 }\r
2924                 else\r
2925                 {\r
2926                         p_desc->local_ds[j].vaddr =\r
2927                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2928                         p_desc->local_ds[j].length =\r
2929                                 p_sgl->Elements[i].Length - offset;\r
2930                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2931                         i++;\r
2932                         j++;\r
2933                         break;\r
2934                 }\r
2935         }\r
2936         /* Now fill in the rest of the local data segments. */\r
2937         while( i < p_sgl->NumberOfElements )\r
2938         {\r
2939                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2940                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2941                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2942                 i++;\r
2943                 j++;\r
2944         }\r
2945 \r
2946         /* Set the number of data segments. */\r
2947         p_desc->wr.num_ds = j;\r
2948 \r
2949         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2950         return NDIS_STATUS_SUCCESS;\r
2951 }\r
2952 #endif\r
2953 \r
2954 \r
2955 static NDIS_STATUS\r
2956 __send_mgr_filter_ip(\r
2957         IN                              ipoib_port_t* const                     p_port,\r
2958         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2959         IN                              NDIS_BUFFER*                            p_buf,\r
2960         IN                              size_t                                          buf_len,\r
2961         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2962 {\r
2963         NDIS_STATUS             status;\r
2964         ip_hdr_t                *p_ip_hdr;\r
2965 \r
2966         PERF_DECLARE( QueryIp );\r
2967         PERF_DECLARE( SendTcp );\r
2968         PERF_DECLARE( FilterUdp );\r
2969 \r
2970         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2971 \r
2972         if( !buf_len )\r
2973         {\r
2974                 cl_perf_start( QueryIp );\r
2975                 NdisGetNextBuffer( p_buf, &p_buf );\r
2976                 if( !p_buf )\r
2977                 {\r
2978                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2979                                 ("Failed to get IP header buffer.\n") );\r
2980                         return NDIS_STATUS_FAILURE;\r
2981                 }\r
2982                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2983                 if( !p_ip_hdr )\r
2984                 {\r
2985                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2986                                 ("Failed to query IP header buffer.\n") );\r
2987                         return NDIS_STATUS_FAILURE;\r
2988                 }\r
2989                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2990         }\r
2991         else\r
2992         {\r
2993                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2994         }\r
2995         if( buf_len < sizeof(ip_hdr_t) )\r
2996         {\r
2997                 /* This buffer is done for.  Get the next buffer. */\r
2998                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2999                         ("Buffer too small for IP packet.\n") );\r
3000                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
3001         }\r
3002 \r
3003         if( p_ip_hdr->offset ||\r
3004                 p_ip_hdr->prot != IP_PROT_UDP )\r
3005         {\r
3006                 /* Not a UDP packet. */\r
3007                 cl_perf_start( SendTcp );\r
3008                 status = __send_gen( p_port, p_desc );\r
3009                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
3010                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
3011                 return status;\r
3012         }\r
3013 \r
3014         buf_len -= sizeof(ip_hdr_t);\r
3015 \r
3016         cl_perf_start( FilterUdp );\r
3017         status = __send_mgr_filter_udp(\r
3018                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
3019         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
3020 \r
3021         IPOIB_EXIT( IPOIB_DBG_SEND );\r
3022         return status;\r
3023 }\r
3024 \r