[ipoib] mcast garbage collector and igmp V2 support.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
5  *\r
6  * This software is available to you under the OpenIB.org BSD license\r
7  * below:\r
8  *\r
9  *     Redistribution and use in source and binary forms, with or\r
10  *     without modification, are permitted provided that the following\r
11  *     conditions are met:\r
12  *\r
13  *      - Redistributions of source code must retain the above\r
14  *        copyright notice, this list of conditions and the following\r
15  *        disclaimer.\r
16  *\r
17  *      - Redistributions in binary form must reproduce the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer in the documentation and/or other materials\r
20  *        provided with the distribution.\r
21  *\r
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
23  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
26  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
27  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
28  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
29  * SOFTWARE.\r
30  *\r
31  * $Id$\r
32  */\r
33 \r
34 \r
35 \r
36 #include "ipoib_port.h"\r
37 #include "ipoib_adapter.h"\r
38 #include "ipoib_debug.h"\r
39 #if defined(EVENT_TRACING)\r
40 #ifdef offsetof\r
41 #undef offsetof\r
42 #endif\r
43 #include "ipoib_port.tmh"\r
44 #endif\r
45 \r
46 \r
47 /* Amount of physical memory to register. */\r
48 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
49 \r
50 /* Number of work completions to chain for send and receive polling. */\r
51 #define MAX_SEND_WC             8\r
52 #define MAX_RECV_WC             16\r
53 \r
54 \r
55 ib_gid_t        bcast_mgid_template = {\r
56         0xff,                                                           /* multicast field */\r
57         0x12,                                                           /* scope (to be filled in) */\r
58         0x40, 0x1b,                                                     /* IPv4 signature */\r
59         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
60         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
61         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
62 };\r
63 \r
64 \r
65 #ifdef _DEBUG_\r
66 /* Handy pointer for debug use. */\r
67 ipoib_port_t    *gp_ipoib_port;\r
68 #endif\r
69 \r
70 static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
71 static void __port_do_mcast_garbage(ipoib_port_t* const p_port );\r
72 /******************************************************************************\r
73 *\r
74 * Declarations\r
75 *\r
76 ******************************************************************************/\r
77 static void\r
78 __port_construct(\r
79         IN                              ipoib_port_t* const                     p_port );\r
80 \r
81 static ib_api_status_t\r
82 __port_init(\r
83         IN                              ipoib_port_t* const                     p_port,\r
84         IN                              ipoib_adapter_t* const          p_adapter,\r
85         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
86 \r
87 static void\r
88 __port_destroying(\r
89         IN                              cl_obj_t* const                         p_obj );\r
90 \r
91 static void\r
92 __port_cleanup(\r
93         IN                              cl_obj_t* const                         p_obj );\r
94 \r
95 static void\r
96 __port_free(\r
97         IN                              cl_obj_t* const                         p_obj );\r
98 \r
99 \r
100 /******************************************************************************\r
101 *\r
102 * IB resource manager operations\r
103 *\r
104 ******************************************************************************/\r
105 static void\r
106 __ib_mgr_construct(\r
107         IN                              ipoib_port_t* const                     p_port );\r
108 \r
109 static ib_api_status_t\r
110 __ib_mgr_init(\r
111         IN                              ipoib_port_t* const                     p_port );\r
112 \r
113 static void\r
114 __ib_mgr_destroy(\r
115         IN                              ipoib_port_t* const                     p_port );\r
116 \r
117 static void\r
118 __qp_event(\r
119         IN                              ib_async_event_rec_t            *p_event_rec );\r
120 \r
121 static void\r
122 __cq_event(\r
123         IN                              ib_async_event_rec_t            *p_event_rec );\r
124 \r
125 static ib_api_status_t\r
126 __ib_mgr_activate(\r
127         IN                              ipoib_port_t* const                     p_port );\r
128 \r
129 /******************************************************************************\r
130 *\r
131 * Buffer manager operations.\r
132 *\r
133 ******************************************************************************/\r
134 static void\r
135 __buf_mgr_construct(\r
136         IN                              ipoib_port_t* const                     p_port );\r
137 \r
138 static ib_api_status_t\r
139 __buf_mgr_init(\r
140         IN                              ipoib_port_t* const                     p_port );\r
141 \r
142 static void\r
143 __buf_mgr_destroy(\r
144         IN                              ipoib_port_t* const                     p_port );\r
145 \r
146 static cl_status_t\r
147 __recv_ctor(\r
148         IN                              void* const                                     p_object,\r
149         IN                              void*                                           context,\r
150                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
151 \r
152 #if !IPOIB_INLINE_RECV\r
153 static void\r
154 __recv_dtor(\r
155         IN              const   cl_pool_item_t* const           p_pool_item,\r
156         IN                              void                                            *context );\r
157 #endif  /* IPOIB_INLINE_RECV */\r
158 \r
159 static inline ipoib_send_desc_t*\r
160 __buf_mgr_get_send(\r
161         IN                              ipoib_port_t* const                     p_port );\r
162 \r
163 static inline void\r
164 __buf_mgr_put_send(\r
165         IN                              ipoib_port_t* const                     p_port,\r
166         IN                              ipoib_send_desc_t* const        p_desc );\r
167 \r
168 static inline ipoib_recv_desc_t*\r
169 __buf_mgr_get_recv(\r
170         IN                              ipoib_port_t* const                     p_port );\r
171 \r
172 static inline void\r
173 __buf_mgr_put_recv(\r
174         IN                              ipoib_port_t* const                     p_port,\r
175         IN                              ipoib_recv_desc_t* const        p_desc,\r
176         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
177 \r
178 static inline void\r
179 __buf_mgr_put_recv_list(\r
180         IN                              ipoib_port_t* const                     p_port,\r
181         IN                              cl_qlist_t* const                       p_list );\r
182 \r
183 static inline NDIS_PACKET*\r
184 __buf_mgr_get_ndis_pkt(\r
185         IN                              ipoib_port_t* const                     p_port,\r
186         IN                              ipoib_recv_desc_t* const        p_desc );\r
187 \r
188 \r
189 /******************************************************************************\r
190 *\r
191 * Receive manager operations.\r
192 *\r
193 ******************************************************************************/\r
194 static void\r
195 __recv_mgr_construct(\r
196         IN                              ipoib_port_t* const                     p_port );\r
197 \r
198 static ib_api_status_t\r
199 __recv_mgr_init(\r
200         IN                              ipoib_port_t* const                     p_port );\r
201 \r
202 static void\r
203 __recv_mgr_destroy(\r
204         IN                              ipoib_port_t* const                     p_port );\r
205 \r
206 /* Posts receive buffers to the receive queue. */\r
207 static ib_api_status_t\r
208 __recv_mgr_repost(\r
209         IN                              ipoib_port_t* const                     p_port );\r
210 \r
211 static void\r
212 __recv_cb(\r
213         IN              const   ib_cq_handle_t                          h_cq,\r
214         IN                              void                                            *cq_context );\r
215 \r
216 static void\r
217 __recv_get_endpts(\r
218         IN                              ipoib_port_t* const                     p_port,\r
219         IN                              ipoib_recv_desc_t* const        p_desc,\r
220         IN                              ib_wc_t* const                          p_wc,\r
221                 OUT                     ipoib_endpt_t** const           pp_src,\r
222                 OUT                     ipoib_endpt_t** const           pp_dst );\r
223 \r
224 static int32_t\r
225 __recv_mgr_filter(\r
226         IN                              ipoib_port_t* const                     p_port,\r
227         IN                              ib_wc_t* const                          p_done_wc_list,\r
228                 OUT                     cl_qlist_t* const                       p_done_list,\r
229                 OUT                     cl_qlist_t* const                       p_bad_list );\r
230 \r
231 static ib_api_status_t\r
232 __recv_gen(\r
233         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
234                 OUT                     eth_pkt_t* const                        p_eth,\r
235         IN                              ipoib_endpt_t* const            p_src,\r
236         IN                              ipoib_endpt_t* const            p_dst );\r
237 \r
238 static ib_api_status_t\r
239 __recv_dhcp(\r
240         IN                              ipoib_port_t* const                     p_port,\r
241         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
242                 OUT                     eth_pkt_t* const                        p_eth,\r
243         IN                              ipoib_endpt_t* const            p_src,\r
244         IN                              ipoib_endpt_t* const            p_dst );\r
245 \r
246 static ib_api_status_t\r
247 __recv_arp(\r
248         IN                              ipoib_port_t* const                     p_port,\r
249         IN                              ib_wc_t* const                          p_wc,\r
250         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
251                 OUT                     eth_pkt_t* const                        p_eth,\r
252         IN                              ipoib_endpt_t** const           p_src,\r
253         IN                              ipoib_endpt_t* const            p_dst );\r
254 \r
255 static ib_api_status_t\r
256 __recv_mgr_prepare_pkt(\r
257         IN                              ipoib_port_t* const                     p_port,\r
258         IN                              ipoib_recv_desc_t* const        p_desc,\r
259                 OUT                     NDIS_PACKET** const                     pp_packet );\r
260 \r
261 static uint32_t\r
262 __recv_mgr_build_pkt_array(\r
263         IN                              ipoib_port_t* const                     p_port,\r
264         IN                              int32_t                                         shortage,\r
265                 OUT                     cl_qlist_t* const                       p_done_list,\r
266                 OUT                     int32_t* const                          p_discarded );\r
267 \r
268 /******************************************************************************\r
269 *\r
270 * Send manager operations.\r
271 *\r
272 ******************************************************************************/\r
273 static void\r
274 __send_mgr_construct(\r
275         IN                              ipoib_port_t* const                     p_port );\r
276 \r
277 static void\r
278 __send_mgr_destroy(\r
279         IN                              ipoib_port_t* const                     p_port );\r
280 \r
281 static NDIS_STATUS\r
282 __send_gen(\r
283         IN                              ipoib_port_t* const                     p_port,\r
284         IN                              ipoib_send_desc_t* const        p_desc );\r
285 \r
286 static NDIS_STATUS\r
287 __send_mgr_filter_ip(\r
288         IN                              ipoib_port_t* const                     p_port,\r
289         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
290         IN                              NDIS_BUFFER*                            p_buf,\r
291         IN                              size_t                                          buf_len,\r
292         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
293 \r
294 static NDIS_STATUS\r
295 __send_mgr_filter_igmp_v2(\r
296         IN                              ipoib_port_t* const                     p_port,\r
297     IN          const   ip_hdr_t* const                         p_ip_hdr,\r
298         IN                              size_t                                          iph_options_size,\r
299         IN                              NDIS_BUFFER*                            p_buf,\r
300         IN                              size_t                                          buf_len );\r
301 \r
302 static NDIS_STATUS\r
303 __send_mgr_filter_udp(\r
304         IN                              ipoib_port_t* const                     p_port,\r
305         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
306         IN                              NDIS_BUFFER*                            p_buf,\r
307         IN                              size_t                                          buf_len,\r
308         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
309 \r
310 static NDIS_STATUS\r
311 __send_mgr_filter_dhcp(\r
312         IN                              ipoib_port_t* const                     p_port,\r
313         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
314         IN                              NDIS_BUFFER*                            p_buf,\r
315         IN                              size_t                                          buf_len,\r
316         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
317 \r
318 static NDIS_STATUS\r
319 __send_mgr_filter_arp(\r
320         IN                              ipoib_port_t* const                     p_port,\r
321         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
322         IN                              NDIS_BUFFER*                            p_buf,\r
323         IN                              size_t                                          buf_len,\r
324         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
325 \r
326 static void\r
327 __process_failed_send(\r
328         IN                              ipoib_port_t* const                     p_port,\r
329         IN                              ipoib_send_desc_t* const        p_desc,\r
330         IN              const   NDIS_STATUS                                     status );\r
331 \r
332 static void\r
333 __send_cb(\r
334         IN              const   ib_cq_handle_t                          h_cq,\r
335         IN                              void                                            *cq_context );\r
336 \r
337 \r
338 /******************************************************************************\r
339 *\r
340 * Endpoint manager operations\r
341 *\r
342 ******************************************************************************/\r
343 static void\r
344 __endpt_mgr_construct(\r
345         IN                              ipoib_port_t* const                     p_port );\r
346 \r
347 static ib_api_status_t\r
348 __endpt_mgr_init(\r
349         IN                              ipoib_port_t* const                     p_port );\r
350 \r
351 static void\r
352 __endpt_mgr_destroy(\r
353         IN                              ipoib_port_t* const                     p_port );\r
354 \r
355 /****f* IPoIB/__endpt_mgr_remove_all\r
356 * NAME\r
357 *       __endpt_mgr_remove_all\r
358 *\r
359 * DESCRIPTION\r
360 *       Removes all enpoints from the port, dereferencing them to initiate\r
361 *       destruction.\r
362 *\r
363 * SYNOPSIS\r
364 */\r
365 static void\r
366 __endpt_mgr_remove_all(\r
367         IN                              ipoib_port_t* const                     p_port );\r
368 /*\r
369 ********/\r
370 \r
371 static void\r
372 __endpt_mgr_remove(\r
373         IN                              ipoib_port_t* const                     p_port,\r
374         IN                              ipoib_endpt_t* const            p_endpt );\r
375 \r
376 static void\r
377 __endpt_mgr_reset_all(\r
378         IN                              ipoib_port_t* const                     p_port );\r
379 \r
380 static inline NDIS_STATUS\r
381 __endpt_mgr_ref(\r
382         IN                              ipoib_port_t* const                     p_port,\r
383         IN              const   mac_addr_t                                      mac,\r
384                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
385 \r
386 static inline NDIS_STATUS\r
387 __endpt_mgr_get_gid_qpn(\r
388         IN                              ipoib_port_t* const                     p_port,\r
389         IN              const   mac_addr_t                                      mac,\r
390                 OUT                     ib_gid_t* const                         p_gid,\r
391                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
392 \r
393 static inline ipoib_endpt_t*\r
394 __endpt_mgr_get_by_gid(\r
395         IN                              ipoib_port_t* const                     p_port,\r
396         IN              const   ib_gid_t* const                         p_gid );\r
397 \r
398 static inline ipoib_endpt_t*\r
399 __endpt_mgr_get_by_lid(\r
400         IN                              ipoib_port_t* const                     p_port,\r
401         IN              const   net16_t                                         lid );\r
402 \r
403 static inline ib_api_status_t\r
404 __endpt_mgr_insert_locked(\r
405         IN                              ipoib_port_t* const                     p_port,\r
406         IN              const   mac_addr_t                                      mac,\r
407         IN                              ipoib_endpt_t* const            p_endpt );\r
408 \r
409 static inline ib_api_status_t\r
410 __endpt_mgr_insert(\r
411         IN                              ipoib_port_t* const                     p_port,\r
412         IN              const   mac_addr_t                                      mac,\r
413         IN                              ipoib_endpt_t* const            p_endpt );\r
414 \r
415 static ib_api_status_t\r
416 __endpt_mgr_add_local(\r
417         IN                              ipoib_port_t* const                     p_port,\r
418         IN                              ib_port_info_t* const           p_port_info );\r
419 \r
420 static ib_api_status_t\r
421 __endpt_mgr_add_bcast(\r
422         IN                              ipoib_port_t* const                     p_port,\r
423         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
424 \r
425 /******************************************************************************\r
426 *\r
427 * MCast operations.\r
428 *\r
429 ******************************************************************************/\r
430 static ib_api_status_t\r
431 __port_get_bcast(\r
432         IN                              ipoib_port_t* const                     p_port );\r
433 \r
434 static ib_api_status_t\r
435 __port_join_bcast(\r
436         IN                              ipoib_port_t* const                     p_port,\r
437         IN                              ib_member_rec_t* const          p_member_rec );\r
438 \r
439 static ib_api_status_t\r
440 __port_create_bcast(\r
441         IN                              ipoib_port_t* const                     p_port );\r
442 \r
443 static void\r
444 __port_info_cb(\r
445         IN                              ib_query_rec_t                          *p_query_rec );\r
446 \r
447 \r
448 static void\r
449 __bcast_get_cb(\r
450         IN                              ib_query_rec_t                          *p_query_rec );\r
451 \r
452 \r
453 static void\r
454 __bcast_cb(\r
455         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
456 \r
457 \r
458 static void\r
459 __mcast_cb(\r
460         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
461 \r
462 void\r
463 __leave_error_mcast_cb(\r
464         IN                              void                            *context );\r
465 \r
466 \r
467 static intn_t\r
468 __gid_cmp(\r
469         IN              const   void* const                                     p_key1,\r
470         IN              const   void* const                                     p_key2 )\r
471 {\r
472         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
473 }\r
474 \r
475 \r
476 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
477 {\r
478         cl_obj_ref( &p_port->obj );\r
479 #if DBG\r
480         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
481         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
482                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
483 #else\r
484         UNREFERENCED_PARAMETER(type);\r
485 #endif\r
486 }\r
487 \r
488 \r
489 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
490 {\r
491         cl_obj_deref( &p_port->obj );\r
492 \r
493 #if DBG\r
494         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
495         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
496                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
497 #else\r
498         UNREFERENCED_PARAMETER(type);\r
499 #endif\r
500 }\r
501 \r
502 /* function returns pointer to payload that is going after IP header.\r
503 *  asssuming that payload and IP header are in the same buffer\r
504 */\r
505 static void* GetIpPayloadPtr(const      ip_hdr_t* const p_ip_hdr)\r
506 {\r
507         return (void*)((uint8_t*)p_ip_hdr + 4*(p_ip_hdr->ver_hl & 0xf));\r
508 }\r
509 \r
510 /******************************************************************************\r
511 *\r
512 * Implementation\r
513 *\r
514 ******************************************************************************/\r
515 ib_api_status_t\r
516 ipoib_create_port(\r
517         IN                              ipoib_adapter_t* const          p_adapter,\r
518         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
519                 OUT                     ipoib_port_t** const            pp_port )\r
520 {\r
521         ib_api_status_t         status;\r
522         ipoib_port_t            *p_port;\r
523 \r
524         IPOIB_ENTER( IPOIB_DBG_INIT );\r
525 \r
526         CL_ASSERT( !p_adapter->p_port );\r
527 \r
528         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
529                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
530         if( !p_port )\r
531         {\r
532                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
533                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
534                         sizeof(ipoib_port_t)) );\r
535                 return IB_INSUFFICIENT_MEMORY;\r
536         }\r
537 \r
538 #ifdef _DEBUG_\r
539         gp_ipoib_port = p_port;\r
540 #endif\r
541 \r
542         __port_construct( p_port );\r
543 \r
544         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
545         if( status != IB_SUCCESS )\r
546         {\r
547                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
548                         ("ipoib_port_init returned %s.\n", \r
549                         p_adapter->p_ifc->get_err_str( status )) );\r
550                 __port_cleanup( &p_port->obj );\r
551                 __port_free( &p_port->obj );\r
552                 return status;\r
553         }\r
554 \r
555         *pp_port = p_port;\r
556         IPOIB_EXIT( IPOIB_DBG_INIT );\r
557         return IB_SUCCESS;\r
558 }\r
559 \r
560 \r
561 void\r
562 ipoib_port_destroy(\r
563         IN                              ipoib_port_t* const                     p_port )\r
564 {\r
565         IPOIB_ENTER( IPOIB_DBG_INIT );\r
566 \r
567         CL_ASSERT( p_port );\r
568         CL_ASSERT( p_port->p_adapter );\r
569         CL_ASSERT( !p_port->p_adapter->p_port );\r
570 \r
571         cl_obj_destroy( &p_port->obj );\r
572 \r
573         IPOIB_EXIT( IPOIB_DBG_INIT );\r
574 }\r
575 \r
576 \r
577 static void\r
578 __port_construct(\r
579         IN                              ipoib_port_t* const                     p_port )\r
580 {\r
581         IPOIB_ENTER( IPOIB_DBG_INIT );\r
582 \r
583         p_port->state = IB_QPS_RESET;\r
584 \r
585         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
586         cl_spinlock_construct( &p_port->send_lock );\r
587         cl_spinlock_construct( &p_port->recv_lock );\r
588         __ib_mgr_construct( p_port );\r
589         __buf_mgr_construct( p_port );\r
590 \r
591         __recv_mgr_construct( p_port );\r
592         __send_mgr_construct( p_port );\r
593 \r
594         __endpt_mgr_construct( p_port );\r
595 \r
596         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
597         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
598         \r
599         IPOIB_EXIT( IPOIB_DBG_INIT );\r
600 }\r
601 \r
602 \r
603 static ib_api_status_t\r
604 __port_init(\r
605         IN                              ipoib_port_t* const                     p_port,\r
606         IN                              ipoib_adapter_t* const          p_adapter,\r
607         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
608 {\r
609         cl_status_t                     cl_status;\r
610         ib_api_status_t         status;\r
611 \r
612         IPOIB_ENTER( IPOIB_DBG_INIT );\r
613 \r
614         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
615         p_port->p_adapter = p_adapter;\r
616 \r
617         cl_status = cl_spinlock_init( &p_port->send_lock );\r
618         if( cl_status != CL_SUCCESS )\r
619         {\r
620                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
621                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
622                 return IB_ERROR;\r
623         }\r
624 \r
625         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
626         if( cl_status != CL_SUCCESS )\r
627         {\r
628                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
629                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
630                 return IB_ERROR;\r
631         }\r
632 \r
633         /* Initialize the IB resource manager. */\r
634         status = __ib_mgr_init( p_port );\r
635         if( status != IB_SUCCESS )\r
636         {\r
637                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
638                         ("__ib_mgr_init returned %s\n", \r
639                         p_adapter->p_ifc->get_err_str( status )) );\r
640                 return status;\r
641         }\r
642 \r
643         /* Initialize the buffer manager. */\r
644         status = __buf_mgr_init( p_port );\r
645         if( status != IB_SUCCESS )\r
646         {\r
647                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
648                         ("__buf_mgr_init returned %s\n", \r
649                         p_adapter->p_ifc->get_err_str( status )) );\r
650                 return status;\r
651         }\r
652 \r
653         /* Initialize the receive manager. */\r
654         status = __recv_mgr_init( p_port );\r
655         if( status != IB_SUCCESS )\r
656         {\r
657                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
658                         ("__recv_mgr_init returned %s\n", \r
659                         p_adapter->p_ifc->get_err_str( status )) );\r
660                 return status;\r
661         }\r
662 \r
663         /* Initialize the endpoint manager. */\r
664         status = __endpt_mgr_init( p_port );\r
665         if( status != IB_SUCCESS )\r
666         {\r
667                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
668                         ("__endpt_mgr_init returned %s\n", \r
669                         p_adapter->p_ifc->get_err_str( status )) );\r
670                 return status;\r
671         }\r
672          /* Initialize multicast garbage collector timer and DPC object */\r
673          KeInitializeDpc(&p_port->gc_dpc,(PKDEFERRED_ROUTINE)__port_mcast_garbage_dpc,p_port);\r
674          KeInitializeTimerEx(&p_port->gc_timer,SynchronizationTimer);\r
675 \r
676         /* We only ever destroy from the PnP callback thread. */\r
677         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
678                 __port_destroying, __port_cleanup, __port_free );\r
679 \r
680 #if DBG\r
681         cl_atomic_inc( &p_port->ref[ref_init] );\r
682         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
683                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
684 #endif\r
685 \r
686         if( cl_status != CL_SUCCESS )\r
687         {\r
688                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
689                         ("cl_obj_init returned %#x\n", cl_status) );\r
690                 return IB_ERROR;\r
691         }\r
692 \r
693         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
694         if( cl_status != CL_SUCCESS )\r
695         {\r
696                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
697                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
698                 cl_obj_destroy( &p_port->obj );\r
699                 return IB_ERROR;\r
700         }\r
701 \r
702 #if DBG\r
703         cl_atomic_inc( &p_port->ref[ref_init] );\r
704         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
705                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
706 #endif\r
707 \r
708         IPOIB_EXIT( IPOIB_DBG_INIT );\r
709         return IB_SUCCESS;\r
710 }\r
711 \r
712 \r
713 static void\r
714 __port_destroying(\r
715         IN                              cl_obj_t* const                         p_obj )\r
716 {\r
717         ipoib_port_t    *p_port;\r
718 \r
719         IPOIB_ENTER( IPOIB_DBG_INIT );\r
720 \r
721         CL_ASSERT( p_obj );\r
722 \r
723         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
724 \r
725         ipoib_port_down( p_port );\r
726 \r
727         __endpt_mgr_remove_all( p_port );\r
728 \r
729         ipoib_port_resume( p_port );\r
730 \r
731         IPOIB_EXIT( IPOIB_DBG_INIT );\r
732 }\r
733 \r
734 \r
735 static void\r
736 __port_cleanup(\r
737         IN                              cl_obj_t* const                         p_obj )\r
738 {\r
739         ipoib_port_t    *p_port;\r
740 \r
741         IPOIB_ENTER( IPOIB_DBG_INIT );\r
742 \r
743         CL_ASSERT( p_obj );\r
744 \r
745         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
746 \r
747         /* Wait for all sends and receives to get flushed. */\r
748         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
749                 cl_thread_suspend( 0 );\r
750 \r
751         /* Destroy the send and receive managers before closing the CA. */\r
752         __ib_mgr_destroy( p_port );\r
753 \r
754         IPOIB_EXIT( IPOIB_DBG_INIT );\r
755 }\r
756 \r
757 \r
758 static void\r
759 __port_free(\r
760         IN                              cl_obj_t* const                         p_obj )\r
761 {\r
762         ipoib_port_t    *p_port;\r
763 \r
764         IPOIB_ENTER( IPOIB_DBG_INIT );\r
765 \r
766         CL_ASSERT( p_obj );\r
767 \r
768         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
769 \r
770         KeCancelTimer(&p_port->gc_timer);\r
771         KeFlushQueuedDpcs();\r
772         __endpt_mgr_destroy( p_port );\r
773         __recv_mgr_destroy( p_port );\r
774         __send_mgr_destroy( p_port );\r
775         __buf_mgr_destroy( p_port );\r
776 \r
777         cl_spinlock_destroy( &p_port->send_lock );\r
778         cl_spinlock_destroy( &p_port->recv_lock );\r
779 \r
780         cl_obj_deinit( p_obj );\r
781 \r
782         cl_free( p_port );\r
783 \r
784         IPOIB_EXIT( IPOIB_DBG_INIT );\r
785 }\r
786 \r
787 \r
788 \r
789 /******************************************************************************\r
790 *\r
791 * IB resource manager implementation.\r
792 *\r
793 ******************************************************************************/\r
794 static void\r
795 __ib_mgr_construct(\r
796         IN                              ipoib_port_t* const                     p_port )\r
797 {\r
798         IPOIB_ENTER( IPOIB_DBG_INIT );\r
799 \r
800         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
801 \r
802         IPOIB_EXIT( IPOIB_DBG_INIT );\r
803 }\r
804 \r
805 \r
806 static ib_api_status_t\r
807 __ib_mgr_init(\r
808         IN                              ipoib_port_t* const                     p_port )\r
809 {\r
810         ib_api_status_t         status;\r
811         ib_cq_create_t          cq_create;\r
812         ib_qp_create_t          qp_create;\r
813         ib_phys_create_t        phys_create;\r
814         ib_phys_range_t         phys_range;\r
815         uint64_t                        vaddr;\r
816         net32_t                         rkey;\r
817         ib_qp_attr_t            qp_attr;\r
818         ib_ca_attr_t            *ca_attr;\r
819         uint32_t                        ca_size;\r
820 \r
821         IPOIB_ENTER( IPOIB_DBG_INIT );\r
822 \r
823         /* Open the CA. */\r
824         status = p_port->p_adapter->p_ifc->open_ca(\r
825                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
826                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
827         if( status != IB_SUCCESS )\r
828         {\r
829                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
830                         EVENT_IPOIB_OPEN_CA, 1, status );\r
831                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
832                         ("ib_open_ca returned %s\n", \r
833                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
834                 return status;\r
835         }\r
836 \r
837         /* Query the CA for Pkey table */\r
838         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, NULL, &ca_size);\r
839         if(status != IB_INSUFFICIENT_MEMORY)\r
840         {\r
841                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
842                         ("ib_query_ca failed\n"));\r
843                         return status;\r
844         }\r
845 \r
846         ca_attr = (ib_ca_attr_t*)cl_zalloc(ca_size);\r
847         if      (!ca_attr)\r
848         {\r
849                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
850                         ("cl_zalloc can't allocate %d\n",ca_size));\r
851                 return IB_INSUFFICIENT_RESOURCES;\r
852         }\r
853 \r
854         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, ca_attr,&ca_size);     \r
855         if( status != IB_SUCCESS )\r
856         {\r
857                 cl_free(ca_attr);\r
858                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
859                         ("ib_query_ca returned %s\n", \r
860                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
861                 return status;\r
862         }\r
863         if( ca_attr->p_port_attr->link_state == IB_LINK_ACTIVE)\r
864         {\r
865                 uint16_t index;\r
866                 CL_ASSERT(ca_attr->p_port_attr->p_pkey_table[0] == IB_DEFAULT_PKEY);\r
867                 for(index = 0; index < ca_attr->p_port_attr->num_pkeys; index++)\r
868                 {\r
869                         if(p_port->p_adapter->guids.port_guid.pkey == ca_attr->p_port_attr->p_pkey_table[index])\r
870                                 break;\r
871                 }\r
872                 if(index >= ca_attr->p_port_attr->num_pkeys)\r
873                 {\r
874                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
875                         ("Pkey table is invalid, index not found\n"));\r
876                     return IB_NOT_FOUND;\r
877                 }\r
878                 p_port->pkey_index = index;\r
879                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_IB,\r
880                         ("for PKEY = 0x%04X got index = %d\n",p_port->p_adapter->guids.port_guid.pkey,index));\r
881         }\r
882         cl_free(ca_attr);\r
883         /* Allocate the PD. */\r
884         status = p_port->p_adapter->p_ifc->alloc_pd(\r
885                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
886         if( status != IB_SUCCESS )\r
887         {\r
888                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
889                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
890                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
891                         ("ib_alloc_pd returned %s\n", \r
892                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
893                 return status;\r
894         }\r
895 \r
896         /* Allocate receive CQ. */\r
897         cq_create.size = p_port->p_adapter->params.rq_depth;\r
898         cq_create.pfn_comp_cb = __recv_cb;\r
899         cq_create.h_wait_obj = NULL;\r
900 \r
901         status = p_port->p_adapter->p_ifc->create_cq(\r
902                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
903                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
904         if( status != IB_SUCCESS )\r
905         {\r
906                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
907                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
908                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
909                         ("ib_create_cq returned %s.\n", \r
910                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
911                 return status;\r
912         }\r
913 \r
914         /* Allocate send CQ. */\r
915         cq_create.size = p_port->p_adapter->params.sq_depth;\r
916         cq_create.pfn_comp_cb = __send_cb;\r
917 \r
918         status = p_port->p_adapter->p_ifc->create_cq(\r
919                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
920                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
921         if( status != IB_SUCCESS )\r
922         {\r
923                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
924                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
925                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
926                         ("ib_create_cq returned %s.\n", \r
927                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
928                 return status;\r
929         }\r
930 \r
931         /* Allocate the QP. */\r
932         cl_memclr( &qp_create, sizeof(qp_create) );\r
933         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
934         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
935         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
936         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
937         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
938         //TODO: Figure out the right number of SGE entries for sends.\r
939         qp_create.sq_sge = MAX_SEND_SGE;\r
940         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
941         qp_create.sq_signaled = TRUE;\r
942         status = p_port->p_adapter->p_ifc->create_qp(\r
943                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
944                 __qp_event, &p_port->ib_mgr.h_qp );\r
945         if( status != IB_SUCCESS )\r
946         {\r
947                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
948                         EVENT_IPOIB_CREATE_QP, 1, status );\r
949                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
950                         ("ib_create_qp returned %s\n", \r
951                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
952                 return status;\r
953         }\r
954         /* Query the QP so we can get our QPN. */\r
955         status = p_port->p_adapter->p_ifc->query_qp(\r
956                 p_port->ib_mgr.h_qp, &qp_attr );\r
957         if( status != IB_SUCCESS )\r
958         {\r
959                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
960                         EVENT_IPOIB_QUERY_QP, 1, status );\r
961                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
962                         ("ib_query_qp returned %s\n", \r
963                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
964                 return status;\r
965         }\r
966         p_port->ib_mgr.qpn = qp_attr.num;\r
967 \r
968         /* Register all of physical memory */\r
969         phys_create.length = MEM_REG_SIZE;\r
970         phys_create.num_ranges = 1;\r
971         phys_create.range_array = &phys_range;\r
972         phys_create.buf_offset = 0;\r
973         phys_create.hca_page_size = PAGE_SIZE;\r
974         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
975         phys_range.base_addr = 0;\r
976         phys_range.size = MEM_REG_SIZE;\r
977         vaddr = 0;\r
978         status = p_port->p_adapter->p_ifc->reg_phys(\r
979                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
980                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
981         if( status != IB_SUCCESS )\r
982         {\r
983                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
984                         EVENT_IPOIB_REG_PHYS, 1, status );\r
985                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
986                         ("ib_reg_phys returned %s\n", \r
987                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
988                 return status;\r
989         }\r
990 \r
991         IPOIB_EXIT( IPOIB_DBG_INIT );\r
992         return IB_SUCCESS;\r
993 }\r
994 \r
995 \r
996 static void\r
997 __ib_mgr_destroy(\r
998         IN                              ipoib_port_t* const                     p_port )\r
999 {\r
1000         ib_api_status_t status;\r
1001 \r
1002         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1003 \r
1004         if( p_port->ib_mgr.h_ca )\r
1005         {\r
1006                 status =\r
1007                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
1008                 CL_ASSERT( status == IB_SUCCESS );\r
1009                 p_port->ib_mgr.h_ca = NULL;\r
1010         }\r
1011 \r
1012         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1013 }\r
1014 \r
1015 \r
1016 \r
1017 /******************************************************************************\r
1018 *\r
1019 * Buffer manager implementation.\r
1020 *\r
1021 ******************************************************************************/\r
1022 static void\r
1023 __buf_mgr_construct(\r
1024         IN                              ipoib_port_t* const                     p_port )\r
1025 {\r
1026         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1027 \r
1028         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1029 \r
1030         p_port->buf_mgr.h_packet_pool = NULL;\r
1031         p_port->buf_mgr.h_buffer_pool = NULL;\r
1032 \r
1033         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1034                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1035 \r
1036         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1037         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1038 \r
1039         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1040 }\r
1041 \r
1042 \r
1043 static ib_api_status_t\r
1044 __buf_mgr_init(\r
1045         IN                              ipoib_port_t* const                     p_port )\r
1046 {\r
1047         cl_status_t             cl_status;\r
1048         NDIS_STATUS             ndis_status;\r
1049         ipoib_params_t  *p_params;\r
1050 \r
1051         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1052 \r
1053         CL_ASSERT( p_port );\r
1054         CL_ASSERT( p_port->p_adapter );\r
1055 \r
1056         p_params = &p_port->p_adapter->params;\r
1057 \r
1058         /* Allocate the receive descriptor pool */\r
1059         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1060                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1061 #if IPOIB_INLINE_RECV\r
1062                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1063 #else   /* IPOIB_INLINE_RECV */\r
1064                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1065 #endif  /* IPOIB_INLINE_RECV */\r
1066         if( cl_status != CL_SUCCESS )\r
1067         {\r
1068                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1069                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1070                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1071                         ("cl_qpool_init for recvs returned %#x\n",\r
1072                         cl_status) );\r
1073                 return IB_INSUFFICIENT_MEMORY;\r
1074         }\r
1075 \r
1076         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1077         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1078                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1079         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1080         {\r
1081                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1082                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1083                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1084                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1085                 return IB_INSUFFICIENT_RESOURCES;\r
1086         }\r
1087 \r
1088         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1089                 p_params->rq_depth );\r
1090         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1091         {\r
1092                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1093                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1094                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1095                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1096                 return IB_INSUFFICIENT_RESOURCES;\r
1097         }\r
1098 \r
1099         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1100         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1101                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1102         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1103         {\r
1104                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1105                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1106                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1107                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1108                 return IB_INSUFFICIENT_RESOURCES;\r
1109         }\r
1110 \r
1111         NdisAllocateBufferPool( &ndis_status,\r
1112                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1113         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1114         {\r
1115                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1116                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1117                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1118                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1119                 return IB_INSUFFICIENT_RESOURCES;\r
1120         }\r
1121 \r
1122         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1123         return IB_SUCCESS;\r
1124 }\r
1125 \r
1126 \r
1127 static void\r
1128 __buf_mgr_destroy(\r
1129         IN                              ipoib_port_t* const                     p_port )\r
1130 {\r
1131         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1132 \r
1133         CL_ASSERT( p_port );\r
1134 \r
1135         /* Destroy the send packet and buffer pools. */\r
1136         if( p_port->buf_mgr.h_send_buf_pool )\r
1137                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1138         if( p_port->buf_mgr.h_send_pkt_pool )\r
1139                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1140 \r
1141         /* Destroy the receive packet and buffer pools. */\r
1142         if( p_port->buf_mgr.h_buffer_pool )\r
1143                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1144         if( p_port->buf_mgr.h_packet_pool )\r
1145                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1146 \r
1147         /* Free the receive and send descriptors. */\r
1148         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1149 \r
1150         /* Free the lookaside list of scratch buffers. */\r
1151         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1152 \r
1153         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1154 }\r
1155 \r
1156 \r
1157 static cl_status_t\r
1158 __recv_ctor(\r
1159         IN                              void* const                                     p_object,\r
1160         IN                              void*                                           context,\r
1161                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1162 {\r
1163         ipoib_recv_desc_t       *p_desc;\r
1164         ipoib_port_t            *p_port;\r
1165         uint32_t                        ds0_len;\r
1166 \r
1167         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1168 \r
1169         CL_ASSERT( p_object );\r
1170         CL_ASSERT( context );\r
1171 \r
1172         p_desc = (ipoib_recv_desc_t*)p_object;\r
1173         p_port = (ipoib_port_t*)context;\r
1174 \r
1175         /* Setup the work request. */\r
1176         p_desc->wr.ds_array = p_desc->local_ds;\r
1177         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1178 \r
1179 #if IPOIB_INLINE_RECV\r
1180         /* Sanity check on the receive buffer layout */\r
1181         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1182                 (void*)&p_desc->buf.ib.pkt.type );\r
1183         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1184 \r
1185         /* Setup the local data segment. */\r
1186         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1187         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1188         ds0_len =\r
1189                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1190         if( ds0_len >= sizeof(recv_buf_t) )\r
1191         {\r
1192                 /* The whole buffer is within a page. */\r
1193                 p_desc->local_ds[0].length = ds0_len;\r
1194                 p_desc->wr.num_ds = 1;\r
1195         }\r
1196         else\r
1197         {\r
1198                 /* The buffer crosses page boundaries. */\r
1199                 p_desc->local_ds[0].length = ds0_len;\r
1200                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1201                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1202                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1203                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1204                 p_desc->wr.num_ds = 2;\r
1205         }\r
1206 #else   /* IPOIB_INLINE_RECV */\r
1207         /* Allocate the receive buffer. */\r
1208         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1209         if( !p_desc->p_buf )\r
1210         {\r
1211                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1212                         ("Failed to allocate receive buffer.\n") );\r
1213                 return CL_INSUFFICIENT_MEMORY;\r
1214         }\r
1215 \r
1216         /* Sanity check on the receive buffer layout */\r
1217         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1218                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1219 \r
1220         /* Setup the local data segment. */\r
1221         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1222         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1223         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1224 #endif  /* IPOIB_INLINE_RECV */\r
1225 \r
1226         *pp_pool_item = &p_desc->item;\r
1227 \r
1228         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1229         return CL_SUCCESS;\r
1230 }\r
1231 \r
1232 \r
1233 #if !IPOIB_INLINE_RECV\r
1234 static void\r
1235 __recv_dtor(\r
1236         IN              const   cl_pool_item_t* const           p_pool_item,\r
1237         IN                              void                                            *context )\r
1238 {\r
1239         ipoib_recv_desc_t       *p_desc;\r
1240 \r
1241         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1242 \r
1243         UNUSED_PARAM( context );\r
1244 \r
1245         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1246 \r
1247         if( p_desc->p_buf )\r
1248                 cl_free( p_desc->p_buf );\r
1249 \r
1250         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1251 }\r
1252 #endif\r
1253 \r
1254 \r
1255 static inline ipoib_recv_desc_t*\r
1256 __buf_mgr_get_recv(\r
1257         IN                              ipoib_port_t* const                     p_port )\r
1258 {\r
1259         ipoib_recv_desc_t       *p_desc;\r
1260         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1261         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1262         /* Reference the port object for the send. */\r
1263         if( p_desc )\r
1264         {\r
1265                 ipoib_port_ref( p_port, ref_get_recv );\r
1266                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1267 #if IPOIB_INLINE_RECV\r
1268                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1269                         cl_get_physaddr( &p_desc->buf ) );\r
1270 #else   /* IPOIB_INLINE_RECV */\r
1271                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1272                         cl_get_physaddr( p_desc->p_buf ) );\r
1273                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1274                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1275 #endif  /* IPOIB_INLINE_RECV */\r
1276                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1277         }\r
1278         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1279         return p_desc;\r
1280 }\r
1281 \r
1282 \r
1283 static inline void\r
1284 __buf_mgr_put_recv(\r
1285         IN                              ipoib_port_t* const                     p_port,\r
1286         IN                              ipoib_recv_desc_t* const        p_desc,\r
1287         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1288 {\r
1289         NDIS_BUFFER             *p_buf;\r
1290 \r
1291         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1292 \r
1293         if( p_packet )\r
1294         {\r
1295                 /* Unchain the NDIS buffer. */\r
1296                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1297                 CL_ASSERT( p_buf );\r
1298                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1299                 NdisDprFreePacketNonInterlocked( p_packet );\r
1300                 NdisFreeBuffer( p_buf );\r
1301         }\r
1302 \r
1303         /* Return the descriptor to its pools. */\r
1304         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1305 \r
1306         /*\r
1307          * Dereference the port object since the receive is no longer outstanding.\r
1308          */\r
1309         ipoib_port_deref( p_port, ref_get_recv );\r
1310         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1311 }\r
1312 \r
1313 \r
1314 static inline void\r
1315 __buf_mgr_put_recv_list(\r
1316         IN                              ipoib_port_t* const                     p_port,\r
1317         IN                              cl_qlist_t* const                       p_list )\r
1318 {\r
1319         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1320         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1321         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1322 }\r
1323 \r
1324 \r
1325 static inline NDIS_PACKET*\r
1326 __buf_mgr_get_ndis_pkt(\r
1327         IN                              ipoib_port_t* const                     p_port,\r
1328         IN                              ipoib_recv_desc_t* const        p_desc )\r
1329 {\r
1330         NDIS_STATUS                             status;\r
1331         NDIS_PACKET                             *p_packet;\r
1332         NDIS_BUFFER                             *p_buffer;\r
1333 \r
1334         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1335 \r
1336         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1337                 p_port->buf_mgr.h_packet_pool );\r
1338         if( status != NDIS_STATUS_SUCCESS )\r
1339         {\r
1340                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1341                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1342                 return NULL;\r
1343         }\r
1344 \r
1345         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1346         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1347 \r
1348         NdisAllocateBuffer( &status, &p_buffer,\r
1349 #if IPOIB_INLINE_RECV\r
1350                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1351 #else   /* IPOIB_INLINE_RECV */\r
1352                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1353 #endif  /* IPOIB_INLINE_RECV */\r
1354         if( status != NDIS_STATUS_SUCCESS )\r
1355         {\r
1356                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1357                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1358                 NdisDprFreePacketNonInterlocked( p_packet );\r
1359                 return NULL;\r
1360         }\r
1361 \r
1362         NdisChainBufferAtFront( p_packet, p_buffer );\r
1363         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1364 \r
1365         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1366         return p_packet;\r
1367 }\r
1368 \r
1369 /******************************************************************************\r
1370 *\r
1371 * Receive manager implementation.\r
1372 *\r
1373 ******************************************************************************/\r
1374 static void\r
1375 __recv_mgr_construct(\r
1376         IN                              ipoib_port_t* const                     p_port )\r
1377 {\r
1378         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1379 \r
1380         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1381 \r
1382         p_port->recv_mgr.recv_pkt_array = NULL;\r
1383 \r
1384         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1385 }\r
1386 \r
1387 \r
1388 static ib_api_status_t\r
1389 __recv_mgr_init(\r
1390         IN                              ipoib_port_t* const                     p_port )\r
1391 {\r
1392         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1393 \r
1394         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1395         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1396                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1397         if( !p_port->recv_mgr.recv_pkt_array )\r
1398         {\r
1399                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1400                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1401                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1402                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1403                 return IB_INSUFFICIENT_MEMORY;\r
1404         }\r
1405 \r
1406         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1407         return IB_SUCCESS;\r
1408 }\r
1409 \r
1410 \r
1411 static void\r
1412 __recv_mgr_destroy(\r
1413         IN                              ipoib_port_t* const                     p_port )\r
1414 {\r
1415         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1416 \r
1417         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1418         CL_ASSERT( !p_port->recv_mgr.depth );\r
1419 \r
1420         if( p_port->recv_mgr.recv_pkt_array )\r
1421                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1422 \r
1423         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1424 }\r
1425 \r
1426 \r
1427 /*\r
1428  * Posts receive buffers to the receive queue and returns the number\r
1429  * of receives needed to bring the RQ to its low water mark.  Note\r
1430  * that the value is signed, and can go negative.  All tests must\r
1431  * be for > 0.\r
1432  */\r
1433 static int32_t\r
1434 __recv_mgr_repost(\r
1435         IN                              ipoib_port_t* const                     p_port )\r
1436 {\r
1437         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1438         ib_api_status_t         status;\r
1439         ib_recv_wr_t            *p_failed;\r
1440         PERF_DECLARE( GetRecv );\r
1441         PERF_DECLARE( PostRecv );\r
1442 \r
1443         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1444 \r
1445         CL_ASSERT( p_port );\r
1446         cl_obj_lock( &p_port->obj );\r
1447         if( p_port->state != IB_QPS_RTS )\r
1448         {\r
1449                 cl_obj_unlock( &p_port->obj );\r
1450                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1451                         ("Port in invalid state.  Not reposting.\n") );\r
1452                 return 0;\r
1453         }\r
1454         ipoib_port_ref( p_port, ref_repost );\r
1455         cl_obj_unlock( &p_port->obj );\r
1456 \r
1457         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1458         {\r
1459                 /* Pull receives out of the pool and chain them up. */\r
1460                 cl_perf_start( GetRecv );\r
1461                 p_next = __buf_mgr_get_recv( p_port );\r
1462                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1463                 if( !p_next )\r
1464                 {\r
1465                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1466                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1467                         break;\r
1468                 }\r
1469 \r
1470                 if( !p_tail )\r
1471                 {\r
1472                         p_tail = p_next;\r
1473                         p_next->wr.p_next = NULL;\r
1474                 }\r
1475                 else\r
1476                 {\r
1477                         p_next->wr.p_next = &p_head->wr;\r
1478                 }\r
1479 \r
1480                 p_head = p_next;\r
1481 \r
1482                 p_port->recv_mgr.depth++;\r
1483         }\r
1484 \r
1485         if( p_head )\r
1486         {\r
1487                 cl_perf_start( PostRecv );\r
1488                 status = p_port->p_adapter->p_ifc->post_recv(\r
1489                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1490                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1491 \r
1492                 if( status != IB_SUCCESS )\r
1493                 {\r
1494                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1495                                 ("ip_post_recv returned %s\n", \r
1496                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1497                         /* return the descriptors to the pool */\r
1498                         while( p_failed )\r
1499                         {\r
1500                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1501                                 p_failed = p_failed->p_next;\r
1502 \r
1503                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1504                                 p_port->recv_mgr.depth--;\r
1505                         }\r
1506                 }\r
1507         }\r
1508 \r
1509         ipoib_port_deref( p_port, ref_repost );\r
1510         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1511         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1512 }\r
1513 \r
1514 \r
1515 void\r
1516 ipoib_return_packet(\r
1517         IN                              NDIS_HANDLE                                     adapter_context,\r
1518         IN                              NDIS_PACKET                                     *p_packet )\r
1519 {\r
1520         cl_list_item_t          *p_item;\r
1521         ipoib_port_t            *p_port;\r
1522         ipoib_recv_desc_t       *p_desc;\r
1523         ib_api_status_t         status = IB_NOT_DONE;\r
1524         int32_t                         shortage;\r
1525         PERF_DECLARE( ReturnPacket );\r
1526         PERF_DECLARE( ReturnPutRecv );\r
1527         PERF_DECLARE( ReturnRepostRecv );\r
1528         PERF_DECLARE( ReturnPreparePkt );\r
1529         PERF_DECLARE( ReturnNdisIndicate );\r
1530 \r
1531         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1532 \r
1533         UNUSED_PARAM( adapter_context );\r
1534         CL_ASSERT( p_packet );\r
1535 \r
1536         cl_perf_start( ReturnPacket );\r
1537 \r
1538         /* Get the port and descriptor from the packet. */\r
1539         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1540         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1541 \r
1542         cl_spinlock_acquire( &p_port->recv_lock );\r
1543 \r
1544         cl_perf_start( ReturnPutRecv );\r
1545         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1546         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1547 \r
1548         /* Repost buffers. */\r
1549         cl_perf_start( ReturnRepostRecv );\r
1550         shortage = __recv_mgr_repost( p_port );\r
1551         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1552 \r
1553         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1554                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1555                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1556         {\r
1557                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1558 \r
1559                 cl_perf_start( ReturnPreparePkt );\r
1560                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1561                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1562                 if( status == IB_SUCCESS )\r
1563                 {\r
1564                         if( shortage > 0 )\r
1565                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1566                         else\r
1567                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1568 \r
1569                         cl_spinlock_release( &p_port->recv_lock );\r
1570                         cl_perf_start( ReturnNdisIndicate );\r
1571                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1572                                 &p_packet, 1 );\r
1573                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1574                         cl_spinlock_acquire( &p_port->recv_lock );\r
1575 \r
1576                         if( shortage > 0 )\r
1577                         {\r
1578                                 cl_perf_start( ReturnPutRecv );\r
1579                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1580                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1581 \r
1582                                 /* Repost buffers. */\r
1583                                 cl_perf_start( ReturnRepostRecv );\r
1584                                 shortage = __recv_mgr_repost( p_port );\r
1585                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1586                         }\r
1587                 }\r
1588                 else if( status != IB_NOT_DONE )\r
1589                 {\r
1590                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1591                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1592                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1593                         /* Return the item to the head of the list. */\r
1594                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1595                         break;\r
1596                 }\r
1597         }\r
1598         cl_spinlock_release( &p_port->recv_lock );\r
1599         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1600 \r
1601         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1602 }\r
1603 \r
1604 \r
1605 static void\r
1606 __recv_cb(\r
1607         IN              const   ib_cq_handle_t                          h_cq,\r
1608         IN                              void                                            *cq_context )\r
1609 {\r
1610         ipoib_port_t            *p_port;\r
1611         ib_api_status_t         status;\r
1612         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1613         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1614         cl_qlist_t                      done_list, bad_list;\r
1615         size_t                          i;\r
1616         PERF_DECLARE( RecvCompBundle );\r
1617         PERF_DECLARE( RecvCb );\r
1618         PERF_DECLARE( PollRecv );\r
1619         PERF_DECLARE( RepostRecv );\r
1620         PERF_DECLARE( FilterRecv );\r
1621         PERF_DECLARE( BuildPktArray );\r
1622         PERF_DECLARE( RecvNdisIndicate );\r
1623         PERF_DECLARE( RearmRecv );\r
1624         PERF_DECLARE( PutRecvList );\r
1625 \r
1626         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1627 \r
1628         cl_perf_clr( RecvCompBundle );\r
1629 \r
1630         cl_perf_start( RecvCb );\r
1631 \r
1632         UNUSED_PARAM( h_cq );\r
1633 \r
1634         p_port = (ipoib_port_t*)cq_context;\r
1635 \r
1636         cl_qlist_init( &done_list );\r
1637         cl_qlist_init( &bad_list );\r
1638 \r
1639         ipoib_port_ref( p_port, ref_recv_cb );\r
1640         for( i = 0; i < MAX_RECV_WC; i++ )\r
1641                 wc[i].p_next = &wc[i + 1];\r
1642         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1643 \r
1644         /*\r
1645          * We'll be accessing the endpoint map so take a reference\r
1646          * on it to prevent modifications.\r
1647          */\r
1648         cl_obj_lock( &p_port->obj );\r
1649         cl_atomic_inc( &p_port->endpt_rdr );\r
1650         cl_obj_unlock( &p_port->obj );\r
1651 \r
1652         do\r
1653         {\r
1654                 /* If we get here, then the list of WCs is intact. */\r
1655                 p_free = wc;\r
1656 \r
1657                 cl_perf_start( PollRecv );\r
1658                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1659                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1660                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1661                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1662 \r
1663                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1664                 cl_perf_start( FilterRecv );\r
1665                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1666                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1667 \r
1668         } while( !p_free );\r
1669 \r
1670         /* We're done looking at the endpoint map, release the reference. */\r
1671         cl_atomic_dec( &p_port->endpt_rdr );\r
1672 \r
1673         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1674 \r
1675         cl_spinlock_acquire( &p_port->recv_lock );\r
1676 \r
1677         /* Update our posted depth. */\r
1678         p_port->recv_mgr.depth -= recv_cnt;\r
1679 \r
1680         /* Return any discarded receives to the pool */\r
1681         cl_perf_start( PutRecvList );\r
1682         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1683         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1684 \r
1685         do\r
1686         {\r
1687                 /* Repost ASAP so we don't starve the RQ. */\r
1688                 cl_perf_start( RepostRecv );\r
1689                 shortage = __recv_mgr_repost( p_port );\r
1690                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1691 \r
1692                 cl_perf_start( BuildPktArray );\r
1693                 /* Notify NDIS of any and all possible receive buffers. */\r
1694                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1695                         p_port, shortage, &done_list, &discarded );\r
1696                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1697 \r
1698                 /* Only indicate receives if we actually had any. */\r
1699                 if( discarded && shortage > 0 )\r
1700                 {\r
1701                         /* We may have thrown away packets, and have a shortage */\r
1702                         cl_perf_start( RepostRecv );\r
1703                         __recv_mgr_repost( p_port );\r
1704                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1705                 }\r
1706 \r
1707                 if( !pkt_cnt )\r
1708                         break;\r
1709 \r
1710                 cl_spinlock_release( &p_port->recv_lock );\r
1711 \r
1712                 cl_perf_start( RecvNdisIndicate );\r
1713                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1714                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1715                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1716 \r
1717                 /*\r
1718                  * Cap the number of receives to put back to what we just indicated\r
1719                  * with NDIS_STATUS_RESOURCES.\r
1720                  */\r
1721                 if( shortage > 0 )\r
1722                 {\r
1723                         if( pkt_cnt < shortage )\r
1724                                 shortage = pkt_cnt;\r
1725 \r
1726                         /* Return all but the last packet to the pool. */\r
1727                         cl_spinlock_acquire( &p_port->recv_lock );\r
1728                         while( shortage-- > 1 )\r
1729                         {\r
1730                                 __buf_mgr_put_recv( p_port,\r
1731                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1732                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1733                         }\r
1734                         cl_spinlock_release( &p_port->recv_lock );\r
1735 \r
1736                         /*\r
1737                          * Return the last packet as if NDIS returned it, so that we repost\r
1738                          * and report any other pending receives.\r
1739                          */\r
1740                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1741                 }\r
1742                 cl_spinlock_acquire( &p_port->recv_lock );\r
1743 \r
1744         } while( pkt_cnt );\r
1745         cl_spinlock_release( &p_port->recv_lock );\r
1746 \r
1747         /*\r
1748          * Rearm after filtering to prevent contention on the enpoint maps\r
1749          * and eliminate the possibility of having a call to\r
1750          * __endpt_mgr_insert find a duplicate.\r
1751          */\r
1752         cl_perf_start( RearmRecv );\r
1753         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1754                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1755         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1756         CL_ASSERT( status == IB_SUCCESS );\r
1757 \r
1758         ipoib_port_deref( p_port, ref_recv_cb );\r
1759 \r
1760         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1761 \r
1762         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1763 }\r
1764 \r
1765 \r
1766 static void\r
1767 __recv_get_endpts(\r
1768         IN                              ipoib_port_t* const                     p_port,\r
1769         IN                              ipoib_recv_desc_t* const        p_desc,\r
1770         IN                              ib_wc_t* const                          p_wc,\r
1771                 OUT                     ipoib_endpt_t** const           pp_src,\r
1772                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1773 {\r
1774         ib_api_status_t         status;\r
1775         mac_addr_t                      mac;\r
1776         PERF_DECLARE( GetEndptByGid );\r
1777         PERF_DECLARE( GetEndptByLid );\r
1778         PERF_DECLARE( EndptInsert );\r
1779 \r
1780         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1781 \r
1782         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1783         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1784         {\r
1785                 /* Lookup the source endpoints based on GID. */\r
1786                 cl_perf_start( GetEndptByGid );\r
1787                 *pp_src =\r
1788 #if IPOIB_INLINE_RECV\r
1789                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1790 #else   /* IPOIB_INLINE_RECV */\r
1791                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1792 #endif  /* IPOIB_INLINE_RECV */\r
1793                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1794 \r
1795                 /*\r
1796                  * Lookup the destination endpoint based on GID.\r
1797                  * This is used along with the packet filter to determine\r
1798                  * whether to report this to NDIS.\r
1799                  */\r
1800                 cl_perf_start( GetEndptByGid );\r
1801                 *pp_dst =\r
1802 #if IPOIB_INLINE_RECV\r
1803                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1804 #else   /* IPOIB_INLINE_RECV */\r
1805                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1806 #endif  /* IPOIB_INLINE_RECV */\r
1807                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1808 \r
1809                 /*\r
1810                  * Create the source endpoint if it does not exist.  Note that we\r
1811                  * can only do this for globally routed traffic since we need the\r
1812                  * information from the GRH to generate the MAC.\r
1813                  */\r
1814                 if( !*pp_src )\r
1815                 {\r
1816                         status = ipoib_mac_from_guid(\r
1817 #if IPOIB_INLINE_RECV\r
1818                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1819 #else   /* IPOIB_INLINE_RECV */\r
1820                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1821 #endif  /* IPOIB_INLINE_RECV */\r
1822                         if( status != IB_SUCCESS )\r
1823                         {\r
1824                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1825                                         ("ipoib_mac_from_guid returned %s\n",\r
1826                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1827                                 return;\r
1828                         }\r
1829 \r
1830                         /* Create the endpoint. */\r
1831 #if IPOIB_INLINE_RECV\r
1832                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1833 #else   /* IPOIB_INLINE_RECV */\r
1834                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1835 #endif  /* IPOIB_INLINE_RECV */\r
1836                                 0, p_wc->recv.ud.remote_qp );\r
1837                         if( !*pp_src )\r
1838                         {\r
1839                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1840                                         ("ipoib_endpt_create failed\n") );\r
1841                                 return;\r
1842                         }\r
1843                         cl_perf_start( EndptInsert );\r
1844                         cl_obj_lock( &p_port->obj );\r
1845                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1846                         if( status != IB_SUCCESS )\r
1847                         {\r
1848                                 cl_obj_unlock( &p_port->obj );\r
1849                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1850                                         ("__endpt_mgr_insert returned %s\n",\r
1851                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1852                                 return;\r
1853                         }\r
1854                         cl_obj_unlock( &p_port->obj );\r
1855                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1856                 }\r
1857         }\r
1858         else\r
1859         {\r
1860                 /*\r
1861                  * Lookup the remote endpoint based on LID.  Note that only\r
1862                  * unicast traffic can be LID routed.\r
1863                  */\r
1864                 cl_perf_start( GetEndptByLid );\r
1865                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1866                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1867                 *pp_dst = p_port->p_local_endpt;\r
1868                 CL_ASSERT( *pp_dst );\r
1869         }\r
1870 \r
1871         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1872                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1873         {\r
1874                 /* Update the QPN for the endpoint. */\r
1875                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1876                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1877                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1878                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1879                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1880                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1881         }\r
1882 \r
1883         if( *pp_src && *pp_dst )\r
1884         {\r
1885                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1886                         ("Recv:\n"\r
1887                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1888                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1889                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1890                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1891                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1892                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1893                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1894                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1895         }\r
1896 \r
1897         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1898 }\r
1899 \r
1900 \r
1901 static int32_t\r
1902 __recv_mgr_filter(\r
1903         IN                              ipoib_port_t* const                     p_port,\r
1904         IN                              ib_wc_t* const                          p_done_wc_list,\r
1905                 OUT                     cl_qlist_t* const                       p_done_list,\r
1906                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1907 {\r
1908         ipoib_recv_desc_t               *p_desc;\r
1909         ib_wc_t                                 *p_wc;\r
1910         ipoib_pkt_t                             *p_ipoib;\r
1911         eth_pkt_t                               *p_eth;\r
1912         ipoib_endpt_t                   *p_src, *p_dst;\r
1913         ib_api_status_t                 status;\r
1914         uint32_t                                len;\r
1915         int32_t                                 recv_cnt = 0;\r
1916         PERF_DECLARE( GetRecvEndpts );\r
1917         PERF_DECLARE( RecvGen );\r
1918         PERF_DECLARE( RecvTcp );\r
1919         PERF_DECLARE( RecvUdp );\r
1920         PERF_DECLARE( RecvDhcp );\r
1921         PERF_DECLARE( RecvArp );\r
1922 \r
1923         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1924 \r
1925         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1926         {\r
1927                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
1928                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1929                 recv_cnt++;\r
1930 \r
1931                 if( p_wc->status != IB_WCS_SUCCESS )\r
1932                 {\r
1933                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1934                         {\r
1935                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1936                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1937                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1938                                         (int)p_wc->vendor_specific) );\r
1939                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1940                         }\r
1941                         else\r
1942                         {\r
1943                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1944                                         ("Flushed completion %s\n",\r
1945                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1946                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1947                         }\r
1948                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1949                         /* Dereference the port object on behalf of the failed receive. */\r
1950                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1951                         continue;\r
1952                 }\r
1953 \r
1954                 len = p_wc->length - sizeof(ib_grh_t);\r
1955 \r
1956                 if( len < sizeof(ipoib_hdr_t) )\r
1957                 {\r
1958                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1959                                 ("Received ETH packet < min size\n") );\r
1960                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1961                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1962                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1963                         continue;\r
1964                 }\r
1965 \r
1966                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1967                 {\r
1968                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1969                                 ("Received ETH packet > payload MTU (%d)\n",\r
1970                                 p_port->p_adapter->params.payload_mtu) );\r
1971                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1972                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1973                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1974                         continue;\r
1975                         \r
1976                 }\r
1977                 /* Successful completion.  Get the receive information. */\r
1978                 p_desc->ndis_csum.Value = (ULONG) p_wc->csum_ok;\r
1979                 cl_perf_start( GetRecvEndpts );\r
1980                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1981                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1982 \r
1983 #if IPOIB_INLINE_RECV\r
1984                 p_ipoib = &p_desc->buf.ib.pkt;\r
1985                 p_eth = &p_desc->buf.eth.pkt;\r
1986 #else   /* IPOIB_INLINE_RECV */\r
1987                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1988                 p_eth = &p_desc->p_buf->eth.pkt;\r
1989 #endif  /*IPOIB_INLINE_RECV */\r
1990 \r
1991                 if( p_src )\r
1992                 {\r
1993                         /* Don't report loopback traffic - we requested SW loopback. */\r
1994                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1995                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1996                         {\r
1997                                 /*\r
1998                                  * "This is not the packet you're looking for" - don't update\r
1999                                  * receive statistics, the packet never happened.\r
2000                                  */\r
2001                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2002                                 /* Dereference the port object on behalf of the failed recv. */\r
2003                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
2004                                 continue;\r
2005                         }\r
2006                 }\r
2007 \r
2008                 switch( p_ipoib->hdr.type )\r
2009                 {\r
2010                 case ETH_PROT_TYPE_IP:\r
2011                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
2012                         {\r
2013                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2014                                         ("Received IP packet < min size\n") );\r
2015                                 status = IB_INVALID_SETTING;\r
2016                                 break;\r
2017                         }\r
2018 \r
2019                         if( p_ipoib->type.ip.hdr.offset ||\r
2020                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
2021                         {\r
2022                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2023                                 cl_perf_start( RecvTcp );\r
2024                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2025                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2026                                 break;\r
2027                         }\r
2028 \r
2029                         /* First packet of a UDP transfer. */\r
2030                         if( len <\r
2031                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2032                         {\r
2033                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2034                                         ("Received UDP packet < min size\n") );\r
2035                                 status = IB_INVALID_SETTING;\r
2036                                 break;\r
2037                         }\r
2038 \r
2039                         /* Check if DHCP conversion is required. */\r
2040                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2041                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2042                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2043                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
2044                         {\r
2045                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2046                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2047                                 {\r
2048                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2049                                                 ("Received DHCP < min size\n") );\r
2050                                         status = IB_INVALID_SETTING;\r
2051                                         break;\r
2052                                 }\r
2053                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2054                                         // If there are IP options in this message, we are in trouble in any case\r
2055                                         status = IB_INVALID_SETTING;\r
2056                                         break;                                  \r
2057                                 }\r
2058                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2059                                 cl_perf_start( RecvDhcp );\r
2060                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2061                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2062                         }\r
2063                         else\r
2064                         {\r
2065                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2066                                 cl_perf_start( RecvUdp );\r
2067                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2068                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2069                         }\r
2070                         break;\r
2071 \r
2072                 case ETH_PROT_TYPE_ARP:\r
2073                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2074                         {\r
2075                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2076                                         ("Received ARP < min size\n") );\r
2077                                 status = IB_INVALID_SETTING;\r
2078                                 break;\r
2079                         }\r
2080                         cl_perf_start( RecvArp );\r
2081                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2082                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2083                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2084                         break;\r
2085 \r
2086                 default:\r
2087                         /* Unfiltered.  Setup the ethernet header and report. */\r
2088                         cl_perf_start( RecvGen );\r
2089                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2090                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2091                 }\r
2092 \r
2093                 if( status != IB_SUCCESS )\r
2094                 {\r
2095                         /* Update stats. */\r
2096                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2097                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2098                         /* Dereference the port object on behalf of the failed receive. */\r
2099                         ipoib_port_deref( p_port, ref_recv_filter );\r
2100                 }\r
2101                 else\r
2102                 {\r
2103                         p_desc->len =\r
2104                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2105                         if( p_dst->h_mcast)\r
2106                         {\r
2107                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2108                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2109                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2110                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2111                                 {\r
2112                                         p_desc->type = PKT_TYPE_BCAST;\r
2113                                 }\r
2114                                 else\r
2115                                 {\r
2116                                         p_desc->type = PKT_TYPE_MCAST;\r
2117                                 }\r
2118                         }\r
2119                         else\r
2120                         {\r
2121                                 p_desc->type = PKT_TYPE_UCAST;\r
2122                         }\r
2123                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2124                 }\r
2125         }\r
2126 \r
2127         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2128         return recv_cnt;\r
2129 }\r
2130 \r
2131 \r
2132 static ib_api_status_t\r
2133 __recv_gen(\r
2134         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2135                 OUT                     eth_pkt_t* const                        p_eth,\r
2136         IN                              ipoib_endpt_t* const            p_src,\r
2137         IN                              ipoib_endpt_t* const            p_dst )\r
2138 {\r
2139         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2140 \r
2141         if( !p_src || !p_dst )\r
2142         {\r
2143                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2144                         ("Received packet with no matching endpoints.\n") );\r
2145                 return IB_NOT_DONE;\r
2146         }\r
2147 \r
2148         /*\r
2149          * Fill in the ethernet header.  Note that doing so will overwrite\r
2150          * the IPoIB header, so start by moving the information from the IPoIB\r
2151          * header.\r
2152          */\r
2153         p_eth->hdr.type = p_ipoib->hdr.type;\r
2154         p_eth->hdr.src = p_src->mac;\r
2155         p_eth->hdr.dst = p_dst->mac;\r
2156 \r
2157         if (p_dst->h_mcast) {\r
2158                 p_dst->is_in_use = TRUE;\r
2159         }\r
2160         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2161         return IB_SUCCESS;\r
2162 }\r
2163 \r
2164 \r
2165 static ib_api_status_t\r
2166 __recv_dhcp(\r
2167         IN                              ipoib_port_t* const                     p_port,\r
2168         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2169                 OUT                     eth_pkt_t* const                        p_eth,\r
2170         IN                              ipoib_endpt_t* const            p_src,\r
2171         IN                              ipoib_endpt_t* const            p_dst )\r
2172 {\r
2173         ib_api_status_t         status;\r
2174         dhcp_pkt_t                      *p_dhcp;\r
2175         uint8_t                         *p_option;\r
2176         uint8_t                         *p_cid = NULL;\r
2177         ib_gid_t                        gid;\r
2178         uint8_t                         msg = 0;\r
2179 \r
2180         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2181 \r
2182         UNUSED_PARAM( p_port );\r
2183 \r
2184         /* Create the ethernet header. */\r
2185         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2186         if( status != IB_SUCCESS )\r
2187         {\r
2188                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2189                         ("__recv_gen returned %s.\n", \r
2190                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2191                 return status;\r
2192         }\r
2193 \r
2194         /* Fixup the payload. */\r
2195         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2196         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2197         {\r
2198                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2199                         ("Invalid DHCP op code.\n") );\r
2200                 return IB_INVALID_SETTING;\r
2201         }\r
2202 \r
2203         /*\r
2204          * Find the client identifier option, making sure to skip\r
2205          * the "magic cookie".\r
2206          */\r
2207         p_option = &p_dhcp->options[0];\r
2208         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2209         {\r
2210                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2211                         ("DHCP cookie corrupted.\n") );\r
2212                 return IB_INVALID_PARAMETER;\r
2213         }\r
2214 \r
2215         p_option = &p_dhcp->options[4];\r
2216         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2217         {\r
2218                 switch( *p_option )\r
2219                 {\r
2220                 case DHCP_OPT_PAD:\r
2221                         p_option++;\r
2222                         break;\r
2223 \r
2224                 case DHCP_OPT_MSG:\r
2225                         msg = p_option[2];\r
2226                         p_option += 3;\r
2227                         break;\r
2228 \r
2229                 case DHCP_OPT_CLIENT_ID:\r
2230                         p_cid = p_option;\r
2231                         /* Fall through. */\r
2232 \r
2233                 default:\r
2234                         /*\r
2235                          * All other options have a length byte following the option code.\r
2236                          * Offset by the length to get to the next option.\r
2237                          */\r
2238                         p_option += (p_option[1] + 2);\r
2239                 }\r
2240         }\r
2241 \r
2242         switch( msg )\r
2243         {\r
2244         /* message from client */\r
2245         case DHCPDISCOVER:\r
2246         case DHCPREQUEST:\r
2247         case DHCPDECLINE:\r
2248         case DHCPRELEASE:\r
2249         case DHCPINFORM:\r
2250                 if( !p_cid )\r
2251                 {\r
2252                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2253                                 ("Failed to find required Client-identifier option.\n") );\r
2254                         return IB_INVALID_SETTING;\r
2255                 }\r
2256                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2257                 {\r
2258                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2259                                 ("Invalid hardware address type.\n") );\r
2260                         return IB_INVALID_SETTING;\r
2261                 }\r
2262                 break;\r
2263         /* message from DHCP server */\r
2264         case DHCPOFFER:\r
2265         case DHCPACK:\r
2266         case DHCPNAK:\r
2267                 break;\r
2268 \r
2269         default:\r
2270                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2271                         ("Invalide message type.\n") );\r
2272                 return IB_INVALID_PARAMETER;\r
2273         }\r
2274         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2275         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2276         p_dhcp->hlen = HW_ADDR_LEN;\r
2277 \r
2278         if( p_cid ) /* from client */\r
2279         {\r
2280                 /* Validate that the length and type of the option is as required. */\r
2281                 if( p_cid[1] != 21 )\r
2282                 {\r
2283                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2284                                 ("Client-identifier length not 21 as required.\n") );\r
2285                         return IB_INVALID_SETTING;\r
2286                 }\r
2287                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2288                 {\r
2289                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2290                                 ("Client-identifier type is wrong.\n") );\r
2291                         return IB_INVALID_SETTING;\r
2292                 }\r
2293                 /*\r
2294                  * Copy the GID value from the option so that we can make aligned\r
2295                  * accesses to the contents.\r
2296                  * Recover CID to standard type.\r
2297                  */\r
2298                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2299                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2300                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2301                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2302                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2303         }\r
2304         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2305         return status;\r
2306 }\r
2307 \r
2308 \r
2309 static ib_api_status_t\r
2310 __recv_arp(\r
2311         IN                              ipoib_port_t* const                     p_port,\r
2312         IN                              ib_wc_t* const                          p_wc,\r
2313         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2314                 OUT                     eth_pkt_t* const                        p_eth,\r
2315         IN                              ipoib_endpt_t** const           pp_src,\r
2316         IN                              ipoib_endpt_t* const            p_dst )\r
2317 {\r
2318         ib_api_status_t                 status;\r
2319         arp_pkt_t                               *p_arp;\r
2320         const ipoib_arp_pkt_t   *p_ib_arp;\r
2321         ib_gid_t                                gid;\r
2322         mac_addr_t                              mac;\r
2323         ipoib_hw_addr_t                 null_hw = {0};\r
2324 \r
2325         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2326 \r
2327         if( !p_dst )\r
2328         {\r
2329                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2330                         ("Unknown destination endpoint\n") );\r
2331                 return IB_INVALID_SETTING;\r
2332         }\r
2333 \r
2334         p_ib_arp = &p_ipoib->type.arp;\r
2335         p_arp = &p_eth->type.arp;\r
2336 \r
2337         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2338         {\r
2339                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2340                         ("ARP hardware type is not IB\n") );\r
2341                 return IB_INVALID_SETTING;\r
2342         }\r
2343 \r
2344         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2345         {\r
2346                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2347                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2348                 return IB_INVALID_SETTING;\r
2349         }\r
2350 \r
2351         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2352         {\r
2353                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2354                         ("ARP protocal type not IP\n") );\r
2355                 return IB_INVALID_SETTING;\r
2356         }\r
2357 \r
2358         /*\r
2359          * If we don't have a source, lookup the endpoint specified in the payload.\r
2360          */\r
2361         if( !*pp_src )\r
2362                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2363 \r
2364         /*\r
2365          * If the endpoint exists for the GID, make sure\r
2366          * the dlid and qpn match the arp.\r
2367          */\r
2368         if( *pp_src )\r
2369         {\r
2370                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2371                         sizeof(ib_gid_t) ) )\r
2372                 {\r
2373                         /*\r
2374                          * GIDs for the endpoint are different.  The ARP must\r
2375                          * have been proxied.  Dereference it.\r
2376                          */\r
2377                         *pp_src = NULL;\r
2378                 }\r
2379                 else if( (*pp_src)->dlid &&\r
2380                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2381                 {\r
2382                         /* Out of date!  Destroy the endpoint and replace it. */\r
2383                         __endpt_mgr_remove( p_port, *pp_src );\r
2384                         *pp_src = NULL;\r
2385                 }\r
2386                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2387                 {\r
2388                         if( (*pp_src)->qpn !=\r
2389                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2390                                 p_wc->recv.ud.remote_qp !=\r
2391                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2392                         {\r
2393                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2394                                 __endpt_mgr_remove( p_port, *pp_src );\r
2395                                 *pp_src = NULL;\r
2396                         }\r
2397                 }\r
2398                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2399                 {\r
2400                         /* Out of date!  Destroy the endpoint and replace it. */\r
2401                         __endpt_mgr_remove( p_port, *pp_src );\r
2402                         *pp_src = NULL;\r
2403                 }\r
2404         }\r
2405 \r
2406         /* Do we need to create an endpoint for this GID? */\r
2407         if( !*pp_src )\r
2408         {\r
2409                 /* Copy the src GID to allow aligned access */\r
2410                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2411                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2412                 if( status != IB_SUCCESS )\r
2413                 {\r
2414                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2415                                 ("ipoib_mac_from_guid returned %s\n",\r
2416                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2417                         return status;\r
2418                 }\r
2419                 /*\r
2420                  * Create the endpoint.  Note that the LID is left blank and will be\r
2421                  * resolved by a path query as needed.  This is done because the\r
2422                  * remote LID/GID from the work completion may not be the original\r
2423                  * initiator.\r
2424                  */\r
2425                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2426                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2427 \r
2428                 if( !*pp_src )\r
2429                 {\r
2430                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2431                                 ("ipoib_endpt_create failed\n") );\r
2432                         return status;\r
2433                 }\r
2434 \r
2435                 cl_obj_lock( &p_port->obj );\r
2436                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2437                 if( status != IB_SUCCESS )\r
2438                 {\r
2439                         cl_obj_unlock( &p_port->obj );\r
2440                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2441                                 ("__endpt_mgr_insert return %s \n",\r
2442                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2443                         return status;\r
2444                 }\r
2445 \r
2446                 cl_obj_unlock( &p_port->obj );\r
2447         }\r
2448 \r
2449         CL_ASSERT( !cl_memcmp(\r
2450                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2451         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2452                 (*pp_src)->qpn ==\r
2453                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2454         /* Now swizzle the data. */\r
2455         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2456         p_arp->hw_size = sizeof(mac_addr_t);\r
2457         p_arp->src_hw = (*pp_src)->mac;\r
2458         p_arp->src_ip = p_ib_arp->src_ip;\r
2459 \r
2460         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2461         {\r
2462                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2463                 {\r
2464                         /*\r
2465                          * We received bcast ARP packet that means\r
2466                          * remote port lets everyone know it was changed IP/MAC\r
2467                          * or just activated\r
2468                          */\r
2469 \r
2470                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2471 \r
2472                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2473                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2474                         {\r
2475                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2476                                         ("ARP: is not ARP MCAST\n") );\r
2477                                 return IB_INVALID_SETTING;\r
2478                         }\r
2479 \r
2480                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2481                         p_dst->mac = p_port->p_local_endpt->mac;\r
2482                         /*\r
2483                          * we don't care what receiver ip addr is,\r
2484                          * as long as OS' ARP table is global  ???\r
2485                          */\r
2486                         p_arp->dst_ip = (net32_t)0;\r
2487                 }\r
2488                 else /* we've got reply to our ARP request */\r
2489                 {\r
2490                         p_arp->dst_hw = p_dst->mac;\r
2491                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2492                         CL_ASSERT( p_dst->qpn == \r
2493                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2494                 }\r
2495         }\r
2496         else /* we got ARP reqeust */\r
2497         {\r
2498                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2499                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2500         }\r
2501 \r
2502         /*\r
2503          * Create the ethernet header.  Note that this is done last so that\r
2504          * we have a chance to create a new endpoint.\r
2505          */\r
2506         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2507         if( status != IB_SUCCESS )\r
2508         {\r
2509                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2510                         ("__recv_gen returned %s.\n", \r
2511                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2512                 return status;\r
2513         }\r
2514 \r
2515         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2516         return IB_SUCCESS;\r
2517 }\r
2518 \r
2519 \r
2520 static ib_api_status_t\r
2521 __recv_mgr_prepare_pkt(\r
2522         IN                              ipoib_port_t* const                     p_port,\r
2523         IN                              ipoib_recv_desc_t* const        p_desc,\r
2524                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2525 {\r
2526         NDIS_STATUS                                                     status;\r
2527         uint32_t                                                        pkt_filter;\r
2528         ip_stat_sel_t                                           type;\r
2529         PERF_DECLARE( GetNdisPkt );\r
2530 \r
2531         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2532 \r
2533         pkt_filter = p_port->p_adapter->packet_filter;\r
2534         /* Check the packet filter. */\r
2535         switch( p_desc->type )\r
2536         {\r
2537         default:\r
2538         case PKT_TYPE_UCAST:\r
2539                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2540                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2541                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2542                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2543                 {\r
2544                         /* OK to report. */\r
2545                         type = IP_STAT_UCAST_BYTES;\r
2546                         status = NDIS_STATUS_SUCCESS;\r
2547                 }\r
2548                 else\r
2549                 {\r
2550                         type = IP_STAT_DROPPED;\r
2551                         status = NDIS_STATUS_FAILURE;\r
2552                 }\r
2553                 break;\r
2554         case PKT_TYPE_BCAST:\r
2555                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2556                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2557                 {\r
2558                         /* OK to report. */\r
2559                         type = IP_STAT_BCAST_BYTES;\r
2560                         status = NDIS_STATUS_SUCCESS;\r
2561                 }\r
2562                 else\r
2563                 {\r
2564                         type = IP_STAT_DROPPED;\r
2565                         status = NDIS_STATUS_FAILURE;\r
2566                 }\r
2567                 break;\r
2568         case PKT_TYPE_MCAST:\r
2569                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2570                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2571                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2572                 {\r
2573                         /* OK to report. */\r
2574                         type = IP_STAT_MCAST_BYTES;\r
2575                         status = NDIS_STATUS_SUCCESS;\r
2576                 }\r
2577                 else\r
2578                 {\r
2579                         type = IP_STAT_DROPPED;\r
2580                         status = NDIS_STATUS_FAILURE;\r
2581                 }\r
2582                 break;\r
2583         }\r
2584 \r
2585         if( status != NDIS_STATUS_SUCCESS )\r
2586         {\r
2587                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2588                 /* Return the receive descriptor to the pool. */\r
2589                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2590                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2591                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2592                 /*\r
2593                  * Return IB_NOT_DONE since the packet has been completed,\r
2594                  * but has not consumed an array entry.\r
2595                  */\r
2596                 return IB_NOT_DONE;\r
2597         }\r
2598 \r
2599         cl_perf_start( GetNdisPkt );\r
2600         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2601         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2602         if( !*pp_packet )\r
2603         {\r
2604                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2605                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2606                 return IB_INSUFFICIENT_RESOURCES;\r
2607         }\r
2608 \r
2609         /* Get the checksums directly from packet information. */\r
2610         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
2611                 (PVOID) (uintn_t) (p_desc->ndis_csum.Value);\r
2612         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2613 \r
2614         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2615         return IB_SUCCESS;\r
2616 }\r
2617 \r
2618 \r
2619 static uint32_t\r
2620 __recv_mgr_build_pkt_array(\r
2621         IN                              ipoib_port_t* const                     p_port,\r
2622         IN                              int32_t                                         shortage,\r
2623                 OUT                     cl_qlist_t* const                       p_done_list,\r
2624                 OUT                     int32_t* const                          p_discarded )\r
2625 {\r
2626         cl_list_item_t                  *p_item;\r
2627         ipoib_recv_desc_t               *p_desc;\r
2628         uint32_t                                i = 0;\r
2629         ib_api_status_t                 status;\r
2630         PERF_DECLARE( PreparePkt );\r
2631 \r
2632         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2633 \r
2634         *p_discarded = 0;\r
2635 \r
2636         /* Move any existing receives to the head to preserve ordering. */\r
2637         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2638         p_item = cl_qlist_remove_head( p_done_list );\r
2639         while( p_item != cl_qlist_end( p_done_list ) )\r
2640         {\r
2641                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2642 \r
2643                 cl_perf_start( PreparePkt );\r
2644                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2645                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2646                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2647                 if( status == IB_SUCCESS )\r
2648                 {\r
2649                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2650                         if( shortage-- > 0 )\r
2651                         {\r
2652                                 NDIS_SET_PACKET_STATUS(\r
2653                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2654                         }\r
2655                         else\r
2656                         {\r
2657                                 NDIS_SET_PACKET_STATUS(\r
2658                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2659                         }\r
2660                         i++;\r
2661                 }\r
2662                 else if( status == IB_NOT_DONE )\r
2663                 {\r
2664                         (*p_discarded)++;\r
2665                 }\r
2666                 else\r
2667                 {\r
2668                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2669                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2670                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2671                         /* Put all completed receives on the port's done list. */\r
2672                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2673                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2674                         break;\r
2675                 }\r
2676 \r
2677                 p_item = cl_qlist_remove_head( p_done_list );\r
2678         }\r
2679 \r
2680         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2681         return i;\r
2682 }\r
2683 \r
2684 \r
2685 \r
2686 \r
2687 /******************************************************************************\r
2688 *\r
2689 * Send manager implementation.\r
2690 *\r
2691 ******************************************************************************/\r
2692 static void\r
2693 __send_mgr_construct(\r
2694         IN                              ipoib_port_t* const                     p_port )\r
2695 {\r
2696         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2697         p_port->send_mgr.depth = 0;\r
2698         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2699         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2700 }\r
2701 \r
2702 \r
2703 static void \r
2704 __pending_list_destroy(\r
2705         IN                              ipoib_port_t* const                     p_port )\r
2706 {\r
2707         cl_list_item_t  *p_item;\r
2708         NDIS_PACKET             *p_packet;\r
2709         \r
2710         cl_spinlock_acquire( &p_port->send_lock );\r
2711         /* Complete any pending packets. */\r
2712         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2713                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2714                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2715         {\r
2716                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2717                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2718                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2719         }\r
2720         cl_spinlock_release( &p_port->send_lock );\r
2721 }\r
2722 \r
2723 static void\r
2724 __send_mgr_destroy(\r
2725         IN                              ipoib_port_t* const                     p_port )\r
2726 {\r
2727         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2728         __pending_list_destroy(p_port);\r
2729 \r
2730         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2731 }\r
2732 \r
2733 \r
2734 static NDIS_STATUS\r
2735 __send_mgr_filter(\r
2736         IN                              ipoib_port_t* const                     p_port,\r
2737         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2738         IN                              NDIS_BUFFER* const                      p_buf,\r
2739         IN                              size_t                                          buf_len,\r
2740         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2741 {\r
2742         NDIS_STATUS             status;\r
2743 \r
2744         PERF_DECLARE( FilterIp );\r
2745         PERF_DECLARE( FilterArp );\r
2746         PERF_DECLARE( SendGen );\r
2747 \r
2748         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2749 \r
2750         /*\r
2751          * We already checked the ethernet header length, so we know it's safe\r
2752          * to decrement the buf_len without underflowing.\r
2753          */\r
2754         buf_len -= sizeof(eth_hdr_t);\r
2755 \r
2756         switch( p_eth_hdr->type )\r
2757         {\r
2758         case ETH_PROT_TYPE_IP:\r
2759                 cl_perf_start( FilterIp );\r
2760                 status = __send_mgr_filter_ip(\r
2761                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2762                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2763                 break;\r
2764 \r
2765         case ETH_PROT_TYPE_ARP:\r
2766                 cl_perf_start( FilterArp );\r
2767                 status = __send_mgr_filter_arp(\r
2768                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2769                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2770                 break;\r
2771 \r
2772         default:\r
2773                 /*\r
2774                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2775                  * Just send the payload and hope for the best.\r
2776                  */\r
2777                 cl_perf_start( SendGen );\r
2778                 status = __send_gen( p_port, p_desc );\r
2779                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2780                 break;\r
2781         }\r
2782 \r
2783         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2784         return status;\r
2785 }\r
2786 \r
2787 \r
2788 static NDIS_STATUS\r
2789 __send_copy(\r
2790         IN                              ipoib_port_t* const                     p_port,\r
2791         IN                              ipoib_send_desc_t* const        p_desc )\r
2792 {\r
2793         NDIS_PACKET                             *p_packet;\r
2794         NDIS_BUFFER                             *p_buf;\r
2795         NDIS_STATUS                             status;\r
2796         UINT                                    tot_len, bytes_copied = 0;\r
2797 \r
2798         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2799 \r
2800         p_desc->p_buf = \r
2801                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2802         if( !p_desc->p_buf )\r
2803         {\r
2804                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2805                         ("Failed to allocate buffer for packet copy.\n") );\r
2806                 return NDIS_STATUS_RESOURCES;\r
2807         }\r
2808 \r
2809         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2810         if( status != NDIS_STATUS_SUCCESS )\r
2811         {\r
2812                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2813                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2814                 return status;\r
2815         }\r
2816 \r
2817         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2818                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2819         if( status != NDIS_STATUS_SUCCESS )\r
2820         {\r
2821                 NdisFreePacket( p_packet );\r
2822                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2823                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2824                 return status;\r
2825         }\r
2826 \r
2827         NdisChainBufferAtFront( p_packet, p_buf );\r
2828 \r
2829         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2830 \r
2831         /* Setup the work request. */\r
2832         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2833                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2834         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2835         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2836         p_desc->wr.num_ds = 2;\r
2837 \r
2838         /* Copy the packet. */\r
2839         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2840                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2841                 NormalPagePriority );\r
2842 \r
2843         /* Free our temp packet now that the data is copied. */\r
2844         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2845         NdisFreeBuffer( p_buf );\r
2846         NdisFreePacket( p_packet );\r
2847 \r
2848         if( bytes_copied != tot_len )\r
2849         {\r
2850                 /* Something went wrong.  Drop the packet. */\r
2851                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2852                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2853                         bytes_copied, tot_len) );\r
2854                 return NDIS_STATUS_RESOURCES;\r
2855         }\r
2856 \r
2857         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2858         return NDIS_STATUS_SUCCESS;\r
2859 }\r
2860 \r
2861 \r
2862 #if !IPOIB_USE_DMA\r
2863 /* Send using the MDL's page information rather than the SGL. */\r
2864 static ib_api_status_t\r
2865 __send_gen(\r
2866         IN                              ipoib_port_t* const                     p_port,\r
2867         IN                              ipoib_send_desc_t* const        p_desc )\r
2868 {\r
2869         uint32_t                                i, j = 1;\r
2870         ULONG                                   offset;\r
2871         MDL                                             *p_mdl;\r
2872         UINT                                    num_pages, tot_len;\r
2873         ULONG                                   buf_len;\r
2874         PPFN_NUMBER                             page_array;\r
2875         boolean_t                               hdr_done = FALSE;\r
2876         ib_api_status_t                 status;\r
2877 \r
2878         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2879 \r
2880         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2881                 &tot_len );\r
2882 \r
2883         if( !p_mdl )\r
2884         {\r
2885                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2886                         ("No buffers associated with packet.\n") );\r
2887                 return IB_ERROR;\r
2888         }\r
2889 \r
2890         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2891         if( num_pages >= MAX_SEND_SGE )\r
2892         {\r
2893                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2894                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2895                 status = __send_copy( p_port, p_desc );\r
2896                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2897                 return status;\r
2898         }\r
2899 \r
2900         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2901         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2902         /*\r
2903          * Assume that the ethernet header is always fully contained\r
2904          * in the first page of the first MDL.  This makes for much\r
2905          * simpler code.\r
2906          */\r
2907         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2908         CL_ASSERT( offset <= PAGE_SIZE );\r
2909 \r
2910         while( tot_len )\r
2911         {\r
2912                 buf_len = MmGetMdlByteCount( p_mdl );\r
2913                 page_array = MmGetMdlPfnArray( p_mdl );\r
2914                 CL_ASSERT( page_array );\r
2915                 i = 0;\r
2916                 if( !hdr_done )\r
2917                 {\r
2918                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2919                         /* Skip the ethernet header. */\r
2920                         buf_len -= sizeof(eth_hdr_t);\r
2921                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2922                         if( buf_len )\r
2923                         {\r
2924                                 /* The ethernet header is a subset of this MDL. */\r
2925                                 CL_ASSERT( i == 0 );\r
2926                                 if( offset < PAGE_SIZE )\r
2927                                 {\r
2928                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2929                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2930                                         /* Add the byte offset since we're on the 1st page. */\r
2931                                         p_desc->local_ds[j].vaddr += offset;\r
2932                                         if( offset + buf_len > PAGE_SIZE )\r
2933                                         {\r
2934                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2935                                                 buf_len -= p_desc->local_ds[j].length;\r
2936                                         }\r
2937                                         else\r
2938                                         {\r
2939                                                 p_desc->local_ds[j].length = buf_len;\r
2940                                                 buf_len = 0;\r
2941                                         }\r
2942                                         /* This data segment is done.  Move to the next. */\r
2943                                         j++;\r
2944                                 }\r
2945                                 /* This page is done.  Move to the next. */\r
2946                                 i++;\r
2947                         }\r
2948                         /* Done handling the ethernet header. */\r
2949                         hdr_done = TRUE;\r
2950                 }\r
2951 \r
2952                 /* Finish this MDL */\r
2953                 while( buf_len )\r
2954                 {\r
2955                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2956                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2957                         /* Add the first page's offset if we're on the first page. */\r
2958                         if( i == 0 )\r
2959                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2960 \r
2961                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2962                         {\r
2963                                 /* Buffers spans pages. */\r
2964                                 p_desc->local_ds[j].length =\r
2965                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2966                                 buf_len -= p_desc->local_ds[j].length;\r
2967                                 /* This page is done.  Move to the next. */\r
2968                                 i++;\r
2969                         }\r
2970                         else\r
2971                         {\r
2972                                 /* Last page of the buffer. */\r
2973                                 p_desc->local_ds[j].length = buf_len;\r
2974                                 buf_len = 0;\r
2975                         }\r
2976                         /* This data segment is done.  Move to the next. */\r
2977                         j++;\r
2978                 }\r
2979 \r
2980                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2981                 if( !tot_len )\r
2982                         break;\r
2983 \r
2984                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2985                 if( !p_mdl )\r
2986                 {\r
2987                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2988                                 ("Failed to get next buffer.\n") );\r
2989                         return IB_ERROR;\r
2990                 }\r
2991         }\r
2992 \r
2993         /* Set the number of data segments. */\r
2994         p_desc->wr.num_ds = j;\r
2995 \r
2996         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2997         return IB_SUCCESS;\r
2998 }\r
2999 \r
3000 #else\r
3001 \r
3002 static NDIS_STATUS\r
3003 __send_gen(\r
3004         IN                              ipoib_port_t* const                     p_port,\r
3005         IN                              ipoib_send_desc_t* const        p_desc )\r
3006 {\r
3007         ib_api_status_t                 status;\r
3008         SCATTER_GATHER_LIST             *p_sgl;\r