f73f263899504f9e7b85ddcdcb2693abb3e4e35d
[mirror/winof/.git] / ulp / ipoib_NDIS6_CM / kernel / ipoib_port.cpp
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
5  *\r
6  * This software is available to you under the OpenIB.org BSD license\r
7  * below:\r
8  *\r
9  *     Redistribution and use in source and binary forms, with or\r
10  *     without modification, are permitted provided that the following\r
11  *     conditions are met:\r
12  *\r
13  *      - Redistributions of source code must retain the above\r
14  *        copyright notice, this list of conditions and the following\r
15  *        disclaimer.\r
16  *\r
17  *      - Redistributions in binary form must reproduce the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer in the documentation and/or other materials\r
20  *        provided with the distribution.\r
21  *\r
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
23  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
26  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
27  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
28  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
29  * SOFTWARE.\r
30  *\r
31  * $Id: ipoib_port.c 4506 2009-06-23 14:40:54Z xalex $\r
32  */\r
33 \r
34 #include "precompile.h"\r
35 \r
36 #if defined(EVENT_TRACING)\r
37 #ifdef offsetof\r
38 #undef offsetof\r
39 #endif\r
40 #include "ipoib_port.tmh"\r
41 #endif\r
42 #include <offload.h>\r
43 \r
44 #include "wdm.h"\r
45 #include <ntddk.h>\r
46 \r
47 extern ULONG g_ipoib_send;\r
48 extern ULONG g_ipoib_send_ack;\r
49 extern ULONG g_ipoib_send_SW;\r
50 extern ULONG g_ipoib_send_SG;\r
51 extern ULONG g_ipoib_send_SW_in_loop;\r
52 extern ULONG g_ipoib_send_SG_pending;\r
53 extern ULONG g_ipoib_send_SG_real;\r
54 extern ULONG g_ipoib_send_SG_failed;\r
55 \r
56 \r
57 \r
58 \r
59 ib_gid_t        bcast_mgid_template = {\r
60         0xff,                                                           /* multicast field */\r
61         0x12,                                                           /* scope (to be filled in) */\r
62         0x40, 0x1b,                                                     /* IPv4 signature */\r
63         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
64         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
65         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
66 };\r
67 \r
68 \r
69 #ifdef _DEBUG_\r
70 /* Handy pointer for debug use. */\r
71 ipoib_port_t    *gp_ipoib_port;\r
72 #endif\r
73 \r
74 static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
75 static void __port_do_mcast_garbage(ipoib_port_t* const p_port );\r
76 \r
77 \r
78 static void __recv_cb_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
79 \r
80 \r
81 #ifndef _IPOIB_DEBUG_NDIS6\r
82 #define _IPOIB_DEBUG_NDIS6\r
83 CL_INLINE void CL_API\r
84 cl_qlist_check_validity(\r
85         IN      cl_qlist_t* const       p_list )\r
86 {\r
87         cl_list_item_t  *p_item;\r
88         size_t cnt = 0;\r
89 \r
90         /* CL_ASSERT that a non-null pointer is provided. */\r
91         CL_ASSERT( p_list );\r
92         /* CL_ASSERT that the list was initialized. */\r
93         CL_ASSERT( p_list->state == CL_INITIALIZED );\r
94         p_item = cl_qlist_head(p_list);\r
95         while(p_item != cl_qlist_end(p_list)) {\r
96                 ++cnt;\r
97                 CL_ASSERT(p_item->p_list == p_list);\r
98                 p_item = p_item->p_next;\r
99                 CL_ASSERT (cnt <= p_list->count);\r
100         }\r
101         CL_ASSERT (cnt == p_list->count);\r
102         return;\r
103 }\r
104 #endif\r
105 \r
106 /******************************************************************************\r
107 *\r
108 * Declarations\r
109 *\r
110 ******************************************************************************/\r
111 static void\r
112 __port_construct(\r
113         IN                              ipoib_port_t* const                     p_port );\r
114 \r
115 static ib_api_status_t\r
116 __port_init(\r
117         IN                              ipoib_port_t* const                     p_port,\r
118         IN                              ipoib_adapter_t* const          p_adapter,\r
119         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
120 \r
121 static void\r
122 __port_destroying(\r
123         IN                              cl_obj_t* const                         p_obj );\r
124 \r
125 static void\r
126 __port_cleanup(\r
127         IN                              cl_obj_t* const                         p_obj );\r
128 \r
129 static void\r
130 __port_free(\r
131         IN                              cl_obj_t* const                         p_obj );\r
132 \r
133 static ib_api_status_t\r
134 __port_query_ca_attrs( \r
135         IN              ipoib_port_t* const                                     p_port,\r
136         IN              ib_ca_attr_t**                                          pp_ca_attrs );\r
137 \r
138 static void\r
139 __srq_async_event_cb(\r
140 IN                      ib_async_event_rec_t            *p_event_rec );\r
141 \r
142 /******************************************************************************\r
143 *\r
144 * IB resource manager operations\r
145 *\r
146 ******************************************************************************/\r
147 static void\r
148 __ib_mgr_construct(\r
149         IN                              ipoib_port_t* const                     p_port );\r
150 \r
151 static ib_api_status_t\r
152 __ib_mgr_init(\r
153         IN                              ipoib_port_t* const                     p_port );\r
154 \r
155 static void\r
156 __ib_mgr_destroy(\r
157         IN                              ipoib_port_t* const                     p_port );\r
158 \r
159 static void\r
160 __qp_event(\r
161         IN                              ib_async_event_rec_t            *p_event_rec );\r
162 \r
163 static void\r
164 __cq_event(\r
165         IN                              ib_async_event_rec_t            *p_event_rec );\r
166 \r
167 static ib_api_status_t\r
168 __ib_mgr_activate(\r
169         IN                              ipoib_port_t* const                     p_port );\r
170 \r
171 /******************************************************************************\r
172 *\r
173 * Buffer manager operations.\r
174 *\r
175 ******************************************************************************/\r
176 static void\r
177 __buf_mgr_construct(\r
178         IN                              ipoib_port_t* const                     p_port );\r
179 \r
180 static ib_api_status_t\r
181 __buf_mgr_init(\r
182         IN                              ipoib_port_t* const                     p_port );\r
183 \r
184 static void\r
185 __buf_mgr_destroy(\r
186         IN                              ipoib_port_t* const                     p_port );\r
187 \r
188 static cl_status_t\r
189 __recv_ctor(\r
190         IN                              void* const                                     p_object,\r
191         IN                              void*                                           context,\r
192                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
193 \r
194 #if !IPOIB_INLINE_RECV\r
195 static void\r
196 __recv_dtor(\r
197         IN              const   cl_pool_item_t* const           p_pool_item,\r
198         IN                              void                                            *context );\r
199 #endif  /* IPOIB_INLINE_RECV */\r
200 \r
201 static inline ipoib_send_desc_t*\r
202 __buf_mgr_get_send(\r
203         IN                              ipoib_port_t* const                     p_port );\r
204 \r
205 static inline void\r
206 __buf_mgr_put_send(\r
207         IN                              ipoib_port_t* const                     p_port,\r
208         IN                              ipoib_send_desc_t* const        p_desc );\r
209 \r
210 static inline ipoib_recv_desc_t*\r
211 __buf_mgr_get_recv(\r
212         IN                              ipoib_port_t* const                     p_port );\r
213 \r
214 static inline void\r
215 __buf_mgr_put_recv(\r
216         IN                              ipoib_port_t* const                     p_port,\r
217         IN                              ipoib_recv_desc_t* const        p_desc,\r
218         IN                              NET_BUFFER_LIST* const          p_net_buffer_list OPTIONAL );\r
219 \r
220 static inline void\r
221 __buf_mgr_put_recv_list(\r
222         IN                              ipoib_port_t* const                     p_port,\r
223         IN                              cl_qlist_t* const                       p_list );\r
224 \r
225 //NDIS60\r
226 static inline NET_BUFFER_LIST*\r
227 __buf_mgr_get_NBL(\r
228         IN                              ipoib_port_t* const                     p_port,\r
229         IN                              ipoib_recv_desc_t* const        p_desc );\r
230 \r
231 \r
232 /******************************************************************************\r
233 *\r
234 * Receive manager operations.\r
235 *\r
236 ******************************************************************************/\r
237 static void\r
238 __recv_mgr_construct(\r
239         IN                              ipoib_port_t* const                     p_port );\r
240 \r
241 static ib_api_status_t\r
242 __recv_mgr_init(\r
243         IN                              ipoib_port_t* const                     p_port );\r
244 \r
245 static void\r
246 __recv_mgr_destroy(\r
247         IN                              ipoib_port_t* const                     p_port );\r
248 \r
249 /* Posts receive buffers to the receive queue. */\r
250 int32_t\r
251 __recv_mgr_repost(\r
252         IN                              ipoib_port_t* const                     p_port );\r
253 \r
254 static void\r
255 __recv_cb(\r
256         IN              const   ib_cq_handle_t                          h_cq,\r
257         IN                              void                                            *cq_context );\r
258 \r
259 static void\r
260 __recv_get_endpts(\r
261         IN                              ipoib_port_t* const                     p_port,\r
262         IN                              ipoib_recv_desc_t* const        p_desc,\r
263         IN                              ib_wc_t* const                          p_wc,\r
264                 OUT                     ipoib_endpt_t** const           pp_src,\r
265                 OUT                     ipoib_endpt_t** const           pp_dst );\r
266 \r
267 static int32_t\r
268 __recv_mgr_filter(\r
269         IN                              ipoib_port_t* const                     p_port,\r
270         IN                              ib_wc_t* const                          p_done_wc_list,\r
271                 OUT                     cl_qlist_t* const                       p_done_list,\r
272                 OUT                     cl_qlist_t* const                       p_bad_list );\r
273 \r
274 static ib_api_status_t\r
275 __recv_gen(\r
276         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
277                 OUT                     eth_pkt_t* const                        p_eth,\r
278         IN                              ipoib_endpt_t* const            p_src,\r
279         IN                              ipoib_endpt_t* const            p_dst );\r
280 \r
281 static ib_api_status_t\r
282 __recv_dhcp(\r
283         IN                              ipoib_port_t* const                     p_port,\r
284         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
285                 OUT                     eth_pkt_t* const                        p_eth,\r
286         IN                              ipoib_endpt_t* const            p_src,\r
287         IN                              ipoib_endpt_t* const            p_dst );\r
288 \r
289 static ib_api_status_t\r
290 __recv_arp(\r
291         IN                              ipoib_port_t* const                     p_port,\r
292         IN                              ib_wc_t* const                          p_wc,\r
293         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
294                 OUT                     eth_pkt_t* const                        p_eth,\r
295         IN                              ipoib_endpt_t** const           p_src,\r
296         IN                              ipoib_endpt_t* const            p_dst );\r
297 \r
298 static ib_api_status_t\r
299 __recv_mgr_prepare_NBL(\r
300         IN                              ipoib_port_t* const                     p_port,\r
301         IN                              ipoib_recv_desc_t* const        p_desc,\r
302                 OUT                     NET_BUFFER_LIST** const         pp_net_buffer_list );\r
303 \r
304 static uint32_t\r
305 __recv_mgr_build_NBL_array(\r
306         IN                              ipoib_port_t* const                     p_port,\r
307                 OUT                     cl_qlist_t*                                     p_done_list,\r
308                 OUT                     int32_t* const                          p_discarded );\r
309 \r
310 /******************************************************************************\r
311 *\r
312 * Send manager operations.\r
313 *\r
314 ******************************************************************************/\r
315 static void\r
316 __send_mgr_construct(\r
317         IN                              ipoib_port_t* const                     p_port );\r
318 \r
319 static ib_api_status_t\r
320 __send_mgr_init(\r
321         IN                              ipoib_port_t* const                     p_port );\r
322 \r
323 \r
324 static void\r
325 __send_mgr_destroy(\r
326         IN                              ipoib_port_t* const                     p_port );\r
327 \r
328 static NDIS_STATUS\r
329 __send_gen(\r
330         IN                              ipoib_port_t* const                     p_port,\r
331         IN                              ipoib_send_desc_t* const        p_desc,\r
332         IN                              SCATTER_GATHER_LIST                     *p_sgl,\r
333         IN                              INT                                             lso_data_index,\r
334         IN                              NET_BUFFER                                      *p_netbuf);\r
335 \r
336 static NDIS_STATUS\r
337 __send_mgr_filter_ip(\r
338         IN                              ipoib_port_t* const                     p_port,\r
339         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
340         IN                              MDL*                                            p_mdl,\r
341         IN                              size_t                                          buf_len,\r
342         IN                              SCATTER_GATHER_LIST                     *p_sgl,\r
343         IN      OUT                     ipoib_send_desc_t* const        p_desc,\r
344         IN                              NET_BUFFER                                      *p_netbuf);\r
345 \r
346 static NDIS_STATUS\r
347 __send_mgr_filter_igmp_v2(\r
348         IN                              ipoib_port_t* const                     p_port,\r
349     IN          const   ip_hdr_t* const                         p_ip_hdr,\r
350         IN                              size_t                                          iph_options_size,\r
351         IN                              MDL*                                            p_mdl,\r
352         IN                              size_t                                          buf_len );\r
353 \r
354 static NDIS_STATUS\r
355 __send_mgr_filter_udp(\r
356         IN                              ipoib_port_t* const                     p_port,\r
357         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
358         IN                              MDL*                                            p_mdl,\r
359         IN                              size_t                                          buf_len,\r
360         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
361 \r
362 static NDIS_STATUS\r
363 __send_mgr_filter_dhcp(\r
364         IN                              ipoib_port_t* const                     p_port,\r
365         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
366         IN                              MDL*                                            p_mdl,\r
367         IN                              size_t                                          buf_len,\r
368         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
369 \r
370 static NDIS_STATUS\r
371 __send_mgr_filter_arp(\r
372         IN                              ipoib_port_t* const                     p_port,\r
373         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
374         IN                              MDL*                                            p_mdl,\r
375         IN                              size_t                                          buf_len,\r
376         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
377 \r
378 static void\r
379 __process_failed_send(\r
380         IN                              ipoib_port_t* const                     p_port,\r
381         IN                              ipoib_send_desc_t* const        p_desc,\r
382         IN              const   NDIS_STATUS                                     status,\r
383         IN              ULONG                                           send_complete_flags );\r
384 \r
385 static inline NDIS_STATUS\r
386 __send_mgr_queue(\r
387         IN                              ipoib_port_t* const                     p_port,\r
388         IN                              eth_hdr_t* const                        p_eth_hdr,\r
389                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
390 \r
391 static NDIS_STATUS\r
392 __build_send_desc(\r
393         IN                              ipoib_port_t* const                     p_port,\r
394         IN                              eth_hdr_t* const                        p_eth_hdr,\r
395         IN                              MDL* const                                      p_mdl,\r
396         IN              const   size_t                                          mdl_len,\r
397         IN                              SCATTER_GATHER_LIST                     *p_sgl,\r
398         IN      OUT                     ipoib_send_desc_t* const        p_desc,\r
399         IN                              NET_BUFFER                                      *p_netbuf,\r
400         IN                              ipoib_send_NB_SG                        *s_buf);\r
401 \r
402 \r
403 static void\r
404 __send_cb(\r
405         IN              const   ib_cq_handle_t                          h_cq,\r
406         IN                              void                                            *cq_context );\r
407 \r
408 static NDIS_STATUS \r
409 GetLsoHeaderSize(\r
410         IN              PNET_BUFFER             pNetBuffer,\r
411         IN              LsoData                 *pLsoData,\r
412         OUT     UINT                    *IndexOfData,\r
413         IN              ipoib_hdr_t *ipoib_hdr );\r
414 \r
415 \r
416 static NDIS_STATUS\r
417 __build_lso_desc(\r
418         IN                              ipoib_port_t* const                     p_port,\r
419         IN      OUT                     ipoib_send_desc_t* const        p_desc,\r
420         IN                              ULONG                                           mss,\r
421         IN                              SCATTER_GATHER_LIST                     *p_sgl,\r
422         IN                              int32_t                                         hdr_idx,\r
423         IN PNDIS_TCP_LARGE_SEND_OFFLOAD_NET_BUFFER_LIST_INFO p_lso_info,\r
424         IN                              NET_BUFFER                                      *p_netbuf);\r
425 \r
426 static NDIS_STATUS\r
427 __send_fragments(\r
428         IN              ipoib_port_t* const                                     p_port,\r
429         IN              ipoib_send_desc_t* const                        p_desc,\r
430         IN              eth_hdr_t* const                                        p_eth_hdr,\r
431         IN              ip_hdr_t* const                                         p_ip_hdr,\r
432         IN              uint32_t                                                        buf_len,\r
433         IN              NDIS_BUFFER*                                            p_ndis_buf );\r
434 \r
435 //TODO CM Restore\r
436 #if 0\r
437 static void\r
438 __update_fragment_ip_hdr(\r
439 IN              ip_hdr_t* const         p_ip_hdr,\r
440 IN              uint16_t                        fragment_size, \r
441 IN              uint16_t                        fragment_offset, \r
442 IN              BOOLEAN                         more_fragments );\r
443 \r
444 static void\r
445 __copy_ip_options(\r
446 IN              uint8_t*                        p_buf,\r
447 IN              uint8_t*                        p_options,\r
448 IN              uint32_t                        options_len,\r
449 IN              BOOLEAN                         copy_all );\r
450 #endif\r
451 /******************************************************************************\r
452 *\r
453 * Endpoint manager operations\r
454 *\r
455 ******************************************************************************/\r
456 static void\r
457 __endpt_mgr_construct(\r
458         IN                              ipoib_port_t* const                     p_port );\r
459 \r
460 static ib_api_status_t\r
461 __endpt_mgr_init(\r
462         IN                              ipoib_port_t* const                     p_port );\r
463 \r
464 static void\r
465 __endpt_mgr_destroy(\r
466         IN                              ipoib_port_t* const                     p_port );\r
467 \r
468 /****f* IPoIB/__endpt_mgr_remove_all\r
469 * NAME\r
470 *       __endpt_mgr_remove_all\r
471 *\r
472 * DESCRIPTION\r
473 *       Removes all enpoints from the port, dereferencing them to initiate\r
474 *       destruction.\r
475 *\r
476 * SYNOPSIS\r
477 */\r
478 static void\r
479 __endpt_mgr_remove_all(\r
480         IN                              ipoib_port_t* const                     p_port );\r
481 /*\r
482 ********/\r
483 \r
484 static void\r
485 __endpt_mgr_remove(\r
486         IN                              ipoib_port_t* const                     p_port,\r
487         IN                              ipoib_endpt_t* const            p_endpt );\r
488 \r
489 static void\r
490 __endpt_mgr_reset_all(\r
491         IN                              ipoib_port_t* const                     p_port );\r
492 \r
493 static inline NDIS_STATUS\r
494 __endpt_mgr_ref(\r
495         IN                              ipoib_port_t* const                     p_port,\r
496         IN              const   mac_addr_t                                      mac,\r
497                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
498 \r
499 static inline NDIS_STATUS\r
500 __endpt_mgr_get_gid_qpn(\r
501         IN                              ipoib_port_t* const                     p_port,\r
502         IN              const   mac_addr_t                                      mac,\r
503                 OUT                     ib_gid_t* const                         p_gid,\r
504                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
505 \r
506 static inline ipoib_endpt_t*\r
507 __endpt_mgr_get_by_gid(\r
508         IN                              ipoib_port_t* const                     p_port,\r
509         IN              const   ib_gid_t* const                         p_gid );\r
510 \r
511 static inline ipoib_endpt_t*\r
512 __endpt_mgr_get_by_lid(\r
513         IN                              ipoib_port_t* const                     p_port,\r
514         IN              const   net16_t                                         lid );\r
515 \r
516 static inline ib_api_status_t\r
517 __endpt_mgr_insert_locked(\r
518         IN                              ipoib_port_t* const                     p_port,\r
519         IN              const   mac_addr_t                                      mac,\r
520         IN                              ipoib_endpt_t* const            p_endpt );\r
521 \r
522 static inline ib_api_status_t\r
523 __endpt_mgr_insert(\r
524         IN                              ipoib_port_t* const                     p_port,\r
525         IN              const   mac_addr_t                                      mac,\r
526         IN                              ipoib_endpt_t* const            p_endpt );\r
527 \r
528 static ib_api_status_t\r
529 __endpt_mgr_add_local(\r
530         IN                              ipoib_port_t* const                     p_port,\r
531         IN                              ib_port_info_t* const           p_port_info );\r
532 \r
533 static ib_api_status_t\r
534 __endpt_mgr_add_bcast(\r
535         IN                              ipoib_port_t* const                     p_port,\r
536         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
537 \r
538 /******************************************************************************\r
539 *\r
540 * MCast operations.\r
541 *\r
542 ******************************************************************************/\r
543 static ib_api_status_t\r
544 __port_get_bcast(\r
545         IN                              ipoib_port_t* const                     p_port );\r
546 \r
547 static ib_api_status_t\r
548 __port_join_bcast(\r
549         IN                              ipoib_port_t* const                     p_port,\r
550         IN                              ib_member_rec_t* const          p_member_rec );\r
551 \r
552 static ib_api_status_t\r
553 __port_create_bcast(\r
554         IN                              ipoib_port_t* const                     p_port );\r
555 \r
556 \r
557 \r
558 static void\r
559 __bcast_get_cb(\r
560         IN                              ib_query_rec_t                          *p_query_rec );\r
561 \r
562 \r
563 static void\r
564 __bcast_cb(\r
565         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
566 \r
567 \r
568 static void\r
569 __mcast_cb(\r
570         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
571 \r
572 void\r
573 __leave_error_mcast_cb(\r
574         IN                              void                            *context );\r
575 \r
576 \r
577 static intn_t\r
578 __gid_cmp(\r
579         IN              const   void* const                                     p_key1,\r
580         IN              const   void* const                                     p_key2 )\r
581 {\r
582         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
583 }\r
584 \r
585 \r
586 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
587 {\r
588         cl_obj_ref( &p_port->obj );\r
589 #if DBG\r
590         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
591         if ((p_port->obj.ref_cnt % 20)==0)\r
592         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
593                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
594 #else\r
595         UNREFERENCED_PARAMETER(type);\r
596 #endif\r
597 }\r
598 \r
599 \r
600 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
601 {\r
602 #if DBG\r
603         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
604         if ((p_port->obj.ref_cnt % 20) == 0)\r
605         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
606                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
607 #else\r
608         UNREFERENCED_PARAMETER(type);\r
609 #endif\r
610         cl_obj_deref( &p_port->obj );\r
611 \r
612 }\r
613 \r
614 /* function returns pointer to payload that is going after IP header.\r
615 *  asssuming that payload and IP header are in the same buffer\r
616 */\r
617 static void* GetIpPayloadPtr(const      ip_hdr_t* const p_ip_hdr)\r
618 {\r
619         return (void*)((uint8_t*)p_ip_hdr + IP_HEADER_LENGTH(p_ip_hdr));\r
620 }\r
621 \r
622 /******************************************************************************\r
623 *\r
624 * Implementation\r
625 *\r
626 ******************************************************************************/\r
627 ib_api_status_t\r
628 ipoib_create_port(\r
629         IN                              ipoib_adapter_t* const          p_adapter,\r
630         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
631                 OUT                     ipoib_port_t** const            pp_port )\r
632 {\r
633         ib_api_status_t         status;\r
634         ipoib_port_t            *p_port;\r
635 \r
636         IPOIB_ENTER( IPOIB_DBG_INIT );\r
637 \r
638         CL_ASSERT( !p_adapter->p_port );\r
639 \r
640         p_port = (ipoib_port_t *) cl_zalloc( sizeof(ipoib_port_t) +\r
641                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
642         if( !p_port )\r
643         {\r
644                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
645                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
646                         sizeof(ipoib_port_t)) );\r
647                 return IB_INSUFFICIENT_MEMORY;\r
648         }\r
649 \r
650 #ifdef _DEBUG_\r
651         gp_ipoib_port = p_port;\r
652 #endif\r
653 \r
654         __port_construct( p_port );\r
655 \r
656         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
657         if( status != IB_SUCCESS )\r
658         {\r
659                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
660                         ("ipoib_port_init returned %s.\n", \r
661                         p_adapter->p_ifc->get_err_str( status )) );\r
662                 __port_cleanup( &p_port->obj );\r
663                 __port_free( &p_port->obj );\r
664                 return status;\r
665         }\r
666 \r
667         *pp_port = p_port;\r
668         IPOIB_EXIT( IPOIB_DBG_INIT );\r
669         return IB_SUCCESS;\r
670 }\r
671 \r
672 \r
673 void\r
674 ipoib_port_destroy(\r
675         IN                              ipoib_port_t* const                     p_port )\r
676 {\r
677         IPOIB_ENTER( IPOIB_DBG_INIT );\r
678 \r
679         CL_ASSERT( p_port );\r
680         CL_ASSERT( p_port->p_adapter );\r
681         CL_ASSERT( !p_port->p_adapter->p_port );\r
682 \r
683         cl_obj_destroy( &p_port->obj );\r
684 \r
685         IPOIB_EXIT( IPOIB_DBG_INIT );\r
686 }\r
687 \r
688 \r
689 static void\r
690 __port_construct(\r
691         IN                              ipoib_port_t* const                     p_port )\r
692 {\r
693         IPOIB_ENTER( IPOIB_DBG_INIT );\r
694 \r
695         p_port->state = IB_QPS_RESET;\r
696 \r
697         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
698         cl_spinlock_construct( &p_port->send_lock );\r
699         cl_spinlock_construct( &p_port->recv_lock );\r
700         __ib_mgr_construct( p_port );\r
701         __buf_mgr_construct( p_port );\r
702 \r
703         __recv_mgr_construct( p_port );\r
704         __send_mgr_construct( p_port );\r
705 \r
706         __endpt_mgr_construct( p_port );\r
707 \r
708         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
709         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
710         \r
711         IPOIB_EXIT( IPOIB_DBG_INIT );\r
712 }\r
713 \r
714 \r
715 static ib_api_status_t\r
716 __port_init(\r
717         IN                              ipoib_port_t* const                     p_port,\r
718         IN                              ipoib_adapter_t* const          p_adapter,\r
719         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
720 {\r
721         cl_status_t                     cl_status;\r
722         ib_api_status_t         status;\r
723 \r
724         IPOIB_ENTER( IPOIB_DBG_INIT );\r
725 \r
726         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
727         p_port->p_adapter = p_adapter;\r
728 \r
729         cl_status = cl_spinlock_init( &p_port->send_lock );\r
730         if( cl_status != CL_SUCCESS )\r
731         {\r
732                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
733                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
734                 return IB_ERROR;\r
735         }\r
736 \r
737         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
738         if( cl_status != CL_SUCCESS )\r
739         {\r
740                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
741                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
742                 return IB_ERROR;\r
743         }\r
744 \r
745         /* Initialize the IB resource manager. */\r
746         status = __ib_mgr_init( p_port );\r
747         if( status != IB_SUCCESS )\r
748         {\r
749                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
750                         ("__ib_mgr_init returned %s\n", \r
751                         p_adapter->p_ifc->get_err_str( status )) );\r
752                 return status;\r
753         }\r
754 \r
755         /* Initialize the buffer manager. */\r
756         status = __buf_mgr_init( p_port );\r
757         if( status != IB_SUCCESS )\r
758         {\r
759                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
760                         ("__buf_mgr_init returned %s\n", \r
761                         p_adapter->p_ifc->get_err_str( status )) );\r
762                 return status;\r
763         }\r
764 \r
765         /* Initialize the receive manager. */\r
766         status = __recv_mgr_init( p_port );\r
767         if( status != IB_SUCCESS )\r
768         {\r
769                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
770                         ("__recv_mgr_init returned %s\n", \r
771                         p_adapter->p_ifc->get_err_str( status )) );\r
772                 return status;\r
773         }\r
774         \r
775         status =__send_mgr_init( p_port );\r
776         if( status != IB_SUCCESS )\r
777         {\r
778                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
779                         ("__send_mgr_init returned %s\n", \r
780                         p_adapter->p_ifc->get_err_str( status )) );\r
781                 return status;\r
782         }\r
783 \r
784         /* Initialize the endpoint manager. */\r
785         status = __endpt_mgr_init( p_port );\r
786         if( status != IB_SUCCESS )\r
787         {\r
788                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
789                         ("__endpt_mgr_init returned %s\n", \r
790                         p_adapter->p_ifc->get_err_str( status )) );\r
791                 return status;\r
792         }\r
793 \r
794          KeInitializeDpc(&p_port->recv_dpc,(PKDEFERRED_ROUTINE)__recv_cb_dpc,p_port);\r
795 \r
796 \r
797          /* Initialize multicast garbage collector timer and DPC object */\r
798          KeInitializeDpc(&p_port->gc_dpc,(PKDEFERRED_ROUTINE)__port_mcast_garbage_dpc,p_port);\r
799          KeInitializeTimerEx(&p_port->gc_timer,SynchronizationTimer);\r
800 \r
801         /* We only ever destroy from the PnP callback thread. */\r
802         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
803                 __port_destroying, __port_cleanup, __port_free );\r
804 \r
805 #if DBG\r
806         cl_atomic_inc( &p_port->ref[ref_init] );\r
807         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
808                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
809 #endif\r
810 \r
811         if( cl_status != CL_SUCCESS )\r
812         {\r
813                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
814                         ("cl_obj_init returned %#x\n", cl_status) );\r
815                 return IB_ERROR;\r
816         }\r
817 \r
818         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
819         if( cl_status != CL_SUCCESS )\r
820         {\r
821                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
822                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
823                 cl_obj_destroy( &p_port->obj );\r
824                 return IB_ERROR;\r
825         }\r
826 \r
827 #if DBG\r
828         cl_atomic_inc( &p_port->ref[ref_init] );\r
829         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
830                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
831 #endif\r
832 \r
833         IPOIB_EXIT( IPOIB_DBG_INIT );\r
834         return IB_SUCCESS;\r
835 }\r
836 \r
837 \r
838 static void\r
839 __port_destroying(\r
840         IN                              cl_obj_t* const                         p_obj )\r
841 {\r
842         ipoib_port_t    *p_port;\r
843 \r
844         IPOIB_ENTER( IPOIB_DBG_INIT );\r
845 \r
846         CL_ASSERT( p_obj );\r
847 \r
848         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
849 \r
850         ipoib_port_down( p_port );\r
851 \r
852         __endpt_mgr_remove_all( p_port );\r
853 \r
854 #if 0\r
855         if( p_port->p_adapter->params.cm_enabled )\r
856         {\r
857                 endpt_cm_buf_mgr_destroy(  p_port );\r
858                 ipoib_port_srq_destroy( p_port );\r
859                 p_port->endpt_mgr.thread_is_done = 1;\r
860                 cl_event_signal( &p_port->endpt_mgr.event );\r
861         }\r
862 #endif\r
863         cl_spinlock_acquire(&p_port->send_lock);\r
864         ipoib_port_resume( p_port, FALSE );\r
865         cl_spinlock_release(&p_port->send_lock);\r
866 \r
867         IPOIB_EXIT( IPOIB_DBG_INIT );\r
868 }\r
869 \r
870 \r
871 static void\r
872 __port_cleanup(\r
873         IN                              cl_obj_t* const                         p_obj )\r
874 {\r
875         ipoib_port_t    *p_port;\r
876 \r
877         IPOIB_ENTER( IPOIB_DBG_INIT );\r
878 \r
879         CL_ASSERT( p_obj );\r
880 \r
881         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
882 \r
883         /* Wait for all sends and receives to get flushed. */\r
884         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
885                 cl_thread_suspend( 0 );\r
886 \r
887         /* Destroy the send and receive managers before closing the CA. */\r
888         __ib_mgr_destroy( p_port );\r
889 \r
890         IPOIB_EXIT( IPOIB_DBG_INIT );\r
891 }\r
892 \r
893 \r
894 static void\r
895 __port_free(\r
896         IN                              cl_obj_t* const                         p_obj )\r
897 {\r
898         ipoib_port_t    *p_port;\r
899 \r
900         IPOIB_ENTER( IPOIB_DBG_INIT );\r
901 \r
902         CL_ASSERT( p_obj );\r
903 \r
904         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
905 \r
906         KeCancelTimer(&p_port->gc_timer);\r
907         KeFlushQueuedDpcs();\r
908         __endpt_mgr_destroy( p_port );\r
909         __recv_mgr_destroy( p_port );\r
910         __send_mgr_destroy( p_port );\r
911         __buf_mgr_destroy( p_port );\r
912 \r
913         cl_spinlock_destroy( &p_port->send_lock );\r
914         cl_spinlock_destroy( &p_port->recv_lock );\r
915 \r
916         cl_obj_deinit( p_obj );\r
917         if( p_port->p_ca_attrs )\r
918         {\r
919                 cl_free ( p_port->p_ca_attrs );\r
920         }\r
921         cl_free( p_port );\r
922 \r
923         IPOIB_EXIT( IPOIB_DBG_INIT );\r
924 }\r
925 \r
926 \r
927 \r
928 /******************************************************************************\r
929 *\r
930 * IB resource manager implementation.\r
931 *\r
932 ******************************************************************************/\r
933 static void\r
934 __ib_mgr_construct(\r
935         IN                              ipoib_port_t* const                     p_port )\r
936 {\r
937         IPOIB_ENTER( IPOIB_DBG_INIT );\r
938 \r
939         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
940 \r
941         IPOIB_EXIT( IPOIB_DBG_INIT );\r
942 }\r
943 \r
944 \r
945 static ib_api_status_t\r
946 __ib_mgr_init(\r
947         IN                              ipoib_port_t* const                     p_port )\r
948 {\r
949         ib_api_status_t         status;\r
950         ib_cq_create_t          cq_create;\r
951         ib_qp_create_t          qp_create;\r
952         ib_phys_create_t        phys_create;\r
953         ib_phys_range_t         phys_range;\r
954         uint64_t                        vaddr;\r
955         net32_t                         rkey;\r
956         ib_qp_attr_t            qp_attr;\r
957 \r
958         IPOIB_ENTER( IPOIB_DBG_INIT );\r
959 \r
960         /* Open the CA. */\r
961         status = p_port->p_adapter->p_ifc->open_ca(\r
962                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
963                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
964         if( status != IB_SUCCESS )\r
965         {\r
966                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
967                         EVENT_IPOIB_OPEN_CA, 1, status );\r
968                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
969                         ("ib_open_ca returned %s\n", \r
970                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
971                 return status;\r
972         }\r
973 \r
974         status = __port_query_ca_attrs( p_port, &p_port->p_ca_attrs );\r
975         if( status != IB_SUCCESS )\r
976         {\r
977                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
978                         ("Query CA attributes failed\n" ) );\r
979                 return status;\r
980         }\r
981 #if 0\r
982         if( p_port->p_adapter->params.cm_enabled )\r
983         {\r
984                 uint32_t payload_mtu = __port_attr_to_mtu_size( \r
985                         p_port->p_ca_attrs->p_port_attr[p_port->port_num - 1].mtu )\r
986                         - sizeof(ipoib_hdr_t);\r
987                 /* adjust ipoib UD payload MTU to actual port MTU size. */\r
988                 p_port->p_adapter->params.payload_mtu = \r
989                                 max( DEFAULT_PAYLOAD_MTU, payload_mtu );\r
990                 p_port->p_adapter->params.xfer_block_size = \r
991                         (sizeof(eth_hdr_t) + p_port->p_adapter->params.payload_mtu);\r
992         }\r
993 #endif\r
994 #if IPOIB_USE_DMA\r
995         /* init DMA only once while running MiniportInitialize */\r
996         if ( !p_port->p_adapter->reset )\r
997         {\r
998                 ULONG max_phys_mapping;\r
999                 if( p_port->p_adapter->params.cm_enabled )\r
1000                 {\r
1001                         max_phys_mapping = p_port->p_adapter->params.cm_xfer_block_size;\r
1002                 }\r
1003                 else if( p_port->p_adapter->params.lso )\r
1004                 {\r
1005                         max_phys_mapping = LARGE_SEND_OFFLOAD_SIZE;\r
1006                 }\r
1007                 else\r
1008                 {\r
1009                         max_phys_mapping = p_port->p_adapter->params.xfer_block_size;\r
1010                 }\r
1011                 /*if( NdisMInitializeScatterGatherDma( p_port->p_adapter->h_adapter,\r
1012                                                         TRUE, max_phys_mapping )!= NDIS_STATUS_SUCCESS )\r
1013                 {\r
1014                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1015                                 ("NdisMInitializeScatterGatherDma failed\n" ) );\r
1016                         return IB_INSUFFICIENT_RESOURCES;\r
1017                 }*/\r
1018         }\r
1019 #endif\r
1020 \r
1021         /* Allocate the PD. */\r
1022         status = p_port->p_adapter->p_ifc->alloc_pd(\r
1023                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
1024         if( status != IB_SUCCESS )\r
1025         {\r
1026                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1027                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
1028                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1029                         ("ib_alloc_pd returned %s\n", \r
1030                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1031                 return status;\r
1032         }\r
1033 \r
1034         /* Allocate receive CQ. */\r
1035         cq_create.size = p_port->p_adapter->params.rq_depth;\r
1036         cq_create.pfn_comp_cb = __recv_cb;\r
1037         cq_create.h_wait_obj = NULL;\r
1038 \r
1039         status = p_port->p_adapter->p_ifc->create_cq(\r
1040                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
1041                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
1042         if( status != IB_SUCCESS )\r
1043         {\r
1044                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1045                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
1046                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1047                         ("ib_create_cq returned %s.\n", \r
1048                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1049                 return status;\r
1050         }\r
1051 \r
1052         /* Allocate send CQ. */\r
1053         cq_create.size = p_port->p_adapter->params.sq_depth;\r
1054         cq_create.pfn_comp_cb = __send_cb;\r
1055 \r
1056         status = p_port->p_adapter->p_ifc->create_cq(\r
1057                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
1058                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
1059         if( status != IB_SUCCESS )\r
1060         {\r
1061                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1062                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
1063                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1064                         ("ib_create_cq returned %s.\n", \r
1065                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1066                 return status;\r
1067         }\r
1068         \r
1069         /* Allocate the QP. */\r
1070         cl_memclr( &qp_create, sizeof(qp_create) );\r
1071         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
1072         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
1073         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
1074         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
1075         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
1076 \r
1077 #define UD_QP_USED_SGE 3\r
1078         qp_create.sq_sge = MAX_SEND_SGE < p_port->p_ca_attrs->max_sges ? \r
1079                         MAX_SEND_SGE  : ( p_port->p_ca_attrs->max_sges - UD_QP_USED_SGE );\r
1080         if ( !p_port->p_ca_attrs->ipoib_csum ) \r
1081         { \r
1082                 /* checksum is not supported by device\r
1083                 user must specify BYPASS to explicitly cancel checksum calculation */\r
1084                 if (p_port->p_adapter->params.send_chksum_offload == CSUM_ENABLED)\r
1085                         p_port->p_adapter->params.send_chksum_offload = CSUM_DISABLED;\r
1086                 if (p_port->p_adapter->params.recv_chksum_offload == CSUM_ENABLED)\r
1087                         p_port->p_adapter->params.recv_chksum_offload = CSUM_DISABLED;\r
1088         }\r
1089 \r
1090         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
1091         qp_create.sq_signaled = FALSE;\r
1092         status = p_port->p_adapter->p_ifc->create_qp(\r
1093                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
1094                 __qp_event, &p_port->ib_mgr.h_qp );\r
1095         if( status != IB_SUCCESS )\r
1096         {\r
1097                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1098                         EVENT_IPOIB_CREATE_QP, 1, status );\r
1099                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1100                         ("ib_create_qp returned %s\n", \r
1101                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1102                 return status;\r
1103         }\r
1104         /* Query the QP so we can get our QPN. */\r
1105         status = p_port->p_adapter->p_ifc->query_qp(\r
1106                 p_port->ib_mgr.h_qp, &qp_attr );\r
1107         if( status != IB_SUCCESS )\r
1108         {\r
1109                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1110                         EVENT_IPOIB_QUERY_QP, 1, status );\r
1111                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1112                         ("ib_query_qp returned %s\n", \r
1113                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1114                 return status;\r
1115         }\r
1116         p_port->ib_mgr.qpn = qp_attr.num;\r
1117 \r
1118         /* Register all of physical memory */\r
1119         phys_create.length = MEM_REG_SIZE;\r
1120         phys_create.num_ranges = 1;\r
1121         phys_create.range_array = &phys_range;\r
1122         phys_create.buf_offset = 0;\r
1123         phys_create.hca_page_size = PAGE_SIZE;\r
1124         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
1125         phys_range.base_addr = 0;\r
1126         phys_range.size = MEM_REG_SIZE;\r
1127         vaddr = 0;\r
1128         status = p_port->p_adapter->p_ifc->reg_phys(\r
1129                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
1130                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
1131         if( status != IB_SUCCESS )\r
1132         {\r
1133                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1134                         EVENT_IPOIB_REG_PHYS, 1, status );\r
1135                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1136                         ("ib_reg_phys returned %s\n", \r
1137                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1138                 return status;\r
1139         }\r
1140 \r
1141         status = ipoib_port_srq_init( p_port );\r
1142         if( status != IB_SUCCESS )\r
1143         {\r
1144                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1145                         ("ipoib_port_srq_init failed %s\n",\r
1146                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1147                 /* disable further CM initialization */\r
1148                 p_port->p_adapter->params.cm_enabled = FALSE;\r
1149 \r
1150                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1151                                 EVENT_IPOIB_CONNECTED_MODE_ERR, 1, 0xbadc0de1 );\r
1152 \r
1153         }\r
1154 //CM\r
1155 #if 0\r
1156         if( p_port->p_adapter->params.cm_enabled )\r
1157         {\r
1158                 status = endpt_cm_buf_mgr_init( p_port );\r
1159                 if( status != IB_SUCCESS )\r
1160                 {\r
1161                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1162                                 ("CM Init buf mgr failed status %#x\n", status ) );\r
1163                         ipoib_port_srq_destroy( p_port );\r
1164                         p_port->p_adapter->params.cm_enabled = FALSE;\r
1165 \r
1166                         NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1167                                 EVENT_IPOIB_CONNECTED_MODE_ERR, 1, 0xbadc0de2 );\r
1168                 }\r
1169                 else \r
1170                 {\r
1171                         if ( p_port->p_adapter->params.send_chksum_offload )\r
1172                         p_port->p_adapter->params.send_chksum_offload = CSUM_DISABLED;\r
1173                 }\r
1174         }\r
1175 #endif\r
1176         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1177         return IB_SUCCESS;\r
1178 }\r
1179 \r
1180 static void\r
1181 __srq_async_event_cb(\r
1182 IN                      ib_async_event_rec_t            *p_event_rec )\r
1183 {\r
1184         ipoib_port_t* p_port = \r
1185                 (ipoib_port_t *)p_event_rec->context;\r
1186 \r
1187         switch( p_event_rec->code )\r
1188         {\r
1189         case IB_AE_SRQ_LIMIT_REACHED:\r
1190                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1191                         ("SRQ ASYNC EVENT CODE %d: %s\n", \r
1192                         p_event_rec->code, "IB_AE_SRQ_LIMIT_REACHED" ) );\r
1193                         break;\r
1194         case IB_AE_SRQ_CATAS_ERROR:\r
1195                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1196                                 ("SRQ ASYNC EVENT CODE %d: %s\n", \r
1197                                 p_event_rec->code, "IB_AE_SRQ_CATAS_ERROR" ) );\r
1198                         /*SRQ is in err state, must reinitialize */\r
1199                         p_port->p_adapter->hung = TRUE;\r
1200                         break;\r
1201         case IB_AE_SRQ_QP_LAST_WQE_REACHED:\r
1202                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1203                                 ("SRQ ASYNC EVENT CODE %d: %s\n", \r
1204                                 p_event_rec->code, "IB_AE_SRQ_QP_LAST_WQE_REACHED" ) );\r
1205                         /*SRQ is in err state, must reinitialize */\r
1206                         p_port->p_adapter->hung = TRUE;\r
1207                         break;\r
1208         default:\r
1209                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1210                                 ("ASYNC EVENT CODE ARRIVED %d(%#x)\n", \r
1211                                 p_event_rec->code, p_event_rec->code ) );\r
1212         }\r
1213 }\r
1214 \r
1215 ib_api_status_t\r
1216 ipoib_port_srq_init(\r
1217         IN                              ipoib_port_t* const                     p_port )\r
1218 {\r
1219         ib_api_status_t         ib_status;\r
1220         ib_srq_handle_t         h_srq;\r
1221         ib_srq_attr_t           srq_attr;\r
1222 \r
1223         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1224         \r
1225         if( !p_port->p_adapter->params.cm_enabled )\r
1226                 return IB_SUCCESS;\r
1227 \r
1228         srq_attr.max_sge = min( 2, p_port->p_ca_attrs->max_srq_sges );\r
1229         srq_attr.srq_limit = 10;\r
1230         srq_attr.max_wr = \r
1231                 min( (uint32_t)p_port->p_adapter->params.rq_depth * 8,\r
1232                                 p_port->p_ca_attrs->max_srq_wrs/2 );\r
1233 \r
1234         ib_status = p_port->p_adapter->p_ifc->create_srq( \r
1235                                                                 p_port->ib_mgr.h_pd, \r
1236                                                                 &srq_attr, \r
1237                                                                 p_port, \r
1238                                                                 __srq_async_event_cb, \r
1239                                                                 &h_srq );\r
1240         if( ib_status != IB_SUCCESS )\r
1241         {\r
1242                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1243                         EVENT_IPOIB_CREATE_QP, 1, ib_status );\r
1244                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1245                         ("ib_create_srq failed status %s\n", \r
1246                         p_port->p_adapter->p_ifc->get_err_str( ib_status )) );\r
1247                 return ib_status;\r
1248         }\r
1249         p_port->ib_mgr.h_srq = h_srq;\r
1250 \r
1251         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1252 \r
1253         return ib_status;\r
1254 }\r
1255 \r
1256 /*  __port_query_ca_attrs() \r
1257  *  returns a pointer to allocated memory.\r
1258  *  must be released by caller.\r
1259  */\r
1260 static ib_api_status_t\r
1261 __port_query_ca_attrs( \r
1262         IN              ipoib_port_t* const             p_port,\r
1263         IN              ib_ca_attr_t**                  pp_ca_attrs )\r
1264 {\r
1265         ib_api_status_t         ib_status;\r
1266         uint32_t                        attr_size;\r
1267         ib_ca_attr_t*           p_ca_attrs;\r
1268 \r
1269         *pp_ca_attrs = NULL;\r
1270 \r
1271         ib_status = \r
1272                 p_port->p_adapter->p_ifc->query_ca( p_port->ib_mgr.h_ca, NULL , &attr_size );\r
1273         if( ib_status != IB_INSUFFICIENT_MEMORY )\r
1274         {\r
1275                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1276                         ("ib_query_ca failed status %s\n",\r
1277                         p_port->p_adapter->p_ifc->get_err_str( ib_status )) );\r
1278                 goto done;\r
1279         }\r
1280         CL_ASSERT( attr_size );\r
1281 \r
1282         p_ca_attrs = (ib_ca_attr_t *) cl_zalloc( attr_size );\r
1283         if ( p_ca_attrs == NULL )\r
1284         {\r
1285                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1286                         ("Allocate %d bytes failed for CA Attributes\n", attr_size ));\r
1287                 ib_status = IB_INSUFFICIENT_MEMORY;\r
1288                 goto done;\r
1289         }\r
1290 \r
1291         ib_status = \r
1292                 p_port->p_adapter->p_ifc->query_ca( p_port->ib_mgr.h_ca, p_ca_attrs , &attr_size );\r
1293         if ( ib_status != IB_SUCCESS )\r
1294         {\r
1295                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1296                         ("CA attributes query failed\n") );\r
1297                 cl_free ( p_ca_attrs );\r
1298                 goto done;\r
1299         }\r
1300 \r
1301         *pp_ca_attrs = p_ca_attrs;\r
1302 done:\r
1303         return ib_status;\r
1304 }\r
1305 \r
1306 void\r
1307 ipoib_port_srq_destroy( \r
1308         IN                              ipoib_port_t* const                     p_port )\r
1309 {\r
1310         ib_api_status_t status;\r
1311 \r
1312         if( p_port->ib_mgr.h_srq )\r
1313         {\r
1314                 status =\r
1315                         p_port->p_adapter->p_ifc->destroy_srq( p_port->ib_mgr.h_srq, NULL );\r
1316                 CL_ASSERT( status == IB_SUCCESS );\r
1317                 p_port->ib_mgr.h_srq = NULL;\r
1318         }\r
1319 }\r
1320 \r
1321 static void\r
1322 __ib_mgr_destroy(\r
1323         IN                              ipoib_port_t* const                     p_port )\r
1324 {\r
1325         ib_api_status_t status;\r
1326 \r
1327         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1328 \r
1329         if( p_port->ib_mgr.h_ca )\r
1330         {\r
1331                 status =\r
1332                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
1333                 CL_ASSERT( status == IB_SUCCESS );\r
1334                 p_port->ib_mgr.h_ca = NULL;\r
1335         }\r
1336 \r
1337         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1338 }\r
1339 \r
1340 \r
1341 \r
1342 /******************************************************************************\r
1343 *\r
1344 * Buffer manager implementation.\r
1345 *\r
1346 ******************************************************************************/\r
1347 static void\r
1348 __buf_mgr_construct(\r
1349         IN                              ipoib_port_t* const                     p_port )\r
1350 {\r
1351         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1352 \r
1353         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1354 \r
1355         p_port->buf_mgr.h_packet_pool = NULL;\r
1356         p_port->buf_mgr.h_buffer_pool = NULL;\r
1357 \r
1358         NdisInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1359                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1360 \r
1361         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1362         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1363 \r
1364         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1365 }\r
1366 \r
1367 \r
1368 static ib_api_status_t\r
1369 __buf_mgr_init(\r
1370         IN                              ipoib_port_t* const                     p_port )\r
1371 {\r
1372         cl_status_t             cl_status;\r
1373         ipoib_params_t  *p_params;\r
1374         NET_BUFFER_LIST_POOL_PARAMETERS pool_parameters;\r
1375         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1376 \r
1377         CL_ASSERT( p_port );\r
1378         CL_ASSERT( p_port->p_adapter );\r
1379 \r
1380         p_params = &p_port->p_adapter->params;\r
1381 \r
1382         /* Allocate the receive descriptor pool */\r
1383         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1384                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1385 #if IPOIB_INLINE_RECV\r
1386                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1387 #else   /* IPOIB_INLINE_RECV */\r
1388                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1389 #endif  /* IPOIB_INLINE_RECV */\r
1390         if( cl_status != CL_SUCCESS )\r
1391         {\r
1392                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1393                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1394                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1395                         ("cl_qpool_init for recvs returned %#x\n",\r
1396                         cl_status) );\r
1397                 return IB_INSUFFICIENT_MEMORY;\r
1398         }\r
1399 \r
1400         /* Allocate the NET BUFFER list pools for receive indication. */\r
1401         NdisZeroMemory(&pool_parameters, sizeof(NET_BUFFER_LIST_POOL_PARAMETERS));\r
1402     pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;\r
1403     pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;\r
1404     pool_parameters.Header.Size = sizeof(pool_parameters);\r
1405     pool_parameters.ProtocolId = 0;\r
1406     pool_parameters.ContextSize = 0;\r
1407     pool_parameters.fAllocateNetBuffer = TRUE;\r
1408     pool_parameters.PoolTag = 'CRPI';\r
1409 \r
1410     p_port->buf_mgr.h_packet_pool = NdisAllocateNetBufferListPool(\r
1411                 p_port->p_adapter->h_adapter,\r
1412                 &pool_parameters); \r
1413 \r
1414         if( !p_port->buf_mgr.h_packet_pool )\r
1415         {\r
1416                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1417                         EVENT_IPOIB_RECV_PKT_POOL, 1, NDIS_STATUS_RESOURCES  );\r
1418                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1419                         ("NdisAllocatePacketPool returned %08X\n", (UINT)NDIS_STATUS_RESOURCES) );\r
1420                 return IB_INSUFFICIENT_RESOURCES;\r
1421         }\r
1422 /*\r
1423         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1424                 p_params->rq_depth );\r
1425         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1426         {\r
1427                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1428                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1429                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1430                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1431                 return IB_INSUFFICIENT_RESOURCES;\r
1432         }\r
1433 */\r
1434         /* Allocate the NET buffer list pool for send formatting. */\r
1435     pool_parameters.PoolTag = 'XTPI';\r
1436 \r
1437     p_port->buf_mgr.h_send_pkt_pool = NdisAllocateNetBufferListPool(\r
1438                 p_port->p_adapter->h_adapter,\r
1439                 &pool_parameters); \r
1440         if( !p_port->buf_mgr.h_send_pkt_pool)\r
1441         {\r
1442                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1443                         EVENT_IPOIB_SEND_PKT_POOL, 1, NDIS_STATUS_RESOURCES );\r
1444                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1445                         ("NdisAllocatePacketPool returned %08X\n", (UINT)NDIS_STATUS_RESOURCES) );\r
1446                 return IB_INSUFFICIENT_RESOURCES;\r
1447         }\r
1448 /*\r
1449         NdisAllocateBufferPool( &ndis_status,\r
1450                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1451         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1452         {\r
1453                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1454                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1455                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1456                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1457                 return IB_INSUFFICIENT_RESOURCES;\r
1458         }\r
1459 */\r
1460         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1461         return IB_SUCCESS;\r
1462 }\r
1463 \r
1464 \r
1465 static void\r
1466 __buf_mgr_destroy(\r
1467         IN                              ipoib_port_t* const                     p_port )\r
1468 {\r
1469         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1470 \r
1471         CL_ASSERT( p_port );\r
1472 \r
1473         /* Destroy the send packet and buffer pools. \r
1474         if( p_port->buf_mgr.h_send_buf_pool )\r
1475                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );*/\r
1476         if( p_port->buf_mgr.h_send_pkt_pool )\r
1477                 NdisFreeNetBufferListPool ( p_port->buf_mgr.h_send_pkt_pool );\r
1478 \r
1479         /* Destroy the receive packet and buffer pools. \r
1480         if( p_port->buf_mgr.h_buffer_pool )\r
1481                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );*/\r
1482         if( p_port->buf_mgr.h_packet_pool )\r
1483                 NdisFreeNetBufferListPool ( p_port->buf_mgr.h_packet_pool );\r
1484 \r
1485         /* Free the receive and send descriptors. */\r
1486         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1487 \r
1488         /* Free the lookaside list of scratch buffers. */\r
1489         NdisDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1490 \r
1491         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1492 }\r
1493 \r
1494 \r
1495 static cl_status_t\r
1496 __recv_ctor(\r
1497         IN                              void* const                                     p_object,\r
1498         IN                              void*                                           context,\r
1499                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1500 {\r
1501         ipoib_recv_desc_t       *p_desc;\r
1502         ipoib_port_t            *p_port;\r
1503 \r
1504 #if IPOIB_INLINE_RECV\r
1505         uint32_t                        ds0_len;\r
1506 #endif\r
1507 \r
1508         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1509 \r
1510         CL_ASSERT( p_object );\r
1511         CL_ASSERT( context );\r
1512 \r
1513         p_desc = (ipoib_recv_desc_t*)p_object;\r
1514         p_port = (ipoib_port_t*)context;\r
1515 \r
1516         /* Setup the work request. */\r
1517         p_desc->wr.ds_array = p_desc->local_ds;\r
1518         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1519 \r
1520 #if IPOIB_INLINE_RECV\r
1521         /* Sanity check on the receive buffer layout */\r
1522         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1523                 (void*)&p_desc->buf.ib.pkt.type );\r
1524         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1525 \r
1526         /* Setup the local data segment. */\r
1527         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1528         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1529         ds0_len =\r
1530                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1531         if( ds0_len >= sizeof(recv_buf_t) )\r
1532         {\r
1533                 /* The whole buffer is within a page. */\r
1534                 p_desc->local_ds[0].length = ds0_len;\r
1535                 p_desc->wr.num_ds = 1;\r
1536         }\r
1537         else\r
1538         {\r
1539                 /* The buffer crosses page boundaries. */\r
1540                 p_desc->local_ds[0].length = ds0_len;\r
1541                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1542                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1543                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1544                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1545                 p_desc->wr.num_ds = 2;\r
1546         }\r
1547 #else   /* IPOIB_INLINE_RECV */\r
1548         /* Allocate the receive buffer. */\r
1549         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1550         if( !p_desc->p_buf )\r
1551         {\r
1552                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1553                         ("Failed to allocate receive buffer.\n") );\r
1554                 return CL_INSUFFICIENT_MEMORY;\r
1555         }\r
1556 \r
1557         /* Sanity check on the receive buffer layout */\r
1558         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1559                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1560 \r
1561         /* Setup the local data segment. */\r
1562         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1563         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1564         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1565         p_desc->wr.num_ds = 1;\r
1566 #endif  /* IPOIB_INLINE_RECV */\r
1567 \r
1568         *pp_pool_item = &p_desc->item;\r
1569 \r
1570         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1571         return CL_SUCCESS;\r
1572 }\r
1573 \r
1574 \r
1575 #if !IPOIB_INLINE_RECV\r
1576 static void\r
1577 __recv_dtor(\r
1578         IN              const   cl_pool_item_t* const           p_pool_item,\r
1579         IN                              void                                            *context )\r
1580 {\r
1581         ipoib_recv_desc_t       *p_desc;\r
1582 \r
1583         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1584 \r
1585         UNUSED_PARAM( context );\r
1586 \r
1587         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1588 \r
1589         if( p_desc->p_buf )\r
1590                 cl_free( p_desc->p_buf );\r
1591 \r
1592         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1593 }\r
1594 #endif\r
1595 \r
1596 \r
1597 static inline ipoib_recv_desc_t*\r
1598 __buf_mgr_get_recv(\r
1599         IN                              ipoib_port_t* const                     p_port )\r
1600 {\r
1601         ipoib_recv_desc_t       *p_desc;\r
1602         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1603         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1604         /* Reference the port object for the send. */\r
1605         if( p_desc )\r
1606         {\r
1607                 ipoib_port_ref( p_port, ref_get_recv );\r
1608                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1609 #if IPOIB_INLINE_RECV\r
1610                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1611                         cl_get_physaddr( &p_desc->buf ) );\r
1612 #else   /* IPOIB_INLINE_RECV */\r
1613                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1614                         cl_get_physaddr( p_desc->p_buf ) );\r
1615                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1616                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1617 #endif  /* IPOIB_INLINE_RECV */\r
1618                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1619         }\r
1620         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1621         return p_desc;\r
1622 }\r
1623 \r
1624 \r
1625 //NDIS60\r
1626 static inline void\r
1627 __buf_mgr_put_recv(\r
1628         IN                              ipoib_port_t* const                     p_port,\r
1629         IN                              ipoib_recv_desc_t* const        p_desc,\r
1630         IN                              NET_BUFFER_LIST* const          p_net_buffer_list OPTIONAL )\r
1631 {\r
1632         NET_BUFFER              *p_buf = NULL;\r
1633         MDL                             *p_mdl = NULL;\r
1634         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1635 \r
1636         if( p_net_buffer_list )\r
1637         {\r
1638                 NET_BUFFER_LIST_NEXT_NBL(p_net_buffer_list) = NULL;\r
1639                 p_buf = NET_BUFFER_LIST_FIRST_NB(p_net_buffer_list);\r
1640                 CL_ASSERT( p_buf );\r
1641                 p_mdl = NET_BUFFER_FIRST_MDL(p_buf);\r
1642                 CL_ASSERT( p_mdl );\r
1643                 NdisFreeMdl(p_mdl);\r
1644                 NdisFreeNetBufferList(p_net_buffer_list);\r
1645         }\r
1646 \r
1647         /* Return the descriptor to its pools. */\r
1648         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1649 \r
1650         /*\r
1651          * Dereference the port object since the receive is no longer outstanding.\r
1652          */\r
1653         ipoib_port_deref( p_port, ref_get_recv );\r
1654         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1655 }\r
1656 \r
1657 \r
1658 static inline void\r
1659 __buf_mgr_put_recv_list(\r
1660         IN                              ipoib_port_t* const                     p_port,\r
1661         IN                              cl_qlist_t* const                       p_list )\r
1662 {\r
1663         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1664         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1665         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1666 }\r
1667 \r
1668 \r
1669 static inline NET_BUFFER_LIST*\r
1670 __buf_mgr_get_NBL(\r
1671         IN                              ipoib_port_t* const                     p_port,\r
1672         IN                              ipoib_recv_desc_t* const        p_desc )\r
1673 {\r
1674         NET_BUFFER_LIST                 *p_net_buffer_list;\r
1675         MDL                                             *p_mdl;\r
1676 \r
1677         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1678 \r
1679         p_mdl = NdisAllocateMdl(p_port->p_adapter->h_adapter,\r
1680                                                         &p_desc->buf.eth.pkt,\r
1681                                                         p_desc->len );\r
1682         if( !p_mdl )\r
1683         {\r
1684                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1685                         ("Failed to allocate MDL\n") );\r
1686                 return NULL;\r
1687         }\r
1688 \r
1689         p_net_buffer_list = NdisAllocateNetBufferAndNetBufferList(\r
1690                                                 p_port->buf_mgr.h_packet_pool,\r
1691                                                 0,\r
1692                                                 0,\r
1693                                                 p_mdl,\r
1694                                                 0,\r
1695                                                 0);\r
1696 \r
1697         if( !p_net_buffer_list )\r
1698         {\r
1699                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1700                         ("Failed to allocate NET_BUFFER_LIST\n") );\r
1701                 NdisFreeMdl(p_mdl);\r
1702                 return NULL;\r
1703         }\r
1704 \r
1705         NET_BUFFER_LIST_NEXT_NBL(p_net_buffer_list) = NULL;\r
1706         IPOIB_PORT_FROM_NBL( p_net_buffer_list ) = p_port;\r
1707         IPOIB_RECV_FROM_NBL( p_net_buffer_list ) = p_desc;\r
1708         p_net_buffer_list->SourceHandle = p_port->p_adapter->h_adapter;\r
1709 \r
1710         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1711         return p_net_buffer_list;\r
1712 }\r
1713 \r
1714 \r
1715 /******************************************************************************\r
1716 *\r
1717 * Receive manager implementation.\r
1718 *\r
1719 ******************************************************************************/\r
1720 static void\r
1721 __recv_mgr_construct(\r
1722         IN                              ipoib_port_t* const                     p_port )\r
1723 {\r
1724         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1725 \r
1726         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1727 \r
1728         p_port->recv_mgr.recv_NBL_array = NULL;\r
1729 \r
1730         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1731 }\r
1732 \r
1733 \r
1734 static ib_api_status_t\r
1735 __recv_mgr_init(\r
1736         IN                              ipoib_port_t* const                     p_port )\r
1737 {\r
1738         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1739 \r
1740         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1741         p_port->recv_mgr.recv_NBL_array = (NET_BUFFER_LIST **)cl_malloc(\r
1742                 sizeof(NET_BUFFER_LIST*) * p_port->p_adapter->params.rq_depth );\r
1743         if( !p_port->recv_mgr.recv_NBL_array )\r
1744         {\r
1745                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1746                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1747                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1748                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1749                 return IB_INSUFFICIENT_MEMORY;\r
1750         }\r
1751 \r
1752         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1753         return IB_SUCCESS;\r
1754 }\r
1755 \r
1756 \r
1757 static void\r
1758 __recv_mgr_destroy(\r
1759         IN                              ipoib_port_t* const                     p_port )\r
1760 {\r
1761         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1762 \r
1763         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1764         CL_ASSERT( !p_port->recv_mgr.depth );\r
1765 \r
1766         if( p_port->recv_mgr.recv_NBL_array )\r
1767                 cl_free( p_port->recv_mgr.recv_NBL_array );\r
1768 \r
1769         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1770 }\r
1771 \r
1772 \r
1773 /*\r
1774  * Posts receive buffers to the receive queue and returns the number\r
1775  * of receives needed to bring the RQ to its low water mark.  Note\r
1776  * that the value is signed, and can go negative.  All tests must\r
1777  * be for > 0.\r
1778  */\r
1779 int32_t\r
1780 __recv_mgr_repost(\r
1781         IN                              ipoib_port_t* const                     p_port )\r
1782 {\r
1783         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1784         ib_api_status_t         status;\r
1785         ib_recv_wr_t            *p_failed;\r
1786         PERF_DECLARE( GetRecv );\r
1787         PERF_DECLARE( PostRecv );\r
1788 \r
1789         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1790 \r
1791         CL_ASSERT( p_port );\r
1792         cl_obj_lock( &p_port->obj );\r
1793         if( p_port->state != IB_QPS_RTS )\r
1794         {\r
1795                 cl_obj_unlock( &p_port->obj );\r
1796                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1797                         ("Port in invalid state.  Not reposting.\n") );\r
1798                 return 0;\r
1799         }\r
1800         ipoib_port_ref( p_port, ref_repost );\r
1801         cl_obj_unlock( &p_port->obj );\r
1802 \r
1803         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1804         {\r
1805                 /* Pull receives out of the pool and chain them up. */\r
1806                 cl_perf_start( GetRecv );\r
1807                 p_next = __buf_mgr_get_recv( p_port );\r
1808                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1809                 if( !p_next )\r
1810                 {\r
1811                         IPOIB_PRINT(TRACE_LEVEL_VERBOSE, IPOIB_DBG_RECV,\r
1812                                 ("Out of receive descriptors! recv queue depth 0x%x\n",p_port->recv_mgr.depth) );\r
1813                         break;\r
1814                 }\r
1815 \r
1816                 if( !p_tail )\r
1817                 {\r
1818                         p_tail = p_next;\r
1819                         p_next->wr.p_next = NULL;\r
1820                 }\r
1821                 else\r
1822                 {\r
1823                         p_next->wr.p_next = &p_head->wr;\r
1824                 }\r
1825 \r
1826                 p_head = p_next;\r
1827 \r
1828                 p_port->recv_mgr.depth++;\r
1829         }\r
1830 \r
1831         if( p_head )\r
1832         {\r
1833                 cl_perf_start( PostRecv );\r
1834                 status = p_port->p_adapter->p_ifc->post_recv(\r
1835                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1836                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1837 \r
1838                 if( status != IB_SUCCESS )\r
1839                 {\r
1840                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1841                                 ("ip_post_recv returned %s\n", \r
1842                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1843                         /* return the descriptors to the pool */\r
1844                         while( p_failed )\r
1845                         {\r
1846                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1847                                 p_failed = p_failed->p_next;\r
1848 \r
1849                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1850                                 p_port->recv_mgr.depth--;\r
1851                         }\r
1852                 }\r
1853         }\r
1854 \r
1855         ipoib_port_deref( p_port, ref_repost );\r
1856         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1857         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1858 }\r
1859 \r
1860 void\r
1861 ipoib_return_net_buffer_list(\r
1862         IN                              NDIS_HANDLE                                     adapter_context,\r
1863         IN                              NET_BUFFER_LIST                         *p_net_buffer_lists,\r
1864         IN                              ULONG                                           return_flags)\r
1865 {\r
1866         ipoib_port_t            *p_port;\r
1867         ipoib_recv_desc_t       *p_desc;\r
1868         NET_BUFFER_LIST         *cur_net_buffer_list,*next_net_buffer_list;\r
1869         int32_t                         shortage;\r
1870         \r
1871         PERF_DECLARE( ReturnPacket );\r
1872         PERF_DECLARE( ReturnPutRecv );\r
1873         PERF_DECLARE( ReturnRepostRecv );\r
1874         PERF_DECLARE( ReturnPreparePkt );\r
1875         PERF_DECLARE( ReturnNdisIndicate );\r
1876 \r
1877         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1878 \r
1879         UNUSED_PARAM( return_flags );\r
1880 \r
1881         p_port = ((ipoib_adapter_t*)adapter_context)->p_port;\r
1882         CL_ASSERT( p_net_buffer_lists );\r
1883 \r
1884         cl_perf_start( ReturnPacket );\r
1885         cl_spinlock_acquire( &p_port->recv_lock );\r
1886         for (cur_net_buffer_list = p_net_buffer_lists;\r
1887                  cur_net_buffer_list != NULL;\r
1888                  cur_net_buffer_list = next_net_buffer_list)\r
1889         {\r
1890                 next_net_buffer_list = NET_BUFFER_LIST_NEXT_NBL(cur_net_buffer_list);\r
1891 \r
1892                 /* Get the port and descriptor from the NET_BUFFER_LIST. */\r
1893                 CL_ASSERT(p_port == IPOIB_PORT_FROM_NBL( cur_net_buffer_list ));\r
1894                 p_desc = IPOIB_RECV_FROM_NBL( cur_net_buffer_list );\r
1895 \r
1896                 \r
1897                 //TODO: NDIS60, rewrite this block\r
1898                 \r
1899 #if 0 //TODO CM flow\r
1900                 if( p_desc->type == PKT_TYPE_CM_UCAST )\r
1901                 {\r
1902                         int32_t                         discarded;\r
1903                         uint32_t                        NBL_cnt = 0;\r
1904                         \r
1905                         ib_api_status_t         status = IB_NOT_DONE;\r
1906 \r
1907                 \r
1908                         NDIS_BUFFER             *p_buf;\r
1909 \r
1910                         /* Unchain the NDIS buffer. */\r
1911                         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1912                         CL_ASSERT( p_buf );\r
1913                         /* Return the NDIS packet and NDIS buffer to their pools. */\r
1914                         NdisDprFreePacketNonInterlocked( p_packet );\r
1915                         NdisFreeBuffer( p_buf );\r
1916 \r
1917                         endpt_cm_buf_mgr_put_recv( &p_port->cm_buf_mgr, (ipoib_cm_desc_t *)p_desc );\r
1918                         status = endpt_cm_post_recv( p_port );\r
1919                         if(  status != IB_SUCCESS )\r
1920                         {\r
1921                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1922                                         ("Post Recv QP failed\n" ) );\r
1923                         }\r
1924                         cl_spinlock_release( &p_port->recv_lock );\r
1925                         return;\r
1926                 }\r
1927 #endif\r
1928 \r
1929                 cl_perf_start( ReturnPutRecv );\r
1930                 __buf_mgr_put_recv( p_port, p_desc, cur_net_buffer_list );\r
1931                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1932         }\r
1933 \r
1934 \r
1935 \r
1936         /* Repost buffers to HW */\r
1937         cl_perf_start( ReturnRepostRecv );\r
1938         shortage = __recv_mgr_repost( p_port );\r
1939         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1940         cl_spinlock_release( &p_port->recv_lock );\r
1941         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1942 \r
1943         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1944 }\r
1945 \r
1946 static void __recv_cb_dpc(KDPC *p_gc_dpc,void *context,void * s_arg1 , void * s_arg2)\r
1947 {\r
1948 \r
1949         ipoib_port_t *p_port = (ipoib_port_t *) context;\r
1950 \r
1951         UNREFERENCED_PARAMETER(p_gc_dpc);\r
1952         UNREFERENCED_PARAMETER(s_arg1);\r
1953         UNREFERENCED_PARAMETER(s_arg2);\r
1954 \r
1955 \r
1956         __recv_cb(NULL, p_port);\r
1957         ipoib_port_deref( p_port, ref_recv_cb );\r
1958 \r
1959 \r
1960 }\r
1961 \r
1962 \r
1963 static void\r
1964 __recv_cb(\r
1965         IN              const   ib_cq_handle_t                          h_cq,\r
1966         IN                              void                                            *cq_context )\r
1967 {\r
1968         ipoib_port_t            *p_port;\r
1969         ib_api_status_t         status;\r
1970         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1971         int32_t                         NBL_cnt, recv_cnt = 0, shortage, discarded;\r
1972         cl_qlist_t                      done_list, bad_list;\r
1973         size_t                          i;\r
1974         ULONG                           recv_complete_flags = 0;\r
1975 \r
1976         PERF_DECLARE( RecvCompBundle );\r
1977         PERF_DECLARE( RecvCb );\r
1978         PERF_DECLARE( PollRecv );\r
1979         PERF_DECLARE( RepostRecv );\r
1980         PERF_DECLARE( FilterRecv );\r
1981         PERF_DECLARE( BuildNBLArray );\r
1982         PERF_DECLARE( RecvNdisIndicate );\r
1983         PERF_DECLARE( RearmRecv );\r
1984         PERF_DECLARE( PutRecvList );\r
1985 \r
1986         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1987 \r
1988         cl_perf_clr( RecvCompBundle );\r
1989 \r
1990         cl_perf_start( RecvCb );\r
1991 \r
1992         UNUSED_PARAM( h_cq );\r
1993 \r
1994         NDIS_SET_SEND_COMPLETE_FLAG(recv_complete_flags, NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL );\r
1995 \r
1996         p_port = (ipoib_port_t*)cq_context;\r
1997 \r
1998         cl_qlist_init( &done_list );\r
1999         cl_qlist_init( &bad_list );\r
2000 \r
2001         ipoib_port_ref( p_port, ref_recv_cb );\r
2002         for( i = 0; i < MAX_RECV_WC; i++ )\r
2003                 wc[i].p_next = &wc[i + 1];\r
2004         wc[MAX_RECV_WC - 1].p_next = NULL;\r
2005 \r
2006         /*\r
2007          * We'll be accessing the endpoint map so take a reference\r
2008          * on it to prevent modifications.\r
2009          */\r
2010         cl_obj_lock( &p_port->obj );\r
2011         cl_atomic_inc( &p_port->endpt_rdr );\r
2012         cl_obj_unlock( &p_port->obj );\r
2013 \r
2014         do\r
2015         {\r
2016                 /* If we get here, then the list of WCs is intact. */\r
2017                 p_free = wc;\r
2018 \r
2019                 cl_perf_start( PollRecv );\r
2020                 status = p_port->p_adapter->p_ifc->poll_cq(\r
2021                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
2022                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
2023                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
2024 \r
2025                 /* Look at the payload now and filter ARP and DHCP packets. */\r
2026                 cl_perf_start( FilterRecv );\r
2027                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
2028                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
2029 \r
2030         } while( ( !p_free ) && ( recv_cnt < 16 )); //TODO restore back to 128\r
2031 \r
2032         /* We're done looking at the endpoint map, release the reference. */\r
2033         cl_atomic_dec( &p_port->endpt_rdr );\r
2034 \r
2035         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
2036 \r
2037         cl_spinlock_acquire( &p_port->recv_lock );\r
2038 \r
2039         /* Update our posted depth. */\r
2040         p_port->recv_mgr.depth -= recv_cnt;\r
2041 \r
2042         /* Return any discarded receives to the pool */\r
2043         cl_perf_start( PutRecvList );\r
2044         __buf_mgr_put_recv_list( p_port, &bad_list );\r
2045         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
2046 \r
2047         do\r
2048         {\r
2049                 //int32_t cnt;\r
2050                 /* Repost ASAP so we don't starve the RQ. */\r
2051                 cl_perf_start( RepostRecv );\r
2052                 shortage = __recv_mgr_repost( p_port );\r
2053                 \r
2054                 if( shortage > 0 )\r
2055                 {                               \r
2056                         recv_complete_flags |= NDIS_RECEIVE_FLAGS_RESOURCES;\r
2057                         cl_dbg_out("Got SHORTAGE=%d\n",shortage);\r
2058                 }\r
2059                 \r
2060                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
2061 \r
2062                 cl_perf_start( BuildNBLArray );\r
2063                 /* Notify NDIS of any and all possible receive buffers. */\r
2064                 NBL_cnt = __recv_mgr_build_NBL_array(\r
2065                         p_port, &done_list, &discarded);\r
2066                 cl_perf_stop( &p_port->p_adapter->perf, BuildNBLArray );\r
2067 \r
2068                 /* Only indicate receives if we actually had any. */\r
2069                 if( discarded && shortage > 0 )\r
2070                 {\r
2071                         /* We may have thrown away packets, and have a shortage */\r
2072                         cl_perf_start( RepostRecv );\r
2073                         __recv_mgr_repost( p_port );\r
2074                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
2075                 }\r
2076 \r
2077                 if( !NBL_cnt ) {\r
2078                         //cl_dbg_out("NBL cnt == 0 :-(\n");\r
2079                         break;\r
2080                 }\r
2081 \r
2082                 cl_spinlock_release( &p_port->recv_lock );\r
2083 \r
2084                 cl_perf_start( RecvNdisIndicate );\r
2085                 \r
2086                 NdisMIndicateReceiveNetBufferLists(\r
2087                         p_port->p_adapter->h_adapter,\r
2088                         p_port->recv_mgr.recv_NBL_array[0],\r
2089                         NDIS_DEFAULT_PORT_NUMBER,\r
2090                         NBL_cnt,\r
2091                         recv_complete_flags);\r
2092 \r
2093                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
2094 \r
2095                 /*\r
2096                  * Cap the number of receives to put back to what we just indicated\r
2097                  * with NDIS_STATUS_RESOURCES.\r
2098                  */\r
2099                 if( shortage > 0 )\r
2100                 {\r
2101 \r
2102                         cl_dbg_out("GOT SHORTAGE <===============\n");\r
2103                         /* Return all but the last packet to the pool. */\r
2104                         cl_spinlock_acquire( &p_port->recv_lock );\r
2105                         //while( shortage-- > 1 )\r
2106                         while ( NBL_cnt-- > 0)\r
2107                         {\r
2108                                 __buf_mgr_put_recv( p_port,\r
2109                                         (ipoib_recv_desc_t *)IPOIB_RECV_FROM_NBL( p_port->recv_mgr.recv_NBL_array[NBL_cnt] ),\r
2110                                         p_port->recv_mgr.recv_NBL_array[NBL_cnt] );\r
2111                         }\r
2112                         __recv_mgr_repost( p_port );\r
2113                         cl_spinlock_release( &p_port->recv_lock );\r
2114 \r
2115                         /*\r
2116                          * Return the last packet as if NDIS returned it, so that we repost\r
2117                          * and report any other pending receives.\r
2118                          */\r
2119                         //ipoib_return_net_buffer_list( p_port, p_port->recv_mgr.recv_NBL_array[0],recv_complete_flags );\r
2120                 }\r
2121                 cl_spinlock_acquire( &p_port->recv_lock );\r
2122 \r
2123         } while( NBL_cnt );\r
2124         cl_spinlock_release( &p_port->recv_lock );\r
2125 \r
2126         if (p_free ) {\r
2127                 /*\r
2128                  * Rearm after filtering to prevent contention on the enpoint maps\r
2129                  * and eliminate the possibility of having a call to\r
2130                  * __endpt_mgr_insert find a duplicate.\r
2131                  */\r
2132                 cl_perf_start( RearmRecv );\r
2133                 status = p_port->p_adapter->p_ifc->rearm_cq(\r
2134                         p_port->ib_mgr.h_recv_cq, FALSE );\r
2135                 cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
2136                 CL_ASSERT( status == IB_SUCCESS );\r
2137 \r
2138                 ipoib_port_deref( p_port, ref_recv_cb );\r
2139         } else {\r
2140                 // Please note the reference is still up\r
2141                 KeInsertQueueDpc(&p_port->recv_dpc, NULL, NULL);\r
2142         }\r
2143 \r
2144         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
2145 \r
2146         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2147 }\r
2148 \r
2149 \r
2150 static void\r
2151 __recv_get_endpts(\r
2152         IN                              ipoib_port_t* const                     p_port,\r
2153         IN                              ipoib_recv_desc_t* const        p_desc,\r
2154         IN                              ib_wc_t* const                          p_wc,\r
2155                 OUT                     ipoib_endpt_t** const           pp_src,\r
2156                 OUT                     ipoib_endpt_t** const           pp_dst )\r
2157 {\r
2158         ib_api_status_t         status;\r
2159         mac_addr_t                      mac;\r
2160         PERF_DECLARE( GetEndptByGid );\r
2161         PERF_DECLARE( GetEndptByLid );\r
2162         PERF_DECLARE( EndptInsert );\r
2163 \r
2164         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2165 \r
2166         /* Setup our shortcut pointers based on whether GRH is valid. */\r
2167         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
2168         {\r
2169                 /* Lookup the source endpoints based on GID. */\r
2170                 cl_perf_start( GetEndptByGid );\r
2171                 *pp_src =\r
2172 #if IPOIB_INLINE_RECV\r
2173                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
2174 #else   /* IPOIB_INLINE_RECV */\r
2175                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
2176 #endif  /* IPOIB_INLINE_RECV */\r
2177                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
2178 \r
2179                 /*\r
2180                  * Lookup the destination endpoint based on GID.\r
2181                  * This is used along with the packet filter to determine\r
2182                  * whether to report this to NDIS.\r
2183                  */\r
2184                 cl_perf_start( GetEndptByGid );\r
2185                 *pp_dst =\r
2186 #if IPOIB_INLINE_RECV\r
2187                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
2188 #else   /* IPOIB_INLINE_RECV */\r
2189                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
2190 #endif  /* IPOIB_INLINE_RECV */\r
2191                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
2192 \r
2193                 /*\r
2194                  * Create the source endpoint if it does not exist.  Note that we\r
2195                  * can only do this for globally routed traffic since we need the\r
2196                  * information from the GRH to generate the MAC.\r
2197                  */\r
2198                 if( !*pp_src )\r
2199                 {\r
2200                         status = ipoib_mac_from_guid(\r
2201 #if IPOIB_INLINE_RECV\r
2202                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
2203 #else   /* IPOIB_INLINE_RECV */\r
2204                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
2205 #endif  /* IPOIB_INLINE_RECV */\r
2206                         if( status != IB_SUCCESS )\r
2207                         {\r
2208                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2209                                         ("ipoib_mac_from_guid returned %s\n",\r
2210                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2211                                 return;\r
2212                         }\r
2213 \r
2214                         /* Create the endpoint. */\r
2215 #if IPOIB_INLINE_RECV\r
2216                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
2217 #else   /* IPOIB_INLINE_RECV */\r
2218                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
2219 #endif  /* IPOIB_INLINE_RECV */\r
2220                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
2221                         if( !*pp_src )\r
2222                         {\r
2223                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2224                                         ("ipoib_endpt_create failed\n") );\r
2225                                 return;\r
2226                         }\r
2227                         cl_perf_start( EndptInsert );\r
2228                         cl_obj_lock( &p_port->obj );\r
2229                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2230                         if( status != IB_SUCCESS )\r
2231                         {\r
2232                                 cl_obj_unlock( &p_port->obj );\r
2233                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2234                                         ("__endpt_mgr_insert returned %s\n",\r
2235                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2236                                 *pp_src = NULL;\r
2237                                 return;\r
2238                         }\r
2239                         cl_obj_unlock( &p_port->obj );\r
2240                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
2241                 }\r
2242         }\r
2243         else\r
2244         {\r
2245                 /*\r
2246                  * Lookup the remote endpoint based on LID.  Note that only\r
2247                  * unicast traffic can be LID routed.\r
2248                  */\r
2249                 cl_perf_start( GetEndptByLid );\r
2250                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
2251                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
2252                 *pp_dst = p_port->p_local_endpt;\r
2253                 CL_ASSERT( *pp_dst );\r
2254         }\r
2255 \r
2256         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2257                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2258         {\r
2259                 /* Update the QPN for the endpoint. */\r
2260                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2261                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
2262                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
2263                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
2264                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
2265 //              (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
2266         }\r
2267 \r
2268         if( *pp_src && *pp_dst )\r
2269         {\r
2270                 IPOIB_PRINT(TRACE_LEVEL_VERBOSE, IPOIB_DBG_RECV,\r
2271                         ("Recv:\n"\r
2272                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
2273                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
2274                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
2275                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
2276                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
2277                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
2278                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
2279                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
2280         }\r
2281 \r
2282         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2283 }\r
2284 \r
2285 \r
2286 static int32_t\r
2287 __recv_mgr_filter(\r
2288         IN                              ipoib_port_t* const                     p_port,\r
2289         IN                              ib_wc_t* const                          p_done_wc_list,\r
2290                 OUT                     cl_qlist_t* const                       p_done_list,\r
2291                 OUT                     cl_qlist_t* const                       p_bad_list )\r
2292 {\r
2293         ipoib_recv_desc_t               *p_desc;\r
2294         ib_wc_t                                 *p_wc;\r
2295         ipoib_pkt_t                             *p_ipoib;\r
2296         eth_pkt_t                               *p_eth;\r
2297         ipoib_endpt_t                   *p_src, *p_dst;\r
2298         ib_api_status_t                 status;\r
2299         uint32_t                                len;\r
2300         int32_t                                 recv_cnt = 0;\r
2301         PERF_DECLARE( GetRecvEndpts );\r
2302         PERF_DECLARE( RecvGen );\r
2303         PERF_DECLARE( RecvTcp );\r
2304         PERF_DECLARE( RecvUdp );\r
2305         PERF_DECLARE( RecvDhcp );\r
2306         PERF_DECLARE( RecvArp );\r
2307 \r
2308         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2309 \r
2310         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
2311         {\r
2312                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
2313                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
2314                 recv_cnt++;\r
2315 \r
2316                 if( p_wc->status != IB_WCS_SUCCESS )\r
2317                 {\r
2318                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
2319                         {\r
2320                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2321                                         ("Failed completion %s  (vendor specific %#x)\n",\r
2322                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
2323                                         (int)p_wc->vendor_specific) );\r
2324                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0, 0 );\r
2325                         }\r
2326                         else\r
2327                         {\r
2328                                 IPOIB_PRINT(TRACE_LEVEL_VERBOSE, IPOIB_DBG_RECV,\r
2329                                         ("Flushed completion %s\n",\r
2330                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
2331                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0, 0 );\r
2332                         }\r
2333                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2334                         /* Dereference the port object on behalf of the failed receive. */\r
2335                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
2336                         continue;\r
2337                 }\r
2338 \r
2339                 len = p_wc->length - sizeof(ib_grh_t);\r
2340 \r
2341                 if( len < sizeof(ipoib_hdr_t) )\r
2342                 {\r
2343                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2344                                 ("Received ETH packet < min size\n") );\r
2345                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0, 0 );\r
2346                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2347                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
2348                         continue;\r
2349                 }\r
2350 \r
2351                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
2352                 {\r
2353                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2354                                 ("Received ETH packet len %d > payload MTU (%d)\n",\r
2355                                 (len - sizeof(ipoib_hdr_t)),\r
2356                                 p_port->p_adapter->params.payload_mtu) );\r
2357                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0, 0 );\r
2358                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2359                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
2360                         continue;\r
2361                         \r
2362                 }\r
2363                 /* Successful completion.  Get the receive information. */\r
2364                 p_desc->ndis_csum.Value = ( ( p_wc->recv.ud.recv_opt & IB_RECV_OPT_CSUM_MASK ) >> 8 );\r
2365                 p_desc->len = len + 14 - 4 ;\r
2366                 cl_perf_start( GetRecvEndpts );\r
2367                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
2368                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
2369 \r
2370 #if IPOIB_INLINE_RECV\r
2371                 p_ipoib = &p_desc->buf.ib.pkt;\r
2372                 p_eth = &p_desc->buf.eth.pkt;\r
2373 #else   /* IPOIB_INLINE_RECV */\r
2374                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
2375                 p_eth = &p_desc->p_buf->eth.pkt;\r
2376 #endif  /*IPOIB_INLINE_RECV */\r
2377 \r
2378                 if( p_src )\r
2379                 {\r
2380                         /* Don't report loopback traffic - we requested SW loopback. */\r
2381                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
2382                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
2383                         {\r
2384                                 /*\r
2385                                  * "This is not the packet you're looking for" - don't update\r
2386                                  * receive statistics, the packet never happened.\r
2387                                  */\r
2388                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2389                                 /* Dereference the port object on behalf of the failed recv. */\r
2390                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
2391                                 continue;\r
2392                         }\r
2393                 }\r
2394 \r
2395                 switch( p_ipoib->hdr.type )\r
2396                 {\r
2397                 case ETH_PROT_TYPE_IP:\r
2398                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
2399                         {\r
2400                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2401                                         ("Received IP packet < min size\n") );\r
2402                                 status = IB_INVALID_SETTING;\r
2403                                 break;\r
2404                         }\r
2405 \r
2406                         if( p_ipoib->type.ip.hdr.offset ||\r
2407                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
2408                         {\r
2409                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2410                                 cl_perf_start( RecvTcp );\r
2411                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2412                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2413                                 break;\r
2414                         }\r
2415 \r
2416                         /* First packet of a UDP transfer. */\r
2417                         if( len <\r
2418                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2419                         {\r
2420                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2421                                         ("Received UDP packet < min size\n") );\r
2422                                 status = IB_INVALID_SETTING;\r
2423                                 break;\r
2424                         }\r
2425 \r
2426                         /* Check if DHCP conversion is required. */\r
2427                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2428                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2429                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2430                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER)||\r
2431                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_PROXY_SERVER &&\r
2432                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2433                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2434                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_PROXY_SERVER))\r
2435                         {\r
2436                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2437                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2438                                 {\r
2439                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2440                                                 ("Received DHCP < min size\n") );\r
2441                                         status = IB_INVALID_SETTING;\r
2442                                         break;\r
2443                                 }\r
2444                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2445                                         // If there are IP options in this message, we are in trouble in any case\r
2446                                         status = IB_INVALID_SETTING;\r
2447                                         break;                                  \r
2448                                 }\r
2449                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2450                                 cl_perf_start( RecvDhcp );\r
2451                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2452                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2453                         }\r
2454                         else\r
2455                         {\r
2456                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2457                                 cl_perf_start( RecvUdp );\r
2458                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2459                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2460                         }\r
2461                         break;\r
2462 \r
2463                 case ETH_PROT_TYPE_ARP:\r
2464                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2465                         {\r
2466                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2467                                         ("Received ARP < min size\n") );\r
2468                                 status = IB_INVALID_SETTING;\r
2469                                 break;\r
2470                         }\r
2471                         cl_perf_start( RecvArp );\r
2472                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2473                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2474                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2475                         break;\r
2476 \r
2477                 default:\r
2478                         /* Unfiltered.  Setup the ethernet header and report. */\r
2479                         cl_perf_start( RecvGen );\r
2480                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2481                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2482                 }\r
2483 \r
2484                 if( status != IB_SUCCESS )\r
2485                 {\r
2486                         /* Update stats. */\r
2487                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0, 0 );\r
2488                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2489                         /* Dereference the port object on behalf of the failed receive. */\r
2490                         ipoib_port_deref( p_port, ref_recv_filter );\r
2491                 }\r
2492                 else\r
2493                 {\r
2494                         ip_stat_sel_t               ip_stat;\r
2495                         p_desc->len =\r
2496                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2497                         if( p_dst->h_mcast)\r
2498                         {\r
2499                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2500                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2501                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2502                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2503                                 {\r
2504                                         p_desc->type = PKT_TYPE_BCAST;\r
2505                                         ip_stat = IP_STAT_BCAST_BYTES;\r
2506                                 }\r
2507                                 else\r
2508                                 {\r
2509                                         p_desc->type = PKT_TYPE_MCAST;\r
2510                                         ip_stat = IP_STAT_MCAST_BYTES;\r
2511                                 }\r
2512                         }\r
2513                         else\r
2514                         {\r
2515                                 p_desc->type = PKT_TYPE_UCAST;\r
2516                                 ip_stat = IP_STAT_UCAST_BYTES;\r
2517                                 \r
2518                         }\r
2519                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2520                         ipoib_inc_recv_stat( p_port->p_adapter, ip_stat, len, 1 );  \r
2521                 }\r
2522         }\r
2523 \r
2524         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2525         return recv_cnt;\r
2526 }\r
2527 \r
2528 \r
2529 static ib_api_status_t\r
2530 __recv_gen(\r
2531         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2532                 OUT                     eth_pkt_t* const                        p_eth,\r
2533         IN                              ipoib_endpt_t* const            p_src,\r
2534         IN                              ipoib_endpt_t* const            p_dst )\r
2535 {\r
2536         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2537 \r
2538         if( !p_src || !p_dst )\r
2539         {\r
2540                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2541                         ("Received packet with no matching endpoints.\n") );\r
2542                 return IB_NOT_DONE;\r
2543         }\r
2544 \r
2545         /*\r
2546          * Fill in the ethernet header.  Note that doing so will overwrite\r
2547          * the IPoIB header, so start by moving the information from the IPoIB\r
2548          * header.\r
2549          */\r
2550         p_eth->hdr.type = p_ipoib->hdr.type;\r
2551         p_eth->hdr.src = p_src->mac;\r
2552         p_eth->hdr.dst = p_dst->mac;\r
2553 \r
2554         if ( p_eth->hdr.dst.addr[0] == 1 && \r
2555                  p_eth->hdr.type == ETH_PROT_TYPE_IP &&\r
2556                  p_eth->hdr.dst.addr[2] == 0x5E)  \r
2557         {\r
2558                 p_eth->hdr.dst.addr[1] = 0;\r
2559                 p_eth->hdr.dst.addr[3] = p_eth->hdr.dst.addr[3] & 0x7f;\r
2560         }\r
2561         if (p_dst->h_mcast)\r
2562                 p_dst->is_in_use = TRUE;\r
2563 \r
2564         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2565         return IB_SUCCESS;\r
2566 }\r
2567 \r
2568 \r
2569 static ib_api_status_t\r
2570 __recv_dhcp(\r
2571         IN                              ipoib_port_t* const                     p_port,\r
2572         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2573                 OUT                     eth_pkt_t* const                        p_eth,\r
2574         IN                              ipoib_endpt_t* const            p_src,\r
2575         IN                              ipoib_endpt_t* const            p_dst )\r
2576 {\r
2577         ib_api_status_t         status;\r
2578         dhcp_pkt_t                      *p_dhcp;\r
2579         uint8_t                         *p_option;\r
2580         uint8_t                         *p_cid = NULL;\r
2581         uint8_t                         msg = 0;\r
2582 \r
2583         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2584 \r
2585         UNUSED_PARAM( p_port );\r
2586 \r
2587         /* Create the ethernet header. */\r
2588         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2589         if( status != IB_SUCCESS )\r
2590         {\r
2591                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2592                         ("__recv_gen returned %s.\n", \r
2593                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2594                 return status;\r
2595         }\r
2596 \r
2597         /* Fixup the payload. */\r
2598         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2599         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2600         {\r
2601                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2602                         ("Invalid DHCP op code.\n") );\r
2603                 return IB_INVALID_SETTING;\r
2604         }\r
2605 \r
2606         /*\r
2607          * Find the client identifier option, making sure to skip\r
2608          * the "magic cookie".\r
2609          */\r
2610         p_option = &p_dhcp->options[0];\r
2611         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2612         {\r
2613                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2614                         ("DHCP cookie corrupted.\n") );\r
2615                 return IB_INVALID_PARAMETER;\r
2616         }\r
2617 \r
2618         p_option = &p_dhcp->options[DHCP_COOKIE_SIZE];\r
2619         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2620         {\r
2621                 switch( *p_option )\r
2622                 {\r
2623                 case DHCP_OPT_PAD:\r
2624                         p_option++;\r
2625                         break;\r
2626 \r
2627                 case DHCP_OPT_MSG:\r
2628                         msg = p_option[2];\r
2629                         p_option += 3;\r
2630                         break;\r
2631 \r
2632                 case DHCP_OPT_CLIENT_ID:\r
2633                         p_cid = p_option;\r
2634                         /* Fall through. */\r
2635 \r
2636                 default:\r
2637                         /*\r
2638                          * All other options have a length byte following the option code.\r
2639                          * Offset by the length to get to the next option.\r
2640                          */\r
2641                         p_option += (p_option[1] + 2);\r
2642                 }\r
2643         }\r
2644 \r
2645         switch( msg )\r
2646         {\r
2647         /* message from client */\r
2648         case DHCPDISCOVER:\r
2649         case DHCPREQUEST:\r
2650         case DHCPDECLINE:\r
2651         case DHCPRELEASE:\r
2652         case DHCPINFORM:\r
2653                 if( !p_cid )\r
2654                 {\r
2655                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2656                                 ("Failed to find required Client-identifier option.\n") );\r
2657                         return IB_INVALID_SETTING;\r
2658                 }\r
2659                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2660                 {\r
2661                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2662                                 ("Invalid hardware address type.\n") );\r
2663                         return IB_INVALID_SETTING;\r
2664                 }\r
2665                 break;\r
2666         /* message from DHCP server */\r
2667         case DHCPOFFER:\r
2668         case DHCPACK:\r
2669         case DHCPNAK:\r
2670                 break;\r
2671 \r
2672         default:\r
2673                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2674                         ("Invalide message type.\n") );\r
2675                 return IB_INVALID_PARAMETER;\r
2676         }\r
2677         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2678         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2679         p_dhcp->hlen = HW_ADDR_LEN;\r
2680 \r
2681         if( p_cid ) /* from client */\r
2682         {\r
2683                 /* Validate that the length and type of the option is as required. */\r
2684                 if( p_cid[1] != 21 )\r
2685                 {\r
2686                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2687                                 ("Client-identifier length not 21 as required.\n") );\r
2688                         return IB_INVALID_SETTING;\r
2689                 }\r
2690                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2691                 {\r
2692                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2693                                 ("Client-identifier type is wrong.\n") );\r
2694                         return IB_INVALID_SETTING;\r
2695                 }\r
2696                 /*\r
2697                  * Copy the GID value from the option so that we can make aligned\r
2698                  * accesses to the contents.\r
2699                  * Recover CID to standard type.\r
2700                  */\r
2701                 p_cid[1] =  sizeof (ib_net64_t) + 1;// CID length \r
2702                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type\r
2703                 RtlMoveMemory( &p_cid[3], &p_cid[15], sizeof (ib_net64_t) );\r
2704                 RtlFillMemory(&p_cid[11], 12, 0);\r
2705 \r
2706                 RtlCopyMemory( p_dhcp->chaddr, &p_src->mac, sizeof(p_src->mac) );\r
2707                 RtlFillMemory( &p_dhcp->chaddr[sizeof(p_src->mac)],\r
2708                                ( sizeof(p_dhcp->chaddr) - sizeof(p_src->mac) ), 0 );\r
2709         }\r
2710         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2711         return status;\r
2712 }\r
2713 \r
2714 \r
2715 static ib_api_status_t\r
2716 __recv_arp(\r
2717         IN                              ipoib_port_t* const                     p_port,\r
2718         IN                              ib_wc_t* const                          p_wc,\r
2719         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2720                 OUT                     eth_pkt_t* const                        p_eth,\r
2721         IN                              ipoib_endpt_t** const           pp_src,\r
2722         IN                              ipoib_endpt_t* const            p_dst )\r
2723 {\r
2724         ib_api_status_t                 status;\r
2725         arp_pkt_t                               *p_arp;\r
2726         const ipoib_arp_pkt_t   *p_ib_arp;\r
2727         ib_gid_t                                gid;\r
2728         mac_addr_t                              mac;\r
2729         ipoib_hw_addr_t                 null_hw = {0};\r
2730         uint8_t                                 cm_capable = 0;\r
2731 \r
2732         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2733 \r
2734         if( !p_dst )\r
2735         {\r
2736                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2737                         ("Unknown destination endpoint\n") );\r
2738                 return IB_INVALID_SETTING;\r
2739         }\r
2740 \r
2741         p_ib_arp = &p_ipoib->type.arp;\r
2742         p_arp = &p_eth->type.arp;\r
2743 \r
2744         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2745         {\r
2746                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2747                         ("ARP hardware type is not IB\n") );\r
2748                 return IB_INVALID_SETTING;\r
2749         }\r
2750 \r
2751         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2752         {\r
2753                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2754                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2755                 return IB_INVALID_SETTING;\r
2756         }\r
2757 \r
2758         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2759         {\r
2760                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2761                         ("ARP protocal type not IP\n") );\r
2762                 return IB_INVALID_SETTING;\r
2763         }\r
2764 \r
2765         cm_capable = ipoib_addr_get_flags( &p_ib_arp->src_hw );\r
2766 \r
2767         /*\r
2768          * If we don't have a source, lookup the endpoint specified in the payload.\r
2769          */\r
2770         if( !*pp_src )\r
2771                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2772 \r
2773         /*\r
2774          * If the endpoint exists for the GID, make sure\r
2775          * the dlid and qpn match the arp.\r
2776          */\r
2777         if( *pp_src )\r
2778         {\r
2779                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2780                         sizeof(ib_gid_t) ) )\r
2781                 {\r
2782                         /*\r
2783                          * GIDs for the endpoint are different.  The ARP must\r
2784                          * have been proxied.  Dereference it.\r
2785                          */\r
2786                         *pp_src = NULL;\r
2787                 }\r
2788                 else if( (*pp_src)->dlid &&\r
2789                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2790                 {\r
2791                         /* Out of date!  Destroy the endpoint and replace it. */\r
2792                         __endpt_mgr_remove( p_port, *pp_src );\r
2793                         *pp_src = NULL;\r
2794                 }\r
2795                 else if ( ! ((*pp_src)->dlid)) {\r
2796                         /* Out of date!  Destroy the endpoint and replace it. */\r
2797                         __endpt_mgr_remove( p_port, *pp_src );\r
2798                         *pp_src = NULL;\r
2799                 }\r
2800                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2801                 {\r
2802                         if( (*pp_src)->qpn != ipoib_addr_get_qpn( &p_ib_arp->src_hw ) &&\r
2803                                  p_wc->recv.ud.remote_qp !=     ipoib_addr_get_qpn( &p_ib_arp->src_hw ) )\r
2804                         {\r
2805                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2806                                 __endpt_mgr_remove( p_port, *pp_src );\r
2807                                 *pp_src = NULL;\r
2808                         }\r
2809                 }\r
2810                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2811                 {\r
2812                         /* Out of date!  Destroy the endpoint and replace it. */\r
2813                         __endpt_mgr_remove( p_port, *pp_src );\r
2814                         *pp_src = NULL;\r
2815                 }\r
2816         }\r
2817 \r
2818         /* Do we need to create an endpoint for this GID? */\r
2819         if( !*pp_src )\r
2820         {\r
2821                 /* Copy the src GID to allow aligned access */\r
2822                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2823                 status = ipoib_mac_from_guid( gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
2824                 if (status == IB_INVALID_GUID_MASK)\r
2825                 {\r
2826                         IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
2827                                 ("Invalid GUID mask received, rejecting it") );\r
2828                         ipoib_create_log(p_port->p_adapter->h_adapter, GUID_MASK_LOG_INDEX, EVENT_IPOIB_WRONG_PARAMETER_WRN);\r
2829                 }\r
2830                 else if( status != IB_SUCCESS )\r
2831                 {\r
2832                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2833                                 ("ipoib_mac_from_guid returned %s\n",\r
2834                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2835                         return status;\r
2836                 }\r
2837                 /*\r
2838                  * Create the endpoint.\r
2839                  */\r
2840                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2841                         p_wc->recv.ud.remote_lid, ipoib_addr_get_qpn( &p_ib_arp->src_hw ) );\r
2842 \r
2843                 if( !*pp_src )\r
2844                 {\r
2845                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2846                                 ("ipoib_endpt_create failed\n") );\r
2847                         return status;\r
2848                 }\r
2849 \r
2850                 cl_obj_lock( &p_port->obj );\r
2851                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2852                 if( status != IB_SUCCESS )\r
2853                 {\r
2854                         cl_obj_unlock( &p_port->obj );\r
2855                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2856                                 ("__endpt_mgr_insert return %s \n",\r
2857                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2858                         return status;\r
2859                 }\r
2860 \r
2861                 cl_obj_unlock( &p_port->obj );\r
2862         }\r
2863 \r
2864         (*pp_src)->cm_flag = cm_capable;\r
2865 \r
2866         CL_ASSERT( !cl_memcmp(\r
2867                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2868         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2869                 (*pp_src)->qpn == ipoib_addr_get_qpn( &p_ib_arp->src_hw ) );\r
2870 #if 0\r
2871         if( p_port->p_adapter->params.cm_enabled &&\r
2872                 p_ib_arp->op == ARP_OP_REQ &&\r
2873                 cm_capable == IPOIB_CM_FLAG_RC )\r
2874         {\r
2875                 /* if we've got ARP request and RC flag is set, \r
2876                 save SID for connect REQ to be sent in ARP reply\r
2877                 when requestor's path get resolved */\r
2878                 if( endpt_cm_get_state( (*pp_src) ) == IPOIB_CM_DISCONNECTED )\r
2879                 {\r
2880                         (*pp_src)->cm_flag = cm_capable;\r
2881                         ipoib_addr_set_sid( \r
2882                                 &(*pp_src)->conn.service_id,\r
2883                                 ipoib_addr_get_qpn( &p_ib_arp->src_hw ) );\r
2884                 }\r
2885         }\r
2886 #endif\r
2887 #if 0 //DBG\r
2888         if( p_port->p_adapter->params.cm_enabled )\r
2889         {\r
2890                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
2891                         (" ARP %s from ENDPT[%p] state %d CM cap: %d QPN: %#x MAC: %02x:%02x:%02x:%02x:%02x:%02x\n",\r
2892                         ((p_ib_arp->op == ARP_OP_REQ )? "REQUEST" : "REPLY"),\r
2893                         *pp_src, endpt_cm_get_state( *pp_src ), \r
2894                         ((cm_capable == IPOIB_CM_FLAG_RC)? 1: 0),\r
2895                         cl_ntoh32( ipoib_addr_get_qpn( &p_ib_arp->src_hw ) ),\r
2896                         (*pp_src)->mac.addr[0], (*pp_src)->mac.addr[1],\r
2897                         (*pp_src)->mac.addr[2], (*pp_src)->mac.addr[3],\r
2898                         (*pp_src)->mac.addr[4], (*pp_src)->mac.addr[5] ));\r
2899         }\r
2900 #endif\r
2901 \r
2902         /* Now swizzle the data. */\r
2903         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2904         p_arp->hw_size = sizeof(mac_addr_t);\r
2905         p_arp->src_hw = (*pp_src)->mac;\r
2906         p_arp->src_ip = p_ib_arp->src_ip;\r
2907 \r
2908         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r