[IPoIB] Use ib_port_info_compute_rate rather than duplicating functionality.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 \r
38 \r
39 /*\r
40  * PR 102492 relates to the SA not accepting wildcard values for MCMemberRecord\r
41  * Sets used to create/join Multicast groups.  Defining this keyword causes all\r
42  * member record fields to be hard coded to well known values.\r
43  */\r
44 //#define PR_102492\r
45 \r
46 /*\r
47  * PR 102801 relates to the SA ignoring the MGID when doing a GET_TABLE\r
48  * request that specifies an MGID.  The ESM returns every McMemberRecord\r
49  * instead.\r
50  */\r
51 //#define PR_102801\r
52 \r
53 \r
54 /* Amount of physical memory to register. */\r
55 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
56 \r
57 /* Number of work completions to chain for send and receive polling. */\r
58 #define MAX_SEND_WC             8\r
59 #define MAX_RECV_WC             16\r
60 \r
61 \r
62 ib_gid_t        bcast_mgid_template = {\r
63         0xff,                                                           /* multicast field */\r
64         0x12,                                                           /* scope (to be filled in) */\r
65         0x40, 0x1b,                                                     /* IPv4 signature */\r
66         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
67         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
68         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
69 };\r
70 \r
71 \r
72 #ifdef _DEBUG_\r
73 /* Handy pointer for debug use. */\r
74 ipoib_port_t    *gp_ipoib_port;\r
75 #endif\r
76 \r
77 \r
78 /******************************************************************************\r
79 *\r
80 * Declarations\r
81 *\r
82 ******************************************************************************/\r
83 static void\r
84 __port_construct(\r
85         IN                              ipoib_port_t* const                     p_port );\r
86 \r
87 static ib_api_status_t\r
88 __port_init(\r
89         IN                              ipoib_port_t* const                     p_port,\r
90         IN                              ipoib_adapter_t* const          p_adapter,\r
91         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
92 \r
93 static void\r
94 __port_destroying(\r
95         IN                              cl_obj_t* const                         p_obj );\r
96 \r
97 static void\r
98 __port_cleanup(\r
99         IN                              cl_obj_t* const                         p_obj );\r
100 \r
101 static void\r
102 __port_free(\r
103         IN                              cl_obj_t* const                         p_obj );\r
104 \r
105 \r
106 /******************************************************************************\r
107 *\r
108 * IB resource manager operations\r
109 *\r
110 ******************************************************************************/\r
111 static void\r
112 __ib_mgr_construct(\r
113         IN                              ipoib_port_t* const                     p_port );\r
114 \r
115 static ib_api_status_t\r
116 __ib_mgr_init(\r
117         IN                              ipoib_port_t* const                     p_port );\r
118 \r
119 static void\r
120 __ib_mgr_destroy(\r
121         IN                              ipoib_port_t* const                     p_port );\r
122 \r
123 static void\r
124 __qp_event(\r
125         IN                              ib_async_event_rec_t            *p_event_rec );\r
126 \r
127 static void\r
128 __cq_event(\r
129         IN                              ib_async_event_rec_t            *p_event_rec );\r
130 \r
131 static ib_api_status_t\r
132 __ib_mgr_activate(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 /******************************************************************************\r
136 *\r
137 * Buffer manager operations.\r
138 *\r
139 ******************************************************************************/\r
140 static void\r
141 __buf_mgr_construct(\r
142         IN                              ipoib_port_t* const                     p_port );\r
143 \r
144 static ib_api_status_t\r
145 __buf_mgr_init(\r
146         IN                              ipoib_port_t* const                     p_port );\r
147 \r
148 static void\r
149 __buf_mgr_destroy(\r
150         IN                              ipoib_port_t* const                     p_port );\r
151 \r
152 static cl_status_t\r
153 __recv_ctor(\r
154         IN                              void* const                                     p_object,\r
155         IN                              void*                                           context,\r
156                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
157 \r
158 #if !IPOIB_INLINE_RECV\r
159 static void\r
160 __recv_dtor(\r
161         IN              const   cl_pool_item_t* const           p_pool_item,\r
162         IN                              void                                            *context );\r
163 #endif  /* IPOIB_INLINE_RECV */\r
164 \r
165 static inline ipoib_send_desc_t*\r
166 __buf_mgr_get_send(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_send(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_send_desc_t* const        p_desc );\r
173 \r
174 static inline ipoib_recv_desc_t*\r
175 __buf_mgr_get_recv(\r
176         IN                              ipoib_port_t* const                     p_port );\r
177 \r
178 static inline void\r
179 __buf_mgr_put_recv(\r
180         IN                              ipoib_port_t* const                     p_port,\r
181         IN                              ipoib_recv_desc_t* const        p_desc,\r
182         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
183 \r
184 static inline void\r
185 __buf_mgr_put_recv_list(\r
186         IN                              ipoib_port_t* const                     p_port,\r
187         IN                              cl_qlist_t* const                       p_list );\r
188 \r
189 static inline NDIS_PACKET*\r
190 __buf_mgr_get_ndis_pkt(\r
191         IN                              ipoib_port_t* const                     p_port,\r
192         IN                              ipoib_recv_desc_t* const        p_desc );\r
193 \r
194 \r
195 /******************************************************************************\r
196 *\r
197 * Receive manager operations.\r
198 *\r
199 ******************************************************************************/\r
200 static void\r
201 __recv_mgr_construct(\r
202         IN                              ipoib_port_t* const                     p_port );\r
203 \r
204 static ib_api_status_t\r
205 __recv_mgr_init(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_mgr_destroy(\r
210         IN                              ipoib_port_t* const                     p_port );\r
211 \r
212 /* Posts receive buffers to the receive queue. */\r
213 static ib_api_status_t\r
214 __recv_mgr_repost(\r
215         IN                              ipoib_port_t* const                     p_port,\r
216         IN                              uint32_t                                        num_wrs );\r
217 \r
218 static void\r
219 __recv_cb(\r
220         IN              const   ib_cq_handle_t                          h_cq,\r
221         IN                              void                                            *cq_context );\r
222 \r
223 static void\r
224 __recv_get_endpts(\r
225         IN                              ipoib_port_t* const                     p_port,\r
226         IN                              ipoib_recv_desc_t* const        p_desc,\r
227         IN                              ib_wc_t* const                          p_wc,\r
228                 OUT                     ipoib_endpt_t** const           pp_src,\r
229                 OUT                     ipoib_endpt_t** const           pp_dst );\r
230 \r
231 static uint32_t\r
232 __recv_mgr_filter(\r
233         IN                              ipoib_port_t* const                     p_port,\r
234         IN                              ib_wc_t* const                          p_done_wc_list,\r
235                 OUT                     cl_qlist_t* const                       p_done_list,\r
236                 OUT                     cl_qlist_t* const                       p_bad_list );\r
237 \r
238 static ib_api_status_t\r
239 __recv_gen(\r
240         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
241                 OUT                     eth_pkt_t* const                        p_eth,\r
242         IN                              ipoib_endpt_t* const            p_src,\r
243         IN                              ipoib_endpt_t* const            p_dst );\r
244 \r
245 static ib_api_status_t\r
246 __recv_dhcp(\r
247         IN                              ipoib_port_t* const                     p_port,\r
248         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
249                 OUT                     eth_pkt_t* const                        p_eth,\r
250         IN                              ipoib_endpt_t* const            p_src,\r
251         IN                              ipoib_endpt_t* const            p_dst );\r
252 \r
253 static ib_api_status_t\r
254 __recv_arp(\r
255         IN                              ipoib_port_t* const                     p_port,\r
256         IN                              ib_wc_t* const                          p_wc,\r
257         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
258                 OUT                     eth_pkt_t* const                        p_eth,\r
259         IN                              ipoib_endpt_t** const           p_src,\r
260         IN                              ipoib_endpt_t* const            p_dst );\r
261 \r
262 static ib_api_status_t\r
263 __recv_mgr_prepare_pkt(\r
264         IN                              ipoib_port_t* const                     p_port,\r
265         IN                              ipoib_recv_desc_t* const        p_desc,\r
266                 OUT                     NDIS_PACKET** const                     pp_packet );\r
267 \r
268 static uint32_t\r
269 __recv_mgr_build_pkt_array(\r
270         IN                              ipoib_port_t* const                     p_port,\r
271                 OUT                     cl_qlist_t* const                       p_done_list );\r
272 \r
273 /******************************************************************************\r
274 *\r
275 * Send manager operations.\r
276 *\r
277 ******************************************************************************/\r
278 static void\r
279 __send_mgr_construct(\r
280         IN                              ipoib_port_t* const                     p_port );\r
281 \r
282 static void\r
283 __send_mgr_destroy(\r
284         IN                              ipoib_port_t* const                     p_port );\r
285 \r
286 static NDIS_STATUS\r
287 __send_gen(\r
288         IN                              ipoib_port_t* const                     p_port,\r
289         IN                              ipoib_send_desc_t* const        p_desc );\r
290 \r
291 static NDIS_STATUS\r
292 __send_mgr_filter_ip(\r
293         IN                              ipoib_port_t* const                     p_port,\r
294         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
295         IN                              NDIS_BUFFER*                            p_buf,\r
296         IN                              size_t                                          buf_len,\r
297         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
298 \r
299 static NDIS_STATUS\r
300 __send_mgr_filter_udp(\r
301         IN                              ipoib_port_t* const                     p_port,\r
302         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
303         IN                              NDIS_BUFFER*                            p_buf,\r
304         IN                              size_t                                          buf_len,\r
305         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
306 \r
307 static NDIS_STATUS\r
308 __send_mgr_filter_dhcp(\r
309         IN                              ipoib_port_t* const                     p_port,\r
310         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
311         IN                              NDIS_BUFFER*                            p_buf,\r
312         IN                              size_t                                          buf_len,\r
313         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
314 \r
315 static NDIS_STATUS\r
316 __send_mgr_filter_arp(\r
317         IN                              ipoib_port_t* const                     p_port,\r
318         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
319         IN                              NDIS_BUFFER*                            p_buf,\r
320         IN                              size_t                                          buf_len,\r
321         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
322 \r
323 static void\r
324 __process_failed_send(\r
325         IN                              ipoib_port_t* const                     p_port,\r
326         IN                              ipoib_send_desc_t* const        p_desc,\r
327         IN              const   NDIS_STATUS                                     status );\r
328 \r
329 static void\r
330 __send_cb(\r
331         IN              const   ib_cq_handle_t                          h_cq,\r
332         IN                              void                                            *cq_context );\r
333 \r
334 \r
335 /******************************************************************************\r
336 *\r
337 * Endpoint manager operations\r
338 *\r
339 ******************************************************************************/\r
340 static void\r
341 __endpt_mgr_construct(\r
342         IN                              ipoib_port_t* const                     p_port );\r
343 \r
344 static ib_api_status_t\r
345 __endpt_mgr_init(\r
346         IN                              ipoib_port_t* const                     p_port );\r
347 \r
348 static void\r
349 __endpt_mgr_destroy(\r
350         IN                              ipoib_port_t* const                     p_port );\r
351 \r
352 /****f* IPoIB/__endpt_mgr_remove_all\r
353 * NAME\r
354 *       __endpt_mgr_remove_all\r
355 *\r
356 * DESCRIPTION\r
357 *       Removes all enpoints from the port, dereferencing them to initiate\r
358 *       destruction.\r
359 *\r
360 * SYNOPSIS\r
361 */\r
362 static void\r
363 __endpt_mgr_remove_all(\r
364         IN                              ipoib_port_t* const                     p_port );\r
365 /*\r
366 ********/\r
367 \r
368 static void\r
369 __endpt_mgr_remove(\r
370         IN                              ipoib_port_t* const                     p_port,\r
371         IN                              ipoib_endpt_t* const            p_endpt );\r
372 \r
373 static inline NDIS_STATUS\r
374 __endpt_mgr_ref(\r
375         IN                              ipoib_port_t* const                     p_port,\r
376         IN              const   mac_addr_t                                      mac,\r
377                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
378 \r
379 static inline NDIS_STATUS\r
380 __endpt_mgr_get_gid_qpn(\r
381         IN                              ipoib_port_t* const                     p_port,\r
382         IN              const   mac_addr_t                                      mac,\r
383                 OUT                     ib_gid_t* const                         p_gid,\r
384                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
385 \r
386 static inline ipoib_endpt_t*\r
387 __endpt_mgr_get_by_gid(\r
388         IN                              ipoib_port_t* const                     p_port,\r
389         IN              const   ib_gid_t* const                         p_gid );\r
390 \r
391 static inline ipoib_endpt_t*\r
392 __endpt_mgr_get_by_lid(\r
393         IN                              ipoib_port_t* const                     p_port,\r
394         IN              const   net16_t                                         lid );\r
395 \r
396 static inline void\r
397 __endpt_mgr_insert_locked(\r
398         IN                              ipoib_port_t* const                     p_port,\r
399         IN              const   mac_addr_t                                      mac,\r
400         IN                              ipoib_endpt_t* const            p_endpt );\r
401 \r
402 static inline void\r
403 __endpt_mgr_insert(\r
404         IN                              ipoib_port_t* const                     p_port,\r
405         IN              const   mac_addr_t                                      mac,\r
406         IN                              ipoib_endpt_t* const            p_endpt );\r
407 \r
408 static ib_api_status_t\r
409 __endpt_mgr_add_local(\r
410         IN                              ipoib_port_t* const                     p_port,\r
411         IN                              ib_port_info_t* const           p_port_info );\r
412 \r
413 static ib_api_status_t\r
414 __endpt_mgr_add_bcast(\r
415         IN                              ipoib_port_t* const                     p_port,\r
416         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
417 \r
418 /******************************************************************************\r
419 *\r
420 * MCast operations.\r
421 *\r
422 ******************************************************************************/\r
423 static void\r
424 __port_get_mcast(\r
425         IN                              ipoib_port_t* const                     p_port );\r
426 \r
427 static void\r
428 __port_join_bcast(\r
429         IN                              ipoib_port_t* const                     p_port,\r
430         IN                              ib_member_rec_t* const          p_member_rec );\r
431 \r
432 static void\r
433 __port_create_bcast(\r
434         IN                              ipoib_port_t* const                     p_port );\r
435 \r
436 static void\r
437 __port_info_cb(\r
438         IN                              ib_query_rec_t                          *p_query_rec );\r
439 \r
440 \r
441 static void\r
442 __bcast_get_cb(\r
443         IN                              ib_query_rec_t                          *p_query_rec );\r
444 \r
445 \r
446 static void\r
447 __bcast_cb(\r
448         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
449 \r
450 \r
451 static void\r
452 __mcast_cb(\r
453         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
454 \r
455 \r
456 static intn_t\r
457 __gid_cmp(\r
458         IN              const   void* const                                     p_key1,\r
459         IN              const   void* const                                     p_key2 )\r
460 {\r
461         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
462 }\r
463 \r
464 \r
465 /******************************************************************************\r
466 *\r
467 * Implementation\r
468 *\r
469 ******************************************************************************/\r
470 ib_api_status_t\r
471 ipoib_create_port(\r
472         IN                              ipoib_adapter_t* const          p_adapter,\r
473         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
474                 OUT                     ipoib_port_t** const            pp_port )\r
475 {\r
476         ib_api_status_t         status;\r
477         ipoib_port_t            *p_port;\r
478 \r
479         IPOIB_ENTER( IPOIB_DBG_INIT );\r
480 \r
481         CL_ASSERT( !p_adapter->p_port );\r
482 \r
483         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
484                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
485         if( !p_port )\r
486         {\r
487                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
488                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
489                         sizeof(ipoib_port_t)) );\r
490                 return IB_INSUFFICIENT_MEMORY;\r
491         }\r
492 \r
493 #ifdef _DEBUG_\r
494         gp_ipoib_port = p_port;\r
495 #endif\r
496 \r
497         __port_construct( p_port );\r
498 \r
499         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
500         if( status != IB_SUCCESS )\r
501         {\r
502                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
503                         ("ipoib_port_init returned %s.\n", \r
504                         p_adapter->p_ifc->get_err_str( status )) );\r
505                 __port_cleanup( &p_port->obj );\r
506                 __port_free( &p_port->obj );\r
507                 return status;\r
508         }\r
509 \r
510         *pp_port = p_port;\r
511         IPOIB_EXIT( IPOIB_DBG_INIT );\r
512         return IB_SUCCESS;\r
513 }\r
514 \r
515 \r
516 void\r
517 ipoib_port_destroy(\r
518         IN                              ipoib_port_t* const                     p_port )\r
519 {\r
520         IPOIB_ENTER( IPOIB_DBG_INIT );\r
521 \r
522         CL_ASSERT( p_port );\r
523         CL_ASSERT( p_port->p_adapter );\r
524         CL_ASSERT( !p_port->p_adapter->p_port );\r
525 \r
526         cl_obj_destroy( &p_port->obj );\r
527 \r
528         IPOIB_EXIT( IPOIB_DBG_INIT );\r
529 }\r
530 \r
531 \r
532 static void\r
533 __port_construct(\r
534         IN                              ipoib_port_t* const                     p_port )\r
535 {\r
536         IPOIB_ENTER( IPOIB_DBG_INIT );\r
537 \r
538         p_port->state = IB_QPS_RESET;\r
539 \r
540         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
541         cl_spinlock_construct( &p_port->send_lock );\r
542         cl_spinlock_construct( &p_port->recv_lock );\r
543         __ib_mgr_construct( p_port );\r
544         __buf_mgr_construct( p_port );\r
545 \r
546         __recv_mgr_construct( p_port );\r
547         __send_mgr_construct( p_port );\r
548 \r
549         __endpt_mgr_construct( p_port );\r
550 \r
551         IPOIB_EXIT( IPOIB_DBG_INIT );\r
552 }\r
553 \r
554 \r
555 static ib_api_status_t\r
556 __port_init(\r
557         IN                              ipoib_port_t* const                     p_port,\r
558         IN                              ipoib_adapter_t* const          p_adapter,\r
559         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
560 {\r
561         cl_status_t                     cl_status;\r
562         ib_api_status_t         status;\r
563 \r
564         IPOIB_ENTER( IPOIB_DBG_INIT );\r
565 \r
566         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
567         p_port->p_adapter = p_adapter;\r
568 \r
569         cl_status = cl_spinlock_init( &p_port->send_lock );\r
570         if( cl_status != CL_SUCCESS )\r
571         {\r
572                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
573                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
574                 return IB_ERROR;\r
575         }\r
576 \r
577         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
578         if( cl_status != CL_SUCCESS )\r
579         {\r
580                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
581                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
582                 return IB_ERROR;\r
583         }\r
584 \r
585         /* Initialize the IB resource manager. */\r
586         status = __ib_mgr_init( p_port );\r
587         if( status != IB_SUCCESS )\r
588         {\r
589                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
590                         ("__ib_mgr_init returned %s\n", \r
591                         p_adapter->p_ifc->get_err_str( status )) );\r
592                 return status;\r
593         }\r
594 \r
595         /* Initialize the buffer manager. */\r
596         status = __buf_mgr_init( p_port );\r
597         if( status != IB_SUCCESS )\r
598         {\r
599                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
600                         ("__buf_mgr_init returned %s\n", \r
601                         p_adapter->p_ifc->get_err_str( status )) );\r
602                 return status;\r
603         }\r
604 \r
605         /* Initialize the receive manager. */\r
606         status = __recv_mgr_init( p_port );\r
607         if( status != IB_SUCCESS )\r
608         {\r
609                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
610                         ("__recv_mgr_init returned %s\n", \r
611                         p_adapter->p_ifc->get_err_str( status )) );\r
612                 return status;\r
613         }\r
614 \r
615         /* Initialize the endpoint manager. */\r
616         status = __endpt_mgr_init( p_port );\r
617         if( status != IB_SUCCESS )\r
618         {\r
619                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
620                         ("__endpt_mgr_init returned %s\n", \r
621                         p_adapter->p_ifc->get_err_str( status )) );\r
622                 return status;\r
623         }\r
624 \r
625         /* We only ever destroy from the PnP callback thread. */\r
626         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
627                 __port_destroying, __port_cleanup, __port_free );\r
628         if( cl_status != CL_SUCCESS )\r
629         {\r
630                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
631                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
632                 return IB_ERROR;\r
633         }\r
634 \r
635         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
636 \r
637         IPOIB_EXIT( IPOIB_DBG_INIT );\r
638         return IB_SUCCESS;\r
639 }\r
640 \r
641 \r
642 static void\r
643 __port_destroying(\r
644         IN                              cl_obj_t* const                         p_obj )\r
645 {\r
646         ipoib_port_t    *p_port;\r
647 \r
648         IPOIB_ENTER( IPOIB_DBG_INIT );\r
649 \r
650         CL_ASSERT( p_obj );\r
651 \r
652         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
653 \r
654         ipoib_port_down( p_port );\r
655 \r
656         IPOIB_EXIT( IPOIB_DBG_INIT );\r
657 }\r
658 \r
659 \r
660 static void\r
661 __port_cleanup(\r
662         IN                              cl_obj_t* const                         p_obj )\r
663 {\r
664         ipoib_port_t    *p_port;\r
665 \r
666         IPOIB_ENTER( IPOIB_DBG_INIT );\r
667 \r
668         CL_ASSERT( p_obj );\r
669 \r
670         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
671 \r
672         /* Wait for all sends and receives to get flushed. */\r
673         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
674                 cl_thread_suspend( 0 );\r
675 \r
676         /* Destroy the send and receive managers before closing the CA. */\r
677         __ib_mgr_destroy( p_port );\r
678 \r
679         IPOIB_EXIT( IPOIB_DBG_INIT );\r
680 }\r
681 \r
682 \r
683 static void\r
684 __port_free(\r
685         IN                              cl_obj_t* const                         p_obj )\r
686 {\r
687         ipoib_port_t    *p_port;\r
688 \r
689         IPOIB_ENTER( IPOIB_DBG_INIT );\r
690 \r
691         CL_ASSERT( p_obj );\r
692 \r
693         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
694 \r
695         __endpt_mgr_destroy( p_port );\r
696         __recv_mgr_destroy( p_port );\r
697         __send_mgr_destroy( p_port );\r
698         __buf_mgr_destroy( p_port );\r
699 \r
700         cl_spinlock_destroy( &p_port->send_lock );\r
701         cl_spinlock_destroy( &p_port->recv_lock );\r
702 \r
703         cl_obj_deinit( p_obj );\r
704 \r
705         cl_free( p_port );\r
706 \r
707         IPOIB_EXIT( IPOIB_DBG_INIT );\r
708 }\r
709 \r
710 \r
711 \r
712 /******************************************************************************\r
713 *\r
714 * IB resource manager implementation.\r
715 *\r
716 ******************************************************************************/\r
717 static void\r
718 __ib_mgr_construct(\r
719         IN                              ipoib_port_t* const                     p_port )\r
720 {\r
721         IPOIB_ENTER( IPOIB_DBG_INIT );\r
722 \r
723         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
724 \r
725         IPOIB_EXIT( IPOIB_DBG_INIT );\r
726 }\r
727 \r
728 \r
729 static ib_api_status_t\r
730 __ib_mgr_init(\r
731         IN                              ipoib_port_t* const                     p_port )\r
732 {\r
733         ib_api_status_t         status;\r
734         ib_cq_create_t          cq_create;\r
735         ib_qp_create_t          qp_create;\r
736         ib_phys_create_t        phys_create;\r
737         ib_phys_range_t         phys_range;\r
738         uint64_t                        vaddr;\r
739         net32_t                         rkey;\r
740         ib_qp_attr_t            qp_attr;\r
741 \r
742         IPOIB_ENTER( IPOIB_DBG_INIT );\r
743 \r
744         /* Open the CA. */\r
745         status = p_port->p_adapter->p_ifc->open_ca(\r
746                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
747                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
748         if( status != IB_SUCCESS )\r
749         {\r
750                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
751                         EVENT_IPOIB_OPEN_CA, 1, status );\r
752                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
753                         ("ib_open_ca returned %s\n", \r
754                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
755                 return status;\r
756         }\r
757 \r
758         /* Allocate the PD. */\r
759         status = p_port->p_adapter->p_ifc->alloc_pd(\r
760                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
761         if( status != IB_SUCCESS )\r
762         {\r
763                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
764                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
765                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
766                         ("ib_alloc_pd returned %s\n", \r
767                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
768                 return status;\r
769         }\r
770 \r
771         /* Allocate receive CQ. */\r
772         cq_create.size = p_port->p_adapter->params.rq_depth;\r
773         cq_create.pfn_comp_cb = __recv_cb;\r
774         cq_create.h_wait_obj = NULL;\r
775 \r
776         status = p_port->p_adapter->p_ifc->create_cq(\r
777                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
778                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
779         if( status != IB_SUCCESS )\r
780         {\r
781                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
782                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
783                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
784                         ("ib_create_cq returned %s.\n", \r
785                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
786                 return status;\r
787         }\r
788 \r
789         /* Allocate send CQ. */\r
790         cq_create.size = p_port->p_adapter->params.sq_depth;\r
791         cq_create.pfn_comp_cb = __send_cb;\r
792 \r
793         status = p_port->p_adapter->p_ifc->create_cq(\r
794                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
795                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
796         if( status != IB_SUCCESS )\r
797         {\r
798                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
799                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
800                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
801                         ("ib_create_cq returned %s.\n", \r
802                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
803                 return status;\r
804         }\r
805 \r
806         /* Allocate the QP. */\r
807         cl_memclr( &qp_create, sizeof(qp_create) );\r
808         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
809         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
810         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
811         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
812         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
813         //TODO: Figure out the right number of SGE entries for sends.\r
814         qp_create.sq_sge = MAX_SEND_SGE;\r
815         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
816         qp_create.sq_signaled = TRUE;\r
817         status = p_port->p_adapter->p_ifc->create_qp(\r
818                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
819                 __qp_event, &p_port->ib_mgr.h_qp );\r
820         if( status != IB_SUCCESS )\r
821         {\r
822                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
823                         EVENT_IPOIB_CREATE_QP, 1, status );\r
824                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
825                         ("ib_create_qp returned %s\n", \r
826                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
827                 return status;\r
828         }\r
829         /* Query the QP so we can get our QPN. */\r
830         status = p_port->p_adapter->p_ifc->query_qp(\r
831                 p_port->ib_mgr.h_qp, &qp_attr );\r
832         if( status != IB_SUCCESS )\r
833         {\r
834                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
835                         EVENT_IPOIB_QUERY_QP, 1, status );\r
836                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
837                         ("ib_query_qp returned %s\n", \r
838                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
839                 return status;\r
840         }\r
841         p_port->ib_mgr.qpn = qp_attr.num;\r
842 \r
843         /* Register all of physical memory */\r
844         phys_create.length = MEM_REG_SIZE;\r
845         phys_create.num_ranges = 1;\r
846         phys_create.range_array = &phys_range;\r
847         phys_create.buf_offset = 0;\r
848         phys_create.hca_page_size = PAGE_SIZE;\r
849         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
850         phys_range.base_addr = 0;\r
851         phys_range.size = MEM_REG_SIZE;\r
852         vaddr = 0;\r
853         status = p_port->p_adapter->p_ifc->reg_phys(\r
854                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
855                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
856         if( status != IB_SUCCESS )\r
857         {\r
858                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
859                         EVENT_IPOIB_REG_PHYS, 1, status );\r
860                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
861                         ("ib_reg_phys returned %s\n", \r
862                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
863                 return status;\r
864         }\r
865 \r
866         IPOIB_EXIT( IPOIB_DBG_INIT );\r
867         return IB_SUCCESS;\r
868 }\r
869 \r
870 \r
871 static void\r
872 __ib_mgr_destroy(\r
873         IN                              ipoib_port_t* const                     p_port )\r
874 {\r
875         ib_api_status_t status;\r
876 \r
877         IPOIB_ENTER( IPOIB_DBG_INIT );\r
878 \r
879         if( p_port->ib_mgr.h_ca )\r
880         {\r
881                 status =\r
882                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
883                 CL_ASSERT( status == IB_SUCCESS );\r
884                 p_port->ib_mgr.h_ca = NULL;\r
885         }\r
886 \r
887         IPOIB_EXIT( IPOIB_DBG_INIT );\r
888 }\r
889 \r
890 \r
891 \r
892 /******************************************************************************\r
893 *\r
894 * Buffer manager implementation.\r
895 *\r
896 ******************************************************************************/\r
897 static void\r
898 __buf_mgr_construct(\r
899         IN                              ipoib_port_t* const                     p_port )\r
900 {\r
901         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
902 \r
903         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
904 \r
905         p_port->buf_mgr.h_packet_pool = NULL;\r
906         p_port->buf_mgr.h_buffer_pool = NULL;\r
907 \r
908         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
909                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
910 \r
911         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
912         p_port->buf_mgr.h_send_buf_pool = NULL;\r
913 \r
914         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
915 }\r
916 \r
917 \r
918 static ib_api_status_t\r
919 __buf_mgr_init(\r
920         IN                              ipoib_port_t* const                     p_port )\r
921 {\r
922         cl_status_t             cl_status;\r
923         NDIS_STATUS             ndis_status;\r
924         ipoib_params_t  *p_params;\r
925 \r
926         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
927 \r
928         CL_ASSERT( p_port );\r
929         CL_ASSERT( p_port->p_adapter );\r
930 \r
931         p_params = &p_port->p_adapter->params;\r
932 \r
933         /* Allocate the receive descriptor pool */\r
934         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
935                 p_params->rq_depth * p_params->recv_pool_ratio,\r
936 #if IPOIB_INLINE_RECV\r
937                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
938 #else   /* IPOIB_INLINE_RECV */\r
939                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
940 #endif  /* IPOIB_INLINE_RECV */\r
941         if( cl_status != CL_SUCCESS )\r
942         {\r
943                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
944                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
945                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
946                         ("cl_qpool_init for recvs returned %s\n",\r
947                         cl_status_text[cl_status]) );\r
948                 return IB_INSUFFICIENT_MEMORY;\r
949         }\r
950 \r
951         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
952         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
953                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
954         if( ndis_status != NDIS_STATUS_SUCCESS )\r
955         {\r
956                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
957                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
958                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
959                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
960                 return IB_INSUFFICIENT_RESOURCES;\r
961         }\r
962 \r
963         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
964                 p_params->rq_depth );\r
965         if( ndis_status != NDIS_STATUS_SUCCESS )\r
966         {\r
967                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
968                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
969                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
970                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
971                 return IB_INSUFFICIENT_RESOURCES;\r
972         }\r
973 \r
974         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
975         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
976                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
977         if( ndis_status != NDIS_STATUS_SUCCESS )\r
978         {\r
979                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
980                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
981                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
982                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
983                 return IB_INSUFFICIENT_RESOURCES;\r
984         }\r
985 \r
986         NdisAllocateBufferPool( &ndis_status,\r
987                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
988         if( ndis_status != NDIS_STATUS_SUCCESS )\r
989         {\r
990                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
991                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
992                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
993                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
994                 return IB_INSUFFICIENT_RESOURCES;\r
995         }\r
996 \r
997         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
998         return IB_SUCCESS;\r
999 }\r
1000 \r
1001 \r
1002 static void\r
1003 __buf_mgr_destroy(\r
1004         IN                              ipoib_port_t* const                     p_port )\r
1005 {\r
1006         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
1007 \r
1008         CL_ASSERT( p_port );\r
1009 \r
1010         /* Destroy the send packet and buffer pools. */\r
1011         if( p_port->buf_mgr.h_send_buf_pool )\r
1012                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1013         if( p_port->buf_mgr.h_send_pkt_pool )\r
1014                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1015 \r
1016         /* Destroy the receive packet and buffer pools. */\r
1017         if( p_port->buf_mgr.h_buffer_pool )\r
1018                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1019         if( p_port->buf_mgr.h_packet_pool )\r
1020                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1021 \r
1022         /* Free the receive and send descriptors. */\r
1023         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1024 \r
1025         /* Free the lookaside list of scratch buffers. */\r
1026         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1027 \r
1028         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
1029 }\r
1030 \r
1031 \r
1032 static cl_status_t\r
1033 __recv_ctor(\r
1034         IN                              void* const                                     p_object,\r
1035         IN                              void*                                           context,\r
1036                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1037 {\r
1038         ipoib_recv_desc_t       *p_desc;\r
1039         ipoib_port_t            *p_port;\r
1040         uint32_t                        ds0_len;\r
1041 \r
1042         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1043 \r
1044         CL_ASSERT( p_object );\r
1045         CL_ASSERT( context );\r
1046 \r
1047         p_desc = (ipoib_recv_desc_t*)p_object;\r
1048         p_port = (ipoib_port_t*)context;\r
1049 \r
1050         /* Setup the work request. */\r
1051         p_desc->wr.ds_array = p_desc->local_ds;\r
1052         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1053 \r
1054 #if IPOIB_INLINE_RECV\r
1055         /* Sanity check on the receive buffer layout */\r
1056         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1057                 (void*)&p_desc->buf.ib.pkt.type );\r
1058         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1059 \r
1060         /* Setup the local data segment. */\r
1061         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1062         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1063         ds0_len =\r
1064                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1065         if( ds0_len >= sizeof(recv_buf_t) )\r
1066         {\r
1067                 /* The whole buffer is within a page. */\r
1068                 p_desc->local_ds[0].length = ds0_len;\r
1069                 p_desc->wr.num_ds = 1;\r
1070         }\r
1071         else\r
1072         {\r
1073                 /* The buffer crosses page boundaries. */\r
1074                 p_desc->local_ds[0].length = ds0_len;\r
1075                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1076                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1077                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1078                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1079                 p_desc->wr.num_ds = 2;\r
1080         }\r
1081 #else   /* IPOIB_INLINE_RECV */\r
1082         /* Allocate the receive buffer. */\r
1083         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1084         if( !p_desc->p_buf )\r
1085         {\r
1086                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1087                         ("Failed to allocate receive buffer.\n") );\r
1088                 return CL_INSUFFICIENT_MEMORY;\r
1089         }\r
1090 \r
1091         /* Sanity check on the receive buffer layout */\r
1092         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1093                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1094 \r
1095         /* Setup the local data segment. */\r
1096         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1097         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1098         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1099 #endif  /* IPOIB_INLINE_RECV */\r
1100 \r
1101         *pp_pool_item = &p_desc->item;\r
1102 \r
1103         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1104         return CL_SUCCESS;\r
1105 }\r
1106 \r
1107 \r
1108 #if !IPOIB_INLINE_RECV\r
1109 static void\r
1110 __recv_dtor(\r
1111         IN              const   cl_pool_item_t* const           p_pool_item,\r
1112         IN                              void                                            *context )\r
1113 {\r
1114         ipoib_recv_desc_t       *p_desc;\r
1115 \r
1116         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1117 \r
1118         UNUSED_PARAM( context );\r
1119 \r
1120         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1121 \r
1122         if( p_desc->p_buf )\r
1123                 cl_free( p_desc->p_buf );\r
1124 \r
1125         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1126 }\r
1127 #endif\r
1128 \r
1129 \r
1130 static inline ipoib_recv_desc_t*\r
1131 __buf_mgr_get_recv(\r
1132         IN                              ipoib_port_t* const                     p_port )\r
1133 {\r
1134         ipoib_recv_desc_t       *p_desc;\r
1135         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1136         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1137         /* Reference the port object for the send. */\r
1138         if( p_desc )\r
1139         {\r
1140                 cl_obj_ref( &p_port->obj );\r
1141                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1142 #if IPOIB_INLINE_RECV\r
1143                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1144                         cl_get_physaddr( &p_desc->buf ) );\r
1145 #else   /* IPOIB_INLINE_RECV */\r
1146                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1147                         cl_get_physaddr( p_desc->p_buf ) );\r
1148                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1149                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1150 #endif  /* IPOIB_INLINE_RECV */\r
1151                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1152         }\r
1153         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1154         return p_desc;\r
1155 }\r
1156 \r
1157 \r
1158 static inline void\r
1159 __buf_mgr_put_recv(\r
1160         IN                              ipoib_port_t* const                     p_port,\r
1161         IN                              ipoib_recv_desc_t* const        p_desc,\r
1162         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1163 {\r
1164         NDIS_BUFFER             *p_buf;\r
1165 \r
1166         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1167 \r
1168         if( p_packet )\r
1169         {\r
1170                 /* Unchain the NDIS buffer. */\r
1171                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1172                 CL_ASSERT( p_buf );\r
1173                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1174                 NdisDprFreePacketNonInterlocked( p_packet );\r
1175                 NdisFreeBuffer( p_buf );\r
1176         }\r
1177 \r
1178         /* Return the descriptor to its pools. */\r
1179         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1180 \r
1181         /*\r
1182          * Dereference the port object since the receive is no longer outstanding.\r
1183          */\r
1184         cl_obj_deref( &p_port->obj );\r
1185 \r
1186         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1187 }\r
1188 \r
1189 \r
1190 static inline void\r
1191 __buf_mgr_put_recv_list(\r
1192         IN                              ipoib_port_t* const                     p_port,\r
1193         IN                              cl_qlist_t* const                       p_list )\r
1194 {\r
1195         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1196         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1197         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1198 }\r
1199 \r
1200 \r
1201 static inline NDIS_PACKET*\r
1202 __buf_mgr_get_ndis_pkt(\r
1203         IN                              ipoib_port_t* const                     p_port,\r
1204         IN                              ipoib_recv_desc_t* const        p_desc )\r
1205 {\r
1206         NDIS_STATUS                             status;\r
1207         NDIS_PACKET                             *p_packet;\r
1208         NDIS_BUFFER                             *p_buffer;\r
1209 \r
1210         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1211 \r
1212         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1213                 p_port->buf_mgr.h_packet_pool );\r
1214         if( status != NDIS_STATUS_SUCCESS )\r
1215         {\r
1216                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1217                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1218                 return NULL;\r
1219         }\r
1220 \r
1221         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1222         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1223 \r
1224         NdisAllocateBuffer( &status, &p_buffer,\r
1225 #if IPOIB_INLINE_RECV\r
1226                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1227 #else   /* IPOIB_INLINE_RECV */\r
1228                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1229 #endif  /* IPOIB_INLINE_RECV */\r
1230         if( status != NDIS_STATUS_SUCCESS )\r
1231         {\r
1232                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1233                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1234                 NdisDprFreePacketNonInterlocked( p_packet );\r
1235                 return NULL;\r
1236         }\r
1237 \r
1238         NdisChainBufferAtFront( p_packet, p_buffer );\r
1239         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1240         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1241 \r
1242         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1243         return p_packet;\r
1244 }\r
1245 \r
1246 /******************************************************************************\r
1247 *\r
1248 * Receive manager implementation.\r
1249 *\r
1250 ******************************************************************************/\r
1251 static void\r
1252 __recv_mgr_construct(\r
1253         IN                              ipoib_port_t* const                     p_port )\r
1254 {\r
1255         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1256 \r
1257         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1258 \r
1259         p_port->recv_mgr.recv_pkt_array = NULL;\r
1260 \r
1261         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1262 }\r
1263 \r
1264 \r
1265 static ib_api_status_t\r
1266 __recv_mgr_init(\r
1267         IN                              ipoib_port_t* const                     p_port )\r
1268 {\r
1269         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1270 \r
1271         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1272         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1273                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1274         if( !p_port->recv_mgr.recv_pkt_array )\r
1275         {\r
1276                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1277                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1278                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1279                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1280                 return IB_INSUFFICIENT_MEMORY;\r
1281         }\r
1282 \r
1283         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1284         return IB_SUCCESS;\r
1285 }\r
1286 \r
1287 \r
1288 static void\r
1289 __recv_mgr_destroy(\r
1290         IN                              ipoib_port_t* const                     p_port )\r
1291 {\r
1292         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1293 \r
1294         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1295         CL_ASSERT( !p_port->recv_mgr.depth );\r
1296 \r
1297         if( p_port->recv_mgr.recv_pkt_array )\r
1298                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1299 \r
1300         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1301 }\r
1302 \r
1303 \r
1304 /* Posts receive buffers to the receive queue. */\r
1305 static ib_api_status_t\r
1306 __recv_mgr_repost(\r
1307         IN                              ipoib_port_t* const                     p_port,\r
1308         IN                              uint32_t                                        num_wrs )\r
1309 {\r
1310         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1311         ib_api_status_t         status;\r
1312         ib_recv_wr_t            *p_failed;\r
1313         PERF_DECLARE( GetRecv );\r
1314         PERF_DECLARE( PostRecv );\r
1315 \r
1316         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1317 \r
1318         CL_ASSERT( p_port );\r
1319         CL_ASSERT( num_wrs );\r
1320         cl_obj_lock( &p_port->obj );\r
1321         if( p_port->state == IB_QPS_ERROR )\r
1322         {\r
1323                 cl_obj_unlock( &p_port->obj );\r
1324                 IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
1325                         ("Port in invalid state.  Not reposting.\n") );\r
1326                 return IB_SUCCESS;\r
1327         }\r
1328         cl_obj_ref( &p_port->obj );\r
1329         cl_obj_unlock( &p_port->obj );\r
1330 \r
1331         while( num_wrs-- )\r
1332         {\r
1333                 if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )\r
1334                 {\r
1335                         IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1336                                 ("Receive queue already full\n") );\r
1337                         break;\r
1338                 }\r
1339 \r
1340                 /* Pull receives out of the pool and chain them up. */\r
1341                 cl_perf_start( GetRecv );\r
1342                 p_next = __buf_mgr_get_recv( p_port );\r
1343                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1344                 if( !p_next )\r
1345                 {\r
1346                         IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1347                                 ("Out of receive descriptors!\n") );\r
1348                         break;\r
1349                 }\r
1350 \r
1351                 if( !p_tail )\r
1352                 {\r
1353                         p_tail = p_next;\r
1354                         p_next->wr.p_next = NULL;\r
1355                 }\r
1356                 else\r
1357                 {\r
1358                         p_next->wr.p_next = &p_head->wr;\r
1359                 }\r
1360 \r
1361                 p_head = p_next;\r
1362 \r
1363                 cl_atomic_inc( &p_port->recv_mgr.depth );\r
1364         }\r
1365 \r
1366         if( !p_head )\r
1367         {\r
1368                 cl_obj_deref( &p_port->obj );\r
1369                 IPOIB_EXIT( IPOIB_DBG_RECV );\r
1370                 return IB_SUCCESS;\r
1371         }\r
1372 \r
1373         cl_perf_start( PostRecv );\r
1374         status = p_port->p_adapter->p_ifc->post_recv(\r
1375                 p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1376         cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1377 \r
1378         /*\r
1379          * If we failed, fix up the work completion list and return those\r
1380          * buffers to the pool\r
1381          */\r
1382         if( status != IB_SUCCESS )\r
1383         {\r
1384                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1385                         ("ip_post_recv returned %s\n", \r
1386                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1387                 /* return the descriptors to the pool */\r
1388                 while( p_failed )\r
1389                 {\r
1390                         p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1391                         p_failed = p_failed->p_next;\r
1392 \r
1393                         __buf_mgr_put_recv( p_port, p_head, NULL );\r
1394                         cl_atomic_dec( &p_port->recv_mgr.depth );\r
1395                 }\r
1396         }\r
1397 \r
1398         cl_obj_deref( &p_port->obj );\r
1399 \r
1400         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1401         return status;\r
1402 }\r
1403 \r
1404 \r
1405 void\r
1406 ipoib_return_packet(\r
1407         IN                              NDIS_HANDLE                                     adapter_context,\r
1408         IN                              NDIS_PACKET                                     *p_packet )\r
1409 {\r
1410         cl_list_item_t          *p_item;\r
1411         ipoib_port_t            *p_port;\r
1412         ipoib_recv_desc_t       *p_desc;\r
1413         ib_api_status_t         status = IB_NOT_DONE;\r
1414         PERF_DECLARE( ReturnPacket );\r
1415         PERF_DECLARE( ReturnPutRecv );\r
1416         PERF_DECLARE( ReturnRepostRecv );\r
1417         PERF_DECLARE( ReturnPreparePkt );\r
1418         PERF_DECLARE( ReturnNdisIndicate );\r
1419 \r
1420         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1421 \r
1422         UNUSED_PARAM( adapter_context );\r
1423         CL_ASSERT( p_packet );\r
1424 \r
1425         cl_perf_start( ReturnPacket );\r
1426 \r
1427         /* Get the manager and descriptor from the packet. */\r
1428         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1429         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1430 \r
1431         cl_spinlock_acquire( &p_port->recv_lock );\r
1432         cl_perf_start( ReturnPutRecv );\r
1433         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1434         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1435 \r
1436         /* Repost buffers. */\r
1437         cl_perf_start( ReturnRepostRecv );\r
1438         __recv_mgr_repost( p_port, 0xFFFF );\r
1439         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1440 \r
1441         /* Complete any additional receives waiting for a packet. */\r
1442         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1443         do\r
1444         {\r
1445                 if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
1446                 {\r
1447                         cl_spinlock_release( &p_port->recv_lock );\r
1448                         break;\r
1449                 }\r
1450 \r
1451                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1452 \r
1453                 cl_perf_start( ReturnPreparePkt );\r
1454                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1455                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1456                 if( status == IB_SUCCESS )\r
1457                 {\r
1458                         cl_spinlock_release( &p_port->recv_lock );\r
1459                         cl_perf_start( ReturnNdisIndicate );\r
1460                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1461                                 &p_packet, 1 );\r
1462                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1463                 }\r
1464                 else if( status != IB_NOT_DONE )\r
1465                 {\r
1466                         IPOIB_TRACE( IPOIB_DBG_RECV,\r
1467                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1468                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1469                         /* Return the item to the head of the list. */\r
1470                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1471                         cl_spinlock_release( &p_port->recv_lock );\r
1472                 }\r
1473                 else\r
1474                 {\r
1475                         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1476                 }\r
1477 \r
1478         } while( status == IB_NOT_DONE );\r
1479         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1480 \r
1481         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1482 }\r
1483 \r
1484 \r
1485 static void\r
1486 __recv_cb(\r
1487         IN              const   ib_cq_handle_t                          h_cq,\r
1488         IN                              void                                            *cq_context )\r
1489 {\r
1490         ipoib_port_t            *p_port;\r
1491         ib_api_status_t         status;\r
1492         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1493         uint32_t                        pkt_cnt, recv_cnt = 0;\r
1494         cl_qlist_t                      done_list, bad_list;\r
1495         size_t                          i;\r
1496         PERF_DECLARE( RecvCompBundle );\r
1497         PERF_DECLARE( RecvCb );\r
1498         PERF_DECLARE( PollRecv );\r
1499         PERF_DECLARE( RepostRecv );\r
1500         PERF_DECLARE( FilterRecv );\r
1501         PERF_DECLARE( BuildPktArray );\r
1502         PERF_DECLARE( RecvNdisIndicate );\r
1503         PERF_DECLARE( RearmRecv );\r
1504         PERF_DECLARE( PutRecvList );\r
1505 \r
1506         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1507 \r
1508         cl_perf_clr( RecvCompBundle );\r
1509 \r
1510         cl_perf_start( RecvCb );\r
1511 \r
1512         UNUSED_PARAM( h_cq );\r
1513 \r
1514         p_port = (ipoib_port_t*)cq_context;\r
1515 \r
1516         cl_qlist_init( &done_list );\r
1517         cl_qlist_init( &bad_list );\r
1518 \r
1519         cl_obj_ref( &p_port->obj );\r
1520 \r
1521         for( i = 0; i < MAX_RECV_WC; i++ )\r
1522                 wc[i].p_next = &wc[i + 1];\r
1523         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1524 \r
1525         /*\r
1526          * We'll be accessing the endpoint map so take a reference\r
1527          * on it to prevent modifications.\r
1528          */\r
1529         cl_obj_lock( &p_port->obj );\r
1530         cl_atomic_inc( &p_port->endpt_rdr );\r
1531         cl_obj_unlock( &p_port->obj );\r
1532 \r
1533         do\r
1534         {\r
1535                 /* If we get here, then the list of WCs is intact. */\r
1536                 p_free = wc;\r
1537 \r
1538                 cl_perf_start( PollRecv );\r
1539                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1540                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1541                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1542                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1543 \r
1544                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1545                 cl_perf_start( FilterRecv );\r
1546                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1547                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1548 \r
1549         } while( !p_free );\r
1550 \r
1551         /* We're done looking at the endpoint map, release the reference. */\r
1552         cl_atomic_dec( &p_port->endpt_rdr );\r
1553 \r
1554         /* Update our posted depth. */\r
1555         cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
1556 \r
1557         cl_perf_start( BuildPktArray );\r
1558         /* Notify NDIS of any and all possible receive buffers. */\r
1559         pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
1560         cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1561 \r
1562         /* Only indicate receives if we actually had any. */\r
1563         if( pkt_cnt )\r
1564         {\r
1565                 cl_perf_start( RecvNdisIndicate );\r
1566                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1567                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1568                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1569         }\r
1570 \r
1571         cl_spinlock_acquire( &p_port->recv_lock );\r
1572 \r
1573         /* Return any discarded receives to the pool */\r
1574         cl_perf_start( PutRecvList );\r
1575         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1576         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1577 \r
1578         /* Repost receives. */\r
1579         cl_perf_start( RepostRecv );\r
1580         __recv_mgr_repost( p_port, recv_cnt );\r
1581         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1582 \r
1583         cl_spinlock_release( &p_port->recv_lock );\r
1584 \r
1585         /*\r
1586          * Rearm after filtering to prevent contention on the enpoint maps\r
1587          * and eliminate the possibility of having a call to\r
1588          * __endpt_mgr_insert find a duplicate.\r
1589          */\r
1590         cl_perf_start( RearmRecv );\r
1591         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1592                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1593         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1594         CL_ASSERT( status == IB_SUCCESS );\r
1595 \r
1596         cl_obj_deref( &p_port->obj );\r
1597 \r
1598         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1599         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1600 \r
1601         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1602 }\r
1603 \r
1604 \r
1605 static void\r
1606 __recv_get_endpts(\r
1607         IN                              ipoib_port_t* const                     p_port,\r
1608         IN                              ipoib_recv_desc_t* const        p_desc,\r
1609         IN                              ib_wc_t* const                          p_wc,\r
1610                 OUT                     ipoib_endpt_t** const           pp_src,\r
1611                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1612 {\r
1613         ib_api_status_t         status;\r
1614         mac_addr_t                      mac;\r
1615         PERF_DECLARE( GetEndptByGid );\r
1616         PERF_DECLARE( GetEndptByLid );\r
1617         PERF_DECLARE( EndptInsert );\r
1618 \r
1619         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1620 \r
1621         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1622         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1623         {\r
1624                 /* Lookup the source endpoints based on GID. */\r
1625                 cl_perf_start( GetEndptByGid );\r
1626                 *pp_src =\r
1627 #if IPOIB_INLINE_RECV\r
1628                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1629 #else   /* IPOIB_INLINE_RECV */\r
1630                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1631 #endif  /* IPOIB_INLINE_RECV */\r
1632                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1633 \r
1634                 /*\r
1635                  * Lookup the destination endpoint based on GID.\r
1636                  * This is used along with the packet filter to determine\r
1637                  * whether to report this to NDIS.\r
1638                  */\r
1639                 cl_perf_start( GetEndptByGid );\r
1640                 *pp_dst =\r
1641 #if IPOIB_INLINE_RECV\r
1642                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1643 #else   /* IPOIB_INLINE_RECV */\r
1644                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1645 #endif  /* IPOIB_INLINE_RECV */\r
1646                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1647 \r
1648                 /*\r
1649                  * Create the source endpoint if it does not exist.  Note that we\r
1650                  * can only do this for globally routed traffic since we need the\r
1651                  * information from the GRH to generate the MAC.\r
1652                  */\r
1653                 if( !*pp_src )\r
1654                 {\r
1655                         status = ipoib_mac_from_guid(\r
1656 #if IPOIB_INLINE_RECV\r
1657                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1658 #else   /* IPOIB_INLINE_RECV */\r
1659                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1660 #endif  /* IPOIB_INLINE_RECV */\r
1661                         if( status != IB_SUCCESS )\r
1662                         {\r
1663                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1664                                         ("ipoib_mac_from_guid returned %s\n",\r
1665                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1666                                 return;\r
1667                         }\r
1668 \r
1669                         /* Create the endpoint. */\r
1670 #if IPOIB_INLINE_RECV\r
1671                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1672 #else   /* IPOIB_INLINE_RECV */\r
1673                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1674 #endif  /* IPOIB_INLINE_RECV */\r
1675                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1676                         if( !*pp_src )\r
1677                         {\r
1678                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1679                                         ("ipoib_endpt_create returned %s\n",\r
1680                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1681                                 return;\r
1682                         }\r
1683                         cl_perf_start( EndptInsert );\r
1684                         cl_obj_lock( &p_port->obj );\r
1685                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1686                         cl_obj_unlock( &p_port->obj );\r
1687                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1688                 }\r
1689         }\r
1690         else\r
1691         {\r
1692                 /*\r
1693                  * Lookup the remote endpoint based on LID.  Note that only\r
1694                  * unicast traffic can be LID routed.\r
1695                  */\r
1696                 cl_perf_start( GetEndptByLid );\r
1697                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1698                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1699                 *pp_dst = p_port->p_local_endpt;\r
1700                 CL_ASSERT( *pp_dst );\r
1701         }\r
1702 \r
1703         IPOIB_TRACE( (IPOIB_DBG_RECV | IPOIB_DBG_INFO),\r
1704                 ("Recv:\n"\r
1705                 "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1706                 "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1707                 (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1708                 (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1709                 (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1710                 (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1711                 (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1712                 (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1713 \r
1714         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1715 }\r
1716 \r
1717 \r
1718 static uint32_t\r
1719 __recv_mgr_filter(\r
1720         IN                              ipoib_port_t* const                     p_port,\r
1721         IN                              ib_wc_t* const                          p_done_wc_list,\r
1722                 OUT                     cl_qlist_t* const                       p_done_list,\r
1723                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1724 {\r
1725         ipoib_recv_desc_t               *p_desc;\r
1726         ib_wc_t                                 *p_wc;\r
1727         ipoib_pkt_t                             *p_ipoib;\r
1728         eth_pkt_t                               *p_eth;\r
1729         ipoib_endpt_t                   *p_src, *p_dst;\r
1730         ib_api_status_t                 status;\r
1731         uint32_t                                len, recv_cnt = 0;\r
1732         PERF_DECLARE( GetRecvEndpts );\r
1733         PERF_DECLARE( RecvGen );\r
1734         PERF_DECLARE( RecvTcp );\r
1735         PERF_DECLARE( RecvUdp );\r
1736         PERF_DECLARE( RecvDhcp );\r
1737         PERF_DECLARE( RecvArp );\r
1738 \r
1739         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1740 \r
1741         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1742         {\r
1743                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1744                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1745                 recv_cnt++;\r
1746 \r
1747                 if( p_wc->status != IB_WCS_SUCCESS )\r
1748                 {\r
1749                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1750                         {\r
1751                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1752                                         ("Failed completion %s\n",\r
1753                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1754                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1755                         }\r
1756                         else\r
1757                         {\r
1758                                 IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1759                                         ("Flushed completion %s\n",\r
1760                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1761                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1762                         }\r
1763                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1764                         /* Dereference the port object on behalf of the failed receive. */\r
1765                         cl_obj_deref( &p_port->obj );\r
1766                         continue;\r
1767                 }\r
1768 \r
1769                 len = p_wc->length - sizeof(ib_grh_t);\r
1770 \r
1771                 if( len < sizeof(ipoib_hdr_t) )\r
1772                 {\r
1773                         IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1774                                 ("Received ETH packet < min size\n") );\r
1775                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1776                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1777                         cl_obj_deref( &p_port->obj );\r
1778                         continue;\r
1779                 }\r
1780 \r
1781                 /* Successful completion.  Get the receive information. */\r
1782                 cl_perf_start( GetRecvEndpts );\r
1783                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1784                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1785 \r
1786 #if IPOIB_INLINE_RECV\r
1787                 p_ipoib = &p_desc->buf.ib.pkt;\r
1788                 p_eth = &p_desc->buf.eth.pkt;\r
1789 #else   /* IPOIB_INLINE_RECV */\r
1790                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1791                 p_eth = &p_desc->p_buf->eth.pkt;\r
1792 #endif  /*IPOIB_INLINE_RECV */\r
1793 \r
1794                 /* Don't report loopback traffic - we requested SW loopback. */\r
1795                 if( !cl_memcmp( &p_port->p_adapter->mac, &p_eth->hdr.src,\r
1796                         sizeof(p_port->p_adapter->mac) ) )\r
1797                 {\r
1798                         /*\r
1799                          * "This is not the packet you're looking for" - don't update\r
1800                          * receive statistics, the packet never happened.\r
1801                          */\r
1802                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1803                         /* Dereference the port object on behalf of the failed receive. */\r
1804                         cl_obj_deref( &p_port->obj );\r
1805                         continue;\r
1806                 }\r
1807 \r
1808                 switch( p_ipoib->hdr.type )\r
1809                 {\r
1810                 case ETH_PROT_TYPE_IP:\r
1811                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1812                         {\r
1813                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1814                                         ("Received IP packet < min size\n") );\r
1815                                 status = IB_INVALID_SETTING;\r
1816                                 break;\r
1817                         }\r
1818 \r
1819                         if( p_ipoib->type.ip.hdr.offset ||\r
1820                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1821                         {\r
1822                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1823                                 cl_perf_start( RecvTcp );\r
1824                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1825                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1826                                 break;\r
1827                         }\r
1828 \r
1829                         /* First packet of a UDP transfer. */\r
1830                         if( len <\r
1831                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1832                         {\r
1833                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1834                                         ("Received UDP packet < min size\n") );\r
1835                                 status = IB_INVALID_SETTING;\r
1836                                 break;\r
1837                         }\r
1838 \r
1839                         /* Check if DHCP conversion is required. */\r
1840                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1841                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1842                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1843                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1844                         {\r
1845                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1846                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1847                                 {\r
1848                                         IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1849                                                 ("Received DHCP < min size\n") );\r
1850                                         status = IB_INVALID_SETTING;\r
1851                                         break;\r
1852                                 }\r
1853                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1854                                 cl_perf_start( RecvDhcp );\r
1855                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1856                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1857                         }\r
1858                         else\r
1859                         {\r
1860                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1861                                 cl_perf_start( RecvUdp );\r
1862                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1863                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1864                         }\r
1865                         break;\r
1866 \r
1867                 case ETH_PROT_TYPE_ARP:\r
1868                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1869                         {\r
1870                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1871                                         ("Received ARP < min size\n") );\r
1872                                 status = IB_INVALID_SETTING;\r
1873                                 break;\r
1874                         }\r
1875                         cl_perf_start( RecvArp );\r
1876                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1877                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1878                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1879                         break;\r
1880 \r
1881                 default:\r
1882                         /* Unfiltered.  Setup the ethernet header and report. */\r
1883                         cl_perf_start( RecvGen );\r
1884                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1885                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1886                 }\r
1887 \r
1888                 if( status != IB_SUCCESS )\r
1889                 {\r
1890                         /* Update stats. */\r
1891                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1892                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1893                         /* Dereference the port object on behalf of the failed receive. */\r
1894                         cl_obj_deref( &p_port->obj );\r
1895                 }\r
1896                 else\r
1897                 {\r
1898                         p_desc->len =\r
1899                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1900                         if( p_dst->h_mcast)\r
1901                         {\r
1902                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1903                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1904                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1905                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1906                                 {\r
1907                                         p_desc->type = PKT_TYPE_BCAST;\r
1908                                 }\r
1909                                 else\r
1910                                 {\r
1911                                         p_desc->type = PKT_TYPE_MCAST;\r
1912                                 }\r
1913                         }\r
1914                         else\r
1915                         {\r
1916                                 p_desc->type = PKT_TYPE_UCAST;\r
1917                         }\r
1918                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1919                 }\r
1920         }\r
1921 \r
1922         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1923         return recv_cnt;\r
1924 }\r
1925 \r
1926 \r
1927 static ib_api_status_t\r
1928 __recv_gen(\r
1929         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1930                 OUT                     eth_pkt_t* const                        p_eth,\r
1931         IN                              ipoib_endpt_t* const            p_src,\r
1932         IN                              ipoib_endpt_t* const            p_dst )\r
1933 {\r
1934         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1935 \r
1936         if( !p_src || !p_dst )\r
1937         {\r
1938                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1939                         ("Received packet with no matching endpoints.\n") );\r
1940                 return IB_NOT_DONE;\r
1941         }\r
1942 \r
1943         /*\r
1944          * Fill in the ethernet header.  Note that doing so will overwrite\r
1945          * the IPoIB header, so start by moving the information from the IPoIB\r
1946          * header.\r
1947          */\r
1948         p_eth->hdr.type = p_ipoib->hdr.type;\r
1949         p_eth->hdr.src = p_src->mac;\r
1950         p_eth->hdr.dst = p_dst->mac;\r
1951 \r
1952         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1953         return IB_SUCCESS;\r
1954 }\r
1955 \r
1956 \r
1957 static ib_api_status_t\r
1958 __recv_dhcp(\r
1959         IN                              ipoib_port_t* const                     p_port,\r
1960         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1961                 OUT                     eth_pkt_t* const                        p_eth,\r
1962         IN                              ipoib_endpt_t* const            p_src,\r
1963         IN                              ipoib_endpt_t* const            p_dst )\r
1964 {\r
1965         ib_api_status_t         status;\r
1966         dhcp_pkt_t                      *p_dhcp;\r
1967         uint8_t                         *p_option;\r
1968         uint8_t                         *p_cid = NULL;\r
1969         ib_gid_t                        gid;\r
1970         uint8_t                         msg = 0;\r
1971 \r
1972         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1973 \r
1974         UNUSED_PARAM( p_port );\r
1975 \r
1976         /* Create the ethernet header. */\r
1977         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1978         if( status != IB_SUCCESS )\r
1979         {\r
1980                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1981                         ("__recv_gen returned %s.\n", \r
1982                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1983                 return status;\r
1984         }\r
1985 \r
1986         /* Fixup the payload. */\r
1987         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
1988         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
1989         {\r
1990                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("Invalid DHCP op code.\n") );\r
1991                 return IB_INVALID_SETTING;\r
1992         }\r
1993 \r
1994         /*\r
1995          * Find the client identifier option, making sure to skip\r
1996          * the "magic cookie".\r
1997          */\r
1998         p_option = &p_dhcp->options[0];\r
1999         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2000         {\r
2001                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2002                         ("DHCP cookie corrupted.\n") );\r
2003                 return IB_INVALID_PARAMETER;\r
2004         }\r
2005 \r
2006         p_option = &p_dhcp->options[4];\r
2007         while( *p_option != DHCP_OPT_END )\r
2008         {\r
2009                 switch( *p_option )\r
2010                 {\r
2011                 case DHCP_OPT_PAD:\r
2012                         p_option++;\r
2013                         break;\r
2014 \r
2015                 case DHCP_OPT_MSG:\r
2016                         msg = p_option[2];\r
2017                         p_option += 3;\r
2018                         break;\r
2019 \r
2020                 case DHCP_OPT_CLIENT_ID:\r
2021                         p_cid = p_option;\r
2022                         /* Fall through. */\r
2023 \r
2024                 default:\r
2025                         /*\r
2026                          * All other options have a length byte following the option code.\r
2027                          * Offset by the length to get to the next option.\r
2028                          */\r
2029                         p_option += (p_option[1] + 2);\r
2030                 }\r
2031         }\r
2032 \r
2033         switch( msg )\r
2034         {\r
2035         /* message from client */\r
2036         case DHCPDISCOVER:\r
2037         case DHCPREQUEST:\r
2038         case DHCPDECLINE:\r
2039         case DHCPRELEASE:\r
2040         case DHCPINFORM:\r
2041                 if( !p_cid )\r
2042                 {\r
2043                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2044                                 ("Failed to find required Client-identifier option.\n") );\r
2045                         return IB_INVALID_SETTING;\r
2046                 }\r
2047                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2048                 {\r
2049                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2050                                 ("Invalid hardware address type.\n") );\r
2051                         return IB_INVALID_SETTING;\r
2052                 }\r
2053                 break;\r
2054         /* message from DHCP server */\r
2055         case DHCPOFFER:\r
2056         case DHCPACK:\r
2057         case DHCPNAK:\r
2058                 break;\r
2059 \r
2060         default:\r
2061                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2062                         ("Invalide message type.\n") );\r
2063                 return IB_INVALID_PARAMETER;\r
2064         }\r
2065         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2066         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2067         p_dhcp->hlen = HW_ADDR_LEN;\r
2068 \r
2069         if( p_cid ) /* from client */\r
2070         {\r
2071                 /* Validate that the length and type of the option is as required. */\r
2072                 if( p_cid[1] != 21 )\r
2073                 {\r
2074                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2075                                 ("Client-identifier length not 21 as required.\n") );\r
2076                         return IB_INVALID_SETTING;\r
2077                 }\r
2078                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2079                 {\r
2080                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2081                                 ("Client-identifier type is wrong.\n") );\r
2082                         return IB_INVALID_SETTING;\r
2083                 }\r
2084                 /*\r
2085                  * Copy the GID value from the option so that we can make aligned\r
2086                  * accesses to the contents.\r
2087                  * Recover CID to standard type.\r
2088                  */\r
2089                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2090                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2091                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2092                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2093                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2094         }\r
2095         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2096         return status;\r
2097 }\r
2098 \r
2099 \r
2100 static ib_api_status_t\r
2101 __recv_arp(\r
2102         IN                              ipoib_port_t* const                     p_port,\r
2103         IN                              ib_wc_t* const                          p_wc,\r
2104         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2105                 OUT                     eth_pkt_t* const                        p_eth,\r
2106         IN                              ipoib_endpt_t** const           pp_src,\r
2107         IN                              ipoib_endpt_t* const            p_dst )\r
2108 {\r
2109         ib_api_status_t                 status;\r
2110         arp_pkt_t                               *p_arp;\r
2111         const ipoib_arp_pkt_t   *p_ib_arp;\r
2112         ib_gid_t                                gid;\r
2113         mac_addr_t                              mac;\r
2114         ipoib_hw_addr_t                 null_hw = {0};\r
2115 \r
2116         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2117 \r
2118         if( !p_dst )\r
2119         {\r
2120                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2121                         ("Unknown destination endpoint\n") );\r
2122                 return IB_INVALID_SETTING;\r
2123         }\r
2124 \r
2125         p_ib_arp = &p_ipoib->type.arp;\r
2126         p_arp = &p_eth->type.arp;\r
2127 \r
2128         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2129         {\r
2130                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2131                         ("ARP hardware type is not IB\n") );\r
2132                 return IB_INVALID_SETTING;\r
2133         }\r
2134 \r
2135         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2136         {\r
2137                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2138                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2139                 return IB_INVALID_SETTING;\r
2140         }\r
2141 \r
2142         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2143         {\r
2144                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP protocal type not IP\n") );\r
2145                 return IB_INVALID_SETTING;\r
2146         }\r
2147 \r
2148         /*\r
2149          * If we don't have a source, lookup the endpoint specified in the payload.\r
2150          */\r
2151         if( !*pp_src )\r
2152                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2153 \r
2154         /*\r
2155          * If the endpoint exists for the GID, make sure\r
2156          * the dlid and qpn match the arp.\r
2157          */\r
2158         if( *pp_src )\r
2159         {\r
2160                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2161                         sizeof(ib_gid_t) ) )\r
2162                 {\r
2163                         /*\r
2164                          * GIDs for the endpoint are different.  The ARP must\r
2165                          * have been proxied.  Dereference it.\r
2166                          */\r
2167                         *pp_src = NULL;\r
2168                 }\r
2169                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2170                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2171                 {\r
2172                         /* Out of date!  Destroy the endpoint and replace it. */\r
2173                         __endpt_mgr_remove( p_port, *pp_src );\r
2174                         *pp_src = NULL;\r
2175                 }\r
2176         }\r
2177 \r
2178         /* Do we need to create an endpoint for this GID? */\r
2179         if( !*pp_src )\r
2180         {\r
2181                 /* Copy the src GID to allow aligned access */\r
2182                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2183                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2184                 if( status != IB_SUCCESS )\r
2185                 {\r
2186                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2187                                 ("ipoib_mac_from_guid returned %s\n",\r
2188                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2189                         return status;\r
2190                 }\r
2191                 /*\r
2192                  * Create the endpoint.  Note that the LID is left blank and will be\r
2193                  * resolved by a path query as needed.  This is done because the\r
2194                  * remote LID/GID from the work completion may not be the original\r
2195                  * initiator.\r
2196                  */\r
2197                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2198                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2199                 if( !*pp_src )\r
2200                 {\r
2201                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2202                                 ("ipoib_endpt_create returned %s\n",\r
2203                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2204                         return status;\r
2205                 }\r
2206 \r
2207                 cl_obj_lock( &p_port->obj );\r
2208                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2209                 cl_obj_unlock( &p_port->obj );\r
2210         }\r
2211 \r
2212         CL_ASSERT( !cl_memcmp(\r
2213                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2214         CL_ASSERT( (*pp_src)->qpn ==\r
2215                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2216         /* Now swizzle the data. */\r
2217         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2218         p_arp->hw_size = sizeof(mac_addr_t);\r
2219         p_arp->src_hw = (*pp_src)->mac;\r
2220         p_arp->src_ip = p_ib_arp->src_ip;\r
2221 \r
2222         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2223         {\r
2224 \r
2225                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2226                 {\r
2227                         /*\r
2228                          * We received bcast ARP packet that means\r
2229                          * remote port lets everyone know it was changed IP/MAC\r
2230                          * or just activated\r
2231                          */\r
2232                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2233                         {\r
2234                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP: is not ARP MCAST\n") );\r
2235                                 return IB_INVALID_SETTING;\r
2236                         }\r
2237 \r
2238                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2239                         p_dst->mac = p_port->p_local_endpt->mac;\r
2240                         /*\r
2241                          * we don't care what receiver ip addr is,\r
2242                          * as long as OS' ARP table is global  ???\r
2243                          */\r
2244                         p_arp->dst_ip = (net32_t)0;\r
2245 \r
2246                 }\r
2247                 else /* we've got reply to our ARP request */\r
2248                 {\r
2249                         p_arp->dst_hw = p_dst->mac;\r
2250                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2251                         CL_ASSERT( p_dst->qpn == \r
2252                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2253                 }\r
2254         }\r
2255         else /* we got ARP reqeust */\r
2256         {\r
2257                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2258                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2259         }\r
2260 \r
2261         /*\r
2262          * Create the ethernet header.  Note that this is done last so that\r
2263          * we have a chance to create a new endpoint.\r
2264          */\r
2265         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2266         if( status != IB_SUCCESS )\r
2267         {\r
2268                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2269                         ("__recv_gen returned %s.\n", \r
2270                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2271                 return status;\r
2272         }\r
2273 \r
2274         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2275         return IB_SUCCESS;\r
2276 }\r
2277 \r
2278 \r
2279 static ib_api_status_t\r
2280 __recv_mgr_prepare_pkt(\r
2281         IN                              ipoib_port_t* const                     p_port,\r
2282         IN                              ipoib_recv_desc_t* const        p_desc,\r
2283                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2284 {\r
2285         NDIS_STATUS                                                     status;\r
2286         uint32_t                                                        pkt_filter;\r
2287         ip_stat_sel_t                                           type;\r
2288         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2289         PERF_DECLARE( GetNdisPkt );\r
2290 \r
2291         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2292 \r
2293         pkt_filter = p_port->p_adapter->packet_filter;\r
2294         /* Check the packet filter. */\r
2295         switch( p_desc->type )\r
2296         {\r
2297         default:\r
2298         case PKT_TYPE_UCAST:\r
2299                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2300                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2301                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2302                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2303                 {\r
2304                         /* OK to report. */\r
2305                         type = IP_STAT_UCAST_BYTES;\r
2306                         status = NDIS_STATUS_SUCCESS;\r
2307                 }\r
2308                 else\r
2309                 {\r
2310                         type = IP_STAT_DROPPED;\r
2311                         status = NDIS_STATUS_FAILURE;\r
2312                 }\r
2313                 break;\r
2314         case PKT_TYPE_BCAST:\r
2315                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2316                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2317                 {\r
2318                         /* OK to report. */\r
2319                         type = IP_STAT_BCAST_BYTES;\r
2320                         status = NDIS_STATUS_SUCCESS;\r
2321                 }\r
2322                 else\r
2323                 {\r
2324                         type = IP_STAT_DROPPED;\r
2325                         status = NDIS_STATUS_FAILURE;\r
2326                 }\r
2327                 break;\r
2328         case PKT_TYPE_MCAST:\r
2329                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2330                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2331                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2332                 {\r
2333                         /* OK to report. */\r
2334                         type = IP_STAT_MCAST_BYTES;\r
2335                         status = NDIS_STATUS_SUCCESS;\r
2336                 }\r
2337                 else\r
2338                 {\r
2339                         type = IP_STAT_DROPPED;\r
2340                         status = NDIS_STATUS_FAILURE;\r
2341                 }\r
2342                 break;\r
2343         }\r
2344 \r
2345         if( status != NDIS_STATUS_SUCCESS )\r
2346         {\r
2347                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2348                 /* Return the receive descriptor to the pool. */\r
2349                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2350                 IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
2351                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2352                 /*\r
2353                  * Return IB_NOT_DONE since the packet has been completed,\r
2354                  * but has not consumed an array entry.\r
2355                  */\r
2356                 return IB_NOT_DONE;\r
2357         }\r
2358 \r
2359         cl_perf_start( GetNdisPkt );\r
2360         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2361         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2362         if( !*pp_packet )\r
2363         {\r
2364                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2365                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2366                 return IB_INSUFFICIENT_RESOURCES;\r
2367         }\r
2368 \r
2369         /* Flag the checksums as having been calculated. */\r
2370         chksum.Value = 0;\r
2371         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2372         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2373         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2374         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2375                 (void*)(uintn_t)chksum.Value;\r
2376 \r
2377         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2378 \r
2379         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2380         return IB_SUCCESS;\r
2381 }\r
2382 \r
2383 \r
2384 static uint32_t\r
2385 __recv_mgr_build_pkt_array(\r
2386         IN                              ipoib_port_t* const                     p_port,\r
2387                 OUT                     cl_qlist_t* const                       p_done_list )\r
2388 {\r
2389         cl_list_item_t                  *p_item;\r
2390         ipoib_recv_desc_t               *p_desc;\r
2391         uint32_t                                i = 0;\r
2392         ib_api_status_t                 status;\r
2393         PERF_DECLARE( PreparePkt );\r
2394 \r
2395         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2396 \r
2397         cl_spinlock_acquire( &p_port->recv_lock );\r
2398 \r
2399         /* Move any existing receives to the head to preserve ordering. */\r
2400         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2401         p_item = cl_qlist_remove_head( p_done_list );\r
2402         while( p_item != cl_qlist_end( p_done_list ) )\r
2403         {\r
2404                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2405 \r
2406                 cl_perf_start( PreparePkt );\r
2407                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2408                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2409                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2410                 if( status == IB_SUCCESS )\r
2411                 {\r
2412                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2413                         i++;\r
2414                 }\r
2415                 else if( status != IB_NOT_DONE )\r
2416                 {\r
2417                         IPOIB_TRACE( IPOIB_DBG_RECV,\r
2418                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2419                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2420                         /* Put all completed receives on the port's done list. */\r
2421                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2422                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2423                         break;\r
2424                 }\r
2425 \r
2426                 p_item = cl_qlist_remove_head( p_done_list );\r
2427         }\r
2428 \r
2429         cl_spinlock_release( &p_port->recv_lock );\r
2430 \r
2431         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2432         return i;\r
2433 }\r
2434 \r
2435 \r
2436 \r
2437 \r
2438 /******************************************************************************\r
2439 *\r
2440 * Send manager implementation.\r
2441 *\r
2442 ******************************************************************************/\r
2443 static void\r
2444 __send_mgr_construct(\r
2445         IN                              ipoib_port_t* const                     p_port )\r
2446 {\r
2447         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2448         p_port->send_mgr.depth = 0;\r
2449         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2450         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2451 }\r
2452 \r
2453 \r
2454 static void\r
2455 __send_mgr_destroy(\r
2456         IN                              ipoib_port_t* const                     p_port )\r
2457 {\r
2458         cl_list_item_t  *p_item;\r
2459         NDIS_PACKET             *p_packet;\r
2460 \r
2461         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2462 \r
2463         /* Complete any pending packets. */\r
2464         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2465                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2466                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2467         {\r
2468                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2469                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2470                         NDIS_STATUS_REQUEST_ABORTED );\r
2471         }\r
2472 \r
2473         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2474 }\r
2475 \r
2476 \r
2477 static NDIS_STATUS\r
2478 __send_mgr_filter(\r
2479         IN                              ipoib_port_t* const                     p_port,\r
2480         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2481         IN                              NDIS_BUFFER* const                      p_buf,\r
2482         IN                              size_t                                          buf_len,\r
2483         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2484 {\r
2485         NDIS_STATUS             status;\r
2486 \r
2487         PERF_DECLARE( FilterIp );\r
2488         PERF_DECLARE( FilterArp );\r
2489         PERF_DECLARE( SendGen );\r
2490 \r
2491         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2492 \r
2493         /*\r
2494          * We already checked the ethernet header length, so we know it's safe\r
2495          * to decrement the buf_len without underflowing.\r
2496          */\r
2497         buf_len -= sizeof(eth_hdr_t);\r
2498 \r
2499         switch( p_eth_hdr->type )\r
2500         {\r
2501         case ETH_PROT_TYPE_IP:\r
2502                 cl_perf_start( FilterIp );\r
2503                 status = __send_mgr_filter_ip(\r
2504                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2505                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2506                 break;\r
2507 \r
2508         case ETH_PROT_TYPE_ARP:\r
2509                 cl_perf_start( FilterArp );\r
2510                 status = __send_mgr_filter_arp(\r
2511                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2512                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2513                 break;\r
2514 \r
2515         default:\r
2516                 /*\r
2517                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2518                  * Just send the payload and hope for the best.\r
2519                  */\r
2520                 cl_perf_start( SendGen );\r
2521                 status = __send_gen( p_port, p_desc );\r
2522                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2523                 break;\r
2524         }\r
2525 \r
2526         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2527         return status;\r
2528 }\r
2529 \r
2530 \r
2531 static NDIS_STATUS\r
2532 __send_copy(\r
2533         IN                              ipoib_port_t* const                     p_port,\r
2534         IN                              ipoib_send_desc_t* const        p_desc )\r
2535 {\r
2536         NDIS_PACKET                             *p_packet;\r
2537         NDIS_BUFFER                             *p_buf;\r
2538         NDIS_STATUS                             status;\r
2539         UINT                                    tot_len, bytes_copied = 0;\r
2540 \r
2541         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2542 \r
2543         p_desc->p_buf = \r
2544                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2545         if( !p_desc->p_buf )\r
2546         {\r
2547                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2548                         ("Failed to allocate buffer for packet copy.\n") );\r
2549                 return NDIS_STATUS_RESOURCES;\r
2550         }\r
2551 \r
2552         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2553         if( status != NDIS_STATUS_SUCCESS )\r
2554         {\r
2555                 IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
2556                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2557                 return status;\r
2558         }\r
2559 \r
2560         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2561                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2562         if( status != NDIS_STATUS_SUCCESS )\r
2563         {\r
2564                 NdisFreePacket( p_packet );\r
2565                 IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
2566                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2567                 return status;\r
2568         }\r
2569 \r
2570         NdisChainBufferAtFront( p_packet, p_buf );\r
2571 \r
2572         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2573 \r
2574         /* Setup the work request. */\r
2575         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2576                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2577         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2578         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2579         p_desc->wr.num_ds = 2;\r
2580 \r
2581         /* Copy the packet. */\r
2582         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2583                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2584                 NormalPagePriority );\r
2585 \r
2586         /* Free our temp packet now that the data is copied. */\r
2587         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2588         NdisFreeBuffer( p_buf );\r
2589         NdisFreePacket( p_packet );\r
2590 \r
2591         if( bytes_copied != tot_len )\r
2592         {\r
2593                 /* Something went wrong.  Drop the packet. */\r
2594                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2595                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2596                         bytes_copied, tot_len) );\r
2597                 return NDIS_STATUS_RESOURCES;\r
2598         }\r
2599 \r
2600         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2601         return NDIS_STATUS_SUCCESS;\r
2602 }\r
2603 \r
2604 \r
2605 #if !IPOIB_USE_DMA\r
2606 /* Send using the MDL's page information rather than the SGL. */\r
2607 static ib_api_status_t\r
2608 __send_gen(\r
2609         IN                              ipoib_port_t* const                     p_port,\r
2610         IN                              ipoib_send_desc_t* const        p_desc )\r
2611 {\r
2612         uint32_t                                i, j = 1;\r
2613         ULONG                                   offset;\r
2614         MDL                                             *p_mdl;\r
2615         UINT                                    num_pages, tot_len;\r
2616         ULONG                                   buf_len;\r
2617         PPFN_NUMBER                             page_array;\r
2618         boolean_t                               hdr_done = FALSE;\r
2619         ib_api_status_t                 status;\r
2620 \r
2621         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2622 \r
2623         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2624                 &tot_len );\r
2625 \r
2626         if( !p_mdl )\r
2627         {\r
2628                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2629                         ("No buffers associated with packet.\n") );\r
2630                 return IB_ERROR;\r
2631         }\r
2632 \r
2633         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2634         if( num_pages >= MAX_SEND_SGE )\r
2635         {\r
2636                 IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
2637                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2638                 status = __send_copy( p_port, p_desc );\r
2639                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2640                 return status;\r
2641         }\r
2642 \r
2643         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2644         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2645         /*\r
2646          * Assume that the ethernet header is always fully contained\r
2647          * in the first page of the first MDL.  This makes for much\r
2648          * simpler code.\r
2649          */\r
2650         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2651         CL_ASSERT( offset <= PAGE_SIZE );\r
2652 \r
2653         while( tot_len )\r
2654         {\r
2655                 buf_len = MmGetMdlByteCount( p_mdl );\r
2656                 page_array = MmGetMdlPfnArray( p_mdl );\r
2657                 CL_ASSERT( page_array );\r
2658                 i = 0;\r
2659                 if( !hdr_done )\r
2660                 {\r
2661                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2662                         /* Skip the ethernet header. */\r
2663                         buf_len -= sizeof(eth_hdr_t);\r
2664                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2665                         if( buf_len )\r
2666                         {\r
2667                                 /* The ethernet header is a subset of this MDL. */\r
2668                                 CL_ASSERT( i == 0 );\r
2669                                 if( offset < PAGE_SIZE )\r
2670                                 {\r
2671                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2672                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2673                                         /* Add the byte offset since we're on the 1st page. */\r
2674                                         p_desc->local_ds[j].vaddr += offset;\r
2675                                         if( offset + buf_len > PAGE_SIZE )\r
2676                                         {\r
2677                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2678                                                 buf_len -= p_desc->local_ds[j].length;\r
2679                                         }\r
2680                                         else\r
2681                                         {\r
2682                                                 p_desc->local_ds[j].length = buf_len;\r
2683                                                 buf_len = 0;\r
2684                                         }\r
2685                                         /* This data segment is done.  Move to the next. */\r
2686                                         j++;\r
2687                                 }\r
2688                                 /* This page is done.  Move to the next. */\r
2689                                 i++;\r
2690                         }\r
2691                         /* Done handling the ethernet header. */\r
2692                         hdr_done = TRUE;\r
2693                 }\r
2694 \r
2695                 /* Finish this MDL */\r
2696                 while( buf_len )\r
2697                 {\r
2698                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2699                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2700                         /* Add the first page's offset if we're on the first page. */\r
2701                         if( i == 0 )\r
2702                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2703 \r
2704                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2705                         {\r
2706                                 /* Buffers spans pages. */\r
2707                                 p_desc->local_ds[j].length =\r
2708                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2709                                 buf_len -= p_desc->local_ds[j].length;\r
2710                                 /* This page is done.  Move to the next. */\r
2711                                 i++;\r
2712                         }\r
2713                         else\r
2714                         {\r
2715                                 /* Last page of the buffer. */\r
2716                                 p_desc->local_ds[j].length = buf_len;\r
2717                                 buf_len = 0;\r
2718                         }\r
2719                         /* This data segment is done.  Move to the next. */\r
2720                         j++;\r
2721                 }\r
2722 \r
2723                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2724                 if( !tot_len )\r
2725                         break;\r
2726 \r
2727                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2728                 if( !p_mdl )\r
2729                 {\r
2730                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2731                                 ("Failed to get next buffer.\n") );\r
2732                         return IB_ERROR;\r
2733                 }\r
2734         }\r
2735 \r
2736         /* Set the number of data segments. */\r
2737         p_desc->wr.num_ds = j;\r
2738 \r
2739         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2740         return IB_SUCCESS;\r
2741 }\r
2742 \r
2743 #else\r
2744 \r
2745 static NDIS_STATUS\r
2746 __send_gen(\r
2747         IN                              ipoib_port_t* const                     p_port,\r
2748         IN                              ipoib_send_desc_t* const        p_desc )\r
2749 {\r
2750         ib_api_status_t                 status;\r
2751         SCATTER_GATHER_LIST             *p_sgl;\r
2752         uint32_t                                i, j = 1;\r
2753         uint32_t                                offset = sizeof(eth_hdr_t);\r
2754         PERF_DECLARE( SendCopy );\r
2755 \r
2756         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2757 \r
2758         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2759                 ScatterGatherListPacketInfo );\r
2760         if( !p_sgl )\r
2761         {\r
2762                 ASSERT( p_sgl );\r
2763                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2764                         ("Failed to get SGL from packet.\n") );\r
2765                 return NDIS_STATUS_FAILURE;\r
2766         }\r
2767 \r
2768         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2769         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2770                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2771                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2772                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2773         {\r
2774                 IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
2775                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2776                 cl_perf_start( SendCopy );\r
2777                 status = __send_copy( p_port, p_desc );\r
2778                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2779                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2780                 return status;\r
2781         }\r
2782 \r
2783         /*\r
2784          * Skip the ethernet header.  It is either the first element,\r
2785          * or part of it.\r
2786          */\r
2787         i = 0;\r
2788         while( offset )\r
2789         {\r
2790                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2791                 {\r
2792                         offset -= p_sgl->Elements[i++].Length;\r
2793                 }\r
2794                 else\r
2795                 {\r
2796                         p_desc->local_ds[j].vaddr =\r
2797                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2798                         p_desc->local_ds[j].length =\r
2799                                 p_sgl->Elements[i].Length - offset;\r
2800                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2801                         i++;\r
2802                         j++;\r
2803                         break;\r
2804                 }\r
2805         }\r
2806         /* Now fill in the rest of the local data segments. */\r
2807         while( i < p_sgl->NumberOfElements )\r
2808         {\r
2809                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2810                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2811                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2812                 i++;\r
2813                 j++;\r
2814         }\r
2815 \r
2816         /* Set the number of data segments. */\r
2817         p_desc->wr.num_ds = j;\r
2818 \r
2819         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2820         return NDIS_STATUS_SUCCESS;\r
2821 }\r
2822 #endif\r
2823 \r
2824 \r
2825 static NDIS_STATUS\r
2826 __send_mgr_filter_ip(\r
2827         IN                              ipoib_port_t* const                     p_port,\r
2828         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2829         IN                              NDIS_BUFFER*                            p_buf,\r
2830         IN                              size_t                                          buf_len,\r
2831         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2832 {\r
2833         NDIS_STATUS             status;\r
2834         ip_hdr_t                *p_ip_hdr;\r
2835 \r
2836         PERF_DECLARE( QueryIp );\r
2837         PERF_DECLARE( SendTcp );\r
2838         PERF_DECLARE( FilterUdp );\r
2839 \r
2840         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2841 \r
2842         if( !buf_len )\r
2843         {\r
2844                 cl_perf_start( QueryIp );\r
2845                 NdisGetNextBuffer( p_buf, &p_buf );\r
2846                 if( !p_buf )\r
2847                 {\r
2848                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2849                                 ("Failed to get IP header buffer.\n") );\r
2850                         return NDIS_STATUS_FAILURE;\r
2851                 }\r
2852                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2853                 if( !p_ip_hdr )\r
2854                 {\r
2855                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2856                                 ("Failed to query IP header buffer.\n") );\r
2857                         return NDIS_STATUS_FAILURE;\r
2858                 }\r
2859                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2860         }\r
2861         else\r
2862         {\r
2863                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2864         }\r
2865         if( buf_len < sizeof(ip_hdr_t) )\r
2866         {\r
2867                 /* This buffer is done for.  Get the next buffer. */\r
2868                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2869                         ("Buffer too small for IP packet.\n") );\r
2870                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2871         }\r
2872 \r
2873         if( p_ip_hdr->offset ||\r
2874                 p_ip_hdr->prot != IP_PROT_UDP )\r
2875         {\r
2876                 /* Not a UDP packet. */\r
2877                 cl_perf_start( SendTcp );\r
2878                 status = __send_gen( p_port, p_desc );\r
2879                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2880                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2881                 return status;\r
2882         }\r
2883 \r
2884         buf_len -= sizeof(ip_hdr_t);\r
2885 \r
2886         cl_perf_start( FilterUdp );\r
2887         status = __send_mgr_filter_udp(\r
2888                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2889         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2890 \r
2891         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2892         return status;\r
2893 }\r
2894 \r
2895 \r
2896 static NDIS_STATUS\r
2897 __send_mgr_filter_udp(\r
2898         IN                              ipoib_port_t* const                     p_port,\r
2899         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2900         IN                              NDIS_BUFFER*                            p_buf,\r
2901         IN                              size_t                                          buf_len,\r
2902         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2903 {\r
2904         ib_api_status_t         status;\r
2905         udp_hdr_t                       *p_udp_hdr;\r
2906         PERF_DECLARE( QueryUdp );\r
2907         PERF_DECLARE( SendUdp );\r
2908         PERF_DECLARE( FilterDhcp );\r
2909 \r
2910         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2911 \r
2912         if( !buf_len )\r
2913         {\r
2914                 cl_perf_start( QueryUdp );\r
2915                 NdisGetNextBuffer( p_buf, &p_buf );\r
2916                 if( !p_buf )\r
2917                 {\r
2918                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2919                                 ("Failed to get UDP header buffer.\n") );\r
2920                         return NDIS_STATUS_FAILURE;\r
2921                 }\r
2922                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2923                 if( !p_udp_hdr )\r
2924                 {\r
2925                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2926                                 ("Failed to query UDP header buffer.\n") );\r
2927                         return NDIS_STATUS_FAILURE;\r
2928                 }\r
2929                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
2930         }\r
2931         else\r
2932         {\r
2933                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
2934         }\r
2935         /* Get the UDP header and check the destination port numbers. */\r
2936         if( buf_len < sizeof(udp_hdr_t) )\r
2937         {\r
2938                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2939                         ("Buffer not large enough for UDP packet.\n") );\r
2940                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2941         }\r
2942 \r
2943         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
2944                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r
2945                 (p_udp_hdr->src_port != DHCP_PORT_SERVER ||\r
2946                 p_udp_hdr->dst_port != DHCP_PORT_CLIENT) )\r
2947         {\r
2948                 /* Not a DHCP packet. */\r
2949                 cl_perf_start( SendUdp );\r
2950                 status = __send_gen( p_port, p_desc );\r
2951                 cl_perf_stop( &p_port->p_adapter->perf, SendUdp );\r
2952                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2953                 return status;\r
2954         }\r
2955 \r
2956         buf_len -= sizeof(udp_hdr_t);\r
2957 \r
2958         /* Allocate our scratch buffer. */\r
2959         p_desc->p_buf = (send_buf_t*)\r
2960                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2961         if( !p_desc->p_buf )\r
2962         {\r
2963                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2964                         ("Failed to query DHCP packet buffer.\n") );\r
2965                 return NDIS_STATUS_RESOURCES;\r
2966         }\r
2967         /* Copy the IP and UDP headers. */\r
2968         cl_memcpy( &p_desc->p_buf->ip.hdr, p_ip_hdr , sizeof(ip_hdr_t) );\r
2969         cl_memcpy(\r
2970                 &p_desc->p_buf->ip.prot.udp.hdr, p_udp_hdr, sizeof(udp_hdr_t) );\r
2971 \r
2972         cl_perf_start( FilterDhcp );\r
2973         status = __send_mgr_filter_dhcp(\r
2974                 p_port, p_udp_hdr, p_buf, buf_len, p_desc );\r
2975         cl_perf_stop( &p_port->p_adapter->perf, FilterDhcp );\r
2976 \r
2977         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2978         return status;\r
2979 }\r
2980 \r
2981 \r
2982 static NDIS_STATUS\r
2983 __send_mgr_filter_dhcp(\r
2984         IN                              ipoib_port_t* const                     p_port,\r
2985         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
2986         IN                              NDIS_BUFFER*                            p_buf,\r
2987         IN                              size_t                                          buf_len,\r
2988         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2989 {\r
2990         dhcp_pkt_t                      *p_dhcp;\r
2991         dhcp_pkt_t                      *p_ib_dhcp;\r
2992         uint8_t                         *p_option, *p_cid = NULL;\r
2993         uint8_t                         msg = 0;\r
2994         size_t                          len;\r
2995         ib_gid_t                        gid;\r
2996 \r
2997         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2998 \r
2999         if( !buf_len )\r
3000         {\r
3001                 NdisGetNextBuffer( p_buf, &p_buf );\r
3002                 if( !p_buf )\r
3003                 {\r
3004                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
3005                                 ("Failed to get DHCP buffer.\n") );\r
3006                         return NDIS_STATUS_FAILURE;\r
3007                 }\r
3008                 NdisQueryBufferSafe( p_buf, &p_dhcp, &buf_len, NormalPagePriority );\r
3009                 if( !p_dhcp )\r
3010                 {\r
3011                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
3012                                 ("Failed to query DHCP buffer.\n") );\r
3013                         return NDIS_STATUS_FAILURE;\r
3014                 }\r
3015         }\r
3016         else\r
3017         {\r
3018                 p_dhcp = (dhcp_pkt_t*)(p_udp_hdr + 1);\r
3019         }\r
3020 \r
3021         if( buf_len < DHCP_MIN_SIZE )\r
3022         {\r
3023                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
3024                         ("Buffer not large enough for DHCP packet.\n") );\r
3025                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
3026         }\r
3027 \r
3028         p_ib_dhcp = &p_desc->p_buf->ip.prot.udp.dhcp;\r
3029         cl_memcpy( p_ib_dhcp, p_dhcp, buf_len );\r
3030 \r
3031         /* Now scan through the options looking for the client identifier. */\r
3032         p_option = &p_ib_dhcp->options[4];\r
3033         while( *p_option != DHCP_OPT_END )\r
3034         {\r
3035                 switch( *p_option )\r
3036                 {\r
3037                 case DHCP_OPT_PAD:\r
3038                         p_option++;\r
3039                         break;\r
3040 \r
3041                 case&n