[IPOIB] - clear the endpt lid_list on SM change event
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 \r
45 /* Amount of physical memory to register. */\r
46 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
47 \r
48 /* Number of work completions to chain for send and receive polling. */\r
49 #define MAX_SEND_WC             8\r
50 #define MAX_RECV_WC             16\r
51 \r
52 \r
53 ib_gid_t        bcast_mgid_template = {\r
54         0xff,                                                           /* multicast field */\r
55         0x12,                                                           /* scope (to be filled in) */\r
56         0x40, 0x1b,                                                     /* IPv4 signature */\r
57         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
58         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
59         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
60 };\r
61 \r
62 \r
63 #ifdef _DEBUG_\r
64 /* Handy pointer for debug use. */\r
65 ipoib_port_t    *gp_ipoib_port;\r
66 #endif\r
67 \r
68 \r
69 /******************************************************************************\r
70 *\r
71 * Declarations\r
72 *\r
73 ******************************************************************************/\r
74 static void\r
75 __port_construct(\r
76         IN                              ipoib_port_t* const                     p_port );\r
77 \r
78 static ib_api_status_t\r
79 __port_init(\r
80         IN                              ipoib_port_t* const                     p_port,\r
81         IN                              ipoib_adapter_t* const          p_adapter,\r
82         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
83 \r
84 static void\r
85 __port_destroying(\r
86         IN                              cl_obj_t* const                         p_obj );\r
87 \r
88 static void\r
89 __port_cleanup(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_free(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 \r
97 /******************************************************************************\r
98 *\r
99 * IB resource manager operations\r
100 *\r
101 ******************************************************************************/\r
102 static void\r
103 __ib_mgr_construct(\r
104         IN                              ipoib_port_t* const                     p_port );\r
105 \r
106 static ib_api_status_t\r
107 __ib_mgr_init(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static void\r
111 __ib_mgr_destroy(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __qp_event(\r
116         IN                              ib_async_event_rec_t            *p_event_rec );\r
117 \r
118 static void\r
119 __cq_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static ib_api_status_t\r
123 __ib_mgr_activate(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 /******************************************************************************\r
127 *\r
128 * Buffer manager operations.\r
129 *\r
130 ******************************************************************************/\r
131 static void\r
132 __buf_mgr_construct(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 static ib_api_status_t\r
136 __buf_mgr_init(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static void\r
140 __buf_mgr_destroy(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static cl_status_t\r
144 __recv_ctor(\r
145         IN                              void* const                                     p_object,\r
146         IN                              void*                                           context,\r
147                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
148 \r
149 #if !IPOIB_INLINE_RECV\r
150 static void\r
151 __recv_dtor(\r
152         IN              const   cl_pool_item_t* const           p_pool_item,\r
153         IN                              void                                            *context );\r
154 #endif  /* IPOIB_INLINE_RECV */\r
155 \r
156 static inline ipoib_send_desc_t*\r
157 __buf_mgr_get_send(\r
158         IN                              ipoib_port_t* const                     p_port );\r
159 \r
160 static inline void\r
161 __buf_mgr_put_send(\r
162         IN                              ipoib_port_t* const                     p_port,\r
163         IN                              ipoib_send_desc_t* const        p_desc );\r
164 \r
165 static inline ipoib_recv_desc_t*\r
166 __buf_mgr_get_recv(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_recv(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_recv_desc_t* const        p_desc,\r
173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
174 \r
175 static inline void\r
176 __buf_mgr_put_recv_list(\r
177         IN                              ipoib_port_t* const                     p_port,\r
178         IN                              cl_qlist_t* const                       p_list );\r
179 \r
180 static inline NDIS_PACKET*\r
181 __buf_mgr_get_ndis_pkt(\r
182         IN                              ipoib_port_t* const                     p_port,\r
183         IN                              ipoib_recv_desc_t* const        p_desc );\r
184 \r
185 \r
186 /******************************************************************************\r
187 *\r
188 * Receive manager operations.\r
189 *\r
190 ******************************************************************************/\r
191 static void\r
192 __recv_mgr_construct(\r
193         IN                              ipoib_port_t* const                     p_port );\r
194 \r
195 static ib_api_status_t\r
196 __recv_mgr_init(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static void\r
200 __recv_mgr_destroy(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 /* Posts receive buffers to the receive queue. */\r
204 static ib_api_status_t\r
205 __recv_mgr_repost(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_cb(\r
210         IN              const   ib_cq_handle_t                          h_cq,\r
211         IN                              void                                            *cq_context );\r
212 \r
213 static void\r
214 __recv_get_endpts(\r
215         IN                              ipoib_port_t* const                     p_port,\r
216         IN                              ipoib_recv_desc_t* const        p_desc,\r
217         IN                              ib_wc_t* const                          p_wc,\r
218                 OUT                     ipoib_endpt_t** const           pp_src,\r
219                 OUT                     ipoib_endpt_t** const           pp_dst );\r
220 \r
221 static int32_t\r
222 __recv_mgr_filter(\r
223         IN                              ipoib_port_t* const                     p_port,\r
224         IN                              ib_wc_t* const                          p_done_wc_list,\r
225                 OUT                     cl_qlist_t* const                       p_done_list,\r
226                 OUT                     cl_qlist_t* const                       p_bad_list );\r
227 \r
228 static ib_api_status_t\r
229 __recv_gen(\r
230         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
231                 OUT                     eth_pkt_t* const                        p_eth,\r
232         IN                              ipoib_endpt_t* const            p_src,\r
233         IN                              ipoib_endpt_t* const            p_dst );\r
234 \r
235 static ib_api_status_t\r
236 __recv_dhcp(\r
237         IN                              ipoib_port_t* const                     p_port,\r
238         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
239                 OUT                     eth_pkt_t* const                        p_eth,\r
240         IN                              ipoib_endpt_t* const            p_src,\r
241         IN                              ipoib_endpt_t* const            p_dst );\r
242 \r
243 static ib_api_status_t\r
244 __recv_arp(\r
245         IN                              ipoib_port_t* const                     p_port,\r
246         IN                              ib_wc_t* const                          p_wc,\r
247         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
248                 OUT                     eth_pkt_t* const                        p_eth,\r
249         IN                              ipoib_endpt_t** const           p_src,\r
250         IN                              ipoib_endpt_t* const            p_dst );\r
251 \r
252 static ib_api_status_t\r
253 __recv_mgr_prepare_pkt(\r
254         IN                              ipoib_port_t* const                     p_port,\r
255         IN                              ipoib_recv_desc_t* const        p_desc,\r
256                 OUT                     NDIS_PACKET** const                     pp_packet );\r
257 \r
258 static uint32_t\r
259 __recv_mgr_build_pkt_array(\r
260         IN                              ipoib_port_t* const                     p_port,\r
261         IN                              int32_t                                         shortage,\r
262                 OUT                     cl_qlist_t* const                       p_done_list,\r
263                 OUT                     int32_t* const                          p_discarded );\r
264 \r
265 /******************************************************************************\r
266 *\r
267 * Send manager operations.\r
268 *\r
269 ******************************************************************************/\r
270 static void\r
271 __send_mgr_construct(\r
272         IN                              ipoib_port_t* const                     p_port );\r
273 \r
274 static void\r
275 __send_mgr_destroy(\r
276         IN                              ipoib_port_t* const                     p_port );\r
277 \r
278 static NDIS_STATUS\r
279 __send_gen(\r
280         IN                              ipoib_port_t* const                     p_port,\r
281         IN                              ipoib_send_desc_t* const        p_desc );\r
282 \r
283 static NDIS_STATUS\r
284 __send_mgr_filter_ip(\r
285         IN                              ipoib_port_t* const                     p_port,\r
286         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
287         IN                              NDIS_BUFFER*                            p_buf,\r
288         IN                              size_t                                          buf_len,\r
289         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
290 \r
291 static NDIS_STATUS\r
292 __send_mgr_filter_udp(\r
293         IN                              ipoib_port_t* const                     p_port,\r
294         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
295         IN                              NDIS_BUFFER*                            p_buf,\r
296         IN                              size_t                                          buf_len,\r
297         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
298 \r
299 static NDIS_STATUS\r
300 __send_mgr_filter_dhcp(\r
301         IN                              ipoib_port_t* const                     p_port,\r
302         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
303         IN                              NDIS_BUFFER*                            p_buf,\r
304         IN                              size_t                                          buf_len,\r
305         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
306 \r
307 static NDIS_STATUS\r
308 __send_mgr_filter_arp(\r
309         IN                              ipoib_port_t* const                     p_port,\r
310         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
311         IN                              NDIS_BUFFER*                            p_buf,\r
312         IN                              size_t                                          buf_len,\r
313         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
314 \r
315 static void\r
316 __process_failed_send(\r
317         IN                              ipoib_port_t* const                     p_port,\r
318         IN                              ipoib_send_desc_t* const        p_desc,\r
319         IN              const   NDIS_STATUS                                     status );\r
320 \r
321 static void\r
322 __send_cb(\r
323         IN              const   ib_cq_handle_t                          h_cq,\r
324         IN                              void                                            *cq_context );\r
325 \r
326 \r
327 /******************************************************************************\r
328 *\r
329 * Endpoint manager operations\r
330 *\r
331 ******************************************************************************/\r
332 static void\r
333 __endpt_mgr_construct(\r
334         IN                              ipoib_port_t* const                     p_port );\r
335 \r
336 static ib_api_status_t\r
337 __endpt_mgr_init(\r
338         IN                              ipoib_port_t* const                     p_port );\r
339 \r
340 static void\r
341 __endpt_mgr_destroy(\r
342         IN                              ipoib_port_t* const                     p_port );\r
343 \r
344 /****f* IPoIB/__endpt_mgr_remove_all\r
345 * NAME\r
346 *       __endpt_mgr_remove_all\r
347 *\r
348 * DESCRIPTION\r
349 *       Removes all enpoints from the port, dereferencing them to initiate\r
350 *       destruction.\r
351 *\r
352 * SYNOPSIS\r
353 */\r
354 static void\r
355 __endpt_mgr_remove_all(\r
356         IN                              ipoib_port_t* const                     p_port );\r
357 /*\r
358 ********/\r
359 \r
360 static void\r
361 __endpt_mgr_remove(\r
362         IN                              ipoib_port_t* const                     p_port,\r
363         IN                              ipoib_endpt_t* const            p_endpt );\r
364 \r
365 static void\r
366 __endpt_mgr_reset_all(\r
367         IN                              ipoib_port_t* const                     p_port );\r
368 \r
369 static inline NDIS_STATUS\r
370 __endpt_mgr_ref(\r
371         IN                              ipoib_port_t* const                     p_port,\r
372         IN              const   mac_addr_t                                      mac,\r
373                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
374 \r
375 static inline NDIS_STATUS\r
376 __endpt_mgr_get_gid_qpn(\r
377         IN                              ipoib_port_t* const                     p_port,\r
378         IN              const   mac_addr_t                                      mac,\r
379                 OUT                     ib_gid_t* const                         p_gid,\r
380                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
381 \r
382 static inline ipoib_endpt_t*\r
383 __endpt_mgr_get_by_gid(\r
384         IN                              ipoib_port_t* const                     p_port,\r
385         IN              const   ib_gid_t* const                         p_gid );\r
386 \r
387 static inline ipoib_endpt_t*\r
388 __endpt_mgr_get_by_lid(\r
389         IN                              ipoib_port_t* const                     p_port,\r
390         IN              const   net16_t                                         lid );\r
391 \r
392 static inline void\r
393 __endpt_mgr_insert_locked(\r
394         IN                              ipoib_port_t* const                     p_port,\r
395         IN              const   mac_addr_t                                      mac,\r
396         IN                              ipoib_endpt_t* const            p_endpt );\r
397 \r
398 static inline void\r
399 __endpt_mgr_insert(\r
400         IN                              ipoib_port_t* const                     p_port,\r
401         IN              const   mac_addr_t                                      mac,\r
402         IN                              ipoib_endpt_t* const            p_endpt );\r
403 \r
404 static ib_api_status_t\r
405 __endpt_mgr_add_local(\r
406         IN                              ipoib_port_t* const                     p_port,\r
407         IN                              ib_port_info_t* const           p_port_info );\r
408 \r
409 static ib_api_status_t\r
410 __endpt_mgr_add_bcast(\r
411         IN                              ipoib_port_t* const                     p_port,\r
412         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
413 \r
414 /******************************************************************************\r
415 *\r
416 * MCast operations.\r
417 *\r
418 ******************************************************************************/\r
419 static ib_api_status_t\r
420 __port_get_bcast(\r
421         IN                              ipoib_port_t* const                     p_port );\r
422 \r
423 static ib_api_status_t\r
424 __port_join_bcast(\r
425         IN                              ipoib_port_t* const                     p_port,\r
426         IN                              ib_member_rec_t* const          p_member_rec );\r
427 \r
428 static ib_api_status_t\r
429 __port_create_bcast(\r
430         IN                              ipoib_port_t* const                     p_port );\r
431 \r
432 static void\r
433 __port_info_cb(\r
434         IN                              ib_query_rec_t                          *p_query_rec );\r
435 \r
436 \r
437 static void\r
438 __bcast_get_cb(\r
439         IN                              ib_query_rec_t                          *p_query_rec );\r
440 \r
441 \r
442 static void\r
443 __bcast_cb(\r
444         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
445 \r
446 \r
447 static void\r
448 __mcast_cb(\r
449         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
450 \r
451 \r
452 static intn_t\r
453 __gid_cmp(\r
454         IN              const   void* const                                     p_key1,\r
455         IN              const   void* const                                     p_key2 )\r
456 {\r
457         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
458 }\r
459 \r
460 \r
461 /******************************************************************************\r
462 *\r
463 * Implementation\r
464 *\r
465 ******************************************************************************/\r
466 ib_api_status_t\r
467 ipoib_create_port(\r
468         IN                              ipoib_adapter_t* const          p_adapter,\r
469         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
470                 OUT                     ipoib_port_t** const            pp_port )\r
471 {\r
472         ib_api_status_t         status;\r
473         ipoib_port_t            *p_port;\r
474 \r
475         IPOIB_ENTER( IPOIB_DBG_INIT );\r
476 \r
477         CL_ASSERT( !p_adapter->p_port );\r
478 \r
479         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
480                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
481         if( !p_port )\r
482         {\r
483                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
484                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
485                         sizeof(ipoib_port_t)) );\r
486                 return IB_INSUFFICIENT_MEMORY;\r
487         }\r
488 \r
489 #ifdef _DEBUG_\r
490         gp_ipoib_port = p_port;\r
491 #endif\r
492 \r
493         __port_construct( p_port );\r
494 \r
495         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
496         if( status != IB_SUCCESS )\r
497         {\r
498                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
499                         ("ipoib_port_init returned %s.\n", \r
500                         p_adapter->p_ifc->get_err_str( status )) );\r
501                 __port_cleanup( &p_port->obj );\r
502                 __port_free( &p_port->obj );\r
503                 return status;\r
504         }\r
505 \r
506         *pp_port = p_port;\r
507         IPOIB_EXIT( IPOIB_DBG_INIT );\r
508         return IB_SUCCESS;\r
509 }\r
510 \r
511 \r
512 void\r
513 ipoib_port_destroy(\r
514         IN                              ipoib_port_t* const                     p_port )\r
515 {\r
516         IPOIB_ENTER( IPOIB_DBG_INIT );\r
517 \r
518         CL_ASSERT( p_port );\r
519         CL_ASSERT( p_port->p_adapter );\r
520         CL_ASSERT( !p_port->p_adapter->p_port );\r
521 \r
522         cl_obj_destroy( &p_port->obj );\r
523 \r
524         IPOIB_EXIT( IPOIB_DBG_INIT );\r
525 }\r
526 \r
527 \r
528 static void\r
529 __port_construct(\r
530         IN                              ipoib_port_t* const                     p_port )\r
531 {\r
532         IPOIB_ENTER( IPOIB_DBG_INIT );\r
533 \r
534         p_port->state = IB_QPS_RESET;\r
535 \r
536         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
537         cl_spinlock_construct( &p_port->send_lock );\r
538         cl_spinlock_construct( &p_port->recv_lock );\r
539         __ib_mgr_construct( p_port );\r
540         __buf_mgr_construct( p_port );\r
541 \r
542         __recv_mgr_construct( p_port );\r
543         __send_mgr_construct( p_port );\r
544 \r
545         __endpt_mgr_construct( p_port );\r
546 \r
547         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
548 \r
549         IPOIB_EXIT( IPOIB_DBG_INIT );\r
550 }\r
551 \r
552 \r
553 static ib_api_status_t\r
554 __port_init(\r
555         IN                              ipoib_port_t* const                     p_port,\r
556         IN                              ipoib_adapter_t* const          p_adapter,\r
557         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
558 {\r
559         cl_status_t                     cl_status;\r
560         ib_api_status_t         status;\r
561 \r
562         IPOIB_ENTER( IPOIB_DBG_INIT );\r
563 \r
564         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
565         p_port->p_adapter = p_adapter;\r
566 \r
567         cl_status = cl_spinlock_init( &p_port->send_lock );\r
568         if( cl_status != CL_SUCCESS )\r
569         {\r
570                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
571                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
572                 return IB_ERROR;\r
573         }\r
574 \r
575         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
576         if( cl_status != CL_SUCCESS )\r
577         {\r
578                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
579                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
580                 return IB_ERROR;\r
581         }\r
582 \r
583         /* Initialize the IB resource manager. */\r
584         status = __ib_mgr_init( p_port );\r
585         if( status != IB_SUCCESS )\r
586         {\r
587                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
588                         ("__ib_mgr_init returned %s\n", \r
589                         p_adapter->p_ifc->get_err_str( status )) );\r
590                 return status;\r
591         }\r
592 \r
593         /* Initialize the buffer manager. */\r
594         status = __buf_mgr_init( p_port );\r
595         if( status != IB_SUCCESS )\r
596         {\r
597                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
598                         ("__buf_mgr_init returned %s\n", \r
599                         p_adapter->p_ifc->get_err_str( status )) );\r
600                 return status;\r
601         }\r
602 \r
603         /* Initialize the receive manager. */\r
604         status = __recv_mgr_init( p_port );\r
605         if( status != IB_SUCCESS )\r
606         {\r
607                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
608                         ("__recv_mgr_init returned %s\n", \r
609                         p_adapter->p_ifc->get_err_str( status )) );\r
610                 return status;\r
611         }\r
612 \r
613         /* Initialize the endpoint manager. */\r
614         status = __endpt_mgr_init( p_port );\r
615         if( status != IB_SUCCESS )\r
616         {\r
617                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
618                         ("__endpt_mgr_init returned %s\n", \r
619                         p_adapter->p_ifc->get_err_str( status )) );\r
620                 return status;\r
621         }\r
622 \r
623         /* We only ever destroy from the PnP callback thread. */\r
624         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
625                 __port_destroying, __port_cleanup, __port_free );\r
626         if( cl_status != CL_SUCCESS )\r
627         {\r
628                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
629                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
630                 return IB_ERROR;\r
631         }\r
632 \r
633         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
634 \r
635         IPOIB_EXIT( IPOIB_DBG_INIT );\r
636         return IB_SUCCESS;\r
637 }\r
638 \r
639 \r
640 static void\r
641 __port_destroying(\r
642         IN                              cl_obj_t* const                         p_obj )\r
643 {\r
644         ipoib_port_t    *p_port;\r
645 \r
646         IPOIB_ENTER( IPOIB_DBG_INIT );\r
647 \r
648         CL_ASSERT( p_obj );\r
649 \r
650         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
651 \r
652         ipoib_port_down( p_port );\r
653 \r
654         __endpt_mgr_remove_all( p_port );\r
655 \r
656         IPOIB_EXIT( IPOIB_DBG_INIT );\r
657 }\r
658 \r
659 \r
660 static void\r
661 __port_cleanup(\r
662         IN                              cl_obj_t* const                         p_obj )\r
663 {\r
664         ipoib_port_t    *p_port;\r
665 \r
666         IPOIB_ENTER( IPOIB_DBG_INIT );\r
667 \r
668         CL_ASSERT( p_obj );\r
669 \r
670         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
671 \r
672         /* Wait for all sends and receives to get flushed. */\r
673         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
674                 cl_thread_suspend( 0 );\r
675 \r
676         /* Destroy the send and receive managers before closing the CA. */\r
677         __ib_mgr_destroy( p_port );\r
678 \r
679         IPOIB_EXIT( IPOIB_DBG_INIT );\r
680 }\r
681 \r
682 \r
683 static void\r
684 __port_free(\r
685         IN                              cl_obj_t* const                         p_obj )\r
686 {\r
687         ipoib_port_t    *p_port;\r
688 \r
689         IPOIB_ENTER( IPOIB_DBG_INIT );\r
690 \r
691         CL_ASSERT( p_obj );\r
692 \r
693         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
694 \r
695         __endpt_mgr_destroy( p_port );\r
696         __recv_mgr_destroy( p_port );\r
697         __send_mgr_destroy( p_port );\r
698         __buf_mgr_destroy( p_port );\r
699 \r
700         cl_spinlock_destroy( &p_port->send_lock );\r
701         cl_spinlock_destroy( &p_port->recv_lock );\r
702 \r
703         cl_obj_deinit( p_obj );\r
704 \r
705         cl_free( p_port );\r
706 \r
707         IPOIB_EXIT( IPOIB_DBG_INIT );\r
708 }\r
709 \r
710 \r
711 \r
712 /******************************************************************************\r
713 *\r
714 * IB resource manager implementation.\r
715 *\r
716 ******************************************************************************/\r
717 static void\r
718 __ib_mgr_construct(\r
719         IN                              ipoib_port_t* const                     p_port )\r
720 {\r
721         IPOIB_ENTER( IPOIB_DBG_INIT );\r
722 \r
723         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
724 \r
725         IPOIB_EXIT( IPOIB_DBG_INIT );\r
726 }\r
727 \r
728 \r
729 static ib_api_status_t\r
730 __ib_mgr_init(\r
731         IN                              ipoib_port_t* const                     p_port )\r
732 {\r
733         ib_api_status_t         status;\r
734         ib_cq_create_t          cq_create;\r
735         ib_qp_create_t          qp_create;\r
736         ib_phys_create_t        phys_create;\r
737         ib_phys_range_t         phys_range;\r
738         uint64_t                        vaddr;\r
739         net32_t                         rkey;\r
740         ib_qp_attr_t            qp_attr;\r
741 \r
742         IPOIB_ENTER( IPOIB_DBG_INIT );\r
743 \r
744         /* Open the CA. */\r
745         status = p_port->p_adapter->p_ifc->open_ca(\r
746                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
747                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
748         if( status != IB_SUCCESS )\r
749         {\r
750                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
751                         EVENT_IPOIB_OPEN_CA, 1, status );\r
752                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
753                         ("ib_open_ca returned %s\n", \r
754                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
755                 return status;\r
756         }\r
757 \r
758         /* Allocate the PD. */\r
759         status = p_port->p_adapter->p_ifc->alloc_pd(\r
760                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
761         if( status != IB_SUCCESS )\r
762         {\r
763                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
764                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
765                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
766                         ("ib_alloc_pd returned %s\n", \r
767                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
768                 return status;\r
769         }\r
770 \r
771         /* Allocate receive CQ. */\r
772         cq_create.size = p_port->p_adapter->params.rq_depth;\r
773         cq_create.pfn_comp_cb = __recv_cb;\r
774         cq_create.h_wait_obj = NULL;\r
775 \r
776         status = p_port->p_adapter->p_ifc->create_cq(\r
777                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
778                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
779         if( status != IB_SUCCESS )\r
780         {\r
781                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
782                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
783                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
784                         ("ib_create_cq returned %s.\n", \r
785                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
786                 return status;\r
787         }\r
788 \r
789         /* Allocate send CQ. */\r
790         cq_create.size = p_port->p_adapter->params.sq_depth;\r
791         cq_create.pfn_comp_cb = __send_cb;\r
792 \r
793         status = p_port->p_adapter->p_ifc->create_cq(\r
794                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
795                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
796         if( status != IB_SUCCESS )\r
797         {\r
798                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
799                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
800                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
801                         ("ib_create_cq returned %s.\n", \r
802                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
803                 return status;\r
804         }\r
805 \r
806         /* Allocate the QP. */\r
807         cl_memclr( &qp_create, sizeof(qp_create) );\r
808         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
809         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
810         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
811         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
812         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
813         //TODO: Figure out the right number of SGE entries for sends.\r
814         qp_create.sq_sge = MAX_SEND_SGE;\r
815         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
816         qp_create.sq_signaled = TRUE;\r
817         status = p_port->p_adapter->p_ifc->create_qp(\r
818                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
819                 __qp_event, &p_port->ib_mgr.h_qp );\r
820         if( status != IB_SUCCESS )\r
821         {\r
822                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
823                         EVENT_IPOIB_CREATE_QP, 1, status );\r
824                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
825                         ("ib_create_qp returned %s\n", \r
826                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
827                 return status;\r
828         }\r
829         /* Query the QP so we can get our QPN. */\r
830         status = p_port->p_adapter->p_ifc->query_qp(\r
831                 p_port->ib_mgr.h_qp, &qp_attr );\r
832         if( status != IB_SUCCESS )\r
833         {\r
834                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
835                         EVENT_IPOIB_QUERY_QP, 1, status );\r
836                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
837                         ("ib_query_qp returned %s\n", \r
838                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
839                 return status;\r
840         }\r
841         p_port->ib_mgr.qpn = qp_attr.num;\r
842 \r
843         /* Register all of physical memory */\r
844         phys_create.length = MEM_REG_SIZE;\r
845         phys_create.num_ranges = 1;\r
846         phys_create.range_array = &phys_range;\r
847         phys_create.buf_offset = 0;\r
848         phys_create.hca_page_size = PAGE_SIZE;\r
849         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
850         phys_range.base_addr = 0;\r
851         phys_range.size = MEM_REG_SIZE;\r
852         vaddr = 0;\r
853         status = p_port->p_adapter->p_ifc->reg_phys(\r
854                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
855                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
856         if( status != IB_SUCCESS )\r
857         {\r
858                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
859                         EVENT_IPOIB_REG_PHYS, 1, status );\r
860                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
861                         ("ib_reg_phys returned %s\n", \r
862                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
863                 return status;\r
864         }\r
865 \r
866         IPOIB_EXIT( IPOIB_DBG_INIT );\r
867         return IB_SUCCESS;\r
868 }\r
869 \r
870 \r
871 static void\r
872 __ib_mgr_destroy(\r
873         IN                              ipoib_port_t* const                     p_port )\r
874 {\r
875         ib_api_status_t status;\r
876 \r
877         IPOIB_ENTER( IPOIB_DBG_INIT );\r
878 \r
879         if( p_port->ib_mgr.h_ca )\r
880         {\r
881                 status =\r
882                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
883                 CL_ASSERT( status == IB_SUCCESS );\r
884                 p_port->ib_mgr.h_ca = NULL;\r
885         }\r
886 \r
887         IPOIB_EXIT( IPOIB_DBG_INIT );\r
888 }\r
889 \r
890 \r
891 \r
892 /******************************************************************************\r
893 *\r
894 * Buffer manager implementation.\r
895 *\r
896 ******************************************************************************/\r
897 static void\r
898 __buf_mgr_construct(\r
899         IN                              ipoib_port_t* const                     p_port )\r
900 {\r
901         IPOIB_ENTER( IPOIB_DBG_INIT );\r
902 \r
903         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
904 \r
905         p_port->buf_mgr.h_packet_pool = NULL;\r
906         p_port->buf_mgr.h_buffer_pool = NULL;\r
907 \r
908         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
909                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
910 \r
911         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
912         p_port->buf_mgr.h_send_buf_pool = NULL;\r
913 \r
914         IPOIB_EXIT( IPOIB_DBG_INIT );\r
915 }\r
916 \r
917 \r
918 static ib_api_status_t\r
919 __buf_mgr_init(\r
920         IN                              ipoib_port_t* const                     p_port )\r
921 {\r
922         cl_status_t             cl_status;\r
923         NDIS_STATUS             ndis_status;\r
924         ipoib_params_t  *p_params;\r
925 \r
926         IPOIB_ENTER(IPOIB_DBG_INIT );\r
927 \r
928         CL_ASSERT( p_port );\r
929         CL_ASSERT( p_port->p_adapter );\r
930 \r
931         p_params = &p_port->p_adapter->params;\r
932 \r
933         /* Allocate the receive descriptor pool */\r
934         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
935                 p_params->rq_depth * p_params->recv_pool_ratio,\r
936 #if IPOIB_INLINE_RECV\r
937                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
938 #else   /* IPOIB_INLINE_RECV */\r
939                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
940 #endif  /* IPOIB_INLINE_RECV */\r
941         if( cl_status != CL_SUCCESS )\r
942         {\r
943                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
944                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
945                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
946                         ("cl_qpool_init for recvs returned %s\n",\r
947                         cl_status_text[cl_status]) );\r
948                 return IB_INSUFFICIENT_MEMORY;\r
949         }\r
950 \r
951         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
952         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
953                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
954         if( ndis_status != NDIS_STATUS_SUCCESS )\r
955         {\r
956                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
957                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
958                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
959                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
960                 return IB_INSUFFICIENT_RESOURCES;\r
961         }\r
962 \r
963         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
964                 p_params->rq_depth );\r
965         if( ndis_status != NDIS_STATUS_SUCCESS )\r
966         {\r
967                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
968                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
969                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
970                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
971                 return IB_INSUFFICIENT_RESOURCES;\r
972         }\r
973 \r
974         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
975         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
976                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
977         if( ndis_status != NDIS_STATUS_SUCCESS )\r
978         {\r
979                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
980                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
981                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
982                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
983                 return IB_INSUFFICIENT_RESOURCES;\r
984         }\r
985 \r
986         NdisAllocateBufferPool( &ndis_status,\r
987                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
988         if( ndis_status != NDIS_STATUS_SUCCESS )\r
989         {\r
990                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
991                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
992                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
993                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
994                 return IB_INSUFFICIENT_RESOURCES;\r
995         }\r
996 \r
997         IPOIB_EXIT( IPOIB_DBG_INIT );\r
998         return IB_SUCCESS;\r
999 }\r
1000 \r
1001 \r
1002 static void\r
1003 __buf_mgr_destroy(\r
1004         IN                              ipoib_port_t* const                     p_port )\r
1005 {\r
1006         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1007 \r
1008         CL_ASSERT( p_port );\r
1009 \r
1010         /* Destroy the send packet and buffer pools. */\r
1011         if( p_port->buf_mgr.h_send_buf_pool )\r
1012                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1013         if( p_port->buf_mgr.h_send_pkt_pool )\r
1014                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1015 \r
1016         /* Destroy the receive packet and buffer pools. */\r
1017         if( p_port->buf_mgr.h_buffer_pool )\r
1018                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1019         if( p_port->buf_mgr.h_packet_pool )\r
1020                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1021 \r
1022         /* Free the receive and send descriptors. */\r
1023         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1024 \r
1025         /* Free the lookaside list of scratch buffers. */\r
1026         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1027 \r
1028         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1029 }\r
1030 \r
1031 \r
1032 static cl_status_t\r
1033 __recv_ctor(\r
1034         IN                              void* const                                     p_object,\r
1035         IN                              void*                                           context,\r
1036                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1037 {\r
1038         ipoib_recv_desc_t       *p_desc;\r
1039         ipoib_port_t            *p_port;\r
1040         uint32_t                        ds0_len;\r
1041 \r
1042         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1043 \r
1044         CL_ASSERT( p_object );\r
1045         CL_ASSERT( context );\r
1046 \r
1047         p_desc = (ipoib_recv_desc_t*)p_object;\r
1048         p_port = (ipoib_port_t*)context;\r
1049 \r
1050         /* Setup the work request. */\r
1051         p_desc->wr.ds_array = p_desc->local_ds;\r
1052         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1053 \r
1054 #if IPOIB_INLINE_RECV\r
1055         /* Sanity check on the receive buffer layout */\r
1056         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1057                 (void*)&p_desc->buf.ib.pkt.type );\r
1058         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1059 \r
1060         /* Setup the local data segment. */\r
1061         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1062         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1063         ds0_len =\r
1064                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1065         if( ds0_len >= sizeof(recv_buf_t) )\r
1066         {\r
1067                 /* The whole buffer is within a page. */\r
1068                 p_desc->local_ds[0].length = ds0_len;\r
1069                 p_desc->wr.num_ds = 1;\r
1070         }\r
1071         else\r
1072         {\r
1073                 /* The buffer crosses page boundaries. */\r
1074                 p_desc->local_ds[0].length = ds0_len;\r
1075                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1076                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1077                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1078                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1079                 p_desc->wr.num_ds = 2;\r
1080         }\r
1081 #else   /* IPOIB_INLINE_RECV */\r
1082         /* Allocate the receive buffer. */\r
1083         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1084         if( !p_desc->p_buf )\r
1085         {\r
1086                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1087                         ("Failed to allocate receive buffer.\n") );\r
1088                 return CL_INSUFFICIENT_MEMORY;\r
1089         }\r
1090 \r
1091         /* Sanity check on the receive buffer layout */\r
1092         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1093                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1094 \r
1095         /* Setup the local data segment. */\r
1096         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1097         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1098         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1099 #endif  /* IPOIB_INLINE_RECV */\r
1100 \r
1101         *pp_pool_item = &p_desc->item;\r
1102 \r
1103         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1104         return CL_SUCCESS;\r
1105 }\r
1106 \r
1107 \r
1108 #if !IPOIB_INLINE_RECV\r
1109 static void\r
1110 __recv_dtor(\r
1111         IN              const   cl_pool_item_t* const           p_pool_item,\r
1112         IN                              void                                            *context )\r
1113 {\r
1114         ipoib_recv_desc_t       *p_desc;\r
1115 \r
1116         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1117 \r
1118         UNUSED_PARAM( context );\r
1119 \r
1120         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1121 \r
1122         if( p_desc->p_buf )\r
1123                 cl_free( p_desc->p_buf );\r
1124 \r
1125         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1126 }\r
1127 #endif\r
1128 \r
1129 \r
1130 static inline ipoib_recv_desc_t*\r
1131 __buf_mgr_get_recv(\r
1132         IN                              ipoib_port_t* const                     p_port )\r
1133 {\r
1134         ipoib_recv_desc_t       *p_desc;\r
1135         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1136         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1137         /* Reference the port object for the send. */\r
1138         if( p_desc )\r
1139         {\r
1140                 cl_obj_ref( &p_port->obj );\r
1141                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1142 #if IPOIB_INLINE_RECV\r
1143                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1144                         cl_get_physaddr( &p_desc->buf ) );\r
1145 #else   /* IPOIB_INLINE_RECV */\r
1146                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1147                         cl_get_physaddr( p_desc->p_buf ) );\r
1148                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1149                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1150 #endif  /* IPOIB_INLINE_RECV */\r
1151                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1152         }\r
1153         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1154         return p_desc;\r
1155 }\r
1156 \r
1157 \r
1158 static inline void\r
1159 __buf_mgr_put_recv(\r
1160         IN                              ipoib_port_t* const                     p_port,\r
1161         IN                              ipoib_recv_desc_t* const        p_desc,\r
1162         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1163 {\r
1164         NDIS_BUFFER             *p_buf;\r
1165 \r
1166         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1167 \r
1168         if( p_packet )\r
1169         {\r
1170                 /* Unchain the NDIS buffer. */\r
1171                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1172                 CL_ASSERT( p_buf );\r
1173                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1174                 NdisDprFreePacketNonInterlocked( p_packet );\r
1175                 NdisFreeBuffer( p_buf );\r
1176         }\r
1177 \r
1178         /* Return the descriptor to its pools. */\r
1179         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1180 \r
1181         /*\r
1182          * Dereference the port object since the receive is no longer outstanding.\r
1183          */\r
1184         cl_obj_deref( &p_port->obj );\r
1185 \r
1186         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1187 }\r
1188 \r
1189 \r
1190 static inline void\r
1191 __buf_mgr_put_recv_list(\r
1192         IN                              ipoib_port_t* const                     p_port,\r
1193         IN                              cl_qlist_t* const                       p_list )\r
1194 {\r
1195         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1196         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1197         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1198 }\r
1199 \r
1200 \r
1201 static inline NDIS_PACKET*\r
1202 __buf_mgr_get_ndis_pkt(\r
1203         IN                              ipoib_port_t* const                     p_port,\r
1204         IN                              ipoib_recv_desc_t* const        p_desc )\r
1205 {\r
1206         NDIS_STATUS                             status;\r
1207         NDIS_PACKET                             *p_packet;\r
1208         NDIS_BUFFER                             *p_buffer;\r
1209 \r
1210         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1211 \r
1212         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1213                 p_port->buf_mgr.h_packet_pool );\r
1214         if( status != NDIS_STATUS_SUCCESS )\r
1215         {\r
1216                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1217                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1218                 return NULL;\r
1219         }\r
1220 \r
1221         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1222         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1223 \r
1224         NdisAllocateBuffer( &status, &p_buffer,\r
1225 #if IPOIB_INLINE_RECV\r
1226                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1227 #else   /* IPOIB_INLINE_RECV */\r
1228                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1229 #endif  /* IPOIB_INLINE_RECV */\r
1230         if( status != NDIS_STATUS_SUCCESS )\r
1231         {\r
1232                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1233                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1234                 NdisDprFreePacketNonInterlocked( p_packet );\r
1235                 return NULL;\r
1236         }\r
1237 \r
1238         NdisChainBufferAtFront( p_packet, p_buffer );\r
1239         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1240 \r
1241         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1242         return p_packet;\r
1243 }\r
1244 \r
1245 /******************************************************************************\r
1246 *\r
1247 * Receive manager implementation.\r
1248 *\r
1249 ******************************************************************************/\r
1250 static void\r
1251 __recv_mgr_construct(\r
1252         IN                              ipoib_port_t* const                     p_port )\r
1253 {\r
1254         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1255 \r
1256         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1257 \r
1258         p_port->recv_mgr.recv_pkt_array = NULL;\r
1259 \r
1260         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1261 }\r
1262 \r
1263 \r
1264 static ib_api_status_t\r
1265 __recv_mgr_init(\r
1266         IN                              ipoib_port_t* const                     p_port )\r
1267 {\r
1268         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1269 \r
1270         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1271         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1272                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1273         if( !p_port->recv_mgr.recv_pkt_array )\r
1274         {\r
1275                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1276                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1277                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1278                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1279                 return IB_INSUFFICIENT_MEMORY;\r
1280         }\r
1281 \r
1282         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1283         return IB_SUCCESS;\r
1284 }\r
1285 \r
1286 \r
1287 static void\r
1288 __recv_mgr_destroy(\r
1289         IN                              ipoib_port_t* const                     p_port )\r
1290 {\r
1291         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1292 \r
1293         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1294         CL_ASSERT( !p_port->recv_mgr.depth );\r
1295 \r
1296         if( p_port->recv_mgr.recv_pkt_array )\r
1297                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1298 \r
1299         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1300 }\r
1301 \r
1302 \r
1303 /*\r
1304  * Posts receive buffers to the receive queue and returns the number\r
1305  * of receives needed to bring the RQ to its low water mark.  Note\r
1306  * that the value is signed, and can go negative.  All tests must\r
1307  * be for > 0.\r
1308  */\r
1309 static int32_t\r
1310 __recv_mgr_repost(\r
1311         IN                              ipoib_port_t* const                     p_port )\r
1312 {\r
1313         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1314         ib_api_status_t         status;\r
1315         ib_recv_wr_t            *p_failed;\r
1316         PERF_DECLARE( GetRecv );\r
1317         PERF_DECLARE( PostRecv );\r
1318 \r
1319         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1320 \r
1321         CL_ASSERT( p_port );\r
1322         cl_obj_lock( &p_port->obj );\r
1323         if( p_port->state != IB_QPS_RTS )\r
1324         {\r
1325                 cl_obj_unlock( &p_port->obj );\r
1326                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1327                         ("Port in invalid state.  Not reposting.\n") );\r
1328                 return 0;\r
1329         }\r
1330         cl_obj_ref( &p_port->obj );\r
1331         cl_obj_unlock( &p_port->obj );\r
1332 \r
1333         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1334         {\r
1335                 /* Pull receives out of the pool and chain them up. */\r
1336                 cl_perf_start( GetRecv );\r
1337                 p_next = __buf_mgr_get_recv( p_port );\r
1338                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1339                 if( !p_next )\r
1340                 {\r
1341                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1342                                 ("Out of receive descriptors!\n") );\r
1343                         break;\r
1344                 }\r
1345 \r
1346                 if( !p_tail )\r
1347                 {\r
1348                         p_tail = p_next;\r
1349                         p_next->wr.p_next = NULL;\r
1350                 }\r
1351                 else\r
1352                 {\r
1353                         p_next->wr.p_next = &p_head->wr;\r
1354                 }\r
1355 \r
1356                 p_head = p_next;\r
1357 \r
1358                 p_port->recv_mgr.depth++;\r
1359         }\r
1360 \r
1361         if( p_head )\r
1362         {\r
1363                 cl_perf_start( PostRecv );\r
1364                 status = p_port->p_adapter->p_ifc->post_recv(\r
1365                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1366                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1367 \r
1368                 if( status != IB_SUCCESS )\r
1369                 {\r
1370                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1371                                 ("ip_post_recv returned %s\n", \r
1372                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1373                         /* return the descriptors to the pool */\r
1374                         while( p_failed )\r
1375                         {\r
1376                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1377                                 p_failed = p_failed->p_next;\r
1378 \r
1379                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1380                                 p_port->recv_mgr.depth--;\r
1381                         }\r
1382                 }\r
1383         }\r
1384 \r
1385         cl_obj_deref( &p_port->obj );\r
1386 \r
1387         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1388         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1389 }\r
1390 \r
1391 \r
1392 void\r
1393 ipoib_return_packet(\r
1394         IN                              NDIS_HANDLE                                     adapter_context,\r
1395         IN                              NDIS_PACKET                                     *p_packet )\r
1396 {\r
1397         cl_list_item_t          *p_item;\r
1398         ipoib_port_t            *p_port;\r
1399         ipoib_recv_desc_t       *p_desc;\r
1400         ib_api_status_t         status = IB_NOT_DONE;\r
1401         int32_t                         shortage;\r
1402         PERF_DECLARE( ReturnPacket );\r
1403         PERF_DECLARE( ReturnPutRecv );\r
1404         PERF_DECLARE( ReturnRepostRecv );\r
1405         PERF_DECLARE( ReturnPreparePkt );\r
1406         PERF_DECLARE( ReturnNdisIndicate );\r
1407 \r
1408         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1409 \r
1410         UNUSED_PARAM( adapter_context );\r
1411         CL_ASSERT( p_packet );\r
1412 \r
1413         cl_perf_start( ReturnPacket );\r
1414 \r
1415         /* Get the port and descriptor from the packet. */\r
1416         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1417         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1418 \r
1419         cl_spinlock_acquire( &p_port->recv_lock );\r
1420 \r
1421         cl_perf_start( ReturnPutRecv );\r
1422         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1423         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1424 \r
1425         /* Repost buffers. */\r
1426         cl_perf_start( ReturnRepostRecv );\r
1427         shortage = __recv_mgr_repost( p_port );\r
1428         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1429 \r
1430         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1431                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1432                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1433         {\r
1434                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1435 \r
1436                 cl_perf_start( ReturnPreparePkt );\r
1437                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1438                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1439                 if( status == IB_SUCCESS )\r
1440                 {\r
1441                         if( shortage > 0 )\r
1442                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1443                         else\r
1444                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1445 \r
1446                         cl_spinlock_release( &p_port->recv_lock );\r
1447                         cl_perf_start( ReturnNdisIndicate );\r
1448                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1449                                 &p_packet, 1 );\r
1450                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1451                         cl_spinlock_acquire( &p_port->recv_lock );\r
1452 \r
1453                         if( shortage > 0 )\r
1454                         {\r
1455                                 cl_perf_start( ReturnPutRecv );\r
1456                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1457                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1458 \r
1459                                 /* Repost buffers. */\r
1460                                 cl_perf_start( ReturnRepostRecv );\r
1461                                 shortage = __recv_mgr_repost( p_port );\r
1462                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1463                         }\r
1464                 }\r
1465                 else if( status != IB_NOT_DONE )\r
1466                 {\r
1467                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1468                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1469                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1470                         /* Return the item to the head of the list. */\r
1471                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1472                         break;\r
1473                 }\r
1474         }\r
1475         cl_spinlock_release( &p_port->recv_lock );\r
1476         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1477 \r
1478         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1479 }\r
1480 \r
1481 \r
1482 static void\r
1483 __recv_cb(\r
1484         IN              const   ib_cq_handle_t                          h_cq,\r
1485         IN                              void                                            *cq_context )\r
1486 {\r
1487         ipoib_port_t            *p_port;\r
1488         ib_api_status_t         status;\r
1489         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1490         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1491         cl_qlist_t                      done_list, bad_list;\r
1492         size_t                          i;\r
1493         PERF_DECLARE( RecvCompBundle );\r
1494         PERF_DECLARE( RecvCb );\r
1495         PERF_DECLARE( PollRecv );\r
1496         PERF_DECLARE( RepostRecv );\r
1497         PERF_DECLARE( FilterRecv );\r
1498         PERF_DECLARE( BuildPktArray );\r
1499         PERF_DECLARE( RecvNdisIndicate );\r
1500         PERF_DECLARE( RearmRecv );\r
1501         PERF_DECLARE( PutRecvList );\r
1502 \r
1503         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1504 \r
1505         cl_perf_clr( RecvCompBundle );\r
1506 \r
1507         cl_perf_start( RecvCb );\r
1508 \r
1509         UNUSED_PARAM( h_cq );\r
1510 \r
1511         p_port = (ipoib_port_t*)cq_context;\r
1512 \r
1513         cl_qlist_init( &done_list );\r
1514         cl_qlist_init( &bad_list );\r
1515 \r
1516         cl_obj_ref( &p_port->obj );\r
1517 \r
1518         for( i = 0; i < MAX_RECV_WC; i++ )\r
1519                 wc[i].p_next = &wc[i + 1];\r
1520         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1521 \r
1522         /*\r
1523          * We'll be accessing the endpoint map so take a reference\r
1524          * on it to prevent modifications.\r
1525          */\r
1526         cl_obj_lock( &p_port->obj );\r
1527         cl_atomic_inc( &p_port->endpt_rdr );\r
1528         cl_obj_unlock( &p_port->obj );\r
1529 \r
1530         do\r
1531         {\r
1532                 /* If we get here, then the list of WCs is intact. */\r
1533                 p_free = wc;\r
1534 \r
1535                 cl_perf_start( PollRecv );\r
1536                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1537                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1538                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1539                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1540 \r
1541                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1542                 cl_perf_start( FilterRecv );\r
1543                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1544                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1545 \r
1546         } while( !p_free );\r
1547 \r
1548         /* We're done looking at the endpoint map, release the reference. */\r
1549         cl_atomic_dec( &p_port->endpt_rdr );\r
1550 \r
1551         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1552 \r
1553         cl_spinlock_acquire( &p_port->recv_lock );\r
1554 \r
1555         /* Update our posted depth. */\r
1556         p_port->recv_mgr.depth -= recv_cnt;\r
1557 \r
1558         /* Return any discarded receives to the pool */\r
1559         cl_perf_start( PutRecvList );\r
1560         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1561         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1562 \r
1563         do\r
1564         {\r
1565                 /* Repost ASAP so we don't starve the RQ. */\r
1566                 cl_perf_start( RepostRecv );\r
1567                 shortage = __recv_mgr_repost( p_port );\r
1568                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1569 \r
1570                 cl_perf_start( BuildPktArray );\r
1571                 /* Notify NDIS of any and all possible receive buffers. */\r
1572                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1573                         p_port, shortage, &done_list, &discarded );\r
1574                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1575 \r
1576                 /* Only indicate receives if we actually had any. */\r
1577                 if( discarded && shortage > 0 )\r
1578                 {\r
1579                         /* We may have thrown away packets, and have a shortage */\r
1580                         cl_perf_start( RepostRecv );\r
1581                         __recv_mgr_repost( p_port );\r
1582                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1583                 }\r
1584 \r
1585                 if( !pkt_cnt )\r
1586                         break;\r
1587 \r
1588                 cl_spinlock_release( &p_port->recv_lock );\r
1589 \r
1590                 cl_perf_start( RecvNdisIndicate );\r
1591                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1592                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1593                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1594 \r
1595                 /*\r
1596                  * Cap the number of receives to put back to what we just indicated\r
1597                  * with NDIS_STATUS_RESOURCES.\r
1598                  */\r
1599                 if( shortage > 0 )\r
1600                 {\r
1601                         if( pkt_cnt < shortage )\r
1602                                 shortage = pkt_cnt;\r
1603 \r
1604                         /* Return all but the last packet to the pool. */\r
1605                         cl_spinlock_acquire( &p_port->recv_lock );\r
1606                         while( shortage-- > 1 )\r
1607                         {\r
1608                                 __buf_mgr_put_recv( p_port,\r
1609                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1610                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1611                         }\r
1612                         cl_spinlock_release( &p_port->recv_lock );\r
1613 \r
1614                         /*\r
1615                          * Return the last packet as if NDIS returned it, so that we repost\r
1616                          * and report any other pending receives.\r
1617                          */\r
1618                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1619                 }\r
1620                 cl_spinlock_acquire( &p_port->recv_lock );\r
1621 \r
1622         } while( pkt_cnt );\r
1623         cl_spinlock_release( &p_port->recv_lock );\r
1624 \r
1625         /*\r
1626          * Rearm after filtering to prevent contention on the enpoint maps\r
1627          * and eliminate the possibility of having a call to\r
1628          * __endpt_mgr_insert find a duplicate.\r
1629          */\r
1630         cl_perf_start( RearmRecv );\r
1631         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1632                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1633         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1634         CL_ASSERT( status == IB_SUCCESS );\r
1635 \r
1636         cl_obj_deref( &p_port->obj );\r
1637 \r
1638         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1639 \r
1640         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1641 }\r
1642 \r
1643 \r
1644 static void\r
1645 __recv_get_endpts(\r
1646         IN                              ipoib_port_t* const                     p_port,\r
1647         IN                              ipoib_recv_desc_t* const        p_desc,\r
1648         IN                              ib_wc_t* const                          p_wc,\r
1649                 OUT                     ipoib_endpt_t** const           pp_src,\r
1650                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1651 {\r
1652         ib_api_status_t         status;\r
1653         mac_addr_t                      mac;\r
1654         PERF_DECLARE( GetEndptByGid );\r
1655         PERF_DECLARE( GetEndptByLid );\r
1656         PERF_DECLARE( EndptInsert );\r
1657 \r
1658         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1659 \r
1660         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1661         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1662         {\r
1663                 /* Lookup the source endpoints based on GID. */\r
1664                 cl_perf_start( GetEndptByGid );\r
1665                 *pp_src =\r
1666 #if IPOIB_INLINE_RECV\r
1667                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1668 #else   /* IPOIB_INLINE_RECV */\r
1669                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1670 #endif  /* IPOIB_INLINE_RECV */\r
1671                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1672 \r
1673                 /*\r
1674                  * Lookup the destination endpoint based on GID.\r
1675                  * This is used along with the packet filter to determine\r
1676                  * whether to report this to NDIS.\r
1677                  */\r
1678                 cl_perf_start( GetEndptByGid );\r
1679                 *pp_dst =\r
1680 #if IPOIB_INLINE_RECV\r
1681                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1682 #else   /* IPOIB_INLINE_RECV */\r
1683                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1684 #endif  /* IPOIB_INLINE_RECV */\r
1685                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1686 \r
1687                 /*\r
1688                  * Create the source endpoint if it does not exist.  Note that we\r
1689                  * can only do this for globally routed traffic since we need the\r
1690                  * information from the GRH to generate the MAC.\r
1691                  */\r
1692                 if( !*pp_src )\r
1693                 {\r
1694                         status = ipoib_mac_from_guid(\r
1695 #if IPOIB_INLINE_RECV\r
1696                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1697 #else   /* IPOIB_INLINE_RECV */\r
1698                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1699 #endif  /* IPOIB_INLINE_RECV */\r
1700                         if( status != IB_SUCCESS )\r
1701                         {\r
1702                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1703                                         ("ipoib_mac_from_guid returned %s\n",\r
1704                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1705                                 return;\r
1706                         }\r
1707 \r
1708                         /* Create the endpoint. */\r
1709 #if IPOIB_INLINE_RECV\r
1710                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1711 #else   /* IPOIB_INLINE_RECV */\r
1712                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1713 #endif  /* IPOIB_INLINE_RECV */\r
1714                                 0, p_wc->recv.ud.remote_qp );\r
1715                         if( !*pp_src )\r
1716                         {\r
1717                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1718                                         ("ipoib_endpt_create returned %s\n",\r
1719                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1720                                 return;\r
1721                         }\r
1722                         cl_perf_start( EndptInsert );\r
1723                         cl_obj_lock( &p_port->obj );\r
1724                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1725                         cl_obj_unlock( &p_port->obj );\r
1726                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1727                 }\r
1728         }\r
1729         else\r
1730         {\r
1731                 /*\r
1732                  * Lookup the remote endpoint based on LID.  Note that only\r
1733                  * unicast traffic can be LID routed.\r
1734                  */\r
1735                 cl_perf_start( GetEndptByLid );\r
1736                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1737                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1738                 *pp_dst = p_port->p_local_endpt;\r
1739                 CL_ASSERT( *pp_dst );\r
1740         }\r
1741 \r
1742         if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1743         {\r
1744                 /* Update the QPN for the endpoint. */\r
1745                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1746                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1747                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1748                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1749                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1750                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1751         }\r
1752 \r
1753         if( *pp_src && *pp_dst )\r
1754         {\r
1755                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1756                         ("Recv:\n"\r
1757                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1758                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1759                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1760                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1761                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1762                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1763                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1764                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1765         }\r
1766 \r
1767         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1768 }\r
1769 \r
1770 \r
1771 static int32_t\r
1772 __recv_mgr_filter(\r
1773         IN                              ipoib_port_t* const                     p_port,\r
1774         IN                              ib_wc_t* const                          p_done_wc_list,\r
1775                 OUT                     cl_qlist_t* const                       p_done_list,\r
1776                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1777 {\r
1778         ipoib_recv_desc_t               *p_desc;\r
1779         ib_wc_t                                 *p_wc;\r
1780         ipoib_pkt_t                             *p_ipoib;\r
1781         eth_pkt_t                               *p_eth;\r
1782         ipoib_endpt_t                   *p_src, *p_dst;\r
1783         ib_api_status_t                 status;\r
1784         uint32_t                                len;\r
1785         int32_t                                 recv_cnt = 0;\r
1786         PERF_DECLARE( GetRecvEndpts );\r
1787         PERF_DECLARE( RecvGen );\r
1788         PERF_DECLARE( RecvTcp );\r
1789         PERF_DECLARE( RecvUdp );\r
1790         PERF_DECLARE( RecvDhcp );\r
1791         PERF_DECLARE( RecvArp );\r
1792 \r
1793         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1794 \r
1795         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1796         {\r
1797                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1798                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1799                 recv_cnt++;\r
1800 \r
1801                 if( p_wc->status != IB_WCS_SUCCESS )\r
1802                 {\r
1803                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1804                         {\r
1805                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1806                                         ("Failed completion %s\n",\r
1807                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1808                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1809                         }\r
1810                         else\r
1811                         {\r
1812                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1813                                         ("Flushed completion %s\n",\r
1814                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1815                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1816                         }\r
1817                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1818                         /* Dereference the port object on behalf of the failed receive. */\r
1819                         cl_obj_deref( &p_port->obj );\r
1820                         continue;\r
1821                 }\r
1822 \r
1823                 len = p_wc->length - sizeof(ib_grh_t);\r
1824 \r
1825                 if( len < sizeof(ipoib_hdr_t) )\r
1826                 {\r
1827                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1828                                 ("Received ETH packet < min size\n") );\r
1829                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1830                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1831                         cl_obj_deref( &p_port->obj );\r
1832                         continue;\r
1833                 }\r
1834 \r
1835                 /* Successful completion.  Get the receive information. */\r
1836                 cl_perf_start( GetRecvEndpts );\r
1837                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1838                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1839 \r
1840 #if IPOIB_INLINE_RECV\r
1841                 p_ipoib = &p_desc->buf.ib.pkt;\r
1842                 p_eth = &p_desc->buf.eth.pkt;\r
1843 #else   /* IPOIB_INLINE_RECV */\r
1844                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1845                 p_eth = &p_desc->p_buf->eth.pkt;\r
1846 #endif  /*IPOIB_INLINE_RECV */\r
1847 \r
1848                 if( p_src )\r
1849                 {\r
1850                         /* Don't report loopback traffic - we requested SW loopback. */\r
1851                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1852                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1853                         {\r
1854                                 /*\r
1855                                  * "This is not the packet you're looking for" - don't update\r
1856                                  * receive statistics, the packet never happened.\r
1857                                  */\r
1858                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1859                                 /* Dereference the port object on behalf of the failed recv. */\r
1860                                 cl_obj_deref( &p_port->obj );\r
1861                                 continue;\r
1862                         }\r
1863                 }\r
1864 \r
1865                 switch( p_ipoib->hdr.type )\r
1866                 {\r
1867                 case ETH_PROT_TYPE_IP:\r
1868                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1869                         {\r
1870                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1871                                         ("Received IP packet < min size\n") );\r
1872                                 status = IB_INVALID_SETTING;\r
1873                                 break;\r
1874                         }\r
1875 \r
1876                         if( p_ipoib->type.ip.hdr.offset ||\r
1877                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1878                         {\r
1879                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1880                                 cl_perf_start( RecvTcp );\r
1881                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1882                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1883                                 break;\r
1884                         }\r
1885 \r
1886                         /* First packet of a UDP transfer. */\r
1887                         if( len <\r
1888                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1889                         {\r
1890                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1891                                         ("Received UDP packet < min size\n") );\r
1892                                 status = IB_INVALID_SETTING;\r
1893                                 break;\r
1894                         }\r
1895 \r
1896                         /* Check if DHCP conversion is required. */\r
1897                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1898                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1899                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1900                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1901                         {\r
1902                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1903                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1904                                 {\r
1905                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1906                                                 ("Received DHCP < min size\n") );\r
1907                                         status = IB_INVALID_SETTING;\r
1908                                         break;\r
1909                                 }\r
1910                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1911                                 cl_perf_start( RecvDhcp );\r
1912                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1913                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1914                         }\r
1915                         else\r
1916                         {\r
1917                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1918                                 cl_perf_start( RecvUdp );\r
1919                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1920                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1921                         }\r
1922                         break;\r
1923 \r
1924                 case ETH_PROT_TYPE_ARP:\r
1925                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1926                         {\r
1927                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1928                                         ("Received ARP < min size\n") );\r
1929                                 status = IB_INVALID_SETTING;\r
1930                                 break;\r
1931                         }\r
1932                         cl_perf_start( RecvArp );\r
1933                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1934                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1935                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1936                         break;\r
1937 \r
1938                 default:\r
1939                         /* Unfiltered.  Setup the ethernet header and report. */\r
1940                         cl_perf_start( RecvGen );\r
1941                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1942                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1943                 }\r
1944 \r
1945                 if( status != IB_SUCCESS )\r
1946                 {\r
1947                         /* Update stats. */\r
1948                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1949                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1950                         /* Dereference the port object on behalf of the failed receive. */\r
1951                         cl_obj_deref( &p_port->obj );\r
1952                 }\r
1953                 else\r
1954                 {\r
1955                         p_desc->len =\r
1956                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1957                         if( p_dst->h_mcast)\r
1958                         {\r
1959                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1960                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1961                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1962                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1963                                 {\r
1964                                         p_desc->type = PKT_TYPE_BCAST;\r
1965                                 }\r
1966                                 else\r
1967                                 {\r
1968                                         p_desc->type = PKT_TYPE_MCAST;\r
1969                                 }\r
1970                         }\r
1971                         else\r
1972                         {\r
1973                                 p_desc->type = PKT_TYPE_UCAST;\r
1974                         }\r
1975                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1976                 }\r
1977         }\r
1978 \r
1979         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1980         return recv_cnt;\r
1981 }\r
1982 \r
1983 \r
1984 static ib_api_status_t\r
1985 __recv_gen(\r
1986         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1987                 OUT                     eth_pkt_t* const                        p_eth,\r
1988         IN                              ipoib_endpt_t* const            p_src,\r
1989         IN                              ipoib_endpt_t* const            p_dst )\r
1990 {\r
1991         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1992 \r
1993         if( !p_src || !p_dst )\r
1994         {\r
1995                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1996                         ("Received packet with no matching endpoints.\n") );\r
1997                 return IB_NOT_DONE;\r
1998         }\r
1999 \r
2000         /*\r
2001          * Fill in the ethernet header.  Note that doing so will overwrite\r
2002          * the IPoIB header, so start by moving the information from the IPoIB\r
2003          * header.\r
2004          */\r
2005         p_eth->hdr.type = p_ipoib->hdr.type;\r
2006         p_eth->hdr.src = p_src->mac;\r
2007         p_eth->hdr.dst = p_dst->mac;\r
2008 \r
2009         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2010         return IB_SUCCESS;\r
2011 }\r
2012 \r
2013 \r
2014 static ib_api_status_t\r
2015 __recv_dhcp(\r
2016         IN                              ipoib_port_t* const                     p_port,\r
2017         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2018                 OUT                     eth_pkt_t* const                        p_eth,\r
2019         IN                              ipoib_endpt_t* const            p_src,\r
2020         IN                              ipoib_endpt_t* const            p_dst )\r
2021 {\r
2022         ib_api_status_t         status;\r
2023         dhcp_pkt_t                      *p_dhcp;\r
2024         uint8_t                         *p_option;\r
2025         uint8_t                         *p_cid = NULL;\r
2026         ib_gid_t                        gid;\r
2027         uint8_t                         msg = 0;\r
2028 \r
2029         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2030 \r
2031         UNUSED_PARAM( p_port );\r
2032 \r
2033         /* Create the ethernet header. */\r
2034         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2035         if( status != IB_SUCCESS )\r
2036         {\r
2037                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2038                         ("__recv_gen returned %s.\n", \r
2039                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2040                 return status;\r
2041         }\r
2042 \r
2043         /* Fixup the payload. */\r
2044         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2045         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2046         {\r
2047                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2048                         ("Invalid DHCP op code.\n") );\r
2049                 return IB_INVALID_SETTING;\r
2050         }\r
2051 \r
2052         /*\r
2053          * Find the client identifier option, making sure to skip\r
2054          * the "magic cookie".\r
2055          */\r
2056         p_option = &p_dhcp->options[0];\r
2057         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2058         {\r
2059                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2060                         ("DHCP cookie corrupted.\n") );\r
2061                 return IB_INVALID_PARAMETER;\r
2062         }\r
2063 \r
2064         p_option = &p_dhcp->options[4];\r
2065         while( *p_option != DHCP_OPT_END )\r
2066         {\r
2067                 switch( *p_option )\r
2068                 {\r
2069                 case DHCP_OPT_PAD:\r
2070                         p_option++;\r
2071                         break;\r
2072 \r
2073                 case DHCP_OPT_MSG:\r
2074                         msg = p_option[2];\r
2075                         p_option += 3;\r
2076                         break;\r
2077 \r
2078                 case DHCP_OPT_CLIENT_ID:\r
2079                         p_cid = p_option;\r
2080                         /* Fall through. */\r
2081 \r
2082                 default:\r
2083                         /*\r
2084                          * All other options have a length byte following the option code.\r
2085                          * Offset by the length to get to the next option.\r
2086                          */\r
2087                         p_option += (p_option[1] + 2);\r
2088                 }\r
2089         }\r
2090 \r
2091         switch( msg )\r
2092         {\r
2093         /* message from client */\r
2094         case DHCPDISCOVER:\r
2095         case DHCPREQUEST:\r
2096         case DHCPDECLINE:\r
2097         case DHCPRELEASE:\r
2098         case DHCPINFORM:\r
2099                 if( !p_cid )\r
2100                 {\r
2101                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2102                                 ("Failed to find required Client-identifier option.\n") );\r
2103                         return IB_INVALID_SETTING;\r
2104                 }\r
2105                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2106                 {\r
2107                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2108                                 ("Invalid hardware address type.\n") );\r
2109                         return IB_INVALID_SETTING;\r
2110                 }\r
2111                 break;\r
2112         /* message from DHCP server */\r
2113         case DHCPOFFER:\r
2114         case DHCPACK:\r
2115         case DHCPNAK:\r
2116                 break;\r
2117 \r
2118         default:\r
2119                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2120                         ("Invalide message type.\n") );\r
2121                 return IB_INVALID_PARAMETER;\r
2122         }\r
2123         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2124         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2125         p_dhcp->hlen = HW_ADDR_LEN;\r
2126 \r
2127         if( p_cid ) /* from client */\r
2128         {\r
2129                 /* Validate that the length and type of the option is as required. */\r
2130                 if( p_cid[1] != 21 )\r
2131                 {\r
2132                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2133                                 ("Client-identifier length not 21 as required.\n") );\r
2134                         return IB_INVALID_SETTING;\r
2135                 }\r
2136                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2137                 {\r
2138                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2139                                 ("Client-identifier type is wrong.\n") );\r
2140                         return IB_INVALID_SETTING;\r
2141                 }\r
2142                 /*\r
2143                  * Copy the GID value from the option so that we can make aligned\r
2144                  * accesses to the contents.\r
2145                  * Recover CID to standard type.\r
2146                  */\r
2147                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2148                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2149                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2150                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2151                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2152         }\r
2153         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2154         return status;\r
2155 }\r
2156 \r
2157 \r
2158 static ib_api_status_t\r
2159 __recv_arp(\r
2160         IN                              ipoib_port_t* const                     p_port,\r
2161         IN                              ib_wc_t* const                          p_wc,\r
2162         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2163                 OUT                     eth_pkt_t* const                        p_eth,\r
2164         IN                              ipoib_endpt_t** const           pp_src,\r
2165         IN                              ipoib_endpt_t* const            p_dst )\r
2166 {\r
2167         ib_api_status_t                 status;\r
2168         arp_pkt_t                               *p_arp;\r
2169         const ipoib_arp_pkt_t   *p_ib_arp;\r
2170         ib_gid_t                                gid;\r
2171         mac_addr_t                              mac;\r
2172         ipoib_hw_addr_t                 null_hw = {0};\r
2173 \r
2174         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2175 \r
2176         if( !p_dst )\r
2177         {\r
2178                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2179                         ("Unknown destination endpoint\n") );\r
2180                 return IB_INVALID_SETTING;\r
2181         }\r
2182 \r
2183         p_ib_arp = &p_ipoib->type.arp;\r
2184         p_arp = &p_eth->type.arp;\r
2185 \r
2186         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2187         {\r
2188                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2189                         ("ARP hardware type is not IB\n") );\r
2190                 return IB_INVALID_SETTING;\r
2191         }\r
2192 \r
2193         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2194         {\r
2195                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2196                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2197                 return IB_INVALID_SETTING;\r
2198         }\r
2199 \r
2200         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2201         {\r
2202                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2203                         ("ARP protocal type not IP\n") );\r
2204                 return IB_INVALID_SETTING;\r
2205         }\r
2206 \r
2207         /*\r
2208          * If we don't have a source, lookup the endpoint specified in the payload.\r
2209          */\r
2210         if( !*pp_src )\r
2211                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2212 \r
2213         /*\r
2214          * If the endpoint exists for the GID, make sure\r
2215          * the dlid and qpn match the arp.\r
2216          */\r
2217         if( *pp_src )\r
2218         {\r
2219                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2220                         sizeof(ib_gid_t) ) )\r
2221                 {\r
2222                         /*\r
2223                          * GIDs for the endpoint are different.  The ARP must\r
2224                          * have been proxied.  Dereference it.\r
2225                          */\r
2226                         *pp_src = NULL;\r
2227                 }\r
2228                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2229                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2230                 {\r
2231                         /* Out of date!  Destroy the endpoint and replace it. */\r
2232                         __endpt_mgr_remove( p_port, *pp_src );\r
2233                         *pp_src = NULL;\r
2234                 }\r
2235         }\r
2236 \r
2237         /* Do we need to create an endpoint for this GID? */\r
2238         if( !*pp_src )\r
2239         {\r
2240                 /* Copy the src GID to allow aligned access */\r
2241                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2242                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2243                 if( status != IB_SUCCESS )\r
2244                 {\r
2245                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2246                                 ("ipoib_mac_from_guid returned %s\n",\r
2247                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2248                         return status;\r
2249                 }\r
2250                 /*\r
2251                  * Create the endpoint.  Note that the LID is left blank and will be\r
2252                  * resolved by a path query as needed.  This is done because the\r
2253                  * remote LID/GID from the work completion may not be the original\r
2254                  * initiator.\r
2255                  */\r
2256                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2257                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2258                 if( !*pp_src )\r
2259                 {\r
2260                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2261                                 ("ipoib_endpt_create returned %s\n",\r
2262                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2263                         return status;\r
2264                 }\r
2265 \r
2266                 cl_obj_lock( &p_port->obj );\r
2267                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2268                 cl_obj_unlock( &p_port->obj );\r
2269         }\r
2270 \r
2271         CL_ASSERT( !cl_memcmp(\r
2272                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2273         CL_ASSERT( (*pp_src)->qpn ==\r
2274                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2275         /* Now swizzle the data. */\r
2276         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2277         p_arp->hw_size = sizeof(mac_addr_t);\r
2278         p_arp->src_hw = (*pp_src)->mac;\r
2279         p_arp->src_ip = p_ib_arp->src_ip;\r
2280 \r
2281         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2282         {\r
2283 \r
2284                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2285                 {\r
2286                         /*\r
2287                          * We received bcast ARP packet that means\r
2288                          * remote port lets everyone know it was changed IP/MAC\r
2289                          * or just activated\r
2290                          */\r
2291                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2292                         {\r
2293                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2294                                         ("ARP: is not ARP MCAST\n") );\r
2295                                 return IB_INVALID_SETTING;\r
2296                         }\r
2297 \r
2298                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2299                         p_dst->mac = p_port->p_local_endpt->mac;\r
2300                         /*\r
2301                          * we don't care what receiver ip addr is,\r
2302                          * as long as OS' ARP table is global  ???\r
2303                          */\r
2304                         p_arp->dst_ip = (net32_t)0;\r
2305 \r
2306                 }\r
2307                 else /* we've got reply to our ARP request */\r
2308                 {\r
2309                         p_arp->dst_hw = p_dst->mac;\r
2310                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2311                         CL_ASSERT( p_dst->qpn == \r
2312                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2313                 }\r
2314         }\r
2315         else /* we got ARP reqeust */\r
2316         {\r
2317                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2318                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2319         }\r
2320 \r
2321         /*\r
2322          * Create the ethernet header.  Note that this is done last so that\r
2323          * we have a chance to create a new endpoint.\r
2324          */\r
2325         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2326         if( status != IB_SUCCESS )\r
2327         {\r
2328                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2329                         ("__recv_gen returned %s.\n", \r
2330                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2331                 return status;\r
2332         }\r
2333 \r
2334         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2335         return IB_SUCCESS;\r
2336 }\r
2337 \r
2338 \r
2339 static ib_api_status_t\r
2340 __recv_mgr_prepare_pkt(\r
2341         IN                              ipoib_port_t* const                     p_port,\r
2342         IN                              ipoib_recv_desc_t* const        p_desc,\r
2343                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2344 {\r
2345         NDIS_STATUS                                                     status;\r
2346         uint32_t                                                        pkt_filter;\r
2347         ip_stat_sel_t                                           type;\r
2348         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2349         PERF_DECLARE( GetNdisPkt );\r
2350 \r
2351         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2352 \r
2353         pkt_filter = p_port->p_adapter->packet_filter;\r
2354         /* Check the packet filter. */\r
2355         switch( p_desc->type )\r
2356         {\r
2357         default:\r
2358         case PKT_TYPE_UCAST:\r
2359                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2360                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2361                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2362                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2363                 {\r
2364                         /* OK to report. */\r
2365                         type = IP_STAT_UCAST_BYTES;\r
2366                         status = NDIS_STATUS_SUCCESS;\r
2367                 }\r
2368                 else\r
2369                 {\r
2370                         type = IP_STAT_DROPPED;\r
2371                         status = NDIS_STATUS_FAILURE;\r
2372                 }\r
2373                 break;\r
2374         case PKT_TYPE_BCAST:\r
2375                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2376                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2377                 {\r
2378                         /* OK to report. */\r
2379                         type = IP_STAT_BCAST_BYTES;\r
2380                         status = NDIS_STATUS_SUCCESS;\r
2381                 }\r
2382                 else\r
2383                 {\r
2384                         type = IP_STAT_DROPPED;\r
2385                         status = NDIS_STATUS_FAILURE;\r
2386                 }\r
2387                 break;\r
2388         case PKT_TYPE_MCAST:\r
2389                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2390                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2391                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2392                 {\r
2393                         /* OK to report. */\r
2394                         type = IP_STAT_MCAST_BYTES;\r
2395                         status = NDIS_STATUS_SUCCESS;\r
2396                 }\r
2397                 else\r
2398                 {\r
2399                         type = IP_STAT_DROPPED;\r
2400                         status = NDIS_STATUS_FAILURE;\r
2401                 }\r
2402                 break;\r
2403         }\r
2404 \r
2405         if( status != NDIS_STATUS_SUCCESS )\r
2406         {\r
2407                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2408                 /* Return the receive descriptor to the pool. */\r
2409                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2410                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2411                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2412                 /*\r
2413                  * Return IB_NOT_DONE since the packet has been completed,\r
2414                  * but has not consumed an array entry.\r
2415                  */\r
2416                 return IB_NOT_DONE;\r
2417         }\r
2418 \r
2419         cl_perf_start( GetNdisPkt );\r
2420         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2421         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2422         if( !*pp_packet )\r
2423         {\r
2424                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2425                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2426                 return IB_INSUFFICIENT_RESOURCES;\r
2427         }\r
2428 \r
2429         /* Flag the checksums as having been calculated. */\r
2430         chksum.Value = 0;\r
2431         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2432         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2433         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2434         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2435                 (void*)(uintn_t)chksum.Value;\r
2436 \r
2437         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2438 \r
2439         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2440         return IB_SUCCESS;\r
2441 }\r
2442 \r
2443 \r
2444 static uint32_t\r
2445 __recv_mgr_build_pkt_array(\r
2446         IN                              ipoib_port_t* const                     p_port,\r
2447         IN                              int32_t                                         shortage,\r
2448                 OUT                     cl_qlist_t* const                       p_done_list,\r
2449                 OUT                     int32_t* const                          p_discarded )\r
2450 {\r
2451         cl_list_item_t                  *p_item;\r
2452         ipoib_recv_desc_t               *p_desc;\r
2453         uint32_t                                i = 0;\r
2454         ib_api_status_t                 status;\r
2455         PERF_DECLARE( PreparePkt );\r
2456 \r
2457         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2458 \r
2459         *p_discarded = 0;\r
2460 \r
2461         /* Move any existing receives to the head to preserve ordering. */\r
2462         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2463         p_item = cl_qlist_remove_head( p_done_list );\r
2464         while( p_item != cl_qlist_end( p_done_list ) )\r
2465         {\r
2466                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2467 \r
2468                 cl_perf_start( PreparePkt );\r
2469                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2470                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2471                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2472                 if( status == IB_SUCCESS )\r
2473                 {\r
2474                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2475                         if( shortage-- > 0 )\r
2476                         {\r
2477                                 NDIS_SET_PACKET_STATUS(\r
2478                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2479                         }\r
2480                         else\r
2481                         {\r
2482                                 NDIS_SET_PACKET_STATUS(\r
2483                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2484                         }\r
2485                         i++;\r
2486                 }\r
2487                 else if( status == IB_NOT_DONE )\r
2488                 {\r
2489                         (*p_discarded)++;\r
2490                 }\r
2491                 else\r
2492                 {\r
2493                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2494                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2495                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2496                         /* Put all completed receives on the port's done list. */\r
2497                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2498                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2499                         break;\r
2500                 }\r
2501 \r
2502                 p_item = cl_qlist_remove_head( p_done_list );\r
2503         }\r
2504 \r
2505         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2506         return i;\r
2507 }\r
2508 \r
2509 \r
2510 \r
2511 \r
2512 /******************************************************************************\r
2513 *\r
2514 * Send manager implementation.\r
2515 *\r
2516 ******************************************************************************/\r
2517 static void\r
2518 __send_mgr_construct(\r
2519         IN                              ipoib_port_t* const                     p_port )\r
2520 {\r
2521         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2522         p_port->send_mgr.depth = 0;\r
2523         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2524         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2525 }\r
2526 \r
2527 \r
2528 static void\r
2529 __send_mgr_destroy(\r
2530         IN                              ipoib_port_t* const                     p_port )\r
2531 {\r
2532         cl_list_item_t  *p_item;\r
2533         NDIS_PACKET             *p_packet;\r
2534 \r
2535         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2536 \r
2537         /* Complete any pending packets. */\r
2538         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2539                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2540                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2541         {\r
2542                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2543                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2544                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2545         }\r
2546 \r
2547         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2548 }\r
2549 \r
2550 \r
2551 static NDIS_STATUS\r
2552 __send_mgr_filter(\r
2553         IN                              ipoib_port_t* const                     p_port,\r
2554         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2555         IN                              NDIS_BUFFER* const                      p_buf,\r
2556         IN                              size_t                                          buf_len,\r
2557         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2558 {\r
2559         NDIS_STATUS             status;\r
2560 \r
2561         PERF_DECLARE( FilterIp );\r
2562         PERF_DECLARE( FilterArp );\r
2563         PERF_DECLARE( SendGen );\r
2564 \r
2565         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2566 \r
2567         /*\r
2568          * We already checked the ethernet header length, so we know it's safe\r
2569          * to decrement the buf_len without underflowing.\r
2570          */\r
2571         buf_len -= sizeof(eth_hdr_t);\r
2572 \r
2573         switch( p_eth_hdr->type )\r
2574         {\r
2575         case ETH_PROT_TYPE_IP:\r
2576                 cl_perf_start( FilterIp );\r
2577                 status = __send_mgr_filter_ip(\r
2578                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2579                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2580                 break;\r
2581 \r
2582         case ETH_PROT_TYPE_ARP:\r
2583                 cl_perf_start( FilterArp );\r
2584                 status = __send_mgr_filter_arp(\r
2585                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2586                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2587                 break;\r
2588 \r
2589         default:\r
2590                 /*\r
2591                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2592                  * Just send the payload and hope for the best.\r
2593                  */\r
2594                 cl_perf_start( SendGen );\r
2595                 status = __send_gen( p_port, p_desc );\r
2596                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2597                 break;\r
2598         }\r
2599 \r
2600         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2601         return status;\r
2602 }\r
2603 \r
2604 \r
2605 static NDIS_STATUS\r
2606 __send_copy(\r
2607         IN                              ipoib_port_t* const                     p_port,\r
2608         IN                              ipoib_send_desc_t* const        p_desc )\r
2609 {\r
2610         NDIS_PACKET                             *p_packet;\r
2611         NDIS_BUFFER                             *p_buf;\r
2612         NDIS_STATUS                             status;\r
2613         UINT                                    tot_len, bytes_copied = 0;\r
2614 \r
2615         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2616 \r
2617         p_desc->p_buf = \r
2618                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2619         if( !p_desc->p_buf )\r
2620         {\r
2621                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2622                         ("Failed to allocate buffer for packet copy.\n") );\r
2623                 return NDIS_STATUS_RESOURCES;\r
2624         }\r
2625 \r
2626         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2627         if( status != NDIS_STATUS_SUCCESS )\r
2628         {\r
2629                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2630                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2631                 return status;\r
2632         }\r
2633 \r
2634         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2635                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2636         if( status != NDIS_STATUS_SUCCESS )\r
2637         {\r
2638                 NdisFreePacket( p_packet );\r
2639                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2640                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2641                 return status;\r
2642         }\r
2643 \r
2644         NdisChainBufferAtFront( p_packet, p_buf );\r
2645 \r
2646         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2647 \r
2648         /* Setup the work request. */\r
2649         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2650                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2651         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2652         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2653         p_desc->wr.num_ds = 2;\r
2654 \r
2655         /* Copy the packet. */\r
2656         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2657                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2658                 NormalPagePriority );\r
2659 \r
2660         /* Free our temp packet now that the data is copied. */\r
2661         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2662         NdisFreeBuffer( p_buf );\r
2663         NdisFreePacket( p_packet );\r
2664 \r
2665         if( bytes_copied != tot_len )\r
2666         {\r
2667                 /* Something went wrong.  Drop the packet. */\r
2668                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2669                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2670                         bytes_copied, tot_len) );\r
2671                 return NDIS_STATUS_RESOURCES;\r
2672         }\r
2673 \r
2674         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2675         return NDIS_STATUS_SUCCESS;\r
2676 }\r
2677 \r
2678 \r
2679 #if !IPOIB_USE_DMA\r
2680 /* Send using the MDL's page information rather than the SGL. */\r
2681 static ib_api_status_t\r
2682 __send_gen(\r
2683         IN                              ipoib_port_t* const                     p_port,\r
2684         IN                              ipoib_send_desc_t* const        p_desc )\r
2685 {\r
2686         uint32_t                                i, j = 1;\r
2687         ULONG                                   offset;\r
2688         MDL                                             *p_mdl;\r
2689         UINT                                    num_pages, tot_len;\r
2690         ULONG                                   buf_len;\r
2691         PPFN_NUMBER                             page_array;\r
2692         boolean_t                               hdr_done = FALSE;\r
2693         ib_api_status_t                 status;\r
2694 \r
2695         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2696 \r
2697         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2698                 &tot_len );\r
2699 \r
2700         if( !p_mdl )\r
2701         {\r
2702                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2703                         ("No buffers associated with packet.\n") );\r
2704                 return IB_ERROR;\r
2705         }\r
2706 \r
2707         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2708         if( num_pages >= MAX_SEND_SGE )\r
2709         {\r
2710                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2711                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2712                 status = __send_copy( p_port, p_desc );\r
2713                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2714                 return status;\r
2715         }\r
2716 \r
2717         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2718         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2719         /*\r
2720          * Assume that the ethernet header is always fully contained\r
2721          * in the first page of the first MDL.  This makes for much\r
2722          * simpler code.\r
2723          */\r
2724         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2725         CL_ASSERT( offset <= PAGE_SIZE );\r
2726 \r
2727         while( tot_len )\r
2728         {\r
2729                 buf_len = MmGetMdlByteCount( p_mdl );\r
2730                 page_array = MmGetMdlPfnArray( p_mdl );\r
2731                 CL_ASSERT( page_array );\r
2732                 i = 0;\r
2733                 if( !hdr_done )\r
2734                 {\r
2735                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2736                         /* Skip the ethernet header. */\r
2737                         buf_len -= sizeof(eth_hdr_t);\r
2738                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2739                         if( buf_len )\r
2740                         {\r
2741                                 /* The ethernet header is a subset of this MDL. */\r
2742                                 CL_ASSERT( i == 0 );\r
2743                                 if( offset < PAGE_SIZE )\r
2744                                 {\r
2745                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2746                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2747                                         /* Add the byte offset since we're on the 1st page. */\r
2748                                         p_desc->local_ds[j].vaddr += offset;\r
2749                                         if( offset + buf_len > PAGE_SIZE )\r
2750                                         {\r
2751                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2752                                                 buf_len -= p_desc->local_ds[j].length;\r
2753                                         }\r
2754                                         else\r
2755                                         {\r
2756                                                 p_desc->local_ds[j].length = buf_len;\r
2757                                                 buf_len = 0;\r
2758                                         }\r
2759                                         /* This data segment is done.  Move to the next. */\r
2760                                         j++;\r
2761                                 }\r
2762                                 /* This page is done.  Move to the next. */\r
2763                                 i++;\r
2764                         }\r
2765                         /* Done handling the ethernet header. */\r
2766                         hdr_done = TRUE;\r
2767                 }\r
2768 \r
2769                 /* Finish this MDL */\r
2770                 while( buf_len )\r
2771                 {\r
2772                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2773                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2774                         /* Add the first page's offset if we're on the first page. */\r
2775                         if( i == 0 )\r
2776                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2777 \r
2778                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2779                         {\r
2780                                 /* Buffers spans pages. */\r
2781                                 p_desc->local_ds[j].length =\r
2782                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2783                                 buf_len -= p_desc->local_ds[j].length;\r
2784                                 /* This page is done.  Move to the next. */\r
2785                                 i++;\r
2786                         }\r
2787                         else\r
2788                         {\r
2789                                 /* Last page of the buffer. */\r
2790                                 p_desc->local_ds[j].length = buf_len;\r
2791                                 buf_len = 0;\r
2792                         }\r
2793                         /* This data segment is done.  Move to the next. */\r
2794                         j++;\r
2795                 }\r
2796 \r
2797                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2798                 if( !tot_len )\r
2799                         break;\r
2800 \r
2801                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2802                 if( !p_mdl )\r
2803                 {\r
2804                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2805                                 ("Failed to get next buffer.\n") );\r
2806                         return IB_ERROR;\r
2807                 }\r
2808         }\r
2809 \r
2810         /* Set the number of data segments. */\r
2811         p_desc->wr.num_ds = j;\r
2812 \r
2813         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2814         return IB_SUCCESS;\r
2815 }\r
2816 \r
2817 #else\r
2818 \r
2819 static NDIS_STATUS\r
2820 __send_gen(\r
2821         IN                              ipoib_port_t* const                     p_port,\r
2822         IN                              ipoib_send_desc_t* const        p_desc )\r
2823 {\r
2824         ib_api_status_t                 status;\r
2825         SCATTER_GATHER_LIST             *p_sgl;\r
2826         uint32_t                                i, j = 1;\r
2827         uint32_t                                offset = sizeof(eth_hdr_t);\r
2828         PERF_DECLARE( SendCopy );\r
2829 \r
2830         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2831 \r
2832         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2833                 ScatterGatherListPacketInfo );\r
2834         if( !p_sgl )\r
2835         {\r
2836                 ASSERT( p_sgl );\r
2837                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2838                         ("Failed to get SGL from packet.\n") );\r
2839                 return NDIS_STATUS_FAILURE;\r
2840         }\r
2841 \r
2842         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2843         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2844                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2845                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2846                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2847         {\r
2848                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2849                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2850                 cl_perf_start( SendCopy );\r
2851                 status = __send_copy( p_port, p_desc );\r
2852                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2853                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2854                 return status;\r
2855         }\r
2856 \r
2857         /*\r
2858          * Skip the ethernet header.  It is either the first element,\r
2859          * or part of it.\r
2860          */\r
2861         i = 0;\r
2862         while( offset )\r
2863         {\r
2864                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2865                 {\r
2866                         offset -= p_sgl->Elements[i++].Length;\r
2867                 }\r
2868                 else\r
2869                 {\r
2870                         p_desc->local_ds[j].vaddr =\r
2871                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2872                         p_desc->local_ds[j].length =\r
2873                                 p_sgl->Elements[i].Length - offset;\r
2874                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2875                         i++;\r
2876                         j++;\r
2877                         break;\r
2878                 }\r
2879         }\r
2880         /* Now fill in the rest of the local data segments. */\r
2881         while( i < p_sgl->NumberOfElements )\r
2882         {\r
2883                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2884                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2885                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2886                 i++;\r
2887                 j++;\r
2888         }\r
2889 \r
2890         /* Set the number of data segments. */\r
2891         p_desc->wr.num_ds = j;\r
2892 \r
2893         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2894         return NDIS_STATUS_SUCCESS;\r
2895 }\r
2896 #endif\r
2897 \r
2898 \r
2899 static NDIS_STATUS\r
2900 __send_mgr_filter_ip(\r
2901         IN                              ipoib_port_t* const                     p_port,\r
2902         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2903         IN                              NDIS_BUFFER*                            p_buf,\r
2904         IN                              size_t                                          buf_len,\r
2905         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2906 {\r
2907         NDIS_STATUS             status;\r
2908         ip_hdr_t                *p_ip_hdr;\r
2909 \r
2910         PERF_DECLARE( QueryIp );\r
2911         PERF_DECLARE( SendTcp );\r
2912         PERF_DECLARE( FilterUdp );\r
2913 \r
2914         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2915 \r
2916         if( !buf_len )\r
2917         {\r
2918                 cl_perf_start( QueryIp );\r
2919                 NdisGetNextBuffer( p_buf, &p_buf );\r
2920                 if( !p_buf )\r
2921                 {\r
2922                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2923                                 ("Failed to get IP header buffer.\n") );\r
2924                         return NDIS_STATUS_FAILURE;\r
2925                 }\r
2926                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2927                 if( !p_ip_hdr )\r
2928                 {\r
2929                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2930                                 ("Failed to query IP header buffer.\n") );\r
2931                         return NDIS_STATUS_FAILURE;\r
2932                 }\r
2933                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2934         }\r
2935         else\r
2936         {\r
2937                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2938         }\r
2939         if( buf_len < sizeof(ip_hdr_t) )\r
2940         {\r
2941                 /* This buffer is done for.  Get the next buffer. */\r
2942                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2943                         ("Buffer too small for IP packet.\n") );\r
2944                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2945         }\r
2946 \r
2947         if( p_ip_hdr->offset ||\r
2948                 p_ip_hdr->prot != IP_PROT_UDP )\r
2949         {\r
2950                 /* Not a UDP packet. */\r
2951                 cl_perf_start( SendTcp );\r
2952                 status = __send_gen( p_port, p_desc );\r
2953                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2954                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2955                 return status;\r
2956         }\r
2957 \r
2958         buf_len -= sizeof(ip_hdr_t);\r
2959 \r
2960         cl_perf_start( FilterUdp );\r
2961         status = __send_mgr_filter_udp(\r
2962                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2963         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2964 \r
2965         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2966         return status;\r
2967 }\r
2968 \r
2969 \r
2970 static NDIS_STATUS\r
2971 __send_mgr_filter_udp(\r
2972         IN                              ipoib_port_t* const                     p_port,\r
2973         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2974         IN                              NDIS_BUFFER*                            p_buf,\r
2975         IN                              size_t                                          buf_len,\r
2976         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2977 {\r
2978         ib_api_status_t         status;\r
2979         udp_hdr_t                       *p_udp_hdr;\r
2980         PERF_DECLARE( QueryUdp );\r
2981         PERF_DECLARE( SendUdp );\r
2982         PERF_DECLARE( FilterDhcp );\r
2983 \r
2984         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2985 \r
2986         if( !buf_len )\r
2987         {\r
2988                 cl_perf_start( QueryUdp );\r
2989                 NdisGetNextBuffer( p_buf, &p_buf );\r
2990                 if( !p_buf )\r
2991                 {\r
2992                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2993                                 ("Failed to get UDP header buffer.\n") );\r
2994                         return NDIS_STATUS_FAILURE;\r
2995                 }\r
2996                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2997                 if( !p_udp_hdr )\r
2998                 {\r
2999                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3000                                 ("Failed to query UDP header buffer.\n") );\r
3001                         return NDIS_STATUS_FAILURE;\r
3002                 }\r
3003                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
3004         }\r
3005         else\r
3006         {\r
3007                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
3008         }\r
3009         /* Get the UDP header and check the destination port numbers. */\r
3010         if( buf_len < sizeof(udp_hdr_t) )\r
3011         {\r
3012                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3013                         ("Buffer not large enough for UDP packet.\n") );\r
3014                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
3015         }\r
3016 \r
3017         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
3018                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r