[IBAL\IPOIB\INC\WSD\SDP\MTHCA] change 2 defines to match Linux ib_types.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  *\r
5  * This software is available to you under the OpenIB.org BSD license\r
6  * below:\r
7  *\r
8  *     Redistribution and use in source and binary forms, with or\r
9  *     without modification, are permitted provided that the following\r
10  *     conditions are met:\r
11  *\r
12  *      - Redistributions of source code must retain the above\r
13  *        copyright notice, this list of conditions and the following\r
14  *        disclaimer.\r
15  *\r
16  *      - Redistributions in binary form must reproduce the above\r
17  *        copyright notice, this list of conditions and the following\r
18  *        disclaimer in the documentation and/or other materials\r
19  *        provided with the distribution.\r
20  *\r
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
22  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
24  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
25  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
26  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
27  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
28  * SOFTWARE.\r
29  *\r
30  * $Id$\r
31  */\r
32 \r
33 \r
34 \r
35 #include "ipoib_port.h"\r
36 #include "ipoib_adapter.h"\r
37 #include "ipoib_debug.h"\r
38 #if defined(EVENT_TRACING)\r
39 #ifdef offsetof\r
40 #undef offsetof\r
41 #endif\r
42 #include "ipoib_port.tmh"\r
43 #endif\r
44 \r
45 \r
46 /* Amount of physical memory to register. */\r
47 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
48 \r
49 /* Number of work completions to chain for send and receive polling. */\r
50 #define MAX_SEND_WC             8\r
51 #define MAX_RECV_WC             16\r
52 \r
53 \r
54 ib_gid_t        bcast_mgid_template = {\r
55         0xff,                                                           /* multicast field */\r
56         0x12,                                                           /* scope (to be filled in) */\r
57         0x40, 0x1b,                                                     /* IPv4 signature */\r
58         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
59         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
60         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
61 };\r
62 \r
63 \r
64 #ifdef _DEBUG_\r
65 /* Handy pointer for debug use. */\r
66 ipoib_port_t    *gp_ipoib_port;\r
67 #endif\r
68 \r
69 \r
70 /******************************************************************************\r
71 *\r
72 * Declarations\r
73 *\r
74 ******************************************************************************/\r
75 static void\r
76 __port_construct(\r
77         IN                              ipoib_port_t* const                     p_port );\r
78 \r
79 static ib_api_status_t\r
80 __port_init(\r
81         IN                              ipoib_port_t* const                     p_port,\r
82         IN                              ipoib_adapter_t* const          p_adapter,\r
83         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
84 \r
85 static void\r
86 __port_destroying(\r
87         IN                              cl_obj_t* const                         p_obj );\r
88 \r
89 static void\r
90 __port_cleanup(\r
91         IN                              cl_obj_t* const                         p_obj );\r
92 \r
93 static void\r
94 __port_free(\r
95         IN                              cl_obj_t* const                         p_obj );\r
96 \r
97 \r
98 /******************************************************************************\r
99 *\r
100 * IB resource manager operations\r
101 *\r
102 ******************************************************************************/\r
103 static void\r
104 __ib_mgr_construct(\r
105         IN                              ipoib_port_t* const                     p_port );\r
106 \r
107 static ib_api_status_t\r
108 __ib_mgr_init(\r
109         IN                              ipoib_port_t* const                     p_port );\r
110 \r
111 static void\r
112 __ib_mgr_destroy(\r
113         IN                              ipoib_port_t* const                     p_port );\r
114 \r
115 static void\r
116 __qp_event(\r
117         IN                              ib_async_event_rec_t            *p_event_rec );\r
118 \r
119 static void\r
120 __cq_event(\r
121         IN                              ib_async_event_rec_t            *p_event_rec );\r
122 \r
123 static ib_api_status_t\r
124 __ib_mgr_activate(\r
125         IN                              ipoib_port_t* const                     p_port );\r
126 \r
127 /******************************************************************************\r
128 *\r
129 * Buffer manager operations.\r
130 *\r
131 ******************************************************************************/\r
132 static void\r
133 __buf_mgr_construct(\r
134         IN                              ipoib_port_t* const                     p_port );\r
135 \r
136 static ib_api_status_t\r
137 __buf_mgr_init(\r
138         IN                              ipoib_port_t* const                     p_port );\r
139 \r
140 static void\r
141 __buf_mgr_destroy(\r
142         IN                              ipoib_port_t* const                     p_port );\r
143 \r
144 static cl_status_t\r
145 __recv_ctor(\r
146         IN                              void* const                                     p_object,\r
147         IN                              void*                                           context,\r
148                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
149 \r
150 #if !IPOIB_INLINE_RECV\r
151 static void\r
152 __recv_dtor(\r
153         IN              const   cl_pool_item_t* const           p_pool_item,\r
154         IN                              void                                            *context );\r
155 #endif  /* IPOIB_INLINE_RECV */\r
156 \r
157 static inline ipoib_send_desc_t*\r
158 __buf_mgr_get_send(\r
159         IN                              ipoib_port_t* const                     p_port );\r
160 \r
161 static inline void\r
162 __buf_mgr_put_send(\r
163         IN                              ipoib_port_t* const                     p_port,\r
164         IN                              ipoib_send_desc_t* const        p_desc );\r
165 \r
166 static inline ipoib_recv_desc_t*\r
167 __buf_mgr_get_recv(\r
168         IN                              ipoib_port_t* const                     p_port );\r
169 \r
170 static inline void\r
171 __buf_mgr_put_recv(\r
172         IN                              ipoib_port_t* const                     p_port,\r
173         IN                              ipoib_recv_desc_t* const        p_desc,\r
174         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
175 \r
176 static inline void\r
177 __buf_mgr_put_recv_list(\r
178         IN                              ipoib_port_t* const                     p_port,\r
179         IN                              cl_qlist_t* const                       p_list );\r
180 \r
181 static inline NDIS_PACKET*\r
182 __buf_mgr_get_ndis_pkt(\r
183         IN                              ipoib_port_t* const                     p_port,\r
184         IN                              ipoib_recv_desc_t* const        p_desc );\r
185 \r
186 \r
187 /******************************************************************************\r
188 *\r
189 * Receive manager operations.\r
190 *\r
191 ******************************************************************************/\r
192 static void\r
193 __recv_mgr_construct(\r
194         IN                              ipoib_port_t* const                     p_port );\r
195 \r
196 static ib_api_status_t\r
197 __recv_mgr_init(\r
198         IN                              ipoib_port_t* const                     p_port );\r
199 \r
200 static void\r
201 __recv_mgr_destroy(\r
202         IN                              ipoib_port_t* const                     p_port );\r
203 \r
204 /* Posts receive buffers to the receive queue. */\r
205 static ib_api_status_t\r
206 __recv_mgr_repost(\r
207         IN                              ipoib_port_t* const                     p_port );\r
208 \r
209 static void\r
210 __recv_cb(\r
211         IN              const   ib_cq_handle_t                          h_cq,\r
212         IN                              void                                            *cq_context );\r
213 \r
214 static void\r
215 __recv_get_endpts(\r
216         IN                              ipoib_port_t* const                     p_port,\r
217         IN                              ipoib_recv_desc_t* const        p_desc,\r
218         IN                              ib_wc_t* const                          p_wc,\r
219                 OUT                     ipoib_endpt_t** const           pp_src,\r
220                 OUT                     ipoib_endpt_t** const           pp_dst );\r
221 \r
222 static int32_t\r
223 __recv_mgr_filter(\r
224         IN                              ipoib_port_t* const                     p_port,\r
225         IN                              ib_wc_t* const                          p_done_wc_list,\r
226                 OUT                     cl_qlist_t* const                       p_done_list,\r
227                 OUT                     cl_qlist_t* const                       p_bad_list );\r
228 \r
229 static ib_api_status_t\r
230 __recv_gen(\r
231         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
232                 OUT                     eth_pkt_t* const                        p_eth,\r
233         IN                              ipoib_endpt_t* const            p_src,\r
234         IN                              ipoib_endpt_t* const            p_dst );\r
235 \r
236 static ib_api_status_t\r
237 __recv_dhcp(\r
238         IN                              ipoib_port_t* const                     p_port,\r
239         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
240                 OUT                     eth_pkt_t* const                        p_eth,\r
241         IN                              ipoib_endpt_t* const            p_src,\r
242         IN                              ipoib_endpt_t* const            p_dst );\r
243 \r
244 static ib_api_status_t\r
245 __recv_arp(\r
246         IN                              ipoib_port_t* const                     p_port,\r
247         IN                              ib_wc_t* const                          p_wc,\r
248         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
249                 OUT                     eth_pkt_t* const                        p_eth,\r
250         IN                              ipoib_endpt_t** const           p_src,\r
251         IN                              ipoib_endpt_t* const            p_dst );\r
252 \r
253 static ib_api_status_t\r
254 __recv_mgr_prepare_pkt(\r
255         IN                              ipoib_port_t* const                     p_port,\r
256         IN                              ipoib_recv_desc_t* const        p_desc,\r
257                 OUT                     NDIS_PACKET** const                     pp_packet );\r
258 \r
259 static uint32_t\r
260 __recv_mgr_build_pkt_array(\r
261         IN                              ipoib_port_t* const                     p_port,\r
262         IN                              int32_t                                         shortage,\r
263                 OUT                     cl_qlist_t* const                       p_done_list,\r
264                 OUT                     int32_t* const                          p_discarded );\r
265 \r
266 /******************************************************************************\r
267 *\r
268 * Send manager operations.\r
269 *\r
270 ******************************************************************************/\r
271 static void\r
272 __send_mgr_construct(\r
273         IN                              ipoib_port_t* const                     p_port );\r
274 \r
275 static void\r
276 __send_mgr_destroy(\r
277         IN                              ipoib_port_t* const                     p_port );\r
278 \r
279 static NDIS_STATUS\r
280 __send_gen(\r
281         IN                              ipoib_port_t* const                     p_port,\r
282         IN                              ipoib_send_desc_t* const        p_desc );\r
283 \r
284 static NDIS_STATUS\r
285 __send_mgr_filter_ip(\r
286         IN                              ipoib_port_t* const                     p_port,\r
287         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
288         IN                              NDIS_BUFFER*                            p_buf,\r
289         IN                              size_t                                          buf_len,\r
290         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
291 \r
292 static NDIS_STATUS\r
293 __send_mgr_filter_udp(\r
294         IN                              ipoib_port_t* const                     p_port,\r
295         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
296         IN                              NDIS_BUFFER*                            p_buf,\r
297         IN                              size_t                                          buf_len,\r
298         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
299 \r
300 static NDIS_STATUS\r
301 __send_mgr_filter_dhcp(\r
302         IN                              ipoib_port_t* const                     p_port,\r
303         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
304         IN                              NDIS_BUFFER*                            p_buf,\r
305         IN                              size_t                                          buf_len,\r
306         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
307 \r
308 static NDIS_STATUS\r
309 __send_mgr_filter_arp(\r
310         IN                              ipoib_port_t* const                     p_port,\r
311         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
312         IN                              NDIS_BUFFER*                            p_buf,\r
313         IN                              size_t                                          buf_len,\r
314         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
315 \r
316 static void\r
317 __process_failed_send(\r
318         IN                              ipoib_port_t* const                     p_port,\r
319         IN                              ipoib_send_desc_t* const        p_desc,\r
320         IN              const   NDIS_STATUS                                     status );\r
321 \r
322 static void\r
323 __send_cb(\r
324         IN              const   ib_cq_handle_t                          h_cq,\r
325         IN                              void                                            *cq_context );\r
326 \r
327 \r
328 /******************************************************************************\r
329 *\r
330 * Endpoint manager operations\r
331 *\r
332 ******************************************************************************/\r
333 static void\r
334 __endpt_mgr_construct(\r
335         IN                              ipoib_port_t* const                     p_port );\r
336 \r
337 static ib_api_status_t\r
338 __endpt_mgr_init(\r
339         IN                              ipoib_port_t* const                     p_port );\r
340 \r
341 static void\r
342 __endpt_mgr_destroy(\r
343         IN                              ipoib_port_t* const                     p_port );\r
344 \r
345 /****f* IPoIB/__endpt_mgr_remove_all\r
346 * NAME\r
347 *       __endpt_mgr_remove_all\r
348 *\r
349 * DESCRIPTION\r
350 *       Removes all enpoints from the port, dereferencing them to initiate\r
351 *       destruction.\r
352 *\r
353 * SYNOPSIS\r
354 */\r
355 static void\r
356 __endpt_mgr_remove_all(\r
357         IN                              ipoib_port_t* const                     p_port );\r
358 /*\r
359 ********/\r
360 \r
361 static void\r
362 __endpt_mgr_remove(\r
363         IN                              ipoib_port_t* const                     p_port,\r
364         IN                              ipoib_endpt_t* const            p_endpt );\r
365 \r
366 static void\r
367 __endpt_mgr_reset_all(\r
368         IN                              ipoib_port_t* const                     p_port );\r
369 \r
370 static inline NDIS_STATUS\r
371 __endpt_mgr_ref(\r
372         IN                              ipoib_port_t* const                     p_port,\r
373         IN              const   mac_addr_t                                      mac,\r
374                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
375 \r
376 static inline NDIS_STATUS\r
377 __endpt_mgr_get_gid_qpn(\r
378         IN                              ipoib_port_t* const                     p_port,\r
379         IN              const   mac_addr_t                                      mac,\r
380                 OUT                     ib_gid_t* const                         p_gid,\r
381                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
382 \r
383 static inline ipoib_endpt_t*\r
384 __endpt_mgr_get_by_gid(\r
385         IN                              ipoib_port_t* const                     p_port,\r
386         IN              const   ib_gid_t* const                         p_gid );\r
387 \r
388 static inline ipoib_endpt_t*\r
389 __endpt_mgr_get_by_lid(\r
390         IN                              ipoib_port_t* const                     p_port,\r
391         IN              const   net16_t                                         lid );\r
392 \r
393 static inline ib_api_status_t\r
394 __endpt_mgr_insert_locked(\r
395         IN                              ipoib_port_t* const                     p_port,\r
396         IN              const   mac_addr_t                                      mac,\r
397         IN                              ipoib_endpt_t* const            p_endpt );\r
398 \r
399 static inline ib_api_status_t\r
400 __endpt_mgr_insert(\r
401         IN                              ipoib_port_t* const                     p_port,\r
402         IN              const   mac_addr_t                                      mac,\r
403         IN                              ipoib_endpt_t* const            p_endpt );\r
404 \r
405 static ib_api_status_t\r
406 __endpt_mgr_add_local(\r
407         IN                              ipoib_port_t* const                     p_port,\r
408         IN                              ib_port_info_t* const           p_port_info );\r
409 \r
410 static ib_api_status_t\r
411 __endpt_mgr_add_bcast(\r
412         IN                              ipoib_port_t* const                     p_port,\r
413         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
414 \r
415 /******************************************************************************\r
416 *\r
417 * MCast operations.\r
418 *\r
419 ******************************************************************************/\r
420 static ib_api_status_t\r
421 __port_get_bcast(\r
422         IN                              ipoib_port_t* const                     p_port );\r
423 \r
424 static ib_api_status_t\r
425 __port_join_bcast(\r
426         IN                              ipoib_port_t* const                     p_port,\r
427         IN                              ib_member_rec_t* const          p_member_rec );\r
428 \r
429 static ib_api_status_t\r
430 __port_create_bcast(\r
431         IN                              ipoib_port_t* const                     p_port );\r
432 \r
433 static void\r
434 __port_info_cb(\r
435         IN                              ib_query_rec_t                          *p_query_rec );\r
436 \r
437 \r
438 static void\r
439 __bcast_get_cb(\r
440         IN                              ib_query_rec_t                          *p_query_rec );\r
441 \r
442 \r
443 static void\r
444 __bcast_cb(\r
445         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
446 \r
447 \r
448 static void\r
449 __mcast_cb(\r
450         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
451 \r
452 \r
453 static intn_t\r
454 __gid_cmp(\r
455         IN              const   void* const                                     p_key1,\r
456         IN              const   void* const                                     p_key2 )\r
457 {\r
458         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
459 }\r
460 \r
461 \r
462 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
463 {\r
464         cl_obj_ref( &p_port->obj );\r
465 #if DBG\r
466         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
467         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
468                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
469 #else\r
470         UNREFERENCED_PARAMETER(type);\r
471 #endif\r
472 }\r
473 \r
474 \r
475 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
476 {\r
477         cl_obj_deref( &p_port->obj );\r
478 \r
479 #if DBG\r
480         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
481         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
482                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
483 #else\r
484         UNREFERENCED_PARAMETER(type);\r
485 #endif\r
486 }\r
487 \r
488 \r
489 /******************************************************************************\r
490 *\r
491 * Implementation\r
492 *\r
493 ******************************************************************************/\r
494 ib_api_status_t\r
495 ipoib_create_port(\r
496         IN                              ipoib_adapter_t* const          p_adapter,\r
497         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
498                 OUT                     ipoib_port_t** const            pp_port )\r
499 {\r
500         ib_api_status_t         status;\r
501         ipoib_port_t            *p_port;\r
502 \r
503         IPOIB_ENTER( IPOIB_DBG_INIT );\r
504 \r
505         CL_ASSERT( !p_adapter->p_port );\r
506 \r
507         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
508                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
509         if( !p_port )\r
510         {\r
511                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
512                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
513                         sizeof(ipoib_port_t)) );\r
514                 return IB_INSUFFICIENT_MEMORY;\r
515         }\r
516 \r
517 #ifdef _DEBUG_\r
518         gp_ipoib_port = p_port;\r
519 #endif\r
520 \r
521         __port_construct( p_port );\r
522 \r
523         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
524         if( status != IB_SUCCESS )\r
525         {\r
526                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
527                         ("ipoib_port_init returned %s.\n", \r
528                         p_adapter->p_ifc->get_err_str( status )) );\r
529                 __port_cleanup( &p_port->obj );\r
530                 __port_free( &p_port->obj );\r
531                 return status;\r
532         }\r
533 \r
534         *pp_port = p_port;\r
535         IPOIB_EXIT( IPOIB_DBG_INIT );\r
536         return IB_SUCCESS;\r
537 }\r
538 \r
539 \r
540 void\r
541 ipoib_port_destroy(\r
542         IN                              ipoib_port_t* const                     p_port )\r
543 {\r
544         IPOIB_ENTER( IPOIB_DBG_INIT );\r
545 \r
546         CL_ASSERT( p_port );\r
547         CL_ASSERT( p_port->p_adapter );\r
548         CL_ASSERT( !p_port->p_adapter->p_port );\r
549 \r
550         cl_obj_destroy( &p_port->obj );\r
551 \r
552         IPOIB_EXIT( IPOIB_DBG_INIT );\r
553 }\r
554 \r
555 \r
556 static void\r
557 __port_construct(\r
558         IN                              ipoib_port_t* const                     p_port )\r
559 {\r
560         IPOIB_ENTER( IPOIB_DBG_INIT );\r
561 \r
562         p_port->state = IB_QPS_RESET;\r
563 \r
564         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
565         cl_spinlock_construct( &p_port->send_lock );\r
566         cl_spinlock_construct( &p_port->recv_lock );\r
567         __ib_mgr_construct( p_port );\r
568         __buf_mgr_construct( p_port );\r
569 \r
570         __recv_mgr_construct( p_port );\r
571         __send_mgr_construct( p_port );\r
572 \r
573         __endpt_mgr_construct( p_port );\r
574 \r
575         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
576 \r
577         IPOIB_EXIT( IPOIB_DBG_INIT );\r
578 }\r
579 \r
580 \r
581 static ib_api_status_t\r
582 __port_init(\r
583         IN                              ipoib_port_t* const                     p_port,\r
584         IN                              ipoib_adapter_t* const          p_adapter,\r
585         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
586 {\r
587         cl_status_t                     cl_status;\r
588         ib_api_status_t         status;\r
589 \r
590         IPOIB_ENTER( IPOIB_DBG_INIT );\r
591 \r
592         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
593         p_port->p_adapter = p_adapter;\r
594 \r
595         cl_status = cl_spinlock_init( &p_port->send_lock );\r
596         if( cl_status != CL_SUCCESS )\r
597         {\r
598                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
599                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
600                 return IB_ERROR;\r
601         }\r
602 \r
603         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
604         if( cl_status != CL_SUCCESS )\r
605         {\r
606                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
607                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
608                 return IB_ERROR;\r
609         }\r
610 \r
611         /* Initialize the IB resource manager. */\r
612         status = __ib_mgr_init( p_port );\r
613         if( status != IB_SUCCESS )\r
614         {\r
615                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
616                         ("__ib_mgr_init returned %s\n", \r
617                         p_adapter->p_ifc->get_err_str( status )) );\r
618                 return status;\r
619         }\r
620 \r
621         /* Initialize the buffer manager. */\r
622         status = __buf_mgr_init( p_port );\r
623         if( status != IB_SUCCESS )\r
624         {\r
625                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
626                         ("__buf_mgr_init returned %s\n", \r
627                         p_adapter->p_ifc->get_err_str( status )) );\r
628                 return status;\r
629         }\r
630 \r
631         /* Initialize the receive manager. */\r
632         status = __recv_mgr_init( p_port );\r
633         if( status != IB_SUCCESS )\r
634         {\r
635                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
636                         ("__recv_mgr_init returned %s\n", \r
637                         p_adapter->p_ifc->get_err_str( status )) );\r
638                 return status;\r
639         }\r
640 \r
641         /* Initialize the endpoint manager. */\r
642         status = __endpt_mgr_init( p_port );\r
643         if( status != IB_SUCCESS )\r
644         {\r
645                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
646                         ("__endpt_mgr_init returned %s\n", \r
647                         p_adapter->p_ifc->get_err_str( status )) );\r
648                 return status;\r
649         }\r
650 \r
651         /* We only ever destroy from the PnP callback thread. */\r
652         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
653                 __port_destroying, __port_cleanup, __port_free );\r
654 \r
655 #if DBG\r
656         cl_atomic_inc( &p_port->ref[ref_init] );\r
657         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
658                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
659 #endif\r
660 \r
661         if( cl_status != CL_SUCCESS )\r
662         {\r
663                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
664                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
665                 return IB_ERROR;\r
666         }\r
667 \r
668         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
669         if( cl_status != CL_SUCCESS )\r
670         {\r
671                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
672                         ("cl_obj_insert_rel returned %s\n", cl_status_text[cl_status]) );\r
673                 cl_obj_destroy( &p_port->obj );\r
674                 return IB_ERROR;\r
675         }\r
676 \r
677 #if DBG\r
678         cl_atomic_inc( &p_port->ref[ref_init] );\r
679         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
680                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
681 #endif\r
682 \r
683         IPOIB_EXIT( IPOIB_DBG_INIT );\r
684         return IB_SUCCESS;\r
685 }\r
686 \r
687 \r
688 static void\r
689 __port_destroying(\r
690         IN                              cl_obj_t* const                         p_obj )\r
691 {\r
692         ipoib_port_t    *p_port;\r
693 \r
694         IPOIB_ENTER( IPOIB_DBG_INIT );\r
695 \r
696         CL_ASSERT( p_obj );\r
697 \r
698         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
699 \r
700         ipoib_port_down( p_port );\r
701 \r
702         __endpt_mgr_remove_all( p_port );\r
703 \r
704         ipoib_port_resume( p_port );\r
705 \r
706         IPOIB_EXIT( IPOIB_DBG_INIT );\r
707 }\r
708 \r
709 \r
710 static void\r
711 __port_cleanup(\r
712         IN                              cl_obj_t* const                         p_obj )\r
713 {\r
714         ipoib_port_t    *p_port;\r
715 \r
716         IPOIB_ENTER( IPOIB_DBG_INIT );\r
717 \r
718         CL_ASSERT( p_obj );\r
719 \r
720         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
721 \r
722         /* Wait for all sends and receives to get flushed. */\r
723         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
724                 cl_thread_suspend( 0 );\r
725 \r
726         /* Destroy the send and receive managers before closing the CA. */\r
727         __ib_mgr_destroy( p_port );\r
728 \r
729         IPOIB_EXIT( IPOIB_DBG_INIT );\r
730 }\r
731 \r
732 \r
733 static void\r
734 __port_free(\r
735         IN                              cl_obj_t* const                         p_obj )\r
736 {\r
737         ipoib_port_t    *p_port;\r
738 \r
739         IPOIB_ENTER( IPOIB_DBG_INIT );\r
740 \r
741         CL_ASSERT( p_obj );\r
742 \r
743         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
744 \r
745         __endpt_mgr_destroy( p_port );\r
746         __recv_mgr_destroy( p_port );\r
747         __send_mgr_destroy( p_port );\r
748         __buf_mgr_destroy( p_port );\r
749 \r
750         cl_spinlock_destroy( &p_port->send_lock );\r
751         cl_spinlock_destroy( &p_port->recv_lock );\r
752 \r
753         cl_obj_deinit( p_obj );\r
754 \r
755         cl_free( p_port );\r
756 \r
757         IPOIB_EXIT( IPOIB_DBG_INIT );\r
758 }\r
759 \r
760 \r
761 \r
762 /******************************************************************************\r
763 *\r
764 * IB resource manager implementation.\r
765 *\r
766 ******************************************************************************/\r
767 static void\r
768 __ib_mgr_construct(\r
769         IN                              ipoib_port_t* const                     p_port )\r
770 {\r
771         IPOIB_ENTER( IPOIB_DBG_INIT );\r
772 \r
773         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
774 \r
775         IPOIB_EXIT( IPOIB_DBG_INIT );\r
776 }\r
777 \r
778 \r
779 static ib_api_status_t\r
780 __ib_mgr_init(\r
781         IN                              ipoib_port_t* const                     p_port )\r
782 {\r
783         ib_api_status_t         status;\r
784         ib_cq_create_t          cq_create;\r
785         ib_qp_create_t          qp_create;\r
786         ib_phys_create_t        phys_create;\r
787         ib_phys_range_t         phys_range;\r
788         uint64_t                        vaddr;\r
789         net32_t                         rkey;\r
790         ib_qp_attr_t            qp_attr;\r
791 \r
792         IPOIB_ENTER( IPOIB_DBG_INIT );\r
793 \r
794         /* Open the CA. */\r
795         status = p_port->p_adapter->p_ifc->open_ca(\r
796                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
797                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
798         if( status != IB_SUCCESS )\r
799         {\r
800                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
801                         EVENT_IPOIB_OPEN_CA, 1, status );\r
802                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
803                         ("ib_open_ca returned %s\n", \r
804                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
805                 return status;\r
806         }\r
807 \r
808         /* Allocate the PD. */\r
809         status = p_port->p_adapter->p_ifc->alloc_pd(\r
810                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
811         if( status != IB_SUCCESS )\r
812         {\r
813                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
814                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
815                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
816                         ("ib_alloc_pd returned %s\n", \r
817                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
818                 return status;\r
819         }\r
820 \r
821         /* Allocate receive CQ. */\r
822         cq_create.size = p_port->p_adapter->params.rq_depth;\r
823         cq_create.pfn_comp_cb = __recv_cb;\r
824         cq_create.h_wait_obj = NULL;\r
825 \r
826         status = p_port->p_adapter->p_ifc->create_cq(\r
827                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
828                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
829         if( status != IB_SUCCESS )\r
830         {\r
831                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
832                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
833                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
834                         ("ib_create_cq returned %s.\n", \r
835                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
836                 return status;\r
837         }\r
838 \r
839         /* Allocate send CQ. */\r
840         cq_create.size = p_port->p_adapter->params.sq_depth;\r
841         cq_create.pfn_comp_cb = __send_cb;\r
842 \r
843         status = p_port->p_adapter->p_ifc->create_cq(\r
844                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
845                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
846         if( status != IB_SUCCESS )\r
847         {\r
848                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
849                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
850                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
851                         ("ib_create_cq returned %s.\n", \r
852                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
853                 return status;\r
854         }\r
855 \r
856         /* Allocate the QP. */\r
857         cl_memclr( &qp_create, sizeof(qp_create) );\r
858         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
859         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
860         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
861         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
862         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
863         //TODO: Figure out the right number of SGE entries for sends.\r
864         qp_create.sq_sge = MAX_SEND_SGE;\r
865         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
866         qp_create.sq_signaled = TRUE;\r
867         status = p_port->p_adapter->p_ifc->create_qp(\r
868                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
869                 __qp_event, &p_port->ib_mgr.h_qp );\r
870         if( status != IB_SUCCESS )\r
871         {\r
872                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
873                         EVENT_IPOIB_CREATE_QP, 1, status );\r
874                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
875                         ("ib_create_qp returned %s\n", \r
876                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
877                 return status;\r
878         }\r
879         /* Query the QP so we can get our QPN. */\r
880         status = p_port->p_adapter->p_ifc->query_qp(\r
881                 p_port->ib_mgr.h_qp, &qp_attr );\r
882         if( status != IB_SUCCESS )\r
883         {\r
884                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
885                         EVENT_IPOIB_QUERY_QP, 1, status );\r
886                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
887                         ("ib_query_qp returned %s\n", \r
888                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
889                 return status;\r
890         }\r
891         p_port->ib_mgr.qpn = qp_attr.num;\r
892 \r
893         /* Register all of physical memory */\r
894         phys_create.length = MEM_REG_SIZE;\r
895         phys_create.num_ranges = 1;\r
896         phys_create.range_array = &phys_range;\r
897         phys_create.buf_offset = 0;\r
898         phys_create.hca_page_size = PAGE_SIZE;\r
899         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
900         phys_range.base_addr = 0;\r
901         phys_range.size = MEM_REG_SIZE;\r
902         vaddr = 0;\r
903         status = p_port->p_adapter->p_ifc->reg_phys(\r
904                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
905                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
906         if( status != IB_SUCCESS )\r
907         {\r
908                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
909                         EVENT_IPOIB_REG_PHYS, 1, status );\r
910                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
911                         ("ib_reg_phys returned %s\n", \r
912                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
913                 return status;\r
914         }\r
915 \r
916         IPOIB_EXIT( IPOIB_DBG_INIT );\r
917         return IB_SUCCESS;\r
918 }\r
919 \r
920 \r
921 static void\r
922 __ib_mgr_destroy(\r
923         IN                              ipoib_port_t* const                     p_port )\r
924 {\r
925         ib_api_status_t status;\r
926 \r
927         IPOIB_ENTER( IPOIB_DBG_INIT );\r
928 \r
929         if( p_port->ib_mgr.h_ca )\r
930         {\r
931                 status =\r
932                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
933                 CL_ASSERT( status == IB_SUCCESS );\r
934                 p_port->ib_mgr.h_ca = NULL;\r
935         }\r
936 \r
937         IPOIB_EXIT( IPOIB_DBG_INIT );\r
938 }\r
939 \r
940 \r
941 \r
942 /******************************************************************************\r
943 *\r
944 * Buffer manager implementation.\r
945 *\r
946 ******************************************************************************/\r
947 static void\r
948 __buf_mgr_construct(\r
949         IN                              ipoib_port_t* const                     p_port )\r
950 {\r
951         IPOIB_ENTER( IPOIB_DBG_INIT );\r
952 \r
953         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
954 \r
955         p_port->buf_mgr.h_packet_pool = NULL;\r
956         p_port->buf_mgr.h_buffer_pool = NULL;\r
957 \r
958         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
959                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
960 \r
961         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
962         p_port->buf_mgr.h_send_buf_pool = NULL;\r
963 \r
964         IPOIB_EXIT( IPOIB_DBG_INIT );\r
965 }\r
966 \r
967 \r
968 static ib_api_status_t\r
969 __buf_mgr_init(\r
970         IN                              ipoib_port_t* const                     p_port )\r
971 {\r
972         cl_status_t             cl_status;\r
973         NDIS_STATUS             ndis_status;\r
974         ipoib_params_t  *p_params;\r
975 \r
976         IPOIB_ENTER(IPOIB_DBG_INIT );\r
977 \r
978         CL_ASSERT( p_port );\r
979         CL_ASSERT( p_port->p_adapter );\r
980 \r
981         p_params = &p_port->p_adapter->params;\r
982 \r
983         /* Allocate the receive descriptor pool */\r
984         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
985                 p_params->rq_depth * p_params->recv_pool_ratio,\r
986 #if IPOIB_INLINE_RECV\r
987                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
988 #else   /* IPOIB_INLINE_RECV */\r
989                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
990 #endif  /* IPOIB_INLINE_RECV */\r
991         if( cl_status != CL_SUCCESS )\r
992         {\r
993                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
994                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
995                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
996                         ("cl_qpool_init for recvs returned %s\n",\r
997                         cl_status_text[cl_status]) );\r
998                 return IB_INSUFFICIENT_MEMORY;\r
999         }\r
1000 \r
1001         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1002         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1003                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1004         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1005         {\r
1006                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1007                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1008                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1009                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1010                 return IB_INSUFFICIENT_RESOURCES;\r
1011         }\r
1012 \r
1013         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1014                 p_params->rq_depth );\r
1015         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1016         {\r
1017                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1018                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1019                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1020                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1021                 return IB_INSUFFICIENT_RESOURCES;\r
1022         }\r
1023 \r
1024         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1025         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1026                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1027         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1028         {\r
1029                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1030                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1031                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1032                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1033                 return IB_INSUFFICIENT_RESOURCES;\r
1034         }\r
1035 \r
1036         NdisAllocateBufferPool( &ndis_status,\r
1037                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1038         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1039         {\r
1040                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1041                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1042                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1043                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1044                 return IB_INSUFFICIENT_RESOURCES;\r
1045         }\r
1046 \r
1047         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1048         return IB_SUCCESS;\r
1049 }\r
1050 \r
1051 \r
1052 static void\r
1053 __buf_mgr_destroy(\r
1054         IN                              ipoib_port_t* const                     p_port )\r
1055 {\r
1056         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1057 \r
1058         CL_ASSERT( p_port );\r
1059 \r
1060         /* Destroy the send packet and buffer pools. */\r
1061         if( p_port->buf_mgr.h_send_buf_pool )\r
1062                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1063         if( p_port->buf_mgr.h_send_pkt_pool )\r
1064                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1065 \r
1066         /* Destroy the receive packet and buffer pools. */\r
1067         if( p_port->buf_mgr.h_buffer_pool )\r
1068                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1069         if( p_port->buf_mgr.h_packet_pool )\r
1070                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1071 \r
1072         /* Free the receive and send descriptors. */\r
1073         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1074 \r
1075         /* Free the lookaside list of scratch buffers. */\r
1076         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1077 \r
1078         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1079 }\r
1080 \r
1081 \r
1082 static cl_status_t\r
1083 __recv_ctor(\r
1084         IN                              void* const                                     p_object,\r
1085         IN                              void*                                           context,\r
1086                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1087 {\r
1088         ipoib_recv_desc_t       *p_desc;\r
1089         ipoib_port_t            *p_port;\r
1090         uint32_t                        ds0_len;\r
1091 \r
1092         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1093 \r
1094         CL_ASSERT( p_object );\r
1095         CL_ASSERT( context );\r
1096 \r
1097         p_desc = (ipoib_recv_desc_t*)p_object;\r
1098         p_port = (ipoib_port_t*)context;\r
1099 \r
1100         /* Setup the work request. */\r
1101         p_desc->wr.ds_array = p_desc->local_ds;\r
1102         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1103 \r
1104 #if IPOIB_INLINE_RECV\r
1105         /* Sanity check on the receive buffer layout */\r
1106         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1107                 (void*)&p_desc->buf.ib.pkt.type );\r
1108         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1109 \r
1110         /* Setup the local data segment. */\r
1111         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1112         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1113         ds0_len =\r
1114                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1115         if( ds0_len >= sizeof(recv_buf_t) )\r
1116         {\r
1117                 /* The whole buffer is within a page. */\r
1118                 p_desc->local_ds[0].length = ds0_len;\r
1119                 p_desc->wr.num_ds = 1;\r
1120         }\r
1121         else\r
1122         {\r
1123                 /* The buffer crosses page boundaries. */\r
1124                 p_desc->local_ds[0].length = ds0_len;\r
1125                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1126                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1127                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1128                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1129                 p_desc->wr.num_ds = 2;\r
1130         }\r
1131 #else   /* IPOIB_INLINE_RECV */\r
1132         /* Allocate the receive buffer. */\r
1133         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1134         if( !p_desc->p_buf )\r
1135         {\r
1136                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1137                         ("Failed to allocate receive buffer.\n") );\r
1138                 return CL_INSUFFICIENT_MEMORY;\r
1139         }\r
1140 \r
1141         /* Sanity check on the receive buffer layout */\r
1142         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1143                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1144 \r
1145         /* Setup the local data segment. */\r
1146         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1147         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1148         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1149 #endif  /* IPOIB_INLINE_RECV */\r
1150 \r
1151         *pp_pool_item = &p_desc->item;\r
1152 \r
1153         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1154         return CL_SUCCESS;\r
1155 }\r
1156 \r
1157 \r
1158 #if !IPOIB_INLINE_RECV\r
1159 static void\r
1160 __recv_dtor(\r
1161         IN              const   cl_pool_item_t* const           p_pool_item,\r
1162         IN                              void                                            *context )\r
1163 {\r
1164         ipoib_recv_desc_t       *p_desc;\r
1165 \r
1166         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1167 \r
1168         UNUSED_PARAM( context );\r
1169 \r
1170         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1171 \r
1172         if( p_desc->p_buf )\r
1173                 cl_free( p_desc->p_buf );\r
1174 \r
1175         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1176 }\r
1177 #endif\r
1178 \r
1179 \r
1180 static inline ipoib_recv_desc_t*\r
1181 __buf_mgr_get_recv(\r
1182         IN                              ipoib_port_t* const                     p_port )\r
1183 {\r
1184         ipoib_recv_desc_t       *p_desc;\r
1185         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1186         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1187         /* Reference the port object for the send. */\r
1188         if( p_desc )\r
1189         {\r
1190                 ipoib_port_ref( p_port, ref_get_recv );\r
1191                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1192 #if IPOIB_INLINE_RECV\r
1193                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1194                         cl_get_physaddr( &p_desc->buf ) );\r
1195 #else   /* IPOIB_INLINE_RECV */\r
1196                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1197                         cl_get_physaddr( p_desc->p_buf ) );\r
1198                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1199                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1200 #endif  /* IPOIB_INLINE_RECV */\r
1201                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1202         }\r
1203         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1204         return p_desc;\r
1205 }\r
1206 \r
1207 \r
1208 static inline void\r
1209 __buf_mgr_put_recv(\r
1210         IN                              ipoib_port_t* const                     p_port,\r
1211         IN                              ipoib_recv_desc_t* const        p_desc,\r
1212         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1213 {\r
1214         NDIS_BUFFER             *p_buf;\r
1215 \r
1216         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1217 \r
1218         if( p_packet )\r
1219         {\r
1220                 /* Unchain the NDIS buffer. */\r
1221                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1222                 CL_ASSERT( p_buf );\r
1223                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1224                 NdisDprFreePacketNonInterlocked( p_packet );\r
1225                 NdisFreeBuffer( p_buf );\r
1226         }\r
1227 \r
1228         /* Return the descriptor to its pools. */\r
1229         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1230 \r
1231         /*\r
1232          * Dereference the port object since the receive is no longer outstanding.\r
1233          */\r
1234         ipoib_port_deref( p_port, ref_get_recv );\r
1235         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1236 }\r
1237 \r
1238 \r
1239 static inline void\r
1240 __buf_mgr_put_recv_list(\r
1241         IN                              ipoib_port_t* const                     p_port,\r
1242         IN                              cl_qlist_t* const                       p_list )\r
1243 {\r
1244         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1245         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1246         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1247 }\r
1248 \r
1249 \r
1250 static inline NDIS_PACKET*\r
1251 __buf_mgr_get_ndis_pkt(\r
1252         IN                              ipoib_port_t* const                     p_port,\r
1253         IN                              ipoib_recv_desc_t* const        p_desc )\r
1254 {\r
1255         NDIS_STATUS                             status;\r
1256         NDIS_PACKET                             *p_packet;\r
1257         NDIS_BUFFER                             *p_buffer;\r
1258 \r
1259         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1260 \r
1261         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1262                 p_port->buf_mgr.h_packet_pool );\r
1263         if( status != NDIS_STATUS_SUCCESS )\r
1264         {\r
1265                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1266                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1267                 return NULL;\r
1268         }\r
1269 \r
1270         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1271         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1272 \r
1273         NdisAllocateBuffer( &status, &p_buffer,\r
1274 #if IPOIB_INLINE_RECV\r
1275                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1276 #else   /* IPOIB_INLINE_RECV */\r
1277                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1278 #endif  /* IPOIB_INLINE_RECV */\r
1279         if( status != NDIS_STATUS_SUCCESS )\r
1280         {\r
1281                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1282                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1283                 NdisDprFreePacketNonInterlocked( p_packet );\r
1284                 return NULL;\r
1285         }\r
1286 \r
1287         NdisChainBufferAtFront( p_packet, p_buffer );\r
1288         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1289 \r
1290         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1291         return p_packet;\r
1292 }\r
1293 \r
1294 /******************************************************************************\r
1295 *\r
1296 * Receive manager implementation.\r
1297 *\r
1298 ******************************************************************************/\r
1299 static void\r
1300 __recv_mgr_construct(\r
1301         IN                              ipoib_port_t* const                     p_port )\r
1302 {\r
1303         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1304 \r
1305         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1306 \r
1307         p_port->recv_mgr.recv_pkt_array = NULL;\r
1308 \r
1309         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1310 }\r
1311 \r
1312 \r
1313 static ib_api_status_t\r
1314 __recv_mgr_init(\r
1315         IN                              ipoib_port_t* const                     p_port )\r
1316 {\r
1317         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1318 \r
1319         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1320         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1321                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1322         if( !p_port->recv_mgr.recv_pkt_array )\r
1323         {\r
1324                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1325                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1326                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1327                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1328                 return IB_INSUFFICIENT_MEMORY;\r
1329         }\r
1330 \r
1331         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1332         return IB_SUCCESS;\r
1333 }\r
1334 \r
1335 \r
1336 static void\r
1337 __recv_mgr_destroy(\r
1338         IN                              ipoib_port_t* const                     p_port )\r
1339 {\r
1340         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1341 \r
1342         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1343         CL_ASSERT( !p_port->recv_mgr.depth );\r
1344 \r
1345         if( p_port->recv_mgr.recv_pkt_array )\r
1346                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1347 \r
1348         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1349 }\r
1350 \r
1351 \r
1352 /*\r
1353  * Posts receive buffers to the receive queue and returns the number\r
1354  * of receives needed to bring the RQ to its low water mark.  Note\r
1355  * that the value is signed, and can go negative.  All tests must\r
1356  * be for > 0.\r
1357  */\r
1358 static int32_t\r
1359 __recv_mgr_repost(\r
1360         IN                              ipoib_port_t* const                     p_port )\r
1361 {\r
1362         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1363         ib_api_status_t         status;\r
1364         ib_recv_wr_t            *p_failed;\r
1365         PERF_DECLARE( GetRecv );\r
1366         PERF_DECLARE( PostRecv );\r
1367 \r
1368         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1369 \r
1370         CL_ASSERT( p_port );\r
1371         cl_obj_lock( &p_port->obj );\r
1372         if( p_port->state != IB_QPS_RTS )\r
1373         {\r
1374                 cl_obj_unlock( &p_port->obj );\r
1375                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1376                         ("Port in invalid state.  Not reposting.\n") );\r
1377                 return 0;\r
1378         }\r
1379         ipoib_port_ref( p_port, ref_repost );\r
1380         cl_obj_unlock( &p_port->obj );\r
1381 \r
1382         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1383         {\r
1384                 /* Pull receives out of the pool and chain them up. */\r
1385                 cl_perf_start( GetRecv );\r
1386                 p_next = __buf_mgr_get_recv( p_port );\r
1387                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1388                 if( !p_next )\r
1389                 {\r
1390                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1391                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1392                         break;\r
1393                 }\r
1394 \r
1395                 if( !p_tail )\r
1396                 {\r
1397                         p_tail = p_next;\r
1398                         p_next->wr.p_next = NULL;\r
1399                 }\r
1400                 else\r
1401                 {\r
1402                         p_next->wr.p_next = &p_head->wr;\r
1403                 }\r
1404 \r
1405                 p_head = p_next;\r
1406 \r
1407                 p_port->recv_mgr.depth++;\r
1408         }\r
1409 \r
1410         if( p_head )\r
1411         {\r
1412                 cl_perf_start( PostRecv );\r
1413                 status = p_port->p_adapter->p_ifc->post_recv(\r
1414                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1415                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1416 \r
1417                 if( status != IB_SUCCESS )\r
1418                 {\r
1419                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1420                                 ("ip_post_recv returned %s\n", \r
1421                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1422                         /* return the descriptors to the pool */\r
1423                         while( p_failed )\r
1424                         {\r
1425                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1426                                 p_failed = p_failed->p_next;\r
1427 \r
1428                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1429                                 p_port->recv_mgr.depth--;\r
1430                         }\r
1431                 }\r
1432         }\r
1433 \r
1434         ipoib_port_deref( p_port, ref_repost );\r
1435         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1436         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1437 }\r
1438 \r
1439 \r
1440 void\r
1441 ipoib_return_packet(\r
1442         IN                              NDIS_HANDLE                                     adapter_context,\r
1443         IN                              NDIS_PACKET                                     *p_packet )\r
1444 {\r
1445         cl_list_item_t          *p_item;\r
1446         ipoib_port_t            *p_port;\r
1447         ipoib_recv_desc_t       *p_desc;\r
1448         ib_api_status_t         status = IB_NOT_DONE;\r
1449         int32_t                         shortage;\r
1450         PERF_DECLARE( ReturnPacket );\r
1451         PERF_DECLARE( ReturnPutRecv );\r
1452         PERF_DECLARE( ReturnRepostRecv );\r
1453         PERF_DECLARE( ReturnPreparePkt );\r
1454         PERF_DECLARE( ReturnNdisIndicate );\r
1455 \r
1456         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1457 \r
1458         UNUSED_PARAM( adapter_context );\r
1459         CL_ASSERT( p_packet );\r
1460 \r
1461         cl_perf_start( ReturnPacket );\r
1462 \r
1463         /* Get the port and descriptor from the packet. */\r
1464         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1465         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1466 \r
1467         cl_spinlock_acquire( &p_port->recv_lock );\r
1468 \r
1469         cl_perf_start( ReturnPutRecv );\r
1470         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1471         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1472 \r
1473         /* Repost buffers. */\r
1474         cl_perf_start( ReturnRepostRecv );\r
1475         shortage = __recv_mgr_repost( p_port );\r
1476         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1477 \r
1478         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1479                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1480                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1481         {\r
1482                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1483 \r
1484                 cl_perf_start( ReturnPreparePkt );\r
1485                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1486                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1487                 if( status == IB_SUCCESS )\r
1488                 {\r
1489                         if( shortage > 0 )\r
1490                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1491                         else\r
1492                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1493 \r
1494                         cl_spinlock_release( &p_port->recv_lock );\r
1495                         cl_perf_start( ReturnNdisIndicate );\r
1496                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1497                                 &p_packet, 1 );\r
1498                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1499                         cl_spinlock_acquire( &p_port->recv_lock );\r
1500 \r
1501                         if( shortage > 0 )\r
1502                         {\r
1503                                 cl_perf_start( ReturnPutRecv );\r
1504                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1505                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1506 \r
1507                                 /* Repost buffers. */\r
1508                                 cl_perf_start( ReturnRepostRecv );\r
1509                                 shortage = __recv_mgr_repost( p_port );\r
1510                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1511                         }\r
1512                 }\r
1513                 else if( status != IB_NOT_DONE )\r
1514                 {\r
1515                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1516                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1517                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1518                         /* Return the item to the head of the list. */\r
1519                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1520                         break;\r
1521                 }\r
1522         }\r
1523         cl_spinlock_release( &p_port->recv_lock );\r
1524         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1525 \r
1526         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1527 }\r
1528 \r
1529 \r
1530 static void\r
1531 __recv_cb(\r
1532         IN              const   ib_cq_handle_t                          h_cq,\r
1533         IN                              void                                            *cq_context )\r
1534 {\r
1535         ipoib_port_t            *p_port;\r
1536         ib_api_status_t         status;\r
1537         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1538         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1539         cl_qlist_t                      done_list, bad_list;\r
1540         size_t                          i;\r
1541         PERF_DECLARE( RecvCompBundle );\r
1542         PERF_DECLARE( RecvCb );\r
1543         PERF_DECLARE( PollRecv );\r
1544         PERF_DECLARE( RepostRecv );\r
1545         PERF_DECLARE( FilterRecv );\r
1546         PERF_DECLARE( BuildPktArray );\r
1547         PERF_DECLARE( RecvNdisIndicate );\r
1548         PERF_DECLARE( RearmRecv );\r
1549         PERF_DECLARE( PutRecvList );\r
1550 \r
1551         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1552 \r
1553         cl_perf_clr( RecvCompBundle );\r
1554 \r
1555         cl_perf_start( RecvCb );\r
1556 \r
1557         UNUSED_PARAM( h_cq );\r
1558 \r
1559         p_port = (ipoib_port_t*)cq_context;\r
1560 \r
1561         cl_qlist_init( &done_list );\r
1562         cl_qlist_init( &bad_list );\r
1563 \r
1564         ipoib_port_ref( p_port, ref_recv_cb );\r
1565         for( i = 0; i < MAX_RECV_WC; i++ )\r
1566                 wc[i].p_next = &wc[i + 1];\r
1567         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1568 \r
1569         /*\r
1570          * We'll be accessing the endpoint map so take a reference\r
1571          * on it to prevent modifications.\r
1572          */\r
1573         cl_obj_lock( &p_port->obj );\r
1574         cl_atomic_inc( &p_port->endpt_rdr );\r
1575         cl_obj_unlock( &p_port->obj );\r
1576 \r
1577         do\r
1578         {\r
1579                 /* If we get here, then the list of WCs is intact. */\r
1580                 p_free = wc;\r
1581 \r
1582                 cl_perf_start( PollRecv );\r
1583                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1584                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1585                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1586                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1587 \r
1588                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1589                 cl_perf_start( FilterRecv );\r
1590                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1591                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1592 \r
1593         } while( !p_free );\r
1594 \r
1595         /* We're done looking at the endpoint map, release the reference. */\r
1596         cl_atomic_dec( &p_port->endpt_rdr );\r
1597 \r
1598         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1599 \r
1600         cl_spinlock_acquire( &p_port->recv_lock );\r
1601 \r
1602         /* Update our posted depth. */\r
1603         p_port->recv_mgr.depth -= recv_cnt;\r
1604 \r
1605         /* Return any discarded receives to the pool */\r
1606         cl_perf_start( PutRecvList );\r
1607         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1608         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1609 \r
1610         do\r
1611         {\r
1612                 /* Repost ASAP so we don't starve the RQ. */\r
1613                 cl_perf_start( RepostRecv );\r
1614                 shortage = __recv_mgr_repost( p_port );\r
1615                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1616 \r
1617                 cl_perf_start( BuildPktArray );\r
1618                 /* Notify NDIS of any and all possible receive buffers. */\r
1619                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1620                         p_port, shortage, &done_list, &discarded );\r
1621                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1622 \r
1623                 /* Only indicate receives if we actually had any. */\r
1624                 if( discarded && shortage > 0 )\r
1625                 {\r
1626                         /* We may have thrown away packets, and have a shortage */\r
1627                         cl_perf_start( RepostRecv );\r
1628                         __recv_mgr_repost( p_port );\r
1629                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1630                 }\r
1631 \r
1632                 if( !pkt_cnt )\r
1633                         break;\r
1634 \r
1635                 cl_spinlock_release( &p_port->recv_lock );\r
1636 \r
1637                 cl_perf_start( RecvNdisIndicate );\r
1638                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1639                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1640                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1641 \r
1642                 /*\r
1643                  * Cap the number of receives to put back to what we just indicated\r
1644                  * with NDIS_STATUS_RESOURCES.\r
1645                  */\r
1646                 if( shortage > 0 )\r
1647                 {\r
1648                         if( pkt_cnt < shortage )\r
1649                                 shortage = pkt_cnt;\r
1650 \r
1651                         /* Return all but the last packet to the pool. */\r
1652                         cl_spinlock_acquire( &p_port->recv_lock );\r
1653                         while( shortage-- > 1 )\r
1654                         {\r
1655                                 __buf_mgr_put_recv( p_port,\r
1656                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1657                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1658                         }\r
1659                         cl_spinlock_release( &p_port->recv_lock );\r
1660 \r
1661                         /*\r
1662                          * Return the last packet as if NDIS returned it, so that we repost\r
1663                          * and report any other pending receives.\r
1664                          */\r
1665                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1666                 }\r
1667                 cl_spinlock_acquire( &p_port->recv_lock );\r
1668 \r
1669         } while( pkt_cnt );\r
1670         cl_spinlock_release( &p_port->recv_lock );\r
1671 \r
1672         /*\r
1673          * Rearm after filtering to prevent contention on the enpoint maps\r
1674          * and eliminate the possibility of having a call to\r
1675          * __endpt_mgr_insert find a duplicate.\r
1676          */\r
1677         cl_perf_start( RearmRecv );\r
1678         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1679                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1680         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1681         CL_ASSERT( status == IB_SUCCESS );\r
1682 \r
1683         ipoib_port_deref( p_port, ref_recv_cb );\r
1684 \r
1685         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1686 \r
1687         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1688 }\r
1689 \r
1690 \r
1691 static void\r
1692 __recv_get_endpts(\r
1693         IN                              ipoib_port_t* const                     p_port,\r
1694         IN                              ipoib_recv_desc_t* const        p_desc,\r
1695         IN                              ib_wc_t* const                          p_wc,\r
1696                 OUT                     ipoib_endpt_t** const           pp_src,\r
1697                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1698 {\r
1699         ib_api_status_t         status;\r
1700         mac_addr_t                      mac;\r
1701         PERF_DECLARE( GetEndptByGid );\r
1702         PERF_DECLARE( GetEndptByLid );\r
1703         PERF_DECLARE( EndptInsert );\r
1704 \r
1705         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1706 \r
1707         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1708         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1709         {\r
1710                 /* Lookup the source endpoints based on GID. */\r
1711                 cl_perf_start( GetEndptByGid );\r
1712                 *pp_src =\r
1713 #if IPOIB_INLINE_RECV\r
1714                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1715 #else   /* IPOIB_INLINE_RECV */\r
1716                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1717 #endif  /* IPOIB_INLINE_RECV */\r
1718                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1719 \r
1720                 /*\r
1721                  * Lookup the destination endpoint based on GID.\r
1722                  * This is used along with the packet filter to determine\r
1723                  * whether to report this to NDIS.\r
1724                  */\r
1725                 cl_perf_start( GetEndptByGid );\r
1726                 *pp_dst =\r
1727 #if IPOIB_INLINE_RECV\r
1728                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1729 #else   /* IPOIB_INLINE_RECV */\r
1730                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1731 #endif  /* IPOIB_INLINE_RECV */\r
1732                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1733 \r
1734                 /*\r
1735                  * Create the source endpoint if it does not exist.  Note that we\r
1736                  * can only do this for globally routed traffic since we need the\r
1737                  * information from the GRH to generate the MAC.\r
1738                  */\r
1739                 if( !*pp_src )\r
1740                 {\r
1741                         status = ipoib_mac_from_guid(\r
1742 #if IPOIB_INLINE_RECV\r
1743                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1744 #else   /* IPOIB_INLINE_RECV */\r
1745                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1746 #endif  /* IPOIB_INLINE_RECV */\r
1747                         if( status != IB_SUCCESS )\r
1748                         {\r
1749                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1750                                         ("ipoib_mac_from_guid returned %s\n",\r
1751                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1752                                 return;\r
1753                         }\r
1754 \r
1755                         /* Create the endpoint. */\r
1756 #if IPOIB_INLINE_RECV\r
1757                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1758 #else   /* IPOIB_INLINE_RECV */\r
1759                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1760 #endif  /* IPOIB_INLINE_RECV */\r
1761                                 0, p_wc->recv.ud.remote_qp );\r
1762                         if( !*pp_src )\r
1763                         {\r
1764                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1765                                         ("ipoib_endpt_create failed\n") );\r
1766                                 return;\r
1767                         }\r
1768                         cl_perf_start( EndptInsert );\r
1769                         cl_obj_lock( &p_port->obj );\r
1770                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1771                         if( status != IB_SUCCESS )\r
1772                         {\r
1773                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1774                                         ("__endpt_mgr_insert returned %s\n",\r
1775                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1776                                 return;\r
1777                         }\r
1778                         cl_obj_unlock( &p_port->obj );\r
1779                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1780                 }\r
1781         }\r
1782         else\r
1783         {\r
1784                 /*\r
1785                  * Lookup the remote endpoint based on LID.  Note that only\r
1786                  * unicast traffic can be LID routed.\r
1787                  */\r
1788                 cl_perf_start( GetEndptByLid );\r
1789                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1790                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1791                 *pp_dst = p_port->p_local_endpt;\r
1792                 CL_ASSERT( *pp_dst );\r
1793         }\r
1794 \r
1795         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1796                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1797         {\r
1798                 /* Update the QPN for the endpoint. */\r
1799                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1800                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1801                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1802                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1803                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1804                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1805         }\r
1806 \r
1807         if( *pp_src && *pp_dst )\r
1808         {\r
1809                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1810                         ("Recv:\n"\r
1811                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1812                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1813                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1814                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1815                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1816                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1817                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1818                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1819         }\r
1820 \r
1821         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1822 }\r
1823 \r
1824 \r
1825 static int32_t\r
1826 __recv_mgr_filter(\r
1827         IN                              ipoib_port_t* const                     p_port,\r
1828         IN                              ib_wc_t* const                          p_done_wc_list,\r
1829                 OUT                     cl_qlist_t* const                       p_done_list,\r
1830                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1831 {\r
1832         ipoib_recv_desc_t               *p_desc;\r
1833         ib_wc_t                                 *p_wc;\r
1834         ipoib_pkt_t                             *p_ipoib;\r
1835         eth_pkt_t                               *p_eth;\r
1836         ipoib_endpt_t                   *p_src, *p_dst;\r
1837         ib_api_status_t                 status;\r
1838         uint32_t                                len;\r
1839         int32_t                                 recv_cnt = 0;\r
1840         PERF_DECLARE( GetRecvEndpts );\r
1841         PERF_DECLARE( RecvGen );\r
1842         PERF_DECLARE( RecvTcp );\r
1843         PERF_DECLARE( RecvUdp );\r
1844         PERF_DECLARE( RecvDhcp );\r
1845         PERF_DECLARE( RecvArp );\r
1846 \r
1847         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1848 \r
1849         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1850         {\r
1851                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1852                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1853                 recv_cnt++;\r
1854 \r
1855                 if( p_wc->status != IB_WCS_SUCCESS )\r
1856                 {\r
1857                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1858                         {\r
1859                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1860                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1861                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1862                                         (int)p_wc->vendor_specific) );\r
1863                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1864                         }\r
1865                         else\r
1866                         {\r
1867                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1868                                         ("Flushed completion %s\n",\r
1869                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1870                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1871                         }\r
1872                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1873                         /* Dereference the port object on behalf of the failed receive. */\r
1874                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1875                         continue;\r
1876                 }\r
1877 \r
1878                 len = p_wc->length - sizeof(ib_grh_t);\r
1879 \r
1880                 if( len < sizeof(ipoib_hdr_t) )\r
1881                 {\r
1882                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1883                                 ("Received ETH packet < min size\n") );\r
1884                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1885                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1886                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1887                         continue;\r
1888                 }\r
1889 \r
1890                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1891                 {\r
1892                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1893                                 ("Received ETH packet > payload MTU (%d)\n",\r
1894                                 p_port->p_adapter->params.payload_mtu) );\r
1895                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1896                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1897                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1898                         continue;\r
1899                         \r
1900                 }\r
1901                 /* Successful completion.  Get the receive information. */\r
1902                 cl_perf_start( GetRecvEndpts );\r
1903                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1904                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1905 \r
1906 #if IPOIB_INLINE_RECV\r
1907                 p_ipoib = &p_desc->buf.ib.pkt;\r
1908                 p_eth = &p_desc->buf.eth.pkt;\r
1909 #else   /* IPOIB_INLINE_RECV */\r
1910                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1911                 p_eth = &p_desc->p_buf->eth.pkt;\r
1912 #endif  /*IPOIB_INLINE_RECV */\r
1913 \r
1914                 if( p_src )\r
1915                 {\r
1916                         /* Don't report loopback traffic - we requested SW loopback. */\r
1917                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1918                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1919                         {\r
1920                                 /*\r
1921                                  * "This is not the packet you're looking for" - don't update\r
1922                                  * receive statistics, the packet never happened.\r
1923                                  */\r
1924                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1925                                 /* Dereference the port object on behalf of the failed recv. */\r
1926                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
1927                                 continue;\r
1928                         }\r
1929                 }\r
1930 \r
1931                 switch( p_ipoib->hdr.type )\r
1932                 {\r
1933                 case ETH_PROT_TYPE_IP:\r
1934                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1935                         {\r
1936                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1937                                         ("Received IP packet < min size\n") );\r
1938                                 status = IB_INVALID_SETTING;\r
1939                                 break;\r
1940                         }\r
1941 \r
1942                         if( p_ipoib->type.ip.hdr.offset ||\r
1943                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1944                         {\r
1945                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1946                                 cl_perf_start( RecvTcp );\r
1947                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1948                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1949                                 break;\r
1950                         }\r
1951 \r
1952                         /* First packet of a UDP transfer. */\r
1953                         if( len <\r
1954                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1955                         {\r
1956                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1957                                         ("Received UDP packet < min size\n") );\r
1958                                 status = IB_INVALID_SETTING;\r
1959                                 break;\r
1960                         }\r
1961 \r
1962                         /* Check if DHCP conversion is required. */\r
1963                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1964                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1965                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1966                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1967                         {\r
1968                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1969                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1970                                 {\r
1971                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1972                                                 ("Received DHCP < min size\n") );\r
1973                                         status = IB_INVALID_SETTING;\r
1974                                         break;\r
1975                                 }\r
1976                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1977                                 cl_perf_start( RecvDhcp );\r
1978                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1979                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1980                         }\r
1981                         else\r
1982                         {\r
1983                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1984                                 cl_perf_start( RecvUdp );\r
1985                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1986                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1987                         }\r
1988                         break;\r
1989 \r
1990                 case ETH_PROT_TYPE_ARP:\r
1991                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1992                         {\r
1993                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1994                                         ("Received ARP < min size\n") );\r
1995                                 status = IB_INVALID_SETTING;\r
1996                                 break;\r
1997                         }\r
1998                         cl_perf_start( RecvArp );\r
1999                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2000                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2001                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2002                         break;\r
2003 \r
2004                 default:\r
2005                         /* Unfiltered.  Setup the ethernet header and report. */\r
2006                         cl_perf_start( RecvGen );\r
2007                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2008                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2009                 }\r
2010 \r
2011                 if( status != IB_SUCCESS )\r
2012                 {\r
2013                         /* Update stats. */\r
2014                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2015                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2016                         /* Dereference the port object on behalf of the failed receive. */\r
2017                         ipoib_port_deref( p_port, ref_recv_filter );\r
2018                 }\r
2019                 else\r
2020                 {\r
2021                         p_desc->len =\r
2022                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2023                         if( p_dst->h_mcast)\r
2024                         {\r
2025                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2026                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2027                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2028                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2029                                 {\r
2030                                         p_desc->type = PKT_TYPE_BCAST;\r
2031                                 }\r
2032                                 else\r
2033                                 {\r
2034                                         p_desc->type = PKT_TYPE_MCAST;\r
2035                                 }\r
2036                         }\r
2037                         else\r
2038                         {\r
2039                                 p_desc->type = PKT_TYPE_UCAST;\r
2040                         }\r
2041                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2042                 }\r
2043         }\r
2044 \r
2045         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2046         return recv_cnt;\r
2047 }\r
2048 \r
2049 \r
2050 static ib_api_status_t\r
2051 __recv_gen(\r
2052         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2053                 OUT                     eth_pkt_t* const                        p_eth,\r
2054         IN                              ipoib_endpt_t* const            p_src,\r
2055         IN                              ipoib_endpt_t* const            p_dst )\r
2056 {\r
2057         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2058 \r
2059         if( !p_src || !p_dst )\r
2060         {\r
2061                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2062                         ("Received packet with no matching endpoints.\n") );\r
2063                 return IB_NOT_DONE;\r
2064         }\r
2065 \r
2066         /*\r
2067          * Fill in the ethernet header.  Note that doing so will overwrite\r
2068          * the IPoIB header, so start by moving the information from the IPoIB\r
2069          * header.\r
2070          */\r
2071         p_eth->hdr.type = p_ipoib->hdr.type;\r
2072         p_eth->hdr.src = p_src->mac;\r
2073         p_eth->hdr.dst = p_dst->mac;\r
2074 \r
2075         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2076         return IB_SUCCESS;\r
2077 }\r
2078 \r
2079 \r
2080 static ib_api_status_t\r
2081 __recv_dhcp(\r
2082         IN                              ipoib_port_t* const                     p_port,\r
2083         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2084                 OUT                     eth_pkt_t* const                        p_eth,\r
2085         IN                              ipoib_endpt_t* const            p_src,\r
2086         IN                              ipoib_endpt_t* const            p_dst )\r
2087 {\r
2088         ib_api_status_t         status;\r
2089         dhcp_pkt_t                      *p_dhcp;\r
2090         uint8_t                         *p_option;\r
2091         uint8_t                         *p_cid = NULL;\r
2092         ib_gid_t                        gid;\r
2093         uint8_t                         msg = 0;\r
2094 \r
2095         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2096 \r
2097         UNUSED_PARAM( p_port );\r
2098 \r
2099         /* Create the ethernet header. */\r
2100         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2101         if( status != IB_SUCCESS )\r
2102         {\r
2103                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2104                         ("__recv_gen returned %s.\n", \r
2105                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2106                 return status;\r
2107         }\r
2108 \r
2109         /* Fixup the payload. */\r
2110         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2111         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2112         {\r
2113                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2114                         ("Invalid DHCP op code.\n") );\r
2115                 return IB_INVALID_SETTING;\r
2116         }\r
2117 \r
2118         /*\r
2119          * Find the client identifier option, making sure to skip\r
2120          * the "magic cookie".\r
2121          */\r
2122         p_option = &p_dhcp->options[0];\r
2123         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2124         {\r
2125                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2126                         ("DHCP cookie corrupted.\n") );\r
2127                 return IB_INVALID_PARAMETER;\r
2128         }\r
2129 \r
2130         p_option = &p_dhcp->options[4];\r
2131         while( *p_option != DHCP_OPT_END )\r
2132         {\r
2133                 switch( *p_option )\r
2134                 {\r
2135                 case DHCP_OPT_PAD:\r
2136                         p_option++;\r
2137                         break;\r
2138 \r
2139                 case DHCP_OPT_MSG:\r
2140                         msg = p_option[2];\r
2141                         p_option += 3;\r
2142                         break;\r
2143 \r
2144                 case DHCP_OPT_CLIENT_ID:\r
2145                         p_cid = p_option;\r
2146                         /* Fall through. */\r
2147 \r
2148                 default:\r
2149                         /*\r
2150                          * All other options have a length byte following the option code.\r
2151                          * Offset by the length to get to the next option.\r
2152                          */\r
2153                         p_option += (p_option[1] + 2);\r
2154                 }\r
2155         }\r
2156 \r
2157         switch( msg )\r
2158         {\r
2159         /* message from client */\r
2160         case DHCPDISCOVER:\r
2161         case DHCPREQUEST:\r
2162         case DHCPDECLINE:\r
2163         case DHCPRELEASE:\r
2164         case DHCPINFORM:\r
2165                 if( !p_cid )\r
2166                 {\r
2167                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2168                                 ("Failed to find required Client-identifier option.\n") );\r
2169                         return IB_INVALID_SETTING;\r
2170                 }\r
2171                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2172                 {\r
2173                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2174                                 ("Invalid hardware address type.\n") );\r
2175                         return IB_INVALID_SETTING;\r
2176                 }\r
2177                 break;\r
2178         /* message from DHCP server */\r
2179         case DHCPOFFER:\r
2180         case DHCPACK:\r
2181         case DHCPNAK:\r
2182                 break;\r
2183 \r
2184         default:\r
2185                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2186                         ("Invalide message type.\n") );\r
2187                 return IB_INVALID_PARAMETER;\r
2188         }\r
2189         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2190         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2191         p_dhcp->hlen = HW_ADDR_LEN;\r
2192 \r
2193         if( p_cid ) /* from client */\r
2194         {\r
2195                 /* Validate that the length and type of the option is as required. */\r
2196                 if( p_cid[1] != 21 )\r
2197                 {\r
2198                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2199                                 ("Client-identifier length not 21 as required.\n") );\r
2200                         return IB_INVALID_SETTING;\r
2201                 }\r
2202                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2203                 {\r
2204                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2205                                 ("Client-identifier type is wrong.\n") );\r
2206                         return IB_INVALID_SETTING;\r
2207                 }\r
2208                 /*\r
2209                  * Copy the GID value from the option so that we can make aligned\r
2210                  * accesses to the contents.\r
2211                  * Recover CID to standard type.\r
2212                  */\r
2213                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2214                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2215                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2216                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2217                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2218         }\r
2219         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2220         return status;\r
2221 }\r
2222 \r
2223 \r
2224 static ib_api_status_t\r
2225 __recv_arp(\r
2226         IN                              ipoib_port_t* const                     p_port,\r
2227         IN                              ib_wc_t* const                          p_wc,\r
2228         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2229                 OUT                     eth_pkt_t* const                        p_eth,\r
2230         IN                              ipoib_endpt_t** const           pp_src,\r
2231         IN                              ipoib_endpt_t* const            p_dst )\r
2232 {\r
2233         ib_api_status_t                 status;\r
2234         arp_pkt_t                               *p_arp;\r
2235         const ipoib_arp_pkt_t   *p_ib_arp;\r
2236         ib_gid_t                                gid;\r
2237         mac_addr_t                              mac;\r
2238         ipoib_hw_addr_t                 null_hw = {0};\r
2239 \r
2240         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2241 \r
2242         if( !p_dst )\r
2243         {\r
2244                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2245                         ("Unknown destination endpoint\n") );\r
2246                 return IB_INVALID_SETTING;\r
2247         }\r
2248 \r
2249         p_ib_arp = &p_ipoib->type.arp;\r
2250         p_arp = &p_eth->type.arp;\r
2251 \r
2252         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2253         {\r
2254                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2255                         ("ARP hardware type is not IB\n") );\r
2256                 return IB_INVALID_SETTING;\r
2257         }\r
2258 \r
2259         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2260         {\r
2261                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2262                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2263                 return IB_INVALID_SETTING;\r
2264         }\r
2265 \r
2266         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2267         {\r
2268                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2269                         ("ARP protocal type not IP\n") );\r
2270                 return IB_INVALID_SETTING;\r
2271         }\r
2272 \r
2273         /*\r
2274          * If we don't have a source, lookup the endpoint specified in the payload.\r
2275          */\r
2276         if( !*pp_src )\r
2277                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2278 \r
2279         /*\r
2280          * If the endpoint exists for the GID, make sure\r
2281          * the dlid and qpn match the arp.\r
2282          */\r
2283         if( *pp_src )\r
2284         {\r
2285                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2286                         sizeof(ib_gid_t) ) )\r
2287                 {\r
2288                         /*\r
2289                          * GIDs for the endpoint are different.  The ARP must\r
2290                          * have been proxied.  Dereference it.\r
2291                          */\r
2292                         *pp_src = NULL;\r
2293                 }\r
2294                 else if( (*pp_src)->dlid &&\r
2295                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2296                 {\r
2297                         /* Out of date!  Destroy the endpoint and replace it. */\r
2298                         __endpt_mgr_remove( p_port, *pp_src );\r
2299                         *pp_src = NULL;\r
2300                 }\r
2301                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2302                 {\r
2303                         if( (*pp_src)->qpn !=\r
2304                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2305                                 p_wc->recv.ud.remote_qp !=\r
2306                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2307                         {\r
2308                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2309                                 __endpt_mgr_remove( p_port, *pp_src );\r
2310                                 *pp_src = NULL;\r
2311                         }\r
2312                 }\r
2313                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2314                 {\r
2315                         /* Out of date!  Destroy the endpoint and replace it. */\r
2316                         __endpt_mgr_remove( p_port, *pp_src );\r
2317                         *pp_src = NULL;\r
2318                 }\r
2319         }\r
2320 \r
2321         /* Do we need to create an endpoint for this GID? */\r
2322         if( !*pp_src )\r
2323         {\r
2324                 /* Copy the src GID to allow aligned access */\r
2325                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2326                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2327                 if( status != IB_SUCCESS )\r
2328                 {\r
2329                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2330                                 ("ipoib_mac_from_guid returned %s\n",\r
2331                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2332                         return status;\r
2333                 }\r
2334                 /*\r
2335                  * Create the endpoint.  Note that the LID is left blank and will be\r
2336                  * resolved by a path query as needed.  This is done because the\r
2337                  * remote LID/GID from the work completion may not be the original\r
2338                  * initiator.\r
2339                  */\r
2340                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2341                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2342 \r
2343                 if( !*pp_src )\r
2344                 {\r
2345                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2346                                 ("ipoib_endpt_create failed\n") );\r
2347                         return status;\r
2348                 }\r
2349 \r
2350                 cl_obj_lock( &p_port->obj );\r
2351                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2352                 if( status != IB_SUCCESS )\r
2353                 {\r
2354                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2355                                 ("__endpt_mgr_insert return %s \n",\r
2356                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2357                         return status;\r
2358                 }\r
2359 \r
2360                 cl_obj_unlock( &p_port->obj );\r
2361         }\r
2362 \r
2363         CL_ASSERT( !cl_memcmp(\r
2364                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2365         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2366                 (*pp_src)->qpn ==\r
2367                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2368         /* Now swizzle the data. */\r
2369         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2370         p_arp->hw_size = sizeof(mac_addr_t);\r
2371         p_arp->src_hw = (*pp_src)->mac;\r
2372         p_arp->src_ip = p_ib_arp->src_ip;\r
2373 \r
2374         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2375         {\r
2376                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2377                 {\r
2378                         /*\r
2379                          * We received bcast ARP packet that means\r
2380                          * remote port lets everyone know it was changed IP/MAC\r
2381                          * or just activated\r
2382                          */\r
2383 \r
2384                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2385 \r
2386                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2387                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2388                         {\r
2389                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2390                                         ("ARP: is not ARP MCAST\n") );\r
2391                                 return IB_INVALID_SETTING;\r
2392                         }\r
2393 \r
2394                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2395                         p_dst->mac = p_port->p_local_endpt->mac;\r
2396                         /*\r
2397                          * we don't care what receiver ip addr is,\r
2398                          * as long as OS' ARP table is global  ???\r
2399                          */\r
2400                         p_arp->dst_ip = (net32_t)0;\r
2401                 }\r
2402                 else /* we've got reply to our ARP request */\r
2403                 {\r
2404                         p_arp->dst_hw = p_dst->mac;\r
2405                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2406                         CL_ASSERT( p_dst->qpn == \r
2407                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2408                 }\r
2409         }\r
2410         else /* we got ARP reqeust */\r
2411         {\r
2412                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2413                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2414         }\r
2415 \r
2416         /*\r
2417          * Create the ethernet header.  Note that this is done last so that\r
2418          * we have a chance to create a new endpoint.\r
2419          */\r
2420         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2421         if( status != IB_SUCCESS )\r
2422         {\r
2423                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2424                         ("__recv_gen returned %s.\n", \r
2425                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2426                 return status;\r
2427         }\r
2428 \r
2429         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2430         return IB_SUCCESS;\r
2431 }\r
2432 \r
2433 \r
2434 static ib_api_status_t\r
2435 __recv_mgr_prepare_pkt(\r
2436         IN                              ipoib_port_t* const                     p_port,\r
2437         IN                              ipoib_recv_desc_t* const        p_desc,\r
2438                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2439 {\r
2440         NDIS_STATUS                                                     status;\r
2441         uint32_t                                                        pkt_filter;\r
2442         ip_stat_sel_t                                           type;\r
2443         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2444         PERF_DECLARE( GetNdisPkt );\r
2445 \r
2446         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2447 \r
2448         pkt_filter = p_port->p_adapter->packet_filter;\r
2449         /* Check the packet filter. */\r
2450         switch( p_desc->type )\r
2451         {\r
2452         default:\r
2453         case PKT_TYPE_UCAST:\r
2454                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2455                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2456                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2457                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2458                 {\r
2459                         /* OK to report. */\r
2460                         type = IP_STAT_UCAST_BYTES;\r
2461                         status = NDIS_STATUS_SUCCESS;\r
2462                 }\r
2463                 else\r
2464                 {\r
2465                         type = IP_STAT_DROPPED;\r
2466                         status = NDIS_STATUS_FAILURE;\r
2467                 }\r
2468                 break;\r
2469         case PKT_TYPE_BCAST:\r
2470                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2471                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2472                 {\r
2473                         /* OK to report. */\r
2474                         type = IP_STAT_BCAST_BYTES;\r
2475                         status = NDIS_STATUS_SUCCESS;\r
2476                 }\r
2477                 else\r
2478                 {\r
2479                         type = IP_STAT_DROPPED;\r
2480                         status = NDIS_STATUS_FAILURE;\r
2481                 }\r
2482                 break;\r
2483         case PKT_TYPE_MCAST:\r
2484                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2485                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2486                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2487                 {\r
2488                         /* OK to report. */\r
2489                         type = IP_STAT_MCAST_BYTES;\r
2490                         status = NDIS_STATUS_SUCCESS;\r
2491                 }\r
2492                 else\r
2493                 {\r
2494                         type = IP_STAT_DROPPED;\r
2495                         status = NDIS_STATUS_FAILURE;\r
2496                 }\r
2497                 break;\r
2498         }\r
2499 \r
2500         if( status != NDIS_STATUS_SUCCESS )\r
2501         {\r
2502                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2503                 /* Return the receive descriptor to the pool. */\r
2504                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2505                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2506                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2507                 /*\r
2508                  * Return IB_NOT_DONE since the packet has been completed,\r
2509                  * but has not consumed an array entry.\r
2510                  */\r
2511                 return IB_NOT_DONE;\r
2512         }\r
2513 \r
2514         cl_perf_start( GetNdisPkt );\r
2515         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2516         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2517         if( !*pp_packet )\r
2518         {\r
2519                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2520                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2521                 return IB_INSUFFICIENT_RESOURCES;\r
2522         }\r
2523 \r
2524         /* Flag the checksums as having been calculated. */\r
2525         chksum.Value = 0;\r
2526         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2527         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2528         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2529         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2530                 (void*)(uintn_t)chksum.Value;\r
2531 \r
2532         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2533 \r
2534         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2535         return IB_SUCCESS;\r
2536 }\r
2537 \r
2538 \r
2539 static uint32_t\r
2540 __recv_mgr_build_pkt_array(\r
2541         IN                              ipoib_port_t* const                     p_port,\r
2542         IN                              int32_t                                         shortage,\r
2543                 OUT                     cl_qlist_t* const                       p_done_list,\r
2544                 OUT                     int32_t* const                          p_discarded )\r
2545 {\r
2546         cl_list_item_t                  *p_item;\r
2547         ipoib_recv_desc_t               *p_desc;\r
2548         uint32_t                                i = 0;\r
2549         ib_api_status_t                 status;\r
2550         PERF_DECLARE( PreparePkt );\r
2551 \r
2552         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2553 \r
2554         *p_discarded = 0;\r
2555 \r
2556         /* Move any existing receives to the head to preserve ordering. */\r
2557         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2558         p_item = cl_qlist_remove_head( p_done_list );\r
2559         while( p_item != cl_qlist_end( p_done_list ) )\r
2560         {\r
2561                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2562 \r
2563                 cl_perf_start( PreparePkt );\r
2564                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2565                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2566                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2567                 if( status == IB_SUCCESS )\r
2568                 {\r
2569                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2570                         if( shortage-- > 0 )\r
2571                         {\r
2572                                 NDIS_SET_PACKET_STATUS(\r
2573                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2574                         }\r
2575                         else\r
2576                         {\r
2577                                 NDIS_SET_PACKET_STATUS(\r
2578                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2579                         }\r
2580                         i++;\r
2581                 }\r
2582                 else if( status == IB_NOT_DONE )\r
2583                 {\r
2584                         (*p_discarded)++;\r
2585                 }\r
2586                 else\r
2587                 {\r
2588                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2589                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2590                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2591                         /* Put all completed receives on the port's done list. */\r
2592                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2593                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2594                         break;\r
2595                 }\r
2596 \r
2597                 p_item = cl_qlist_remove_head( p_done_list );\r
2598         }\r
2599 \r
2600         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2601         return i;\r
2602 }\r
2603 \r
2604 \r
2605 \r
2606 \r
2607 /******************************************************************************\r
2608 *\r
2609 * Send manager implementation.\r
2610 *\r
2611 ******************************************************************************/\r
2612 static void\r
2613 __send_mgr_construct(\r
2614         IN                              ipoib_port_t* const                     p_port )\r
2615 {\r
2616         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2617         p_port->send_mgr.depth = 0;\r
2618         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2619         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2620 }\r
2621 \r
2622 \r
2623 static void\r
2624 __send_mgr_destroy(\r
2625         IN                              ipoib_port_t* const                     p_port )\r
2626 {\r
2627         cl_list_item_t  *p_item;\r
2628         NDIS_PACKET             *p_packet;\r
2629 \r
2630         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2631 \r
2632         /* Complete any pending packets. */\r
2633         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2634                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2635                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2636         {\r
2637                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2638                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2639                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2640         }\r
2641 \r
2642         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2643 }\r
2644 \r
2645 \r
2646 static NDIS_STATUS\r
2647 __send_mgr_filter(\r
2648         IN                              ipoib_port_t* const                     p_port,\r
2649         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2650         IN                              NDIS_BUFFER* const                      p_buf,\r
2651         IN                              size_t                                          buf_len,\r
2652         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2653 {\r
2654         NDIS_STATUS             status;\r
2655 \r
2656         PERF_DECLARE( FilterIp );\r
2657         PERF_DECLARE( FilterArp );\r
2658         PERF_DECLARE( SendGen );\r
2659 \r
2660         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2661 \r
2662         /*\r
2663          * We already checked the ethernet header length, so we know it's safe\r
2664          * to decrement the buf_len without underflowing.\r
2665          */\r
2666         buf_len -= sizeof(eth_hdr_t);\r
2667 \r
2668         switch( p_eth_hdr->type )\r
2669         {\r
2670         case ETH_PROT_TYPE_IP:\r
2671                 cl_perf_start( FilterIp );\r
2672                 status = __send_mgr_filter_ip(\r
2673                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2674                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2675                 break;\r
2676 \r
2677         case ETH_PROT_TYPE_ARP:\r
2678                 cl_perf_start( FilterArp );\r
2679                 status = __send_mgr_filter_arp(\r
2680                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2681                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2682                 break;\r
2683 \r
2684         default:\r
2685                 /*\r
2686                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2687                  * Just send the payload and hope for the best.\r
2688                  */\r
2689                 cl_perf_start( SendGen );\r
2690                 status = __send_gen( p_port, p_desc );\r
2691                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2692                 break;\r
2693         }\r
2694 \r
2695         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2696         return status;\r
2697 }\r
2698 \r
2699 \r
2700 static NDIS_STATUS\r
2701 __send_copy(\r
2702         IN                              ipoib_port_t* const                     p_port,\r
2703         IN                              ipoib_send_desc_t* const        p_desc )\r
2704 {\r
2705         NDIS_PACKET                             *p_packet;\r
2706         NDIS_BUFFER                             *p_buf;\r
2707         NDIS_STATUS                             status;\r
2708         UINT                                    tot_len, bytes_copied = 0;\r
2709 \r
2710         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2711 \r
2712         p_desc->p_buf = \r
2713                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2714         if( !p_desc->p_buf )\r
2715         {\r
2716                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2717                         ("Failed to allocate buffer for packet copy.\n") );\r
2718                 return NDIS_STATUS_RESOURCES;\r
2719         }\r
2720 \r
2721         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2722         if( status != NDIS_STATUS_SUCCESS )\r
2723         {\r
2724                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2725                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2726                 return status;\r
2727         }\r
2728 \r
2729         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2730                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2731         if( status != NDIS_STATUS_SUCCESS )\r
2732         {\r
2733                 NdisFreePacket( p_packet );\r
2734                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2735                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2736                 return status;\r
2737         }\r
2738 \r
2739         NdisChainBufferAtFront( p_packet, p_buf );\r
2740 \r
2741         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2742 \r
2743         /* Setup the work request. */\r
2744         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2745                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2746         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2747         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2748         p_desc->wr.num_ds = 2;\r
2749 \r
2750         /* Copy the packet. */\r
2751         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2752                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2753                 NormalPagePriority );\r
2754 \r
2755         /* Free our temp packet now that the data is copied. */\r
2756         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2757         NdisFreeBuffer( p_buf );\r
2758         NdisFreePacket( p_packet );\r
2759 \r
2760         if( bytes_copied != tot_len )\r
2761         {\r
2762                 /* Something went wrong.  Drop the packet. */\r
2763                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2764                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2765                         bytes_copied, tot_len) );\r
2766                 return NDIS_STATUS_RESOURCES;\r
2767         }\r
2768 \r
2769         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2770         return NDIS_STATUS_SUCCESS;\r
2771 }\r
2772 \r
2773 \r
2774 #if !IPOIB_USE_DMA\r
2775 /* Send using the MDL's page information rather than the SGL. */\r
2776 static ib_api_status_t\r
2777 __send_gen(\r
2778         IN                              ipoib_port_t* const                     p_port,\r
2779         IN                              ipoib_send_desc_t* const        p_desc )\r
2780 {\r
2781         uint32_t                                i, j = 1;\r
2782         ULONG                                   offset;\r
2783         MDL                                             *p_mdl;\r
2784         UINT                                    num_pages, tot_len;\r
2785         ULONG                                   buf_len;\r
2786         PPFN_NUMBER                             page_array;\r
2787         boolean_t                               hdr_done = FALSE;\r
2788         ib_api_status_t                 status;\r
2789 \r
2790         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2791 \r
2792         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2793                 &tot_len );\r
2794 \r
2795         if( !p_mdl )\r
2796         {\r
2797                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2798                         ("No buffers associated with packet.\n") );\r
2799                 return IB_ERROR;\r
2800         }\r
2801 \r
2802         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2803         if( num_pages >= MAX_SEND_SGE )\r
2804         {\r
2805                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2806                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2807                 status = __send_copy( p_port, p_desc );\r
2808                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2809                 return status;\r
2810         }\r
2811 \r
2812         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2813         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2814         /*\r
2815          * Assume that the ethernet header is always fully contained\r
2816          * in the first page of the first MDL.  This makes for much\r
2817          * simpler code.\r
2818          */\r
2819         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2820         CL_ASSERT( offset <= PAGE_SIZE );\r
2821 \r
2822         while( tot_len )\r
2823         {\r
2824                 buf_len = MmGetMdlByteCount( p_mdl );\r
2825                 page_array = MmGetMdlPfnArray( p_mdl );\r
2826                 CL_ASSERT( page_array );\r
2827                 i = 0;\r
2828                 if( !hdr_done )\r
2829                 {\r
2830                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2831                         /* Skip the ethernet header. */\r
2832                         buf_len -= sizeof(eth_hdr_t);\r
2833                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2834                         if( buf_len )\r
2835                         {\r
2836                                 /* The ethernet header is a subset of this MDL. */\r
2837                                 CL_ASSERT( i == 0 );\r
2838                                 if( offset < PAGE_SIZE )\r
2839                                 {\r
2840                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2841                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2842                                         /* Add the byte offset since we're on the 1st page. */\r
2843                                         p_desc->local_ds[j].vaddr += offset;\r
2844                                         if( offset + buf_len > PAGE_SIZE )\r
2845                                         {\r
2846                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2847                                                 buf_len -= p_desc->local_ds[j].length;\r
2848                                         }\r
2849                                         else\r
2850                                         {\r
2851                                                 p_desc->local_ds[j].length = buf_len;\r
2852                                                 buf_len = 0;\r
2853                                         }\r
2854                                         /* This data segment is done.  Move to the next. */\r
2855                                         j++;\r
2856                                 }\r
2857                                 /* This page is done.  Move to the next. */\r
2858                                 i++;\r
2859                         }\r
2860                         /* Done handling the ethernet header. */\r
2861                         hdr_done = TRUE;\r
2862                 }\r
2863 \r
2864                 /* Finish this MDL */\r
2865                 while( buf_len )\r
2866                 {\r
2867                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2868                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2869                         /* Add the first page's offset if we're on the first page. */\r
2870                         if( i == 0 )\r
2871                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2872 \r
2873                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2874                         {\r
2875                                 /* Buffers spans pages. */\r
2876                                 p_desc->local_ds[j].length =\r
2877                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2878                                 buf_len -= p_desc->local_ds[j].length;\r
2879                                 /* This page is done.  Move to the next. */\r
2880                                 i++;\r
2881                         }\r
2882                         else\r
2883                         {\r
2884                                 /* Last page of the buffer. */\r
2885                                 p_desc->local_ds[j].length = buf_len;\r
2886                                 buf_len = 0;\r
2887                         }\r
2888                         /* This data segment is done.  Move to the next. */\r
2889                         j++;\r
2890                 }\r
2891 \r
2892                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2893                 if( !tot_len )\r
2894                         break;\r
2895 \r
2896                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2897                 if( !p_mdl )\r
2898                 {\r
2899                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2900                                 ("Failed to get next buffer.\n") );\r
2901                         return IB_ERROR;\r
2902                 }\r
2903         }\r
2904 \r
2905         /* Set the number of data segments. */\r
2906         p_desc->wr.num_ds = j;\r
2907 \r
2908         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2909         return IB_SUCCESS;\r
2910 }\r
2911 \r
2912 #else\r
2913 \r
2914 static NDIS_STATUS\r
2915 __send_gen(\r
2916         IN                              ipoib_port_t* const                     p_port,\r
2917         IN                              ipoib_send_desc_t* const        p_desc )\r
2918 {\r
2919         ib_api_status_t                 status;\r
2920         SCATTER_GATHER_LIST             *p_sgl;\r
2921         uint32_t                                i, j = 1;\r
2922         uint32_t                                offset = sizeof(eth_hdr_t);\r
2923         PERF_DECLARE( SendCopy );\r
2924 \r
2925         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2926 \r
2927         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2928                 ScatterGatherListPacketInfo );\r
2929         if( !p_sgl )\r
2930         {\r
2931                 ASSERT( p_sgl );\r
2932                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2933                         ("Failed to get SGL from packet.\n") );\r
2934                 return NDIS_STATUS_FAILURE;\r
2935         }\r
2936 \r
2937         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2938         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2939                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2940                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2941                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2942         {\r
2943                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2944                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2945                 cl_perf_start( SendCopy );\r
2946                 status = __send_copy( p_port, p_desc );\r
2947                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2948                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2949                 return status;\r
2950         }\r
2951 \r
2952         /*\r
2953          * Skip the ethernet header.  It is either the first element,\r
2954          * or part of it.\r
2955          */\r
2956         i = 0;\r
2957         while( offset )\r
2958         {\r
2959                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2960                 {\r
2961                         offset -= p_sgl->Elements[i++].Length;\r
2962                 }\r
2963                 else\r
2964                 {\r
2965                         p_desc->local_ds[j].vaddr =\r
2966                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2967                         p_desc->local_ds[j].length =\r
2968                                 p_sgl->Elements[i].Length - offset;\r
2969                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2970                         i++;\r
2971                         j++;\r
2972                         break;\r
2973                 }\r
2974         }\r
2975         /* Now fill in the rest of the local data segments. */\r
2976         while( i < p_sgl->NumberOfElements )\r
2977         {\r
2978                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2979                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2980                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2981                 i++;\r
2982                 j++;\r
2983         }\r
2984 \r
2985         /* Set the number of data segments. */\r
2986         p_desc->wr.num_ds = j;\r
2987 \r
2988         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2989         return NDIS_STATUS_SUCCESS;\r
2990 }\r
2991 #endif\r
2992 \r
2993 \r
2994 static NDIS_STATUS\r
2995 __send_mgr_filter_ip(\r
2996         IN                              ipoib_port_t* const                     p_port,\r
2997         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2998         IN                              NDIS_BUFFER*                            p_buf,\r
2999         IN                              size_t                                          buf_len,\r
3000         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
3001 {\r
3002         NDIS_STATUS             status;\r
3003         ip_hdr_t                *p_ip_hdr;\r
3004 \r
3005         PERF_DECLARE( QueryIp );\r
3006         PERF_DECLARE( SendTcp );\r
3007         PERF_DECLARE( FilterUdp );\r
3008 \r
3009         IPOIB_ENTER( IPOIB_DBG_SEND );\r
3010 \r
3011         if( !buf_len )\r
3012         {\r
3013                 cl_perf_start( QueryIp );\r
3014                 NdisGetNextBuffer( p_buf, &p_buf );\r
3015                 if( !p_buf )\r
3016                 {\r
<