[IPoIB] Prevent reposting receives when QP may be in reset state.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 \r
45 /* Amount of physical memory to register. */\r
46 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
47 \r
48 /* Number of work completions to chain for send and receive polling. */\r
49 #define MAX_SEND_WC             8\r
50 #define MAX_RECV_WC             16\r
51 \r
52 \r
53 ib_gid_t        bcast_mgid_template = {\r
54         0xff,                                                           /* multicast field */\r
55         0x12,                                                           /* scope (to be filled in) */\r
56         0x40, 0x1b,                                                     /* IPv4 signature */\r
57         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
58         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
59         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
60 };\r
61 \r
62 \r
63 #ifdef _DEBUG_\r
64 /* Handy pointer for debug use. */\r
65 ipoib_port_t    *gp_ipoib_port;\r
66 #endif\r
67 \r
68 \r
69 /******************************************************************************\r
70 *\r
71 * Declarations\r
72 *\r
73 ******************************************************************************/\r
74 static void\r
75 __port_construct(\r
76         IN                              ipoib_port_t* const                     p_port );\r
77 \r
78 static ib_api_status_t\r
79 __port_init(\r
80         IN                              ipoib_port_t* const                     p_port,\r
81         IN                              ipoib_adapter_t* const          p_adapter,\r
82         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
83 \r
84 static void\r
85 __port_destroying(\r
86         IN                              cl_obj_t* const                         p_obj );\r
87 \r
88 static void\r
89 __port_cleanup(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_free(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 \r
97 /******************************************************************************\r
98 *\r
99 * IB resource manager operations\r
100 *\r
101 ******************************************************************************/\r
102 static void\r
103 __ib_mgr_construct(\r
104         IN                              ipoib_port_t* const                     p_port );\r
105 \r
106 static ib_api_status_t\r
107 __ib_mgr_init(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static void\r
111 __ib_mgr_destroy(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __qp_event(\r
116         IN                              ib_async_event_rec_t            *p_event_rec );\r
117 \r
118 static void\r
119 __cq_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static ib_api_status_t\r
123 __ib_mgr_activate(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 /******************************************************************************\r
127 *\r
128 * Buffer manager operations.\r
129 *\r
130 ******************************************************************************/\r
131 static void\r
132 __buf_mgr_construct(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 static ib_api_status_t\r
136 __buf_mgr_init(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static void\r
140 __buf_mgr_destroy(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static cl_status_t\r
144 __recv_ctor(\r
145         IN                              void* const                                     p_object,\r
146         IN                              void*                                           context,\r
147                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
148 \r
149 #if !IPOIB_INLINE_RECV\r
150 static void\r
151 __recv_dtor(\r
152         IN              const   cl_pool_item_t* const           p_pool_item,\r
153         IN                              void                                            *context );\r
154 #endif  /* IPOIB_INLINE_RECV */\r
155 \r
156 static inline ipoib_send_desc_t*\r
157 __buf_mgr_get_send(\r
158         IN                              ipoib_port_t* const                     p_port );\r
159 \r
160 static inline void\r
161 __buf_mgr_put_send(\r
162         IN                              ipoib_port_t* const                     p_port,\r
163         IN                              ipoib_send_desc_t* const        p_desc );\r
164 \r
165 static inline ipoib_recv_desc_t*\r
166 __buf_mgr_get_recv(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_recv(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_recv_desc_t* const        p_desc,\r
173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
174 \r
175 static inline void\r
176 __buf_mgr_put_recv_list(\r
177         IN                              ipoib_port_t* const                     p_port,\r
178         IN                              cl_qlist_t* const                       p_list );\r
179 \r
180 static inline NDIS_PACKET*\r
181 __buf_mgr_get_ndis_pkt(\r
182         IN                              ipoib_port_t* const                     p_port,\r
183         IN                              ipoib_recv_desc_t* const        p_desc );\r
184 \r
185 \r
186 /******************************************************************************\r
187 *\r
188 * Receive manager operations.\r
189 *\r
190 ******************************************************************************/\r
191 static void\r
192 __recv_mgr_construct(\r
193         IN                              ipoib_port_t* const                     p_port );\r
194 \r
195 static ib_api_status_t\r
196 __recv_mgr_init(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static void\r
200 __recv_mgr_destroy(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 /* Posts receive buffers to the receive queue. */\r
204 static ib_api_status_t\r
205 __recv_mgr_repost(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_cb(\r
210         IN              const   ib_cq_handle_t                          h_cq,\r
211         IN                              void                                            *cq_context );\r
212 \r
213 static void\r
214 __recv_get_endpts(\r
215         IN                              ipoib_port_t* const                     p_port,\r
216         IN                              ipoib_recv_desc_t* const        p_desc,\r
217         IN                              ib_wc_t* const                          p_wc,\r
218                 OUT                     ipoib_endpt_t** const           pp_src,\r
219                 OUT                     ipoib_endpt_t** const           pp_dst );\r
220 \r
221 static uint32_t\r
222 __recv_mgr_filter(\r
223         IN                              ipoib_port_t* const                     p_port,\r
224         IN                              ib_wc_t* const                          p_done_wc_list,\r
225                 OUT                     cl_qlist_t* const                       p_done_list,\r
226                 OUT                     cl_qlist_t* const                       p_bad_list );\r
227 \r
228 static ib_api_status_t\r
229 __recv_gen(\r
230         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
231                 OUT                     eth_pkt_t* const                        p_eth,\r
232         IN                              ipoib_endpt_t* const            p_src,\r
233         IN                              ipoib_endpt_t* const            p_dst );\r
234 \r
235 static ib_api_status_t\r
236 __recv_dhcp(\r
237         IN                              ipoib_port_t* const                     p_port,\r
238         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
239                 OUT                     eth_pkt_t* const                        p_eth,\r
240         IN                              ipoib_endpt_t* const            p_src,\r
241         IN                              ipoib_endpt_t* const            p_dst );\r
242 \r
243 static ib_api_status_t\r
244 __recv_arp(\r
245         IN                              ipoib_port_t* const                     p_port,\r
246         IN                              ib_wc_t* const                          p_wc,\r
247         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
248                 OUT                     eth_pkt_t* const                        p_eth,\r
249         IN                              ipoib_endpt_t** const           p_src,\r
250         IN                              ipoib_endpt_t* const            p_dst );\r
251 \r
252 static ib_api_status_t\r
253 __recv_mgr_prepare_pkt(\r
254         IN                              ipoib_port_t* const                     p_port,\r
255         IN                              ipoib_recv_desc_t* const        p_desc,\r
256                 OUT                     NDIS_PACKET** const                     pp_packet );\r
257 \r
258 static uint32_t\r
259 __recv_mgr_build_pkt_array(\r
260         IN                              ipoib_port_t* const                     p_port,\r
261                 OUT                     cl_qlist_t* const                       p_done_list );\r
262 \r
263 /******************************************************************************\r
264 *\r
265 * Send manager operations.\r
266 *\r
267 ******************************************************************************/\r
268 static void\r
269 __send_mgr_construct(\r
270         IN                              ipoib_port_t* const                     p_port );\r
271 \r
272 static void\r
273 __send_mgr_destroy(\r
274         IN                              ipoib_port_t* const                     p_port );\r
275 \r
276 static NDIS_STATUS\r
277 __send_gen(\r
278         IN                              ipoib_port_t* const                     p_port,\r
279         IN                              ipoib_send_desc_t* const        p_desc );\r
280 \r
281 static NDIS_STATUS\r
282 __send_mgr_filter_ip(\r
283         IN                              ipoib_port_t* const                     p_port,\r
284         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
285         IN                              NDIS_BUFFER*                            p_buf,\r
286         IN                              size_t                                          buf_len,\r
287         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
288 \r
289 static NDIS_STATUS\r
290 __send_mgr_filter_udp(\r
291         IN                              ipoib_port_t* const                     p_port,\r
292         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
293         IN                              NDIS_BUFFER*                            p_buf,\r
294         IN                              size_t                                          buf_len,\r
295         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
296 \r
297 static NDIS_STATUS\r
298 __send_mgr_filter_dhcp(\r
299         IN                              ipoib_port_t* const                     p_port,\r
300         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
301         IN                              NDIS_BUFFER*                            p_buf,\r
302         IN                              size_t                                          buf_len,\r
303         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
304 \r
305 static NDIS_STATUS\r
306 __send_mgr_filter_arp(\r
307         IN                              ipoib_port_t* const                     p_port,\r
308         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
309         IN                              NDIS_BUFFER*                            p_buf,\r
310         IN                              size_t                                          buf_len,\r
311         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
312 \r
313 static void\r
314 __process_failed_send(\r
315         IN                              ipoib_port_t* const                     p_port,\r
316         IN                              ipoib_send_desc_t* const        p_desc,\r
317         IN              const   NDIS_STATUS                                     status );\r
318 \r
319 static void\r
320 __send_cb(\r
321         IN              const   ib_cq_handle_t                          h_cq,\r
322         IN                              void                                            *cq_context );\r
323 \r
324 \r
325 /******************************************************************************\r
326 *\r
327 * Endpoint manager operations\r
328 *\r
329 ******************************************************************************/\r
330 static void\r
331 __endpt_mgr_construct(\r
332         IN                              ipoib_port_t* const                     p_port );\r
333 \r
334 static ib_api_status_t\r
335 __endpt_mgr_init(\r
336         IN                              ipoib_port_t* const                     p_port );\r
337 \r
338 static void\r
339 __endpt_mgr_destroy(\r
340         IN                              ipoib_port_t* const                     p_port );\r
341 \r
342 /****f* IPoIB/__endpt_mgr_remove_all\r
343 * NAME\r
344 *       __endpt_mgr_remove_all\r
345 *\r
346 * DESCRIPTION\r
347 *       Removes all enpoints from the port, dereferencing them to initiate\r
348 *       destruction.\r
349 *\r
350 * SYNOPSIS\r
351 */\r
352 static void\r
353 __endpt_mgr_remove_all(\r
354         IN                              ipoib_port_t* const                     p_port );\r
355 /*\r
356 ********/\r
357 \r
358 static void\r
359 __endpt_mgr_remove(\r
360         IN                              ipoib_port_t* const                     p_port,\r
361         IN                              ipoib_endpt_t* const            p_endpt );\r
362 \r
363 static void\r
364 __endpt_mgr_reset_all(\r
365         IN                              ipoib_port_t* const                     p_port );\r
366 \r
367 static inline NDIS_STATUS\r
368 __endpt_mgr_ref(\r
369         IN                              ipoib_port_t* const                     p_port,\r
370         IN              const   mac_addr_t                                      mac,\r
371                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
372 \r
373 static inline NDIS_STATUS\r
374 __endpt_mgr_get_gid_qpn(\r
375         IN                              ipoib_port_t* const                     p_port,\r
376         IN              const   mac_addr_t                                      mac,\r
377                 OUT                     ib_gid_t* const                         p_gid,\r
378                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
379 \r
380 static inline ipoib_endpt_t*\r
381 __endpt_mgr_get_by_gid(\r
382         IN                              ipoib_port_t* const                     p_port,\r
383         IN              const   ib_gid_t* const                         p_gid );\r
384 \r
385 static inline ipoib_endpt_t*\r
386 __endpt_mgr_get_by_lid(\r
387         IN                              ipoib_port_t* const                     p_port,\r
388         IN              const   net16_t                                         lid );\r
389 \r
390 static inline void\r
391 __endpt_mgr_insert_locked(\r
392         IN                              ipoib_port_t* const                     p_port,\r
393         IN              const   mac_addr_t                                      mac,\r
394         IN                              ipoib_endpt_t* const            p_endpt );\r
395 \r
396 static inline void\r
397 __endpt_mgr_insert(\r
398         IN                              ipoib_port_t* const                     p_port,\r
399         IN              const   mac_addr_t                                      mac,\r
400         IN                              ipoib_endpt_t* const            p_endpt );\r
401 \r
402 static ib_api_status_t\r
403 __endpt_mgr_add_local(\r
404         IN                              ipoib_port_t* const                     p_port,\r
405         IN                              ib_port_info_t* const           p_port_info );\r
406 \r
407 static ib_api_status_t\r
408 __endpt_mgr_add_bcast(\r
409         IN                              ipoib_port_t* const                     p_port,\r
410         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
411 \r
412 /******************************************************************************\r
413 *\r
414 * MCast operations.\r
415 *\r
416 ******************************************************************************/\r
417 static ib_api_status_t\r
418 __port_get_bcast(\r
419         IN                              ipoib_port_t* const                     p_port );\r
420 \r
421 static ib_api_status_t\r
422 __port_join_bcast(\r
423         IN                              ipoib_port_t* const                     p_port,\r
424         IN                              ib_member_rec_t* const          p_member_rec );\r
425 \r
426 static ib_api_status_t\r
427 __port_create_bcast(\r
428         IN                              ipoib_port_t* const                     p_port );\r
429 \r
430 static void\r
431 __port_info_cb(\r
432         IN                              ib_query_rec_t                          *p_query_rec );\r
433 \r
434 \r
435 static void\r
436 __bcast_get_cb(\r
437         IN                              ib_query_rec_t                          *p_query_rec );\r
438 \r
439 \r
440 static void\r
441 __bcast_cb(\r
442         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
443 \r
444 \r
445 static void\r
446 __mcast_cb(\r
447         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
448 \r
449 \r
450 static intn_t\r
451 __gid_cmp(\r
452         IN              const   void* const                                     p_key1,\r
453         IN              const   void* const                                     p_key2 )\r
454 {\r
455         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
456 }\r
457 \r
458 \r
459 /******************************************************************************\r
460 *\r
461 * Implementation\r
462 *\r
463 ******************************************************************************/\r
464 ib_api_status_t\r
465 ipoib_create_port(\r
466         IN                              ipoib_adapter_t* const          p_adapter,\r
467         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
468                 OUT                     ipoib_port_t** const            pp_port )\r
469 {\r
470         ib_api_status_t         status;\r
471         ipoib_port_t            *p_port;\r
472 \r
473         IPOIB_ENTER( IPOIB_DBG_INIT );\r
474 \r
475         CL_ASSERT( !p_adapter->p_port );\r
476 \r
477         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
478                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
479         if( !p_port )\r
480         {\r
481                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
482                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
483                         sizeof(ipoib_port_t)) );\r
484                 return IB_INSUFFICIENT_MEMORY;\r
485         }\r
486 \r
487 #ifdef _DEBUG_\r
488         gp_ipoib_port = p_port;\r
489 #endif\r
490 \r
491         __port_construct( p_port );\r
492 \r
493         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
494         if( status != IB_SUCCESS )\r
495         {\r
496                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
497                         ("ipoib_port_init returned %s.\n", \r
498                         p_adapter->p_ifc->get_err_str( status )) );\r
499                 __port_cleanup( &p_port->obj );\r
500                 __port_free( &p_port->obj );\r
501                 return status;\r
502         }\r
503 \r
504         *pp_port = p_port;\r
505         IPOIB_EXIT( IPOIB_DBG_INIT );\r
506         return IB_SUCCESS;\r
507 }\r
508 \r
509 \r
510 void\r
511 ipoib_port_destroy(\r
512         IN                              ipoib_port_t* const                     p_port )\r
513 {\r
514         IPOIB_ENTER( IPOIB_DBG_INIT );\r
515 \r
516         CL_ASSERT( p_port );\r
517         CL_ASSERT( p_port->p_adapter );\r
518         CL_ASSERT( !p_port->p_adapter->p_port );\r
519 \r
520         cl_obj_destroy( &p_port->obj );\r
521 \r
522         IPOIB_EXIT( IPOIB_DBG_INIT );\r
523 }\r
524 \r
525 \r
526 static void\r
527 __port_construct(\r
528         IN                              ipoib_port_t* const                     p_port )\r
529 {\r
530         IPOIB_ENTER( IPOIB_DBG_INIT );\r
531 \r
532         p_port->state = IB_QPS_RESET;\r
533 \r
534         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
535         cl_spinlock_construct( &p_port->send_lock );\r
536         cl_spinlock_construct( &p_port->recv_lock );\r
537         __ib_mgr_construct( p_port );\r
538         __buf_mgr_construct( p_port );\r
539 \r
540         __recv_mgr_construct( p_port );\r
541         __send_mgr_construct( p_port );\r
542 \r
543         __endpt_mgr_construct( p_port );\r
544 \r
545         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
546 \r
547         IPOIB_EXIT( IPOIB_DBG_INIT );\r
548 }\r
549 \r
550 \r
551 static ib_api_status_t\r
552 __port_init(\r
553         IN                              ipoib_port_t* const                     p_port,\r
554         IN                              ipoib_adapter_t* const          p_adapter,\r
555         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
556 {\r
557         cl_status_t                     cl_status;\r
558         ib_api_status_t         status;\r
559 \r
560         IPOIB_ENTER( IPOIB_DBG_INIT );\r
561 \r
562         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
563         p_port->p_adapter = p_adapter;\r
564 \r
565         cl_status = cl_spinlock_init( &p_port->send_lock );\r
566         if( cl_status != CL_SUCCESS )\r
567         {\r
568                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
569                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
570                 return IB_ERROR;\r
571         }\r
572 \r
573         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
574         if( cl_status != CL_SUCCESS )\r
575         {\r
576                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
577                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
578                 return IB_ERROR;\r
579         }\r
580 \r
581         /* Initialize the IB resource manager. */\r
582         status = __ib_mgr_init( p_port );\r
583         if( status != IB_SUCCESS )\r
584         {\r
585                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
586                         ("__ib_mgr_init returned %s\n", \r
587                         p_adapter->p_ifc->get_err_str( status )) );\r
588                 return status;\r
589         }\r
590 \r
591         /* Initialize the buffer manager. */\r
592         status = __buf_mgr_init( p_port );\r
593         if( status != IB_SUCCESS )\r
594         {\r
595                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
596                         ("__buf_mgr_init returned %s\n", \r
597                         p_adapter->p_ifc->get_err_str( status )) );\r
598                 return status;\r
599         }\r
600 \r
601         /* Initialize the receive manager. */\r
602         status = __recv_mgr_init( p_port );\r
603         if( status != IB_SUCCESS )\r
604         {\r
605                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
606                         ("__recv_mgr_init returned %s\n", \r
607                         p_adapter->p_ifc->get_err_str( status )) );\r
608                 return status;\r
609         }\r
610 \r
611         /* Initialize the endpoint manager. */\r
612         status = __endpt_mgr_init( p_port );\r
613         if( status != IB_SUCCESS )\r
614         {\r
615                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
616                         ("__endpt_mgr_init returned %s\n", \r
617                         p_adapter->p_ifc->get_err_str( status )) );\r
618                 return status;\r
619         }\r
620 \r
621         /* We only ever destroy from the PnP callback thread. */\r
622         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
623                 __port_destroying, __port_cleanup, __port_free );\r
624         if( cl_status != CL_SUCCESS )\r
625         {\r
626                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
627                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
628                 return IB_ERROR;\r
629         }\r
630 \r
631         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
632 \r
633         IPOIB_EXIT( IPOIB_DBG_INIT );\r
634         return IB_SUCCESS;\r
635 }\r
636 \r
637 \r
638 static void\r
639 __port_destroying(\r
640         IN                              cl_obj_t* const                         p_obj )\r
641 {\r
642         ipoib_port_t    *p_port;\r
643 \r
644         IPOIB_ENTER( IPOIB_DBG_INIT );\r
645 \r
646         CL_ASSERT( p_obj );\r
647 \r
648         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
649 \r
650         ipoib_port_down( p_port );\r
651 \r
652         __endpt_mgr_remove_all( p_port );\r
653 \r
654         IPOIB_EXIT( IPOIB_DBG_INIT );\r
655 }\r
656 \r
657 \r
658 static void\r
659 __port_cleanup(\r
660         IN                              cl_obj_t* const                         p_obj )\r
661 {\r
662         ipoib_port_t    *p_port;\r
663 \r
664         IPOIB_ENTER( IPOIB_DBG_INIT );\r
665 \r
666         CL_ASSERT( p_obj );\r
667 \r
668         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
669 \r
670         /* Wait for all sends and receives to get flushed. */\r
671         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
672                 cl_thread_suspend( 0 );\r
673 \r
674         /* Destroy the send and receive managers before closing the CA. */\r
675         __ib_mgr_destroy( p_port );\r
676 \r
677         IPOIB_EXIT( IPOIB_DBG_INIT );\r
678 }\r
679 \r
680 \r
681 static void\r
682 __port_free(\r
683         IN                              cl_obj_t* const                         p_obj )\r
684 {\r
685         ipoib_port_t    *p_port;\r
686 \r
687         IPOIB_ENTER( IPOIB_DBG_INIT );\r
688 \r
689         CL_ASSERT( p_obj );\r
690 \r
691         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
692 \r
693         __endpt_mgr_destroy( p_port );\r
694         __recv_mgr_destroy( p_port );\r
695         __send_mgr_destroy( p_port );\r
696         __buf_mgr_destroy( p_port );\r
697 \r
698         cl_spinlock_destroy( &p_port->send_lock );\r
699         cl_spinlock_destroy( &p_port->recv_lock );\r
700 \r
701         cl_obj_deinit( p_obj );\r
702 \r
703         cl_free( p_port );\r
704 \r
705         IPOIB_EXIT( IPOIB_DBG_INIT );\r
706 }\r
707 \r
708 \r
709 \r
710 /******************************************************************************\r
711 *\r
712 * IB resource manager implementation.\r
713 *\r
714 ******************************************************************************/\r
715 static void\r
716 __ib_mgr_construct(\r
717         IN                              ipoib_port_t* const                     p_port )\r
718 {\r
719         IPOIB_ENTER( IPOIB_DBG_INIT );\r
720 \r
721         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
722 \r
723         IPOIB_EXIT( IPOIB_DBG_INIT );\r
724 }\r
725 \r
726 \r
727 static ib_api_status_t\r
728 __ib_mgr_init(\r
729         IN                              ipoib_port_t* const                     p_port )\r
730 {\r
731         ib_api_status_t         status;\r
732         ib_cq_create_t          cq_create;\r
733         ib_qp_create_t          qp_create;\r
734         ib_phys_create_t        phys_create;\r
735         ib_phys_range_t         phys_range;\r
736         uint64_t                        vaddr;\r
737         net32_t                         rkey;\r
738         ib_qp_attr_t            qp_attr;\r
739 \r
740         IPOIB_ENTER( IPOIB_DBG_INIT );\r
741 \r
742         /* Open the CA. */\r
743         status = p_port->p_adapter->p_ifc->open_ca(\r
744                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
745                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
746         if( status != IB_SUCCESS )\r
747         {\r
748                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
749                         EVENT_IPOIB_OPEN_CA, 1, status );\r
750                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
751                         ("ib_open_ca returned %s\n", \r
752                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
753                 return status;\r
754         }\r
755 \r
756         /* Allocate the PD. */\r
757         status = p_port->p_adapter->p_ifc->alloc_pd(\r
758                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
759         if( status != IB_SUCCESS )\r
760         {\r
761                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
762                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
763                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
764                         ("ib_alloc_pd returned %s\n", \r
765                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
766                 return status;\r
767         }\r
768 \r
769         /* Allocate receive CQ. */\r
770         cq_create.size = p_port->p_adapter->params.rq_depth;\r
771         cq_create.pfn_comp_cb = __recv_cb;\r
772         cq_create.h_wait_obj = NULL;\r
773 \r
774         status = p_port->p_adapter->p_ifc->create_cq(\r
775                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
776                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
777         if( status != IB_SUCCESS )\r
778         {\r
779                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
780                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
781                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
782                         ("ib_create_cq returned %s.\n", \r
783                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
784                 return status;\r
785         }\r
786 \r
787         /* Allocate send CQ. */\r
788         cq_create.size = p_port->p_adapter->params.sq_depth;\r
789         cq_create.pfn_comp_cb = __send_cb;\r
790 \r
791         status = p_port->p_adapter->p_ifc->create_cq(\r
792                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
793                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
794         if( status != IB_SUCCESS )\r
795         {\r
796                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
797                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
798                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
799                         ("ib_create_cq returned %s.\n", \r
800                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
801                 return status;\r
802         }\r
803 \r
804         /* Allocate the QP. */\r
805         cl_memclr( &qp_create, sizeof(qp_create) );\r
806         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
807         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
808         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
809         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
810         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
811         //TODO: Figure out the right number of SGE entries for sends.\r
812         qp_create.sq_sge = MAX_SEND_SGE;\r
813         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
814         qp_create.sq_signaled = TRUE;\r
815         status = p_port->p_adapter->p_ifc->create_qp(\r
816                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
817                 __qp_event, &p_port->ib_mgr.h_qp );\r
818         if( status != IB_SUCCESS )\r
819         {\r
820                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
821                         EVENT_IPOIB_CREATE_QP, 1, status );\r
822                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
823                         ("ib_create_qp returned %s\n", \r
824                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
825                 return status;\r
826         }\r
827         /* Query the QP so we can get our QPN. */\r
828         status = p_port->p_adapter->p_ifc->query_qp(\r
829                 p_port->ib_mgr.h_qp, &qp_attr );\r
830         if( status != IB_SUCCESS )\r
831         {\r
832                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
833                         EVENT_IPOIB_QUERY_QP, 1, status );\r
834                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
835                         ("ib_query_qp returned %s\n", \r
836                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
837                 return status;\r
838         }\r
839         p_port->ib_mgr.qpn = qp_attr.num;\r
840 \r
841         /* Register all of physical memory */\r
842         phys_create.length = MEM_REG_SIZE;\r
843         phys_create.num_ranges = 1;\r
844         phys_create.range_array = &phys_range;\r
845         phys_create.buf_offset = 0;\r
846         phys_create.hca_page_size = PAGE_SIZE;\r
847         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
848         phys_range.base_addr = 0;\r
849         phys_range.size = MEM_REG_SIZE;\r
850         vaddr = 0;\r
851         status = p_port->p_adapter->p_ifc->reg_phys(\r
852                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
853                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
854         if( status != IB_SUCCESS )\r
855         {\r
856                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
857                         EVENT_IPOIB_REG_PHYS, 1, status );\r
858                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
859                         ("ib_reg_phys returned %s\n", \r
860                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
861                 return status;\r
862         }\r
863 \r
864         IPOIB_EXIT( IPOIB_DBG_INIT );\r
865         return IB_SUCCESS;\r
866 }\r
867 \r
868 \r
869 static void\r
870 __ib_mgr_destroy(\r
871         IN                              ipoib_port_t* const                     p_port )\r
872 {\r
873         ib_api_status_t status;\r
874 \r
875         IPOIB_ENTER( IPOIB_DBG_INIT );\r
876 \r
877         if( p_port->ib_mgr.h_ca )\r
878         {\r
879                 status =\r
880                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
881                 CL_ASSERT( status == IB_SUCCESS );\r
882                 p_port->ib_mgr.h_ca = NULL;\r
883         }\r
884 \r
885         IPOIB_EXIT( IPOIB_DBG_INIT );\r
886 }\r
887 \r
888 \r
889 \r
890 /******************************************************************************\r
891 *\r
892 * Buffer manager implementation.\r
893 *\r
894 ******************************************************************************/\r
895 static void\r
896 __buf_mgr_construct(\r
897         IN                              ipoib_port_t* const                     p_port )\r
898 {\r
899         IPOIB_ENTER( IPOIB_DBG_INIT );\r
900 \r
901         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
902 \r
903         p_port->buf_mgr.h_packet_pool = NULL;\r
904         p_port->buf_mgr.h_buffer_pool = NULL;\r
905 \r
906         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
907                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
908 \r
909         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
910         p_port->buf_mgr.h_send_buf_pool = NULL;\r
911 \r
912         IPOIB_EXIT( IPOIB_DBG_INIT );\r
913 }\r
914 \r
915 \r
916 static ib_api_status_t\r
917 __buf_mgr_init(\r
918         IN                              ipoib_port_t* const                     p_port )\r
919 {\r
920         cl_status_t             cl_status;\r
921         NDIS_STATUS             ndis_status;\r
922         ipoib_params_t  *p_params;\r
923 \r
924         IPOIB_ENTER(IPOIB_DBG_INIT );\r
925 \r
926         CL_ASSERT( p_port );\r
927         CL_ASSERT( p_port->p_adapter );\r
928 \r
929         p_params = &p_port->p_adapter->params;\r
930 \r
931         /* Allocate the receive descriptor pool */\r
932         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
933                 p_params->rq_depth * p_params->recv_pool_ratio,\r
934 #if IPOIB_INLINE_RECV\r
935                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
936 #else   /* IPOIB_INLINE_RECV */\r
937                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
938 #endif  /* IPOIB_INLINE_RECV */\r
939         if( cl_status != CL_SUCCESS )\r
940         {\r
941                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
942                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
943                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
944                         ("cl_qpool_init for recvs returned %s\n",\r
945                         cl_status_text[cl_status]) );\r
946                 return IB_INSUFFICIENT_MEMORY;\r
947         }\r
948 \r
949         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
950         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
951                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
952         if( ndis_status != NDIS_STATUS_SUCCESS )\r
953         {\r
954                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
955                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
956                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
957                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
958                 return IB_INSUFFICIENT_RESOURCES;\r
959         }\r
960 \r
961         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
962                 p_params->rq_depth );\r
963         if( ndis_status != NDIS_STATUS_SUCCESS )\r
964         {\r
965                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
966                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
967                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
968                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
969                 return IB_INSUFFICIENT_RESOURCES;\r
970         }\r
971 \r
972         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
973         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
974                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
975         if( ndis_status != NDIS_STATUS_SUCCESS )\r
976         {\r
977                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
978                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
979                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
980                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
981                 return IB_INSUFFICIENT_RESOURCES;\r
982         }\r
983 \r
984         NdisAllocateBufferPool( &ndis_status,\r
985                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
986         if( ndis_status != NDIS_STATUS_SUCCESS )\r
987         {\r
988                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
989                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
990                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
991                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
992                 return IB_INSUFFICIENT_RESOURCES;\r
993         }\r
994 \r
995         IPOIB_EXIT( IPOIB_DBG_INIT );\r
996         return IB_SUCCESS;\r
997 }\r
998 \r
999 \r
1000 static void\r
1001 __buf_mgr_destroy(\r
1002         IN                              ipoib_port_t* const                     p_port )\r
1003 {\r
1004         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1005 \r
1006         CL_ASSERT( p_port );\r
1007 \r
1008         /* Destroy the send packet and buffer pools. */\r
1009         if( p_port->buf_mgr.h_send_buf_pool )\r
1010                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1011         if( p_port->buf_mgr.h_send_pkt_pool )\r
1012                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1013 \r
1014         /* Destroy the receive packet and buffer pools. */\r
1015         if( p_port->buf_mgr.h_buffer_pool )\r
1016                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1017         if( p_port->buf_mgr.h_packet_pool )\r
1018                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1019 \r
1020         /* Free the receive and send descriptors. */\r
1021         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1022 \r
1023         /* Free the lookaside list of scratch buffers. */\r
1024         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1025 \r
1026         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1027 }\r
1028 \r
1029 \r
1030 static cl_status_t\r
1031 __recv_ctor(\r
1032         IN                              void* const                                     p_object,\r
1033         IN                              void*                                           context,\r
1034                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1035 {\r
1036         ipoib_recv_desc_t       *p_desc;\r
1037         ipoib_port_t            *p_port;\r
1038         uint32_t                        ds0_len;\r
1039 \r
1040         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1041 \r
1042         CL_ASSERT( p_object );\r
1043         CL_ASSERT( context );\r
1044 \r
1045         p_desc = (ipoib_recv_desc_t*)p_object;\r
1046         p_port = (ipoib_port_t*)context;\r
1047 \r
1048         /* Setup the work request. */\r
1049         p_desc->wr.ds_array = p_desc->local_ds;\r
1050         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1051 \r
1052 #if IPOIB_INLINE_RECV\r
1053         /* Sanity check on the receive buffer layout */\r
1054         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1055                 (void*)&p_desc->buf.ib.pkt.type );\r
1056         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1057 \r
1058         /* Setup the local data segment. */\r
1059         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1060         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1061         ds0_len =\r
1062                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1063         if( ds0_len >= sizeof(recv_buf_t) )\r
1064         {\r
1065                 /* The whole buffer is within a page. */\r
1066                 p_desc->local_ds[0].length = ds0_len;\r
1067                 p_desc->wr.num_ds = 1;\r
1068         }\r
1069         else\r
1070         {\r
1071                 /* The buffer crosses page boundaries. */\r
1072                 p_desc->local_ds[0].length = ds0_len;\r
1073                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1074                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1075                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1076                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1077                 p_desc->wr.num_ds = 2;\r
1078         }\r
1079 #else   /* IPOIB_INLINE_RECV */\r
1080         /* Allocate the receive buffer. */\r
1081         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1082         if( !p_desc->p_buf )\r
1083         {\r
1084                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1085                         ("Failed to allocate receive buffer.\n") );\r
1086                 return CL_INSUFFICIENT_MEMORY;\r
1087         }\r
1088 \r
1089         /* Sanity check on the receive buffer layout */\r
1090         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1091                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1092 \r
1093         /* Setup the local data segment. */\r
1094         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1095         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1096         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1097 #endif  /* IPOIB_INLINE_RECV */\r
1098 \r
1099         *pp_pool_item = &p_desc->item;\r
1100 \r
1101         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1102         return CL_SUCCESS;\r
1103 }\r
1104 \r
1105 \r
1106 #if !IPOIB_INLINE_RECV\r
1107 static void\r
1108 __recv_dtor(\r
1109         IN              const   cl_pool_item_t* const           p_pool_item,\r
1110         IN                              void                                            *context )\r
1111 {\r
1112         ipoib_recv_desc_t       *p_desc;\r
1113 \r
1114         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1115 \r
1116         UNUSED_PARAM( context );\r
1117 \r
1118         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1119 \r
1120         if( p_desc->p_buf )\r
1121                 cl_free( p_desc->p_buf );\r
1122 \r
1123         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1124 }\r
1125 #endif\r
1126 \r
1127 \r
1128 static inline ipoib_recv_desc_t*\r
1129 __buf_mgr_get_recv(\r
1130         IN                              ipoib_port_t* const                     p_port )\r
1131 {\r
1132         ipoib_recv_desc_t       *p_desc;\r
1133         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1134         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1135         /* Reference the port object for the send. */\r
1136         if( p_desc )\r
1137         {\r
1138                 cl_obj_ref( &p_port->obj );\r
1139                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1140 #if IPOIB_INLINE_RECV\r
1141                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1142                         cl_get_physaddr( &p_desc->buf ) );\r
1143 #else   /* IPOIB_INLINE_RECV */\r
1144                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1145                         cl_get_physaddr( p_desc->p_buf ) );\r
1146                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1147                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1148 #endif  /* IPOIB_INLINE_RECV */\r
1149                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1150         }\r
1151         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1152         return p_desc;\r
1153 }\r
1154 \r
1155 \r
1156 static inline void\r
1157 __buf_mgr_put_recv(\r
1158         IN                              ipoib_port_t* const                     p_port,\r
1159         IN                              ipoib_recv_desc_t* const        p_desc,\r
1160         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1161 {\r
1162         NDIS_BUFFER             *p_buf;\r
1163 \r
1164         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1165 \r
1166         if( p_packet )\r
1167         {\r
1168                 /* Unchain the NDIS buffer. */\r
1169                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1170                 CL_ASSERT( p_buf );\r
1171                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1172                 NdisDprFreePacketNonInterlocked( p_packet );\r
1173                 NdisFreeBuffer( p_buf );\r
1174         }\r
1175 \r
1176         /* Return the descriptor to its pools. */\r
1177         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1178 \r
1179         /*\r
1180          * Dereference the port object since the receive is no longer outstanding.\r
1181          */\r
1182         cl_obj_deref( &p_port->obj );\r
1183 \r
1184         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1185 }\r
1186 \r
1187 \r
1188 static inline void\r
1189 __buf_mgr_put_recv_list(\r
1190         IN                              ipoib_port_t* const                     p_port,\r
1191         IN                              cl_qlist_t* const                       p_list )\r
1192 {\r
1193         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1194         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1195         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1196 }\r
1197 \r
1198 \r
1199 static inline NDIS_PACKET*\r
1200 __buf_mgr_get_ndis_pkt(\r
1201         IN                              ipoib_port_t* const                     p_port,\r
1202         IN                              ipoib_recv_desc_t* const        p_desc )\r
1203 {\r
1204         NDIS_STATUS                             status;\r
1205         NDIS_PACKET                             *p_packet;\r
1206         NDIS_BUFFER                             *p_buffer;\r
1207 \r
1208         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1209 \r
1210         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1211                 p_port->buf_mgr.h_packet_pool );\r
1212         if( status != NDIS_STATUS_SUCCESS )\r
1213         {\r
1214                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1215                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1216                 return NULL;\r
1217         }\r
1218 \r
1219         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1220         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1221 \r
1222         NdisAllocateBuffer( &status, &p_buffer,\r
1223 #if IPOIB_INLINE_RECV\r
1224                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1225 #else   /* IPOIB_INLINE_RECV */\r
1226                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1227 #endif  /* IPOIB_INLINE_RECV */\r
1228         if( status != NDIS_STATUS_SUCCESS )\r
1229         {\r
1230                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1231                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1232                 NdisDprFreePacketNonInterlocked( p_packet );\r
1233                 return NULL;\r
1234         }\r
1235 \r
1236         NdisChainBufferAtFront( p_packet, p_buffer );\r
1237         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1238         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1239 \r
1240         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1241         return p_packet;\r
1242 }\r
1243 \r
1244 /******************************************************************************\r
1245 *\r
1246 * Receive manager implementation.\r
1247 *\r
1248 ******************************************************************************/\r
1249 static void\r
1250 __recv_mgr_construct(\r
1251         IN                              ipoib_port_t* const                     p_port )\r
1252 {\r
1253         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1254 \r
1255         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1256 \r
1257         p_port->recv_mgr.recv_pkt_array = NULL;\r
1258 \r
1259         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1260 }\r
1261 \r
1262 \r
1263 static ib_api_status_t\r
1264 __recv_mgr_init(\r
1265         IN                              ipoib_port_t* const                     p_port )\r
1266 {\r
1267         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1268 \r
1269         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1270         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1271                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1272         if( !p_port->recv_mgr.recv_pkt_array )\r
1273         {\r
1274                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1275                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1276                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1277                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1278                 return IB_INSUFFICIENT_MEMORY;\r
1279         }\r
1280 \r
1281         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1282         return IB_SUCCESS;\r
1283 }\r
1284 \r
1285 \r
1286 static void\r
1287 __recv_mgr_destroy(\r
1288         IN                              ipoib_port_t* const                     p_port )\r
1289 {\r
1290         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1291 \r
1292         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1293         CL_ASSERT( !p_port->recv_mgr.depth );\r
1294 \r
1295         if( p_port->recv_mgr.recv_pkt_array )\r
1296                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1297 \r
1298         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1299 }\r
1300 \r
1301 \r
1302 /* Posts receive buffers to the receive queue. */\r
1303 static ib_api_status_t\r
1304 __recv_mgr_repost(\r
1305         IN                              ipoib_port_t* const                     p_port )\r
1306 {\r
1307         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1308         ib_api_status_t         status;\r
1309         ib_recv_wr_t            *p_failed;\r
1310         PERF_DECLARE( GetRecv );\r
1311         PERF_DECLARE( PostRecv );\r
1312 \r
1313         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1314 \r
1315         CL_ASSERT( p_port );\r
1316         cl_obj_lock( &p_port->obj );\r
1317         if( p_port->state != IB_QPS_RTS )\r
1318         {\r
1319                 cl_obj_unlock( &p_port->obj );\r
1320                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1321                         ("Port in invalid state.  Not reposting.\n") );\r
1322                 return IB_SUCCESS;\r
1323         }\r
1324         cl_obj_ref( &p_port->obj );\r
1325         cl_obj_unlock( &p_port->obj );\r
1326 \r
1327         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1328         {\r
1329                 /* Pull receives out of the pool and chain them up. */\r
1330                 cl_perf_start( GetRecv );\r
1331                 p_next = __buf_mgr_get_recv( p_port );\r
1332                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1333                 if( !p_next )\r
1334                 {\r
1335                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1336                                 ("Out of receive descriptors!\n") );\r
1337                         break;\r
1338                 }\r
1339 \r
1340                 if( !p_tail )\r
1341                 {\r
1342                         p_tail = p_next;\r
1343                         p_next->wr.p_next = NULL;\r
1344                 }\r
1345                 else\r
1346                 {\r
1347                         p_next->wr.p_next = &p_head->wr;\r
1348                 }\r
1349 \r
1350                 p_head = p_next;\r
1351 \r
1352                 cl_atomic_inc( &p_port->recv_mgr.depth );\r
1353         }\r
1354 \r
1355         if( !p_head )\r
1356         {\r
1357                 cl_obj_deref( &p_port->obj );\r
1358                 IPOIB_EXIT( IPOIB_DBG_RECV );\r
1359                 return IB_SUCCESS;\r
1360         }\r
1361 \r
1362         cl_perf_start( PostRecv );\r
1363         status = p_port->p_adapter->p_ifc->post_recv(\r
1364                 p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1365         cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1366 \r
1367         /*\r
1368          * If we failed, fix up the work completion list and return those\r
1369          * buffers to the pool\r
1370          */\r
1371         if( status != IB_SUCCESS )\r
1372         {\r
1373                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1374                         ("ip_post_recv returned %s\n", \r
1375                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1376                 /* return the descriptors to the pool */\r
1377                 while( p_failed )\r
1378                 {\r
1379                         p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1380                         p_failed = p_failed->p_next;\r
1381 \r
1382                         __buf_mgr_put_recv( p_port, p_head, NULL );\r
1383                         cl_atomic_dec( &p_port->recv_mgr.depth );\r
1384                 }\r
1385         }\r
1386 \r
1387         cl_obj_deref( &p_port->obj );\r
1388 \r
1389         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1390         return status;\r
1391 }\r
1392 \r
1393 \r
1394 void\r
1395 ipoib_return_packet(\r
1396         IN                              NDIS_HANDLE                                     adapter_context,\r
1397         IN                              NDIS_PACKET                                     *p_packet )\r
1398 {\r
1399         cl_list_item_t          *p_item;\r
1400         ipoib_port_t            *p_port;\r
1401         ipoib_recv_desc_t       *p_desc;\r
1402         ib_api_status_t         status = IB_NOT_DONE;\r
1403         PERF_DECLARE( ReturnPacket );\r
1404         PERF_DECLARE( ReturnPutRecv );\r
1405         PERF_DECLARE( ReturnRepostRecv );\r
1406         PERF_DECLARE( ReturnPreparePkt );\r
1407         PERF_DECLARE( ReturnNdisIndicate );\r
1408 \r
1409         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1410 \r
1411         UNUSED_PARAM( adapter_context );\r
1412         CL_ASSERT( p_packet );\r
1413 \r
1414         cl_perf_start( ReturnPacket );\r
1415 \r
1416         /* Get the manager and descriptor from the packet. */\r
1417         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1418         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1419 \r
1420         cl_spinlock_acquire( &p_port->recv_lock );\r
1421         cl_perf_start( ReturnPutRecv );\r
1422         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1423         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1424 \r
1425         /* Repost buffers. */\r
1426         cl_perf_start( ReturnRepostRecv );\r
1427         __recv_mgr_repost( p_port );\r
1428         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1429 \r
1430         /* Complete any additional receives waiting for a packet. */\r
1431         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1432         do\r
1433         {\r
1434                 if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
1435                 {\r
1436                         cl_spinlock_release( &p_port->recv_lock );\r
1437                         break;\r
1438                 }\r
1439 \r
1440                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1441 \r
1442                 cl_perf_start( ReturnPreparePkt );\r
1443                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1444                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1445                 if( status == IB_SUCCESS )\r
1446                 {\r
1447                         cl_spinlock_release( &p_port->recv_lock );\r
1448                         cl_perf_start( ReturnNdisIndicate );\r
1449                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1450                                 &p_packet, 1 );\r
1451                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1452                 }\r
1453                 else if( status != IB_NOT_DONE )\r
1454                 {\r
1455                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1456                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1457                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1458                         /* Return the item to the head of the list. */\r
1459                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1460                         cl_spinlock_release( &p_port->recv_lock );\r
1461                 }\r
1462                 else\r
1463                 {\r
1464                         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1465                 }\r
1466 \r
1467         } while( status == IB_NOT_DONE );\r
1468         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1469 \r
1470         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1471 }\r
1472 \r
1473 \r
1474 static void\r
1475 __recv_cb(\r
1476         IN              const   ib_cq_handle_t                          h_cq,\r
1477         IN                              void                                            *cq_context )\r
1478 {\r
1479         ipoib_port_t            *p_port;\r
1480         ib_api_status_t         status;\r
1481         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1482         uint32_t                        pkt_cnt, recv_cnt = 0;\r
1483         cl_qlist_t                      done_list, bad_list;\r
1484         size_t                          i;\r
1485         PERF_DECLARE( RecvCompBundle );\r
1486         PERF_DECLARE( RecvCb );\r
1487         PERF_DECLARE( PollRecv );\r
1488         PERF_DECLARE( RepostRecv );\r
1489         PERF_DECLARE( FilterRecv );\r
1490         PERF_DECLARE( BuildPktArray );\r
1491         PERF_DECLARE( RecvNdisIndicate );\r
1492         PERF_DECLARE( RearmRecv );\r
1493         PERF_DECLARE( PutRecvList );\r
1494 \r
1495         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1496 \r
1497         cl_perf_clr( RecvCompBundle );\r
1498 \r
1499         cl_perf_start( RecvCb );\r
1500 \r
1501         UNUSED_PARAM( h_cq );\r
1502 \r
1503         p_port = (ipoib_port_t*)cq_context;\r
1504 \r
1505         cl_qlist_init( &done_list );\r
1506         cl_qlist_init( &bad_list );\r
1507 \r
1508         cl_obj_ref( &p_port->obj );\r
1509 \r
1510         for( i = 0; i < MAX_RECV_WC; i++ )\r
1511                 wc[i].p_next = &wc[i + 1];\r
1512         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1513 \r
1514         /*\r
1515          * We'll be accessing the endpoint map so take a reference\r
1516          * on it to prevent modifications.\r
1517          */\r
1518         cl_obj_lock( &p_port->obj );\r
1519         cl_atomic_inc( &p_port->endpt_rdr );\r
1520         cl_obj_unlock( &p_port->obj );\r
1521 \r
1522         do\r
1523         {\r
1524                 /* If we get here, then the list of WCs is intact. */\r
1525                 p_free = wc;\r
1526 \r
1527                 cl_perf_start( PollRecv );\r
1528                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1529                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1530                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1531                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1532 \r
1533                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1534                 cl_perf_start( FilterRecv );\r
1535                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1536                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1537 \r
1538         } while( !p_free );\r
1539 \r
1540         /* We're done looking at the endpoint map, release the reference. */\r
1541         cl_atomic_dec( &p_port->endpt_rdr );\r
1542 \r
1543         /* Update our posted depth. */\r
1544         cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
1545 \r
1546         cl_perf_start( BuildPktArray );\r
1547         /* Notify NDIS of any and all possible receive buffers. */\r
1548         pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
1549         cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1550 \r
1551         /* Only indicate receives if we actually had any. */\r
1552         if( pkt_cnt )\r
1553         {\r
1554                 cl_perf_start( RecvNdisIndicate );\r
1555                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1556                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1557                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1558         }\r
1559 \r
1560         cl_spinlock_acquire( &p_port->recv_lock );\r
1561 \r
1562         /* Return any discarded receives to the pool */\r
1563         cl_perf_start( PutRecvList );\r
1564         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1565         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1566 \r
1567         /* Repost receives. */\r
1568         cl_perf_start( RepostRecv );\r
1569         __recv_mgr_repost( p_port );\r
1570         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1571 \r
1572         cl_spinlock_release( &p_port->recv_lock );\r
1573 \r
1574         /*\r
1575          * Rearm after filtering to prevent contention on the enpoint maps\r
1576          * and eliminate the possibility of having a call to\r
1577          * __endpt_mgr_insert find a duplicate.\r
1578          */\r
1579         cl_perf_start( RearmRecv );\r
1580         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1581                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1582         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1583         CL_ASSERT( status == IB_SUCCESS );\r
1584 \r
1585         cl_obj_deref( &p_port->obj );\r
1586 \r
1587         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1588         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1589 \r
1590         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1591 }\r
1592 \r
1593 \r
1594 static void\r
1595 __recv_get_endpts(\r
1596         IN                              ipoib_port_t* const                     p_port,\r
1597         IN                              ipoib_recv_desc_t* const        p_desc,\r
1598         IN                              ib_wc_t* const                          p_wc,\r
1599                 OUT                     ipoib_endpt_t** const           pp_src,\r
1600                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1601 {\r
1602         ib_api_status_t         status;\r
1603         mac_addr_t                      mac;\r
1604         PERF_DECLARE( GetEndptByGid );\r
1605         PERF_DECLARE( GetEndptByLid );\r
1606         PERF_DECLARE( EndptInsert );\r
1607 \r
1608         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1609 \r
1610         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1611         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1612         {\r
1613                 /* Lookup the source endpoints based on GID. */\r
1614                 cl_perf_start( GetEndptByGid );\r
1615                 *pp_src =\r
1616 #if IPOIB_INLINE_RECV\r
1617                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1618 #else   /* IPOIB_INLINE_RECV */\r
1619                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1620 #endif  /* IPOIB_INLINE_RECV */\r
1621                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1622 \r
1623                 /*\r
1624                  * Lookup the destination endpoint based on GID.\r
1625                  * This is used along with the packet filter to determine\r
1626                  * whether to report this to NDIS.\r
1627                  */\r
1628                 cl_perf_start( GetEndptByGid );\r
1629                 *pp_dst =\r
1630 #if IPOIB_INLINE_RECV\r
1631                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1632 #else   /* IPOIB_INLINE_RECV */\r
1633                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1634 #endif  /* IPOIB_INLINE_RECV */\r
1635                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1636 \r
1637                 /*\r
1638                  * Create the source endpoint if it does not exist.  Note that we\r
1639                  * can only do this for globally routed traffic since we need the\r
1640                  * information from the GRH to generate the MAC.\r
1641                  */\r
1642                 if( !*pp_src )\r
1643                 {\r
1644                         status = ipoib_mac_from_guid(\r
1645 #if IPOIB_INLINE_RECV\r
1646                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1647 #else   /* IPOIB_INLINE_RECV */\r
1648                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1649 #endif  /* IPOIB_INLINE_RECV */\r
1650                         if( status != IB_SUCCESS )\r
1651                         {\r
1652                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1653                                         ("ipoib_mac_from_guid returned %s\n",\r
1654                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1655                                 return;\r
1656                         }\r
1657 \r
1658                         /* Create the endpoint. */\r
1659 #if IPOIB_INLINE_RECV\r
1660                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1661 #else   /* IPOIB_INLINE_RECV */\r
1662                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1663 #endif  /* IPOIB_INLINE_RECV */\r
1664                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1665                         if( !*pp_src )\r
1666                         {\r
1667                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1668                                         ("ipoib_endpt_create returned %s\n",\r
1669                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1670                                 return;\r
1671                         }\r
1672                         cl_perf_start( EndptInsert );\r
1673                         cl_obj_lock( &p_port->obj );\r
1674                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1675                         cl_obj_unlock( &p_port->obj );\r
1676                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1677                 }\r
1678         }\r
1679         else\r
1680         {\r
1681                 /*\r
1682                  * Lookup the remote endpoint based on LID.  Note that only\r
1683                  * unicast traffic can be LID routed.\r
1684                  */\r
1685                 cl_perf_start( GetEndptByLid );\r
1686                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1687                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1688                 *pp_dst = p_port->p_local_endpt;\r
1689                 CL_ASSERT( *pp_dst );\r
1690         }\r
1691 \r
1692         if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1693         {\r
1694                 /* Update the QPN for the endpoint. */\r
1695                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1696                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1697                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1698                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1699                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1700                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1701         }\r
1702 \r
1703         if( *pp_src && *pp_dst )\r
1704         {\r
1705                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1706                         ("Recv:\n"\r
1707                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1708                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1709                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1710                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1711                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1712                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1713                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1714                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1715         }\r
1716 \r
1717         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1718 }\r
1719 \r
1720 \r
1721 static uint32_t\r
1722 __recv_mgr_filter(\r
1723         IN                              ipoib_port_t* const                     p_port,\r
1724         IN                              ib_wc_t* const                          p_done_wc_list,\r
1725                 OUT                     cl_qlist_t* const                       p_done_list,\r
1726                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1727 {\r
1728         ipoib_recv_desc_t               *p_desc;\r
1729         ib_wc_t                                 *p_wc;\r
1730         ipoib_pkt_t                             *p_ipoib;\r
1731         eth_pkt_t                               *p_eth;\r
1732         ipoib_endpt_t                   *p_src, *p_dst;\r
1733         ib_api_status_t                 status;\r
1734         uint32_t                                len, recv_cnt = 0;\r
1735         PERF_DECLARE( GetRecvEndpts );\r
1736         PERF_DECLARE( RecvGen );\r
1737         PERF_DECLARE( RecvTcp );\r
1738         PERF_DECLARE( RecvUdp );\r
1739         PERF_DECLARE( RecvDhcp );\r
1740         PERF_DECLARE( RecvArp );\r
1741 \r
1742         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1743 \r
1744         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1745         {\r
1746                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1747                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1748                 recv_cnt++;\r
1749 \r
1750                 if( p_wc->status != IB_WCS_SUCCESS )\r
1751                 {\r
1752                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1753                         {\r
1754                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1755                                         ("Failed completion %s\n",\r
1756                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1757                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1758                         }\r
1759                         else\r
1760                         {\r
1761                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1762                                         ("Flushed completion %s\n",\r
1763                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1764                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1765                         }\r
1766                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1767                         /* Dereference the port object on behalf of the failed receive. */\r
1768                         cl_obj_deref( &p_port->obj );\r
1769                         continue;\r
1770                 }\r
1771 \r
1772                 len = p_wc->length - sizeof(ib_grh_t);\r
1773 \r
1774                 if( len < sizeof(ipoib_hdr_t) )\r
1775                 {\r
1776                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1777                                 ("Received ETH packet < min size\n") );\r
1778                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1779                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1780                         cl_obj_deref( &p_port->obj );\r
1781                         continue;\r
1782                 }\r
1783 \r
1784                 /* Successful completion.  Get the receive information. */\r
1785                 cl_perf_start( GetRecvEndpts );\r
1786                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1787                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1788 \r
1789 #if IPOIB_INLINE_RECV\r
1790                 p_ipoib = &p_desc->buf.ib.pkt;\r
1791                 p_eth = &p_desc->buf.eth.pkt;\r
1792 #else   /* IPOIB_INLINE_RECV */\r
1793                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1794                 p_eth = &p_desc->p_buf->eth.pkt;\r
1795 #endif  /*IPOIB_INLINE_RECV */\r
1796 \r
1797                 if( p_src )\r
1798                 {\r
1799                         /* Don't report loopback traffic - we requested SW loopback. */\r
1800                         if( !cl_memcmp( &p_port->p_adapter->mac, &p_src->mac,\r
1801                                 sizeof(p_port->p_adapter->mac) ) )\r
1802                         {\r
1803                                 /*\r
1804                                  * "This is not the packet you're looking for" - don't update\r
1805                                  * receive statistics, the packet never happened.\r
1806                                  */\r
1807                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1808                                 /* Dereference the port object on behalf of the failed recv. */\r
1809                                 cl_obj_deref( &p_port->obj );\r
1810                                 continue;\r
1811                         }\r
1812                 }\r
1813 \r
1814                 switch( p_ipoib->hdr.type )\r
1815                 {\r
1816                 case ETH_PROT_TYPE_IP:\r
1817                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1818                         {\r
1819                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1820                                         ("Received IP packet < min size\n") );\r
1821                                 status = IB_INVALID_SETTING;\r
1822                                 break;\r
1823                         }\r
1824 \r
1825                         if( p_ipoib->type.ip.hdr.offset ||\r
1826                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1827                         {\r
1828                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1829                                 cl_perf_start( RecvTcp );\r
1830                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1831                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1832                                 break;\r
1833                         }\r
1834 \r
1835                         /* First packet of a UDP transfer. */\r
1836                         if( len <\r
1837                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1838                         {\r
1839                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1840                                         ("Received UDP packet < min size\n") );\r
1841                                 status = IB_INVALID_SETTING;\r
1842                                 break;\r
1843                         }\r
1844 \r
1845                         /* Check if DHCP conversion is required. */\r
1846                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1847                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1848                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1849                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1850                         {\r
1851                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1852                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1853                                 {\r
1854                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1855                                                 ("Received DHCP < min size\n") );\r
1856                                         status = IB_INVALID_SETTING;\r
1857                                         break;\r
1858                                 }\r
1859                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1860                                 cl_perf_start( RecvDhcp );\r
1861                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1862                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1863                         }\r
1864                         else\r
1865                         {\r
1866                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1867                                 cl_perf_start( RecvUdp );\r
1868                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1869                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1870                         }\r
1871                         break;\r
1872 \r
1873                 case ETH_PROT_TYPE_ARP:\r
1874                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1875                         {\r
1876                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1877                                         ("Received ARP < min size\n") );\r
1878                                 status = IB_INVALID_SETTING;\r
1879                                 break;\r
1880                         }\r
1881                         cl_perf_start( RecvArp );\r
1882                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1883                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1884                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1885                         break;\r
1886 \r
1887                 default:\r
1888                         /* Unfiltered.  Setup the ethernet header and report. */\r
1889                         cl_perf_start( RecvGen );\r
1890                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1891                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1892                 }\r
1893 \r
1894                 if( status != IB_SUCCESS )\r
1895                 {\r
1896                         /* Update stats. */\r
1897                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1898                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1899                         /* Dereference the port object on behalf of the failed receive. */\r
1900                         cl_obj_deref( &p_port->obj );\r
1901                 }\r
1902                 else\r
1903                 {\r
1904                         p_desc->len =\r
1905                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1906                         if( p_dst->h_mcast)\r
1907                         {\r
1908                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1909                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1910                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1911                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1912                                 {\r
1913                                         p_desc->type = PKT_TYPE_BCAST;\r
1914                                 }\r
1915                                 else\r
1916                                 {\r
1917                                         p_desc->type = PKT_TYPE_MCAST;\r
1918                                 }\r
1919                         }\r
1920                         else\r
1921                         {\r
1922                                 p_desc->type = PKT_TYPE_UCAST;\r
1923                         }\r
1924                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1925                 }\r
1926         }\r
1927 \r
1928         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1929         return recv_cnt;\r
1930 }\r
1931 \r
1932 \r
1933 static ib_api_status_t\r
1934 __recv_gen(\r
1935         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1936                 OUT                     eth_pkt_t* const                        p_eth,\r
1937         IN                              ipoib_endpt_t* const            p_src,\r
1938         IN                              ipoib_endpt_t* const            p_dst )\r
1939 {\r
1940         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1941 \r
1942         if( !p_src || !p_dst )\r
1943         {\r
1944                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1945                         ("Received packet with no matching endpoints.\n") );\r
1946                 return IB_NOT_DONE;\r
1947         }\r
1948 \r
1949         /*\r
1950          * Fill in the ethernet header.  Note that doing so will overwrite\r
1951          * the IPoIB header, so start by moving the information from the IPoIB\r
1952          * header.\r
1953          */\r
1954         p_eth->hdr.type = p_ipoib->hdr.type;\r
1955         p_eth->hdr.src = p_src->mac;\r
1956         p_eth->hdr.dst = p_dst->mac;\r
1957 \r
1958         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1959         return IB_SUCCESS;\r
1960 }\r
1961 \r
1962 \r
1963 static ib_api_status_t\r
1964 __recv_dhcp(\r
1965         IN                              ipoib_port_t* const                     p_port,\r
1966         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1967                 OUT                     eth_pkt_t* const                        p_eth,\r
1968         IN                              ipoib_endpt_t* const            p_src,\r
1969         IN                              ipoib_endpt_t* const            p_dst )\r
1970 {\r
1971         ib_api_status_t         status;\r
1972         dhcp_pkt_t                      *p_dhcp;\r
1973         uint8_t                         *p_option;\r
1974         uint8_t                         *p_cid = NULL;\r
1975         ib_gid_t                        gid;\r
1976         uint8_t                         msg = 0;\r
1977 \r
1978         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1979 \r
1980         UNUSED_PARAM( p_port );\r
1981 \r
1982         /* Create the ethernet header. */\r
1983         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1984         if( status != IB_SUCCESS )\r
1985         {\r
1986                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1987                         ("__recv_gen returned %s.\n", \r
1988                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1989                 return status;\r
1990         }\r
1991 \r
1992         /* Fixup the payload. */\r
1993         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
1994         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
1995         {\r
1996                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1997                         ("Invalid DHCP op code.\n") );\r
1998                 return IB_INVALID_SETTING;\r
1999         }\r
2000 \r
2001         /*\r
2002          * Find the client identifier option, making sure to skip\r
2003          * the "magic cookie".\r
2004          */\r
2005         p_option = &p_dhcp->options[0];\r
2006         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2007         {\r
2008                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2009                         ("DHCP cookie corrupted.\n") );\r
2010                 return IB_INVALID_PARAMETER;\r
2011         }\r
2012 \r
2013         p_option = &p_dhcp->options[4];\r
2014         while( *p_option != DHCP_OPT_END )\r
2015         {\r
2016                 switch( *p_option )\r
2017                 {\r
2018                 case DHCP_OPT_PAD:\r
2019                         p_option++;\r
2020                         break;\r
2021 \r
2022                 case DHCP_OPT_MSG:\r
2023                         msg = p_option[2];\r
2024                         p_option += 3;\r
2025                         break;\r
2026 \r
2027                 case DHCP_OPT_CLIENT_ID:\r
2028                         p_cid = p_option;\r
2029                         /* Fall through. */\r
2030 \r
2031                 default:\r
2032                         /*\r
2033                          * All other options have a length byte following the option code.\r
2034                          * Offset by the length to get to the next option.\r
2035                          */\r
2036                         p_option += (p_option[1] + 2);\r
2037                 }\r
2038         }\r
2039 \r
2040         switch( msg )\r
2041         {\r
2042         /* message from client */\r
2043         case DHCPDISCOVER:\r
2044         case DHCPREQUEST:\r
2045         case DHCPDECLINE:\r
2046         case DHCPRELEASE:\r
2047         case DHCPINFORM:\r
2048                 if( !p_cid )\r
2049                 {\r
2050                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2051                                 ("Failed to find required Client-identifier option.\n") );\r
2052                         return IB_INVALID_SETTING;\r
2053                 }\r
2054                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2055                 {\r
2056                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2057                                 ("Invalid hardware address type.\n") );\r
2058                         return IB_INVALID_SETTING;\r
2059                 }\r
2060                 break;\r
2061         /* message from DHCP server */\r
2062         case DHCPOFFER:\r
2063         case DHCPACK:\r
2064         case DHCPNAK:\r
2065                 break;\r
2066 \r
2067         default:\r
2068                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2069                         ("Invalide message type.\n") );\r
2070                 return IB_INVALID_PARAMETER;\r
2071         }\r
2072         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2073         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2074         p_dhcp->hlen = HW_ADDR_LEN;\r
2075 \r
2076         if( p_cid ) /* from client */\r
2077         {\r
2078                 /* Validate that the length and type of the option is as required. */\r
2079                 if( p_cid[1] != 21 )\r
2080                 {\r
2081                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2082                                 ("Client-identifier length not 21 as required.\n") );\r
2083                         return IB_INVALID_SETTING;\r
2084                 }\r
2085                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2086                 {\r
2087                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2088                                 ("Client-identifier type is wrong.\n") );\r
2089                         return IB_INVALID_SETTING;\r
2090                 }\r
2091                 /*\r
2092                  * Copy the GID value from the option so that we can make aligned\r
2093                  * accesses to the contents.\r
2094                  * Recover CID to standard type.\r
2095                  */\r
2096                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2097                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2098                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2099                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2100                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2101         }\r
2102         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2103         return status;\r
2104 }\r
2105 \r
2106 \r
2107 static ib_api_status_t\r
2108 __recv_arp(\r
2109         IN                              ipoib_port_t* const                     p_port,\r
2110         IN                              ib_wc_t* const                          p_wc,\r
2111         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2112                 OUT                     eth_pkt_t* const                        p_eth,\r
2113         IN                              ipoib_endpt_t** const           pp_src,\r
2114         IN                              ipoib_endpt_t* const            p_dst )\r
2115 {\r
2116         ib_api_status_t                 status;\r
2117         arp_pkt_t                               *p_arp;\r
2118         const ipoib_arp_pkt_t   *p_ib_arp;\r
2119         ib_gid_t                                gid;\r
2120         mac_addr_t                              mac;\r
2121         ipoib_hw_addr_t                 null_hw = {0};\r
2122 \r
2123         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2124 \r
2125         if( !p_dst )\r
2126         {\r
2127                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2128                         ("Unknown destination endpoint\n") );\r
2129                 return IB_INVALID_SETTING;\r
2130         }\r
2131 \r
2132         p_ib_arp = &p_ipoib->type.arp;\r
2133         p_arp = &p_eth->type.arp;\r
2134 \r
2135         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2136         {\r
2137                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2138                         ("ARP hardware type is not IB\n") );\r
2139                 return IB_INVALID_SETTING;\r
2140         }\r
2141 \r
2142         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2143         {\r
2144                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2145                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2146                 return IB_INVALID_SETTING;\r
2147         }\r
2148 \r
2149         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2150         {\r
2151                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2152                         ("ARP protocal type not IP\n") );\r
2153                 return IB_INVALID_SETTING;\r
2154         }\r
2155 \r
2156         /*\r
2157          * If we don't have a source, lookup the endpoint specified in the payload.\r
2158          */\r
2159         if( !*pp_src )\r
2160                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2161 \r
2162         /*\r
2163          * If the endpoint exists for the GID, make sure\r
2164          * the dlid and qpn match the arp.\r
2165          */\r
2166         if( *pp_src )\r
2167         {\r
2168                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2169                         sizeof(ib_gid_t) ) )\r
2170                 {\r
2171                         /*\r
2172                          * GIDs for the endpoint are different.  The ARP must\r
2173                          * have been proxied.  Dereference it.\r
2174                          */\r
2175                         *pp_src = NULL;\r
2176                 }\r
2177                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2178                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2179                 {\r
2180                         /* Out of date!  Destroy the endpoint and replace it. */\r
2181                         __endpt_mgr_remove( p_port, *pp_src );\r
2182                         *pp_src = NULL;\r
2183                 }\r
2184         }\r
2185 \r
2186         /* Do we need to create an endpoint for this GID? */\r
2187         if( !*pp_src )\r
2188         {\r
2189                 /* Copy the src GID to allow aligned access */\r
2190                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2191                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2192                 if( status != IB_SUCCESS )\r
2193                 {\r
2194                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2195                                 ("ipoib_mac_from_guid returned %s\n",\r
2196                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2197                         return status;\r
2198                 }\r
2199                 /*\r
2200                  * Create the endpoint.  Note that the LID is left blank and will be\r
2201                  * resolved by a path query as needed.  This is done because the\r
2202                  * remote LID/GID from the work completion may not be the original\r
2203                  * initiator.\r
2204                  */\r
2205                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2206                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2207                 if( !*pp_src )\r
2208                 {\r
2209                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2210                                 ("ipoib_endpt_create returned %s\n",\r
2211                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2212                         return status;\r
2213                 }\r
2214 \r
2215                 cl_obj_lock( &p_port->obj );\r
2216                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2217                 cl_obj_unlock( &p_port->obj );\r
2218         }\r
2219 \r
2220         CL_ASSERT( !cl_memcmp(\r
2221                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2222         CL_ASSERT( (*pp_src)->qpn ==\r
2223                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2224         /* Now swizzle the data. */\r
2225         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2226         p_arp->hw_size = sizeof(mac_addr_t);\r
2227         p_arp->src_hw = (*pp_src)->mac;\r
2228         p_arp->src_ip = p_ib_arp->src_ip;\r
2229 \r
2230         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2231         {\r
2232 \r
2233                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2234                 {\r
2235                         /*\r
2236                          * We received bcast ARP packet that means\r
2237                          * remote port lets everyone know it was changed IP/MAC\r
2238                          * or just activated\r
2239                          */\r
2240                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2241                         {\r
2242                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2243                                         ("ARP: is not ARP MCAST\n") );\r
2244                                 return IB_INVALID_SETTING;\r
2245                         }\r
2246 \r
2247                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2248                         p_dst->mac = p_port->p_local_endpt->mac;\r
2249                         /*\r
2250                          * we don't care what receiver ip addr is,\r
2251                          * as long as OS' ARP table is global  ???\r
2252                          */\r
2253                         p_arp->dst_ip = (net32_t)0;\r
2254 \r
2255                 }\r
2256                 else /* we've got reply to our ARP request */\r
2257                 {\r
2258                         p_arp->dst_hw = p_dst->mac;\r
2259                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2260                         CL_ASSERT( p_dst->qpn == \r
2261                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2262                 }\r
2263         }\r
2264         else /* we got ARP reqeust */\r
2265         {\r
2266                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2267                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2268         }\r
2269 \r
2270         /*\r
2271          * Create the ethernet header.  Note that this is done last so that\r
2272          * we have a chance to create a new endpoint.\r
2273          */\r
2274         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2275         if( status != IB_SUCCESS )\r
2276         {\r
2277                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2278                         ("__recv_gen returned %s.\n", \r
2279                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2280                 return status;\r
2281         }\r
2282 \r
2283         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2284         return IB_SUCCESS;\r
2285 }\r
2286 \r
2287 \r
2288 static ib_api_status_t\r
2289 __recv_mgr_prepare_pkt(\r
2290         IN                              ipoib_port_t* const                     p_port,\r
2291         IN                              ipoib_recv_desc_t* const        p_desc,\r
2292                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2293 {\r
2294         NDIS_STATUS                                                     status;\r
2295         uint32_t                                                        pkt_filter;\r
2296         ip_stat_sel_t                                           type;\r
2297         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2298         PERF_DECLARE( GetNdisPkt );\r
2299 \r
2300         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2301 \r
2302         pkt_filter = p_port->p_adapter->packet_filter;\r
2303         /* Check the packet filter. */\r
2304         switch( p_desc->type )\r
2305         {\r
2306         default:\r
2307         case PKT_TYPE_UCAST:\r
2308                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2309                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2310                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2311                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2312                 {\r
2313                         /* OK to report. */\r
2314                         type = IP_STAT_UCAST_BYTES;\r
2315                         status = NDIS_STATUS_SUCCESS;\r
2316                 }\r
2317                 else\r
2318                 {\r
2319                         type = IP_STAT_DROPPED;\r
2320                         status = NDIS_STATUS_FAILURE;\r
2321                 }\r
2322                 break;\r
2323         case PKT_TYPE_BCAST:\r
2324                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2325                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2326                 {\r
2327                         /* OK to report. */\r
2328                         type = IP_STAT_BCAST_BYTES;\r
2329                         status = NDIS_STATUS_SUCCESS;\r
2330                 }\r
2331                 else\r
2332                 {\r
2333                         type = IP_STAT_DROPPED;\r
2334                         status = NDIS_STATUS_FAILURE;\r
2335                 }\r
2336                 break;\r
2337         case PKT_TYPE_MCAST:\r
2338                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2339                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2340                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2341                 {\r
2342                         /* OK to report. */\r
2343                         type = IP_STAT_MCAST_BYTES;\r
2344                         status = NDIS_STATUS_SUCCESS;\r
2345                 }\r
2346                 else\r
2347                 {\r
2348                         type = IP_STAT_DROPPED;\r
2349                         status = NDIS_STATUS_FAILURE;\r
2350                 }\r
2351                 break;\r
2352         }\r
2353 \r
2354         if( status != NDIS_STATUS_SUCCESS )\r
2355         {\r
2356                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2357                 /* Return the receive descriptor to the pool. */\r
2358                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2359                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2360                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2361                 /*\r
2362                  * Return IB_NOT_DONE since the packet has been completed,\r
2363                  * but has not consumed an array entry.\r
2364                  */\r
2365                 return IB_NOT_DONE;\r
2366         }\r
2367 \r
2368         cl_perf_start( GetNdisPkt );\r
2369         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2370         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2371         if( !*pp_packet )\r
2372         {\r
2373                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2374                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2375                 return IB_INSUFFICIENT_RESOURCES;\r
2376         }\r
2377 \r
2378         /* Flag the checksums as having been calculated. */\r
2379         chksum.Value = 0;\r
2380         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2381         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2382         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2383         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2384                 (void*)(uintn_t)chksum.Value;\r
2385 \r
2386         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2387 \r
2388         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2389         return IB_SUCCESS;\r
2390 }\r
2391 \r
2392 \r
2393 static uint32_t\r
2394 __recv_mgr_build_pkt_array(\r
2395         IN                              ipoib_port_t* const                     p_port,\r
2396                 OUT                     cl_qlist_t* const                       p_done_list )\r
2397 {\r
2398         cl_list_item_t                  *p_item;\r
2399         ipoib_recv_desc_t               *p_desc;\r
2400         uint32_t                                i = 0;\r
2401         ib_api_status_t                 status;\r
2402         PERF_DECLARE( PreparePkt );\r
2403 \r
2404         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2405 \r
2406         cl_spinlock_acquire( &p_port->recv_lock );\r
2407 \r
2408         /* Move any existing receives to the head to preserve ordering. */\r
2409         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2410         p_item = cl_qlist_remove_head( p_done_list );\r
2411         while( p_item != cl_qlist_end( p_done_list ) )\r
2412         {\r
2413                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2414 \r
2415                 cl_perf_start( PreparePkt );\r
2416                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2417                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2418                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2419                 if( status == IB_SUCCESS )\r
2420                 {\r
2421                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2422                         i++;\r
2423                 }\r
2424                 else if( status != IB_NOT_DONE )\r
2425                 {\r
2426                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2427                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2428                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2429                         /* Put all completed receives on the port's done list. */\r
2430                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2431                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2432                         break;\r
2433                 }\r
2434 \r
2435                 p_item = cl_qlist_remove_head( p_done_list );\r
2436         }\r
2437 \r
2438         cl_spinlock_release( &p_port->recv_lock );\r
2439 \r
2440         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2441         return i;\r
2442 }\r
2443 \r
2444 \r
2445 \r
2446 \r
2447 /******************************************************************************\r
2448 *\r
2449 * Send manager implementation.\r
2450 *\r
2451 ******************************************************************************/\r
2452 static void\r
2453 __send_mgr_construct(\r
2454         IN                              ipoib_port_t* const                     p_port )\r
2455 {\r
2456         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2457         p_port->send_mgr.depth = 0;\r
2458         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2459         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2460 }\r
2461 \r
2462 \r
2463 static void\r
2464 __send_mgr_destroy(\r
2465         IN                              ipoib_port_t* const                     p_port )\r
2466 {\r
2467         cl_list_item_t  *p_item;\r
2468         NDIS_PACKET             *p_packet;\r
2469 \r
2470         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2471 \r
2472         /* Complete any pending packets. */\r
2473         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2474                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2475                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2476         {\r
2477                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2478                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2479                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2480         }\r
2481 \r
2482         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2483 }\r
2484 \r
2485 \r
2486 static NDIS_STATUS\r
2487 __send_mgr_filter(\r
2488         IN                              ipoib_port_t* const                     p_port,\r
2489         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2490         IN                              NDIS_BUFFER* const                      p_buf,\r
2491         IN                              size_t                                          buf_len,\r
2492         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2493 {\r
2494         NDIS_STATUS             status;\r
2495 \r
2496         PERF_DECLARE( FilterIp );\r
2497         PERF_DECLARE( FilterArp );\r
2498         PERF_DECLARE( SendGen );\r
2499 \r
2500         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2501 \r
2502         /*\r
2503          * We already checked the ethernet header length, so we know it's safe\r
2504          * to decrement the buf_len without underflowing.\r
2505          */\r
2506         buf_len -= sizeof(eth_hdr_t);\r
2507 \r
2508         switch( p_eth_hdr->type )\r
2509         {\r
2510         case ETH_PROT_TYPE_IP:\r
2511                 cl_perf_start( FilterIp );\r
2512                 status = __send_mgr_filter_ip(\r
2513                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2514                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2515                 break;\r
2516 \r
2517         case ETH_PROT_TYPE_ARP:\r
2518                 cl_perf_start( FilterArp );\r
2519                 status = __send_mgr_filter_arp(\r
2520                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2521                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2522                 break;\r
2523 \r
2524         default:\r
2525                 /*\r
2526                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2527                  * Just send the payload and hope for the best.\r
2528                  */\r
2529                 cl_perf_start( SendGen );\r
2530                 status = __send_gen( p_port, p_desc );\r
2531                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2532                 break;\r
2533         }\r
2534 \r
2535         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2536         return status;\r
2537 }\r
2538 \r
2539 \r
2540 static NDIS_STATUS\r
2541 __send_copy(\r
2542         IN                              ipoib_port_t* const                     p_port,\r
2543         IN                              ipoib_send_desc_t* const        p_desc )\r
2544 {\r
2545         NDIS_PACKET                             *p_packet;\r
2546         NDIS_BUFFER                             *p_buf;\r
2547         NDIS_STATUS                             status;\r
2548         UINT                                    tot_len, bytes_copied = 0;\r
2549 \r
2550         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2551 \r
2552         p_desc->p_buf = \r
2553                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2554         if( !p_desc->p_buf )\r
2555         {\r
2556                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2557                         ("Failed to allocate buffer for packet copy.\n") );\r
2558                 return NDIS_STATUS_RESOURCES;\r
2559         }\r
2560 \r
2561         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2562         if( status != NDIS_STATUS_SUCCESS )\r
2563         {\r
2564                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2565                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2566                 return status;\r
2567         }\r
2568 \r
2569         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2570                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2571         if( status != NDIS_STATUS_SUCCESS )\r
2572         {\r
2573                 NdisFreePacket( p_packet );\r
2574                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2575                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2576                 return status;\r
2577         }\r
2578 \r
2579         NdisChainBufferAtFront( p_packet, p_buf );\r
2580 \r
2581         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2582 \r
2583         /* Setup the work request. */\r
2584         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2585                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2586         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2587         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2588         p_desc->wr.num_ds = 2;\r
2589 \r
2590         /* Copy the packet. */\r
2591         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2592                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2593                 NormalPagePriority );\r
2594 \r
2595         /* Free our temp packet now that the data is copied. */\r
2596         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2597         NdisFreeBuffer( p_buf );\r
2598         NdisFreePacket( p_packet );\r
2599 \r
2600         if( bytes_copied != tot_len )\r
2601         {\r
2602                 /* Something went wrong.  Drop the packet. */\r
2603                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2604                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2605                         bytes_copied, tot_len) );\r
2606                 return NDIS_STATUS_RESOURCES;\r
2607         }\r
2608 \r
2609         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2610         return NDIS_STATUS_SUCCESS;\r
2611 }\r
2612 \r
2613 \r
2614 #if !IPOIB_USE_DMA\r
2615 /* Send using the MDL's page information rather than the SGL. */\r
2616 static ib_api_status_t\r
2617 __send_gen(\r
2618         IN                              ipoib_port_t* const                     p_port,\r
2619         IN                              ipoib_send_desc_t* const        p_desc )\r
2620 {\r
2621         uint32_t                                i, j = 1;\r
2622         ULONG                                   offset;\r
2623         MDL                                             *p_mdl;\r
2624         UINT                                    num_pages, tot_len;\r
2625         ULONG                                   buf_len;\r
2626         PPFN_NUMBER                             page_array;\r
2627         boolean_t                               hdr_done = FALSE;\r
2628         ib_api_status_t                 status;\r
2629 \r
2630         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2631 \r
2632         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2633                 &tot_len );\r
2634 \r
2635         if( !p_mdl )\r
2636         {\r
2637                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2638                         ("No buffers associated with packet.\n") );\r
2639                 return IB_ERROR;\r
2640         }\r
2641 \r
2642         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2643         if( num_pages >= MAX_SEND_SGE )\r
2644         {\r
2645                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2646                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2647                 status = __send_copy( p_port, p_desc );\r
2648                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2649                 return status;\r
2650         }\r
2651 \r
2652         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2653         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2654         /*\r
2655          * Assume that the ethernet header is always fully contained\r
2656          * in the first page of the first MDL.  This makes for much\r
2657          * simpler code.\r
2658          */\r
2659         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2660         CL_ASSERT( offset <= PAGE_SIZE );\r
2661 \r
2662         while( tot_len )\r
2663         {\r
2664                 buf_len = MmGetMdlByteCount( p_mdl );\r
2665                 page_array = MmGetMdlPfnArray( p_mdl );\r
2666                 CL_ASSERT( page_array );\r
2667                 i = 0;\r
2668                 if( !hdr_done )\r
2669                 {\r
2670                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2671                         /* Skip the ethernet header. */\r
2672                         buf_len -= sizeof(eth_hdr_t);\r
2673                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2674                         if( buf_len )\r
2675                         {\r
2676                                 /* The ethernet header is a subset of this MDL. */\r
2677                                 CL_ASSERT( i == 0 );\r
2678                                 if( offset < PAGE_SIZE )\r
2679                                 {\r
2680                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2681                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2682                                         /* Add the byte offset since we're on the 1st page. */\r
2683                                         p_desc->local_ds[j].vaddr += offset;\r
2684                                         if( offset + buf_len > PAGE_SIZE )\r
2685                                         {\r
2686                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2687                                                 buf_len -= p_desc->local_ds[j].length;\r
2688                                         }\r
2689                                         else\r
2690                                         {\r
2691                                                 p_desc->local_ds[j].length = buf_len;\r
2692                                                 buf_len = 0;\r
2693                                         }\r
2694                                         /* This data segment is done.  Move to the next. */\r
2695                                         j++;\r
2696                                 }\r
2697                                 /* This page is done.  Move to the next. */\r
2698                                 i++;\r
2699                         }\r
2700                         /* Done handling the ethernet header. */\r
2701                         hdr_done = TRUE;\r
2702                 }\r
2703 \r
2704                 /* Finish this MDL */\r
2705                 while( buf_len )\r
2706                 {\r
2707                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2708                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2709                         /* Add the first page's offset if we're on the first page. */\r
2710                         if( i == 0 )\r
2711                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2712 \r
2713                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2714                         {\r
2715                                 /* Buffers spans pages. */\r
2716                                 p_desc->local_ds[j].length =\r
2717                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2718                                 buf_len -= p_desc->local_ds[j].length;\r
2719                                 /* This page is done.  Move to the next. */\r
2720                                 i++;\r
2721                         }\r
2722                         else\r
2723                         {\r
2724                                 /* Last page of the buffer. */\r
2725                                 p_desc->local_ds[j].length = buf_len;\r
2726                                 buf_len = 0;\r
2727                         }\r
2728                         /* This data segment is done.  Move to the next. */\r
2729                         j++;\r
2730                 }\r
2731 \r
2732                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2733                 if( !tot_len )\r
2734                         break;\r
2735 \r
2736                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2737                 if( !p_mdl )\r
2738                 {\r
2739                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2740                                 ("Failed to get next buffer.\n") );\r
2741                         return IB_ERROR;\r
2742                 }\r
2743         }\r
2744 \r
2745         /* Set the number of data segments. */\r
2746         p_desc->wr.num_ds = j;\r
2747 \r
2748         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2749         return IB_SUCCESS;\r
2750 }\r
2751 \r
2752 #else\r
2753 \r
2754 static NDIS_STATUS\r
2755 __send_gen(\r
2756         IN                              ipoib_port_t* const                     p_port,\r
2757         IN                              ipoib_send_desc_t* const        p_desc )\r
2758 {\r
2759         ib_api_status_t                 status;\r
2760         SCATTER_GATHER_LIST             *p_sgl;\r
2761         uint32_t                                i, j = 1;\r
2762         uint32_t                                offset = sizeof(eth_hdr_t);\r
2763         PERF_DECLARE( SendCopy );\r
2764 \r
2765         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2766 \r
2767         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2768                 ScatterGatherListPacketInfo );\r
2769         if( !p_sgl )\r
2770         {\r
2771                 ASSERT( p_sgl );\r
2772                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2773                         ("Failed to get SGL from packet.\n") );\r
2774                 return NDIS_STATUS_FAILURE;\r
2775         }\r
2776 \r
2777         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2778         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2779                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2780                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2781                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2782         {\r
2783                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2784                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2785                 cl_perf_start( SendCopy );\r
2786                 status = __send_copy( p_port, p_desc );\r
2787                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2788                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2789                 return status;\r
2790         }\r
2791 \r
2792         /*\r
2793          * Skip the ethernet header.  It is either the first element,\r
2794          * or part of it.\r
2795          */\r
2796         i = 0;\r
2797         while( offset )\r
2798         {\r
2799                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2800                 {\r
2801                         offset -= p_sgl->Elements[i++].Length;\r
2802                 }\r
2803                 else\r
2804                 {\r
2805                         p_desc->local_ds[j].vaddr =\r
2806                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2807                         p_desc->local_ds[j].length =\r
2808                                 p_sgl->Elements[i].Length - offset;\r
2809                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2810                         i++;\r
2811                         j++;\r
2812                         break;\r
2813                 }\r
2814         }\r
2815         /* Now fill in the rest of the local data segments. */\r
2816         while( i < p_sgl->NumberOfElements )\r
2817         {\r
2818                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2819                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2820                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2821                 i++;\r
2822                 j++;\r
2823         }\r
2824 \r
2825         /* Set the number of data segments. */\r
2826         p_desc->wr.num_ds = j;\r
2827 \r
2828         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2829         return NDIS_STATUS_SUCCESS;\r
2830 }\r
2831 #endif\r
2832 \r
2833 \r
2834 static NDIS_STATUS\r
2835 __send_mgr_filter_ip(\r
2836         IN                              ipoib_port_t* const                     p_port,\r
2837         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2838         IN                              NDIS_BUFFER*                            p_buf,\r
2839         IN                              size_t                                          buf_len,\r
2840         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2841 {\r
2842         NDIS_STATUS             status;\r
2843         ip_hdr_t                *p_ip_hdr;\r
2844 \r
2845         PERF_DECLARE( QueryIp );\r
2846         PERF_DECLARE( SendTcp );\r
2847         PERF_DECLARE( FilterUdp );\r
2848 \r
2849         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2850 \r
2851         if( !buf_len )\r
2852         {\r
2853                 cl_perf_start( QueryIp );\r
2854                 NdisGetNextBuffer( p_buf, &p_buf );\r
2855                 if( !p_buf )\r
2856                 {\r
2857                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2858                                 ("Failed to get IP header buffer.\n") );\r
2859                         return NDIS_STATUS_FAILURE;\r
2860                 }\r
2861                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2862                 if( !p_ip_hdr )\r
2863                 {\r
2864                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2865                                 ("Failed to query IP header buffer.\n") );\r
2866                         return NDIS_STATUS_FAILURE;\r
2867                 }\r
2868                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2869         }\r
2870         else\r
2871         {\r
2872                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2873         }\r
2874         if( buf_len < sizeof(ip_hdr_t) )\r
2875         {\r
2876                 /* This buffer is done for.  Get the next buffer. */\r
2877                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2878                         ("Buffer too small for IP packet.\n") );\r
2879                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2880         }\r
2881 \r
2882         if( p_ip_hdr->offset ||\r
2883                 p_ip_hdr->prot != IP_PROT_UDP )\r
2884         {\r
2885                 /* Not a UDP packet. */\r
2886                 cl_perf_start( SendTcp );\r
2887                 status = __send_gen( p_port, p_desc );\r
2888                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2889                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2890                 return status;\r
2891         }\r
2892 \r
2893         buf_len -= sizeof(ip_hdr_t);\r
2894 \r
2895         cl_perf_start( FilterUdp );\r
2896         status = __send_mgr_filter_udp(\r
2897                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2898         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2899 \r
2900         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2901         return status;\r
2902 }\r
2903 \r
2904 \r
2905 static NDIS_STATUS\r
2906 __send_mgr_filter_udp(\r
2907         IN                              ipoib_port_t* const                     p_port,\r
2908         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2909         IN                              NDIS_BUFFER*                            p_buf,\r
2910         IN                              size_t                                          buf_len,\r
2911         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2912 {\r
2913         ib_api_status_t         status;\r
2914         udp_hdr_t                       *p_udp_hdr;\r
2915         PERF_DECLARE( QueryUdp );\r
2916         PERF_DECLARE( SendUdp );\r
2917         PERF_DECLARE( FilterDhcp );\r
2918 \r
2919         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2920 \r
2921         if( !buf_len )\r
2922         {\r
2923                 cl_perf_start( QueryUdp );\r
2924                 NdisGetNextBuffer( p_buf, &p_buf );\r
2925                 if( !p_buf )\r
2926                 {\r
2927                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2928                                 ("Failed to get UDP header buffer.\n") );\r
2929                         return NDIS_STATUS_FAILURE;\r
2930                 }\r
2931                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2932                 if( !p_udp_hdr )\r
2933                 {\r
2934                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2935                                 ("Failed to query UDP header buffer.\n") );\r
2936                         return NDIS_STATUS_FAILURE;\r
2937                 }\r
2938                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
2939         }\r
2940         else\r
2941         {\r
2942                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
2943         }\r
2944         /* Get the UDP header and check the destination port numbers. */\r
2945         if( buf_len < sizeof(udp_hdr_t) )\r
2946         {\r
2947                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2948                         ("Buffer not large enough for UDP packet.\n") );\r
2949                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2950         }\r
2951 \r
2952         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
2953                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r
2954                 (p_udp_hdr->src_port != DHCP_PORT_SERVER ||\r
2955                 p_udp_hdr->dst_port != DHCP_PORT_CLIENT) )\r
2956         {\r
2957                 /* Not a DHCP packet. */\r
2958                 cl_perf_start( SendUdp );\r
2959                 status = __send_gen( p_port, p_desc );\r
2960                 cl_perf_stop( &p_port->p_adapter->perf, SendUdp );\r
2961                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2962                 return status;\r
2963         }\r
2964 \r
2965         buf_len -= sizeof(udp_hdr_t);\r
2966 \r
2967         /* Allocate our scratch buffer. */\r
2968         p_desc->p_buf = (send_buf_t*)\r
2969                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2970         if( !p_desc->p_buf )\r
2971         {\r
2972                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2973                         ("Failed to query DHCP packet buffer.\n") );\r
2974                 return NDIS_STATUS_RESOURCES;\r
2975         }\r
2976         /* Copy the IP and UDP headers. */\r
2977         cl_memcpy( &p_desc->p_buf->ip.hdr, p_ip_hdr , sizeof(ip_hdr_t) );\r
2978         cl_memcpy(\r
2979                 &p_desc->p_buf->ip.prot.udp.hdr, p_udp_hdr, sizeof(udp_hdr_t) );\r
2980 \r
2981         cl_perf_start( FilterDhcp );\r
2982         status = __send_mgr_filter_dhcp(\r
2983                 p_port, p_udp_hdr, p_buf, buf_len, p_desc );\r
2984         cl_perf_stop( &p_port->p_adapter->perf, FilterDhcp );\r
2985 \r
2986         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2987         return status;\r
2988 }\r
2989 \r
2990 \r
2991 static NDIS_STATUS\r
2992 __send_mgr_filter_dhcp(\r
2993         IN                              ipoib_port_t* const                     p_port,\r
2994         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
2995         IN                              NDIS_BUFFER*                            p_buf,\r
2996         IN                              size_t                                          buf_len,\r
2997         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2998 {\r
2999         dhcp_pkt_t                      *p_dhcp;\r
3000         dhcp_pkt_t                      *p_ib_dhcp;\r
3001         uint8_t                         *p_option, *p_cid = NULL;\r
3002         uint8_t                         msg = 0;\r
3003         size_t                          len;\r
3004         ib_gid_t                        gid;\r
3005 \r
3006         IPOIB_ENTER( IPOIB_DBG_SEND );\r
3007 \r
3008         if( !buf_len )\r
3009         {\r
3010                 NdisGetNextBuffer( p_buf, &p_buf );\r
3011                 if( !p_buf )\r
3012                 {\r
3013                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3014                                 ("Failed to get DHCP buffer.\n") );\r
3015                         return NDIS_STATUS_FAILURE;\r
3016                 }\r
3017                 NdisQueryBufferSafe( p_buf, &p_dhcp, &buf_len, NormalPagePriority );\r
3018                 if( !p_dhcp )\r
3019                 {\r
3020                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3021                                 ("Failed to query DHCP buffer.\n") );\r
3022                         return NDIS_STATUS_FAILURE;\r
3023                 }\r
3024         }\r
3025         else\r
3026         {\r
3027                 p_dhcp = (dhcp_pkt_t*)(p_udp_hdr + 1);\r
3028         }\r
3029 \r
3030         if( buf_len < DHCP_MIN_SIZE )\r
3031         {\r
3032                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3033                         ("Buffer not large enough for DHCP packet.\n") );\r
3034                 return NDIS_STATUS_BUFFER_TOO_SHORT;