initial implementation
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id:$\r
30  */\r
31 \r
32 /*\r
33         $Revision$\r
34 */\r
35 \r
36 \r
37 #include "ipoib_port.h"\r
38 #include "ipoib_adapter.h"\r
39 #include "ipoib_debug.h"\r
40 \r
41 \r
42 /*\r
43  * PR 102492 relates to the SA not accepting wildcard values for MCMemberRecord\r
44  * Sets used to create/join Multicast groups.  Defining this keyword causes all\r
45  * member record fields to be hard coded to well known values.\r
46  */\r
47 //#define PR_102492\r
48 \r
49 /*\r
50  * PR 102801 relates to the SA ignoring the MGID when doing a GET_TABLE\r
51  * request that specifies an MGID.  The ESM returns every McMemberRecord\r
52  * instead.\r
53  */\r
54 #define PR_102801\r
55 \r
56 \r
57 /* Amount of physical memory to register. */\r
58 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
59 \r
60 /* Number of work completions to chain for send and receive polling. */\r
61 #define MAX_SEND_WC             8\r
62 #define MAX_RECV_WC             16\r
63 \r
64 \r
65 ib_gid_t        bcast_mgid_template = {\r
66         0xff,                                                           /* multicast field */\r
67         0x12,                                                           /* scope (to be filled in) */\r
68         0x40, 0x1b,                                                     /* IPv4 signature */\r
69         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
70         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
71         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
72 };\r
73 \r
74 \r
75 #ifdef _DEBUG_\r
76 /* Handy pointer for debug use. */\r
77 ipoib_port_t    *gp_ipoib_port;\r
78 #endif\r
79 \r
80 \r
81 /******************************************************************************\r
82 *\r
83 * Declarations\r
84 *\r
85 ******************************************************************************/\r
86 static void\r
87 __port_construct(\r
88         IN                              ipoib_port_t* const                     p_port );\r
89 \r
90 static ib_api_status_t\r
91 __port_init(\r
92         IN                              ipoib_port_t* const                     p_port,\r
93         IN                              ipoib_adapter_t* const          p_adapter,\r
94         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
95 \r
96 static void\r
97 __port_destroying(\r
98         IN                              cl_obj_t* const                         p_obj );\r
99 \r
100 static void\r
101 __port_cleanup(\r
102         IN                              cl_obj_t* const                         p_obj );\r
103 \r
104 static void\r
105 __port_free(\r
106         IN                              cl_obj_t* const                         p_obj );\r
107 \r
108 \r
109 /******************************************************************************\r
110 *\r
111 * IB resource manager operations\r
112 *\r
113 ******************************************************************************/\r
114 static void\r
115 __ib_mgr_construct(\r
116         IN                              ipoib_port_t* const                     p_port );\r
117 \r
118 static ib_api_status_t\r
119 __ib_mgr_init(\r
120         IN                              ipoib_port_t* const                     p_port );\r
121 \r
122 static void\r
123 __ib_mgr_destroy(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 static void\r
127 __qp_event(\r
128         IN                              ib_async_event_rec_t            *p_event_rec );\r
129 \r
130 static void\r
131 __cq_event(\r
132         IN                              ib_async_event_rec_t            *p_event_rec );\r
133 \r
134 static ib_api_status_t\r
135 __ib_mgr_activate(\r
136         IN                              ipoib_port_t* const                     p_port );\r
137 \r
138 /******************************************************************************\r
139 *\r
140 * Buffer manager operations.\r
141 *\r
142 ******************************************************************************/\r
143 static void\r
144 __buf_mgr_construct(\r
145         IN                              ipoib_port_t* const                     p_port );\r
146 \r
147 static ib_api_status_t\r
148 __buf_mgr_init(\r
149         IN                              ipoib_port_t* const                     p_port );\r
150 \r
151 static void\r
152 __buf_mgr_destroy(\r
153         IN                              ipoib_port_t* const                     p_port );\r
154 \r
155 static cl_status_t\r
156 __recv_ctor(\r
157         IN                              void* const                                     p_object,\r
158         IN                              void*                                           context,\r
159                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
160 \r
161 #if !IPOIB_INLINE_RECV\r
162 static void\r
163 __recv_dtor(\r
164         IN              const   cl_pool_item_t* const           p_pool_item,\r
165         IN                              void                                            *context );\r
166 #endif  /* IPOIB_INLINE_RECV */\r
167 \r
168 static inline ipoib_send_desc_t*\r
169 __buf_mgr_get_send(\r
170         IN                              ipoib_port_t* const                     p_port );\r
171 \r
172 static inline void\r
173 __buf_mgr_put_send(\r
174         IN                              ipoib_port_t* const                     p_port,\r
175         IN                              ipoib_send_desc_t* const        p_desc );\r
176 \r
177 static inline ipoib_recv_desc_t*\r
178 __buf_mgr_get_recv(\r
179         IN                              ipoib_port_t* const                     p_port );\r
180 \r
181 static inline void\r
182 __buf_mgr_put_recv(\r
183         IN                              ipoib_port_t* const                     p_port,\r
184         IN                              ipoib_recv_desc_t* const        p_desc,\r
185         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
186 \r
187 static inline void\r
188 __buf_mgr_put_recv_list(\r
189         IN                              ipoib_port_t* const                     p_port,\r
190         IN                              cl_qlist_t* const                       p_list );\r
191 \r
192 static inline NDIS_PACKET*\r
193 __buf_mgr_get_ndis_pkt(\r
194         IN                              ipoib_port_t* const                     p_port,\r
195         IN                              ipoib_recv_desc_t* const        p_desc );\r
196 \r
197 \r
198 /******************************************************************************\r
199 *\r
200 * Receive manager operations.\r
201 *\r
202 ******************************************************************************/\r
203 static void\r
204 __recv_mgr_construct(\r
205         IN                              ipoib_port_t* const                     p_port );\r
206 \r
207 static ib_api_status_t\r
208 __recv_mgr_init(\r
209         IN                              ipoib_port_t* const                     p_port );\r
210 \r
211 static void\r
212 __recv_mgr_destroy(\r
213         IN                              ipoib_port_t* const                     p_port );\r
214 \r
215 /* Posts receive buffers to the receive queue. */\r
216 static ib_api_status_t\r
217 __recv_mgr_repost(\r
218         IN                              ipoib_port_t* const                     p_port,\r
219         IN                              uint32_t                                        num_wrs );\r
220 \r
221 static void\r
222 __recv_cb(\r
223         IN              const   ib_cq_handle_t                          h_cq,\r
224         IN                              void                                            *cq_context );\r
225 \r
226 static void\r
227 __recv_get_endpts(\r
228         IN                              ipoib_port_t* const                     p_port,\r
229         IN                              ipoib_recv_desc_t* const        p_desc,\r
230         IN                              ib_wc_t* const                          p_wc,\r
231                 OUT                     ipoib_endpt_t** const           pp_src,\r
232                 OUT                     ipoib_endpt_t** const           pp_dst );\r
233 \r
234 static uint32_t\r
235 __recv_mgr_filter(\r
236         IN                              ipoib_port_t* const                     p_port,\r
237         IN                              ib_wc_t* const                          p_done_wc_list,\r
238                 OUT                     cl_qlist_t* const                       p_done_list,\r
239                 OUT                     cl_qlist_t* const                       p_bad_list );\r
240 \r
241 static ib_api_status_t\r
242 __recv_gen(\r
243         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
244                 OUT                     eth_pkt_t* const                        p_eth,\r
245         IN                              ipoib_endpt_t* const            p_src,\r
246         IN                              ipoib_endpt_t* const            p_dst );\r
247 \r
248 static ib_api_status_t\r
249 __recv_dhcp(\r
250         IN                              ipoib_port_t* const                     p_port,\r
251         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
252                 OUT                     eth_pkt_t* const                        p_eth,\r
253         IN                              ipoib_endpt_t* const            p_src,\r
254         IN                              ipoib_endpt_t* const            p_dst );\r
255 \r
256 static ib_api_status_t\r
257 __recv_arp(\r
258         IN                              ipoib_port_t* const                     p_port,\r
259         IN                              ib_wc_t* const                          p_wc,\r
260         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
261                 OUT                     eth_pkt_t* const                        p_eth,\r
262         IN                              ipoib_endpt_t** const           p_src,\r
263         IN                              ipoib_endpt_t* const            p_dst );\r
264 \r
265 static ib_api_status_t\r
266 __recv_mgr_prepare_pkt(\r
267         IN                              ipoib_port_t* const                     p_port,\r
268         IN                              ipoib_recv_desc_t* const        p_desc,\r
269                 OUT                     NDIS_PACKET** const                     pp_packet );\r
270 \r
271 static uint32_t\r
272 __recv_mgr_build_pkt_array(\r
273         IN                              ipoib_port_t* const                     p_port,\r
274                 OUT                     cl_qlist_t* const                       p_done_list );\r
275 \r
276 /******************************************************************************\r
277 *\r
278 * Send manager operations.\r
279 *\r
280 ******************************************************************************/\r
281 static void\r
282 __send_mgr_construct(\r
283         IN                              ipoib_port_t* const                     p_port );\r
284 \r
285 static void\r
286 __send_mgr_destroy(\r
287         IN                              ipoib_port_t* const                     p_port );\r
288 \r
289 static NDIS_STATUS\r
290 __send_gen(\r
291         IN                              ipoib_port_t* const                     p_port,\r
292         IN                              ipoib_send_desc_t* const        p_desc );\r
293 \r
294 static NDIS_STATUS\r
295 __send_mgr_filter_ip(\r
296         IN                              ipoib_port_t* const                     p_port,\r
297         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
298         IN                              NDIS_BUFFER*                            p_buf,\r
299         IN                              size_t                                          buf_len,\r
300         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
301 \r
302 static NDIS_STATUS\r
303 __send_mgr_filter_udp(\r
304         IN                              ipoib_port_t* const                     p_port,\r
305         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
306         IN                              NDIS_BUFFER*                            p_buf,\r
307         IN                              size_t                                          buf_len,\r
308         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
309 \r
310 static NDIS_STATUS\r
311 __send_mgr_filter_dhcp(\r
312         IN                              ipoib_port_t* const                     p_port,\r
313         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
314         IN                              NDIS_BUFFER*                            p_buf,\r
315         IN                              size_t                                          buf_len,\r
316         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
317 \r
318 static NDIS_STATUS\r
319 __send_mgr_filter_arp(\r
320         IN                              ipoib_port_t* const                     p_port,\r
321         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
322         IN                              NDIS_BUFFER*                            p_buf,\r
323         IN                              size_t                                          buf_len,\r
324         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
325 \r
326 static void\r
327 __process_failed_send(\r
328         IN                              ipoib_port_t* const                     p_port,\r
329         IN                              ipoib_send_desc_t* const        p_desc,\r
330         IN              const   NDIS_STATUS                                     status );\r
331 \r
332 static void\r
333 __send_cb(\r
334         IN              const   ib_cq_handle_t                          h_cq,\r
335         IN                              void                                            *cq_context );\r
336 \r
337 \r
338 /******************************************************************************\r
339 *\r
340 * Endpoint manager operations\r
341 *\r
342 ******************************************************************************/\r
343 static void\r
344 __endpt_mgr_construct(\r
345         IN                              ipoib_port_t* const                     p_port );\r
346 \r
347 static ib_api_status_t\r
348 __endpt_mgr_init(\r
349         IN                              ipoib_port_t* const                     p_port );\r
350 \r
351 static void\r
352 __endpt_mgr_destroy(\r
353         IN                              ipoib_port_t* const                     p_port );\r
354 \r
355 /****f* IPoIB/__endpt_mgr_remove_all\r
356 * NAME\r
357 *       __endpt_mgr_remove_all\r
358 *\r
359 * DESCRIPTION\r
360 *       Removes all enpoints from the port, dereferencing them to initiate\r
361 *       destruction.\r
362 *\r
363 * SYNOPSIS\r
364 */\r
365 static void\r
366 __endpt_mgr_remove_all(\r
367         IN                              ipoib_port_t* const                     p_port );\r
368 /*\r
369 ********/\r
370 \r
371 static void\r
372 __endpt_mgr_remove(\r
373         IN                              ipoib_port_t* const                     p_port,\r
374         IN                              ipoib_endpt_t* const            p_endpt );\r
375 \r
376 static inline NDIS_STATUS\r
377 __endpt_mgr_ref(\r
378         IN                              ipoib_port_t* const                     p_port,\r
379         IN              const   mac_addr_t                                      mac,\r
380                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
381 \r
382 static inline NDIS_STATUS\r
383 __endpt_mgr_get_gid_qpn(\r
384         IN                              ipoib_port_t* const                     p_port,\r
385         IN              const   mac_addr_t                                      mac,\r
386                 OUT                     ib_gid_t* const                         p_gid,\r
387                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
388 \r
389 static inline ipoib_endpt_t*\r
390 __endpt_mgr_get_by_gid(\r
391         IN                              ipoib_port_t* const                     p_port,\r
392         IN              const   ib_gid_t* const                         p_gid );\r
393 \r
394 static inline ipoib_endpt_t*\r
395 __endpt_mgr_get_by_lid(\r
396         IN                              ipoib_port_t* const                     p_port,\r
397         IN              const   net16_t                                         lid );\r
398 \r
399 static inline void\r
400 __endpt_mgr_insert_locked(\r
401         IN                              ipoib_port_t* const                     p_port,\r
402         IN              const   mac_addr_t                                      mac,\r
403         IN                              ipoib_endpt_t* const            p_endpt );\r
404 \r
405 static inline void\r
406 __endpt_mgr_insert(\r
407         IN                              ipoib_port_t* const                     p_port,\r
408         IN              const   mac_addr_t                                      mac,\r
409         IN                              ipoib_endpt_t* const            p_endpt );\r
410 \r
411 static ib_api_status_t\r
412 __endpt_mgr_add_local(\r
413         IN                              ipoib_port_t* const                     p_port,\r
414         IN                              ib_port_info_t* const           p_port_info );\r
415 \r
416 static ib_api_status_t\r
417 __endpt_mgr_add_bcast(\r
418         IN                              ipoib_port_t* const                     p_port,\r
419         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
420 \r
421 /******************************************************************************\r
422 *\r
423 * MCast operations.\r
424 *\r
425 ******************************************************************************/\r
426 static void\r
427 __port_get_mcast(\r
428         IN                              ipoib_port_t* const                     p_port );\r
429 \r
430 static void\r
431 __port_join_bcast(\r
432         IN                              ipoib_port_t* const                     p_port,\r
433         IN                              ib_member_rec_t* const          p_member_rec );\r
434 \r
435 static void\r
436 __port_create_bcast(\r
437         IN                              ipoib_port_t* const                     p_port );\r
438 \r
439 static void\r
440 __port_info_cb(\r
441         IN                              ib_query_rec_t                          *p_query_rec );\r
442 \r
443 \r
444 static void\r
445 __bcast_get_cb(\r
446         IN                              ib_query_rec_t                          *p_query_rec );\r
447 \r
448 \r
449 static void\r
450 __bcast_cb(\r
451         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
452 \r
453 \r
454 static void\r
455 __mcast_cb(\r
456         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
457 \r
458 \r
459 static intn_t\r
460 __gid_cmp(\r
461         IN              const   void* const                                     p_key1,\r
462         IN              const   void* const                                     p_key2 )\r
463 {\r
464         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
465 }\r
466 \r
467 \r
468 /******************************************************************************\r
469 *\r
470 * Implementation\r
471 *\r
472 ******************************************************************************/\r
473 ib_api_status_t\r
474 ipoib_create_port(\r
475         IN                              ipoib_adapter_t* const          p_adapter,\r
476         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
477                 OUT                     ipoib_port_t** const            pp_port )\r
478 {\r
479         ib_api_status_t         status;\r
480         ipoib_port_t            *p_port;\r
481 \r
482         IPOIB_ENTER( IPOIB_DBG_INIT );\r
483 \r
484         CL_ASSERT( !p_adapter->p_port );\r
485 \r
486         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
487                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
488         if( !p_port )\r
489         {\r
490                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
491                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
492                         sizeof(ipoib_port_t)) );\r
493                 return IB_INSUFFICIENT_MEMORY;\r
494         }\r
495 \r
496 #ifdef _DEBUG_\r
497         gp_ipoib_port = p_port;\r
498 #endif\r
499 \r
500         __port_construct( p_port );\r
501 \r
502         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
503         if( status != IB_SUCCESS )\r
504         {\r
505                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
506                         ("ipoib_port_init returned %s.\n", \r
507                         p_adapter->p_ifc->get_err_str( status )) );\r
508                 __port_cleanup( &p_port->obj );\r
509                 __port_free( &p_port->obj );\r
510                 return status;\r
511         }\r
512 \r
513         *pp_port = p_port;\r
514         IPOIB_EXIT( IPOIB_DBG_INIT );\r
515         return IB_SUCCESS;\r
516 }\r
517 \r
518 \r
519 void\r
520 ipoib_port_destroy(\r
521         IN                              ipoib_port_t* const                     p_port )\r
522 {\r
523         IPOIB_ENTER( IPOIB_DBG_INIT );\r
524 \r
525         CL_ASSERT( p_port );\r
526         CL_ASSERT( p_port->p_adapter );\r
527         CL_ASSERT( p_port->p_adapter->p_port == p_port );\r
528 \r
529         cl_obj_destroy( &p_port->obj );\r
530 \r
531         IPOIB_EXIT( IPOIB_DBG_INIT );\r
532 }\r
533 \r
534 \r
535 static void\r
536 __port_construct(\r
537         IN                              ipoib_port_t* const                     p_port )\r
538 {\r
539         IPOIB_ENTER( IPOIB_DBG_INIT );\r
540 \r
541         p_port->state = IB_QPS_RESET;\r
542 \r
543         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
544         cl_spinlock_construct( &p_port->send_lock );\r
545         cl_spinlock_construct( &p_port->recv_lock );\r
546         __ib_mgr_construct( p_port );\r
547         __buf_mgr_construct( p_port );\r
548 \r
549         __recv_mgr_construct( p_port );\r
550         __send_mgr_construct( p_port );\r
551 \r
552         __endpt_mgr_construct( p_port );\r
553 \r
554         IPOIB_EXIT( IPOIB_DBG_INIT );\r
555 }\r
556 \r
557 \r
558 static ib_api_status_t\r
559 __port_init(\r
560         IN                              ipoib_port_t* const                     p_port,\r
561         IN                              ipoib_adapter_t* const          p_adapter,\r
562         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
563 {\r
564         cl_status_t                     cl_status;\r
565         ib_api_status_t         status;\r
566 \r
567         IPOIB_ENTER( IPOIB_DBG_INIT );\r
568 \r
569         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
570         p_port->p_adapter = p_adapter;\r
571 \r
572         cl_status = cl_spinlock_init( &p_port->send_lock );\r
573         if( cl_status != CL_SUCCESS )\r
574         {\r
575                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
576                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
577                 return IB_ERROR;\r
578         }\r
579 \r
580         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
581         if( cl_status != CL_SUCCESS )\r
582         {\r
583                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
584                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
585                 return IB_ERROR;\r
586         }\r
587 \r
588         /* Initialize the IB resource manager. */\r
589         status = __ib_mgr_init( p_port );\r
590         if( status != IB_SUCCESS )\r
591         {\r
592                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
593                         ("__ib_mgr_init returned %s\n", \r
594                         p_adapter->p_ifc->get_err_str( status )) );\r
595                 return status;\r
596         }\r
597 \r
598         /* Initialize the buffer manager. */\r
599         status = __buf_mgr_init( p_port );\r
600         if( status != IB_SUCCESS )\r
601         {\r
602                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
603                         ("__buf_mgr_init returned %s\n", \r
604                         p_adapter->p_ifc->get_err_str( status )) );\r
605                 return status;\r
606         }\r
607 \r
608         /* Initialize the receive manager. */\r
609         status = __recv_mgr_init( p_port );\r
610         if( status != IB_SUCCESS )\r
611         {\r
612                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
613                         ("__recv_mgr_init returned %s\n", \r
614                         p_adapter->p_ifc->get_err_str( status )) );\r
615                 return status;\r
616         }\r
617 \r
618         /* Initialize the endpoint manager. */\r
619         status = __endpt_mgr_init( p_port );\r
620         if( status != IB_SUCCESS )\r
621         {\r
622                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
623                         ("__endpt_mgr_init returned %s\n", \r
624                         p_adapter->p_ifc->get_err_str( status )) );\r
625                 return status;\r
626         }\r
627 \r
628         /* We only ever destroy from the PnP callback thread. */\r
629         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
630                 __port_destroying, __port_cleanup, __port_free );\r
631         if( cl_status != CL_SUCCESS )\r
632         {\r
633                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
634                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
635                 return IB_ERROR;\r
636         }\r
637 \r
638         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
639 \r
640         IPOIB_EXIT( IPOIB_DBG_INIT );\r
641         return IB_SUCCESS;\r
642 }\r
643 \r
644 \r
645 static void\r
646 __port_destroying(\r
647         IN                              cl_obj_t* const                         p_obj )\r
648 {\r
649         ipoib_port_t    *p_port;\r
650 \r
651         IPOIB_ENTER( IPOIB_DBG_INIT );\r
652 \r
653         CL_ASSERT( p_obj );\r
654 \r
655         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
656 \r
657         ipoib_port_down( p_port );\r
658 \r
659         IPOIB_EXIT( IPOIB_DBG_INIT );\r
660 }\r
661 \r
662 \r
663 static void\r
664 __port_cleanup(\r
665         IN                              cl_obj_t* const                         p_obj )\r
666 {\r
667         ipoib_port_t    *p_port;\r
668 \r
669         IPOIB_ENTER( IPOIB_DBG_INIT );\r
670 \r
671         CL_ASSERT( p_obj );\r
672 \r
673         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
674 \r
675         /* Wait for all sends and receives to get flushed. */\r
676         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
677                 cl_thread_suspend( 0 );\r
678 \r
679         /* Destroy the send and receive managers before closing the CA. */\r
680         __ib_mgr_destroy( p_port );\r
681 \r
682         IPOIB_EXIT( IPOIB_DBG_INIT );\r
683 }\r
684 \r
685 \r
686 static void\r
687 __port_free(\r
688         IN                              cl_obj_t* const                         p_obj )\r
689 {\r
690         ipoib_port_t    *p_port;\r
691 \r
692         IPOIB_ENTER( IPOIB_DBG_INIT );\r
693 \r
694         CL_ASSERT( p_obj );\r
695 \r
696         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
697 \r
698         __endpt_mgr_destroy( p_port );\r
699         __recv_mgr_destroy( p_port );\r
700         __send_mgr_destroy( p_port );\r
701         __buf_mgr_destroy( p_port );\r
702 \r
703         cl_spinlock_destroy( &p_port->send_lock );\r
704         cl_spinlock_destroy( &p_port->recv_lock );\r
705 \r
706         cl_obj_deinit( p_obj );\r
707 \r
708         cl_free( p_port );\r
709 \r
710         IPOIB_EXIT( IPOIB_DBG_INIT );\r
711 }\r
712 \r
713 \r
714 \r
715 /******************************************************************************\r
716 *\r
717 * IB resource manager implementation.\r
718 *\r
719 ******************************************************************************/\r
720 static void\r
721 __ib_mgr_construct(\r
722         IN                              ipoib_port_t* const                     p_port )\r
723 {\r
724         IPOIB_ENTER( IPOIB_DBG_INIT );\r
725 \r
726         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
727 \r
728         IPOIB_EXIT( IPOIB_DBG_INIT );\r
729 }\r
730 \r
731 \r
732 static ib_api_status_t\r
733 __ib_mgr_init(\r
734         IN                              ipoib_port_t* const                     p_port )\r
735 {\r
736         ib_api_status_t         status;\r
737         ib_cq_create_t          cq_create;\r
738         ib_qp_create_t          qp_create;\r
739         ib_phys_create_t        phys_create;\r
740         ib_phys_range_t         phys_range;\r
741         uint64_t                        vaddr;\r
742         net32_t                         rkey;\r
743         ib_qp_attr_t            qp_attr;\r
744 \r
745         IPOIB_ENTER( IPOIB_DBG_INIT );\r
746 \r
747         /* Open the CA. */\r
748         status = p_port->p_adapter->p_ifc->open_ca(\r
749                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
750                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
751         if( status != IB_SUCCESS )\r
752         {\r
753                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
754                         ("ib_open_ca returned %s\n", \r
755                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
756                 return status;\r
757         }\r
758 \r
759         /* Allocate the PD. */\r
760         status = p_port->p_adapter->p_ifc->alloc_pd(\r
761                 p_port->ib_mgr.h_ca, IB_PDT_NORMAL, p_port, &p_port->ib_mgr.h_pd );\r
762         if( status != IB_SUCCESS )\r
763         {\r
764                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
765                         ("ib_alloc_pd returned %s\n", \r
766                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
767                 return status;\r
768         }\r
769 \r
770         /* Allocate receive CQ. */\r
771         cq_create.size = p_port->p_adapter->params.rq_depth;\r
772         cq_create.pfn_comp_cb = __recv_cb;\r
773         cq_create.h_wait_obj = NULL;\r
774 \r
775         status = p_port->p_adapter->p_ifc->create_cq(\r
776                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
777                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
778         if( status != IB_SUCCESS )\r
779         {\r
780                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
781                         ("ib_create_cq returned %s.\n", \r
782                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
783                 return status;\r
784         }\r
785 \r
786         /* Allocate send CQ. */\r
787         cq_create.size = p_port->p_adapter->params.sq_depth;\r
788         cq_create.pfn_comp_cb = __send_cb;\r
789 \r
790         status = p_port->p_adapter->p_ifc->create_cq(\r
791                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
792                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
793         if( status != IB_SUCCESS )\r
794         {\r
795                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
796                         ("ib_create_cq returned %s.\n", \r
797                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
798                 return status;\r
799         }\r
800 \r
801         /* Allocate the QP. */\r
802         cl_memclr( &qp_create, sizeof(qp_create) );\r
803         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
804         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
805         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
806         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
807         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
808         //TODO: Figure out the right number of SGE entries for sends.\r
809         qp_create.sq_sge = MAX_SEND_SGE;\r
810         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
811         qp_create.sq_signaled = TRUE;\r
812         status = p_port->p_adapter->p_ifc->create_qp(\r
813                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
814                 __qp_event, &p_port->ib_mgr.h_qp );\r
815         if( status != IB_SUCCESS )\r
816         {\r
817                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
818                         ("ib_create_qp returned %s\n", \r
819                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
820                 return status;\r
821         }\r
822         /* Query the QP so we can get our QPN. */\r
823         status = p_port->p_adapter->p_ifc->query_qp(\r
824                 p_port->ib_mgr.h_qp, &qp_attr );\r
825         if( status != IB_SUCCESS )\r
826         {\r
827                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
828                         ("ib_query_qp returned %s\n", \r
829                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
830                 return status;\r
831         }\r
832         p_port->ib_mgr.qpn = qp_attr.num;\r
833 \r
834         /* Register all of physical memory */\r
835         phys_create.length = MEM_REG_SIZE;\r
836         phys_create.num_ranges = 1;\r
837         phys_create.range_array = &phys_range;\r
838         phys_create.buf_offset = 0;\r
839         phys_create.hca_page_size = PAGE_SIZE;\r
840         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
841         phys_range.base_addr = 0;\r
842         phys_range.size = MEM_REG_SIZE;\r
843         vaddr = 0;\r
844         status = p_port->p_adapter->p_ifc->reg_phys(\r
845                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
846                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
847         if( status != IB_SUCCESS )\r
848         {\r
849                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
850                         ("ib_reg_phys returned %s\n", \r
851                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
852                 return status;\r
853         }\r
854 \r
855         IPOIB_EXIT( IPOIB_DBG_INIT );\r
856         return IB_SUCCESS;\r
857 }\r
858 \r
859 \r
860 static void\r
861 __ib_mgr_destroy(\r
862         IN                              ipoib_port_t* const                     p_port )\r
863 {\r
864         ib_api_status_t status;\r
865 \r
866         IPOIB_ENTER( IPOIB_DBG_INIT );\r
867 \r
868         if( p_port->ib_mgr.h_ca )\r
869         {\r
870                 status =\r
871                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
872                 CL_ASSERT( status == IB_SUCCESS );\r
873                 p_port->ib_mgr.h_ca = NULL;\r
874         }\r
875 \r
876         IPOIB_EXIT( IPOIB_DBG_INIT );\r
877 }\r
878 \r
879 \r
880 \r
881 /******************************************************************************\r
882 *\r
883 * Buffer manager implementation.\r
884 *\r
885 ******************************************************************************/\r
886 static void\r
887 __buf_mgr_construct(\r
888         IN                              ipoib_port_t* const                     p_port )\r
889 {\r
890         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
891 \r
892         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
893 \r
894         p_port->buf_mgr.h_packet_pool = NULL;\r
895         p_port->buf_mgr.h_buffer_pool = NULL;\r
896 \r
897         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
898                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
899 \r
900         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
901         p_port->buf_mgr.h_send_buf_pool = NULL;\r
902 \r
903         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
904 }\r
905 \r
906 \r
907 static ib_api_status_t\r
908 __buf_mgr_init(\r
909         IN                              ipoib_port_t* const                     p_port )\r
910 {\r
911         cl_status_t             cl_status;\r
912         NDIS_STATUS             ndis_status;\r
913         ipoib_params_t  *p_params;\r
914 \r
915         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
916 \r
917         CL_ASSERT( p_port );\r
918         CL_ASSERT( p_port->p_adapter );\r
919 \r
920         p_params = &p_port->p_adapter->params;\r
921 \r
922         /* Allocate the receive descriptor pool */\r
923         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
924                 p_params->rq_depth * p_params->recv_pool_ratio,\r
925 #if IPOIB_INLINE_RECV\r
926                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
927 #else   /* IPOIB_INLINE_RECV */\r
928                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
929 #endif  /* IPOIB_INLINE_RECV */\r
930         if( cl_status != CL_SUCCESS )\r
931         {\r
932                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
933                         ("cl_qpool_init for recvs returned %s\n",\r
934                         cl_status_text[cl_status]) );\r
935                 return IB_INSUFFICIENT_MEMORY;\r
936         }\r
937 \r
938         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
939         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
940                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
941         if( ndis_status != NDIS_STATUS_SUCCESS )\r
942         {\r
943                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
944                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
945                 return IB_INSUFFICIENT_RESOURCES;\r
946         }\r
947 \r
948         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
949                 p_params->rq_depth );\r
950         if( ndis_status != NDIS_STATUS_SUCCESS )\r
951         {\r
952                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
953                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
954                 return IB_INSUFFICIENT_RESOURCES;\r
955         }\r
956 \r
957         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
958         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
959                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
960         if( ndis_status != NDIS_STATUS_SUCCESS )\r
961         {\r
962                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
963                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
964                 return IB_INSUFFICIENT_RESOURCES;\r
965         }\r
966 \r
967         NdisAllocateBufferPool( &ndis_status,\r
968                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
969         if( ndis_status != NDIS_STATUS_SUCCESS )\r
970         {\r
971                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
972                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
973                 return IB_INSUFFICIENT_RESOURCES;\r
974         }\r
975 \r
976         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
977         return IB_SUCCESS;\r
978 }\r
979 \r
980 \r
981 static void\r
982 __buf_mgr_destroy(\r
983         IN                              ipoib_port_t* const                     p_port )\r
984 {\r
985         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
986 \r
987         CL_ASSERT( p_port );\r
988 \r
989         /* Destroy the send packet and buffer pools. */\r
990         if( p_port->buf_mgr.h_send_buf_pool )\r
991                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
992         if( p_port->buf_mgr.h_send_pkt_pool )\r
993                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
994 \r
995         /* Destroy the receive packet and buffer pools. */\r
996         if( p_port->buf_mgr.h_buffer_pool )\r
997                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
998         if( p_port->buf_mgr.h_packet_pool )\r
999                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1000 \r
1001         /* Free the receive and send descriptors. */\r
1002         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1003 \r
1004         /* Free the lookaside list of scratch buffers. */\r
1005         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1006 \r
1007         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
1008 }\r
1009 \r
1010 \r
1011 static cl_status_t\r
1012 __recv_ctor(\r
1013         IN                              void* const                                     p_object,\r
1014         IN                              void*                                           context,\r
1015                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1016 {\r
1017         ipoib_recv_desc_t       *p_desc;\r
1018         ipoib_port_t            *p_port;\r
1019         uint32_t                        ds0_len;\r
1020 \r
1021         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1022 \r
1023         CL_ASSERT( p_object );\r
1024         CL_ASSERT( context );\r
1025 \r
1026         p_desc = (ipoib_recv_desc_t*)p_object;\r
1027         p_port = (ipoib_port_t*)context;\r
1028 \r
1029         /* Setup the work request. */\r
1030         p_desc->wr.ds_array = p_desc->local_ds;\r
1031         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1032 \r
1033 #if IPOIB_INLINE_RECV\r
1034         /* Sanity check on the receive buffer layout */\r
1035         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1036                 (void*)&p_desc->buf.ib.pkt.type );\r
1037         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1038 \r
1039         /* Setup the local data segment. */\r
1040         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1041         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1042         ds0_len =\r
1043                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1044         if( ds0_len >= sizeof(recv_buf_t) )\r
1045         {\r
1046                 /* The whole buffer is within a page. */\r
1047                 p_desc->local_ds[0].length = ds0_len;\r
1048                 p_desc->wr.num_ds = 1;\r
1049         }\r
1050         else\r
1051         {\r
1052                 /* The buffer crosses page boundaries. */\r
1053                 p_desc->local_ds[0].length = ds0_len;\r
1054                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1055                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1056                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1057                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1058                 p_desc->wr.num_ds = 2;\r
1059         }\r
1060 #else   /* IPOIB_INLINE_RECV */\r
1061         /* Allocate the receive buffer. */\r
1062         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1063         if( !p_desc->p_buf )\r
1064         {\r
1065                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1066                         ("Failed to allocate receive buffer.\n") );\r
1067                 return CL_INSUFFICIENT_MEMORY;\r
1068         }\r
1069 \r
1070         /* Sanity check on the receive buffer layout */\r
1071         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1072                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1073 \r
1074         /* Setup the local data segment. */\r
1075         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1076         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1077         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1078 #endif  /* IPOIB_INLINE_RECV */\r
1079 \r
1080         *pp_pool_item = &p_desc->item;\r
1081 \r
1082         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1083         return CL_SUCCESS;\r
1084 }\r
1085 \r
1086 \r
1087 #if !IPOIB_INLINE_RECV\r
1088 static void\r
1089 __recv_dtor(\r
1090         IN              const   cl_pool_item_t* const           p_pool_item,\r
1091         IN                              void                                            *context )\r
1092 {\r
1093         ipoib_recv_desc_t       *p_desc;\r
1094 \r
1095         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1096 \r
1097         UNUSED_PARAM( context );\r
1098 \r
1099         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1100 \r
1101         if( p_desc->p_buf )\r
1102                 cl_free( p_desc->p_buf );\r
1103 \r
1104         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
1105 }\r
1106 #endif\r
1107 \r
1108 \r
1109 static inline ipoib_recv_desc_t*\r
1110 __buf_mgr_get_recv(\r
1111         IN                              ipoib_port_t* const                     p_port )\r
1112 {\r
1113         ipoib_recv_desc_t       *p_desc;\r
1114         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1115         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1116         /* Reference the port object for the send. */\r
1117         if( p_desc )\r
1118         {\r
1119                 cl_obj_ref( &p_port->obj );\r
1120                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1121 #if IPOIB_INLINE_RECV\r
1122                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1123                         cl_get_physaddr( &p_desc->buf ) );\r
1124 #else   /* IPOIB_INLINE_RECV */\r
1125                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1126                         cl_get_physaddr( p_desc->p_buf ) );\r
1127                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1128                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1129 #endif  /* IPOIB_INLINE_RECV */\r
1130                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1131         }\r
1132         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1133         return p_desc;\r
1134 }\r
1135 \r
1136 \r
1137 static inline void\r
1138 __buf_mgr_put_recv(\r
1139         IN                              ipoib_port_t* const                     p_port,\r
1140         IN                              ipoib_recv_desc_t* const        p_desc,\r
1141         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1142 {\r
1143         NDIS_BUFFER             *p_buf;\r
1144 \r
1145         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1146 \r
1147         if( p_packet )\r
1148         {\r
1149                 /* Unchain the NDIS buffer. */\r
1150                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1151                 CL_ASSERT( p_buf );\r
1152                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1153                 NdisDprFreePacketNonInterlocked( p_packet );\r
1154                 NdisFreeBuffer( p_buf );\r
1155         }\r
1156 \r
1157         /* Return the descriptor to its pools. */\r
1158         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1159 \r
1160         /*\r
1161          * Dereference the port object since the receive is no longer outstanding.\r
1162          */\r
1163         cl_obj_deref( &p_port->obj );\r
1164 \r
1165         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1166 }\r
1167 \r
1168 \r
1169 static inline void\r
1170 __buf_mgr_put_recv_list(\r
1171         IN                              ipoib_port_t* const                     p_port,\r
1172         IN                              cl_qlist_t* const                       p_list )\r
1173 {\r
1174         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1175         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1176         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1177 }\r
1178 \r
1179 \r
1180 static inline NDIS_PACKET*\r
1181 __buf_mgr_get_ndis_pkt(\r
1182         IN                              ipoib_port_t* const                     p_port,\r
1183         IN                              ipoib_recv_desc_t* const        p_desc )\r
1184 {\r
1185         NDIS_STATUS                             status;\r
1186         NDIS_PACKET                             *p_packet;\r
1187         NDIS_BUFFER                             *p_buffer;\r
1188 \r
1189         IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1190 \r
1191         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1192                 p_port->buf_mgr.h_packet_pool );\r
1193         if( status != NDIS_STATUS_SUCCESS )\r
1194         {\r
1195                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1196                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1197                 return NULL;\r
1198         }\r
1199 \r
1200         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1201         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1202 \r
1203         NdisAllocateBuffer( &status, &p_buffer,\r
1204 #if IPOIB_INLINE_RECV\r
1205                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1206 #else   /* IPOIB_INLINE_RECV */\r
1207                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1208 #endif  /* IPOIB_INLINE_RECV */\r
1209         if( status != NDIS_STATUS_SUCCESS )\r
1210         {\r
1211                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1212                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1213                 NdisDprFreePacketNonInterlocked( p_packet );\r
1214                 return NULL;\r
1215         }\r
1216 \r
1217         NdisChainBufferAtFront( p_packet, p_buffer );\r
1218         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1219         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1220 \r
1221         IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
1222         return p_packet;\r
1223 }\r
1224 \r
1225 /******************************************************************************\r
1226 *\r
1227 * Receive manager implementation.\r
1228 *\r
1229 ******************************************************************************/\r
1230 static void\r
1231 __recv_mgr_construct(\r
1232         IN                              ipoib_port_t* const                     p_port )\r
1233 {\r
1234         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1235 \r
1236         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1237 \r
1238         p_port->recv_mgr.recv_pkt_array = NULL;\r
1239 \r
1240         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1241 }\r
1242 \r
1243 \r
1244 static ib_api_status_t\r
1245 __recv_mgr_init(\r
1246         IN                              ipoib_port_t* const                     p_port )\r
1247 {\r
1248         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1249 \r
1250         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1251         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1252                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1253         if( !p_port->recv_mgr.recv_pkt_array )\r
1254         {\r
1255                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1256                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1257                 return IB_INSUFFICIENT_MEMORY;\r
1258         }\r
1259 \r
1260         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1261         return IB_SUCCESS;\r
1262 }\r
1263 \r
1264 \r
1265 static void\r
1266 __recv_mgr_destroy(\r
1267         IN                              ipoib_port_t* const                     p_port )\r
1268 {\r
1269         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1270 \r
1271         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1272         CL_ASSERT( !p_port->recv_mgr.depth );\r
1273 \r
1274         if( p_port->recv_mgr.recv_pkt_array )\r
1275                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1276 \r
1277         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1278 }\r
1279 \r
1280 \r
1281 /* Posts receive buffers to the receive queue. */\r
1282 static ib_api_status_t\r
1283 __recv_mgr_repost(\r
1284         IN                              ipoib_port_t* const                     p_port,\r
1285         IN                              uint32_t                                        num_wrs )\r
1286 {\r
1287         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1288         ib_api_status_t         status;\r
1289         ib_recv_wr_t            *p_failed;\r
1290         PERF_DECLARE( GetRecv );\r
1291         PERF_DECLARE( PostRecv );\r
1292 \r
1293         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1294 \r
1295         CL_ASSERT( p_port );\r
1296         CL_ASSERT( num_wrs );\r
1297         cl_obj_lock( &p_port->obj );\r
1298         if( p_port->state == IB_QPS_ERROR )\r
1299         {\r
1300                 cl_obj_unlock( &p_port->obj );\r
1301                 IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
1302                         ("Port in invalid state.  Not reposting.\n") );\r
1303                 return IB_SUCCESS;\r
1304         }\r
1305         cl_obj_ref( &p_port->obj );\r
1306         cl_obj_unlock( &p_port->obj );\r
1307 \r
1308         while( num_wrs-- )\r
1309         {\r
1310                 if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )\r
1311                 {\r
1312                         IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1313                                 ("Receive queue already full\n") );\r
1314                         break;\r
1315                 }\r
1316 \r
1317                 /* Pull receives out of the pool and chain them up. */\r
1318                 cl_perf_start( GetRecv );\r
1319                 p_next = __buf_mgr_get_recv( p_port );\r
1320                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1321                 if( !p_next )\r
1322                 {\r
1323                         IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1324                                 ("Out of receive descriptors!\n") );\r
1325                         break;\r
1326                 }\r
1327 \r
1328                 if( !p_tail )\r
1329                 {\r
1330                         p_tail = p_next;\r
1331                         p_next->wr.p_next = NULL;\r
1332                 }\r
1333                 else\r
1334                 {\r
1335                         p_next->wr.p_next = &p_head->wr;\r
1336                 }\r
1337 \r
1338                 p_head = p_next;\r
1339 \r
1340                 cl_atomic_inc( &p_port->recv_mgr.depth );\r
1341         }\r
1342 \r
1343         if( !p_head )\r
1344         {\r
1345                 cl_obj_deref( &p_port->obj );\r
1346                 IPOIB_EXIT( IPOIB_DBG_RECV );\r
1347                 return IB_SUCCESS;\r
1348         }\r
1349 \r
1350         cl_perf_start( PostRecv );\r
1351         status = p_port->p_adapter->p_ifc->post_recv(\r
1352                 p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1353         cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1354 \r
1355         /*\r
1356          * If we failed, fix up the work completion list and return those\r
1357          * buffers to the pool\r
1358          */\r
1359         if( status != IB_SUCCESS )\r
1360         {\r
1361                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1362                         ("ip_post_recv returned %s\n", \r
1363                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1364                 /* return the descriptors to the pool */\r
1365                 while( p_failed )\r
1366                 {\r
1367                         p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1368                         p_failed = p_failed->p_next;\r
1369 \r
1370                         __buf_mgr_put_recv( p_port, p_head, NULL );\r
1371                         cl_atomic_dec( &p_port->recv_mgr.depth );\r
1372                 }\r
1373         }\r
1374 \r
1375         cl_obj_deref( &p_port->obj );\r
1376 \r
1377         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1378         return status;\r
1379 }\r
1380 \r
1381 \r
1382 void\r
1383 ipoib_return_packet(\r
1384         IN                              NDIS_HANDLE                                     adapter_context,\r
1385         IN                              NDIS_PACKET                                     *p_packet )\r
1386 {\r
1387         cl_list_item_t          *p_item;\r
1388         ipoib_port_t            *p_port;\r
1389         ipoib_recv_desc_t       *p_desc;\r
1390         ib_api_status_t         status = IB_NOT_DONE;\r
1391         PERF_DECLARE( ReturnPacket );\r
1392         PERF_DECLARE( ReturnPutRecv );\r
1393         PERF_DECLARE( ReturnRepostRecv );\r
1394         PERF_DECLARE( ReturnPreparePkt );\r
1395         PERF_DECLARE( ReturnNdisIndicate );\r
1396 \r
1397         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1398 \r
1399         UNUSED_PARAM( adapter_context );\r
1400         CL_ASSERT( p_packet );\r
1401 \r
1402         cl_perf_start( ReturnPacket );\r
1403 \r
1404         /* Get the manager and descriptor from the packet. */\r
1405         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1406         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1407 \r
1408         cl_spinlock_acquire( &p_port->recv_lock );\r
1409         cl_perf_start( ReturnPutRecv );\r
1410         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1411         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1412 \r
1413         /* Repost buffers. */\r
1414         cl_perf_start( ReturnRepostRecv );\r
1415         __recv_mgr_repost( p_port, 0xFFFF );\r
1416         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1417 \r
1418         /* Complete any additional receives waiting for a packet. */\r
1419         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1420         do\r
1421         {\r
1422                 if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
1423                 {\r
1424                         cl_spinlock_release( &p_port->recv_lock );\r
1425                         break;\r
1426                 }\r
1427 \r
1428                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1429 \r
1430                 cl_perf_start( ReturnPreparePkt );\r
1431                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1432                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1433                 if( status == IB_SUCCESS )\r
1434                 {\r
1435                         cl_spinlock_release( &p_port->recv_lock );\r
1436                         cl_perf_start( ReturnNdisIndicate );\r
1437                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1438                                 &p_packet, 1 );\r
1439                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1440                 }\r
1441                 else if( status != IB_NOT_DONE )\r
1442                 {\r
1443                         IPOIB_TRACE( IPOIB_DBG_RECV,\r
1444                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1445                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1446                         /* Return the item to the head of the list. */\r
1447                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1448                         cl_spinlock_release( &p_port->recv_lock );\r
1449                 }\r
1450                 else\r
1451                 {\r
1452                         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1453                 }\r
1454 \r
1455         } while( status == IB_NOT_DONE );\r
1456         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1457 \r
1458         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1459 }\r
1460 \r
1461 \r
1462 static void\r
1463 __recv_cb(\r
1464         IN              const   ib_cq_handle_t                          h_cq,\r
1465         IN                              void                                            *cq_context )\r
1466 {\r
1467         ipoib_port_t            *p_port;\r
1468         ib_api_status_t         status;\r
1469         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1470         uint32_t                        pkt_cnt, recv_cnt = 0;\r
1471         cl_qlist_t                      done_list, bad_list;\r
1472         size_t                          i;\r
1473         PERF_DECLARE( RecvCompBundle );\r
1474         PERF_DECLARE( RecvCb );\r
1475         PERF_DECLARE( PollRecv );\r
1476         PERF_DECLARE( RepostRecv );\r
1477         PERF_DECLARE( FilterRecv );\r
1478         PERF_DECLARE( BuildPktArray );\r
1479         PERF_DECLARE( RecvNdisIndicate );\r
1480         PERF_DECLARE( RearmRecv );\r
1481         PERF_DECLARE( PutRecvList );\r
1482 \r
1483         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1484 \r
1485         cl_perf_clr( RecvCompBundle );\r
1486 \r
1487         cl_perf_start( RecvCb );\r
1488 \r
1489         UNUSED_PARAM( h_cq );\r
1490 \r
1491         p_port = (ipoib_port_t*)cq_context;\r
1492 \r
1493         cl_qlist_init( &done_list );\r
1494         cl_qlist_init( &bad_list );\r
1495 \r
1496         cl_obj_ref( &p_port->obj );\r
1497 \r
1498         for( i = 0; i < MAX_RECV_WC; i++ )\r
1499                 wc[i].p_next = &wc[i + 1];\r
1500         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1501 \r
1502         /*\r
1503          * We'll be accessing the endpoint map so take a reference\r
1504          * on it to prevent modifications.\r
1505          */\r
1506         cl_obj_lock( &p_port->obj );\r
1507         cl_atomic_inc( &p_port->endpt_rdr );\r
1508         cl_obj_unlock( &p_port->obj );\r
1509 \r
1510         do\r
1511         {\r
1512                 /* If we get here, then the list of WCs is intact. */\r
1513                 p_free = wc;\r
1514 \r
1515                 cl_perf_start( PollRecv );\r
1516                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1517                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1518                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1519                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1520 \r
1521                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1522                 cl_perf_start( FilterRecv );\r
1523                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1524                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1525 \r
1526         } while( !p_free );\r
1527 \r
1528         /* We're done looking at the endpoint map, release the reference. */\r
1529         cl_atomic_dec( &p_port->endpt_rdr );\r
1530 \r
1531         /* Update our posted depth. */\r
1532         cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
1533 \r
1534         cl_perf_start( BuildPktArray );\r
1535         /* Notify NDIS of any and all possible receive buffers. */\r
1536         pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
1537         cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1538 \r
1539         /* Only indicate receives if we actually had any. */\r
1540         if( pkt_cnt )\r
1541         {\r
1542                 cl_perf_start( RecvNdisIndicate );\r
1543                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1544                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1545                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1546         }\r
1547 \r
1548         cl_spinlock_acquire( &p_port->recv_lock );\r
1549 \r
1550         /* Return any discarded receives to the pool */\r
1551         cl_perf_start( PutRecvList );\r
1552         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1553         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1554 \r
1555         /* Repost receives. */\r
1556         cl_perf_start( RepostRecv );\r
1557         __recv_mgr_repost( p_port, recv_cnt );\r
1558         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1559 \r
1560         cl_spinlock_release( &p_port->recv_lock );\r
1561 \r
1562         /*\r
1563          * Rearm after filtering to prevent contention on the enpoint maps\r
1564          * and eliminate the possibility of having a call to\r
1565          * __endpt_mgr_insert find a duplicate.\r
1566          */\r
1567         cl_perf_start( RearmRecv );\r
1568         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1569                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1570         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1571         CL_ASSERT( status == IB_SUCCESS );\r
1572 \r
1573         cl_obj_deref( &p_port->obj );\r
1574 \r
1575         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1576         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1577 \r
1578         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1579 }\r
1580 \r
1581 \r
1582 static void\r
1583 __recv_get_endpts(\r
1584         IN                              ipoib_port_t* const                     p_port,\r
1585         IN                              ipoib_recv_desc_t* const        p_desc,\r
1586         IN                              ib_wc_t* const                          p_wc,\r
1587                 OUT                     ipoib_endpt_t** const           pp_src,\r
1588                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1589 {\r
1590         ib_api_status_t         status;\r
1591         mac_addr_t                      mac;\r
1592         PERF_DECLARE( GetEndptByGid );\r
1593         PERF_DECLARE( GetEndptByLid );\r
1594         PERF_DECLARE( EndptInsert );\r
1595 \r
1596         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1597 \r
1598         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1599         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1600         {\r
1601                 /* Lookup the source endpoints based on GID. */\r
1602                 cl_perf_start( GetEndptByGid );\r
1603                 *pp_src =\r
1604 #if IPOIB_INLINE_RECV\r
1605                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1606 #else   /* IPOIB_INLINE_RECV */\r
1607                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1608 #endif  /* IPOIB_INLINE_RECV */\r
1609                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1610 \r
1611                 /*\r
1612                  * Lookup the destination endpoint based on GID.\r
1613                  * This is used along with the packet filter to determine\r
1614                  * whether to report this to NDIS.\r
1615                  */\r
1616                 cl_perf_start( GetEndptByGid );\r
1617                 *pp_dst =\r
1618 #if IPOIB_INLINE_RECV\r
1619                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1620 #else   /* IPOIB_INLINE_RECV */\r
1621                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1622 #endif  /* IPOIB_INLINE_RECV */\r
1623                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1624 \r
1625                 /*\r
1626                  * Create the source endpoint if it does not exist.  Note that we\r
1627                  * can only do this for globally routed traffic since we need the\r
1628                  * information from the GRH to generate the MAC.\r
1629                  */\r
1630                 if( !*pp_src )\r
1631                 {\r
1632                         status = ipoib_mac_from_guid(\r
1633 #if IPOIB_INLINE_RECV\r
1634                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1635 #else   /* IPOIB_INLINE_RECV */\r
1636                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1637 #endif  /* IPOIB_INLINE_RECV */\r
1638                         if( status != IB_SUCCESS )\r
1639                         {\r
1640                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1641                                         ("ipoib_mac_from_guid returned %s\n",\r
1642                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1643                                 return;\r
1644                         }\r
1645 \r
1646                         /* Create the endpoint. */\r
1647 #if IPOIB_INLINE_RECV\r
1648                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1649 #else   /* IPOIB_INLINE_RECV */\r
1650                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1651 #endif  /* IPOIB_INLINE_RECV */\r
1652                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1653                         if( !*pp_src )\r
1654                         {\r
1655                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1656                                         ("ipoib_endpt_create returned %s\n",\r
1657                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1658                                 return;\r
1659                         }\r
1660                         cl_perf_start( EndptInsert );\r
1661                         cl_obj_lock( &p_port->obj );\r
1662                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1663                         cl_obj_unlock( &p_port->obj );\r
1664                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1665                 }\r
1666         }\r
1667         else\r
1668         {\r
1669                 /*\r
1670                  * Lookup the remote endpoint based on LID.  Note that only\r
1671                  * unicast traffic can be LID routed.\r
1672                  */\r
1673                 cl_perf_start( GetEndptByLid );\r
1674                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1675                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1676                 *pp_dst = p_port->p_local_endpt;\r
1677                 CL_ASSERT( *pp_dst );\r
1678         }\r
1679 \r
1680         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1681 }\r
1682 \r
1683 \r
1684 static uint32_t\r
1685 __recv_mgr_filter(\r
1686         IN                              ipoib_port_t* const                     p_port,\r
1687         IN                              ib_wc_t* const                          p_done_wc_list,\r
1688                 OUT                     cl_qlist_t* const                       p_done_list,\r
1689                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1690 {\r
1691         ipoib_recv_desc_t               *p_desc;\r
1692         ib_wc_t                                 *p_wc;\r
1693         ipoib_pkt_t                             *p_ipoib;\r
1694         eth_pkt_t                               *p_eth;\r
1695         ipoib_endpt_t                   *p_src, *p_dst;\r
1696         ib_api_status_t                 status;\r
1697         uint32_t                                len, recv_cnt = 0;\r
1698         PERF_DECLARE( GetRecvEndpts );\r
1699         PERF_DECLARE( RecvGen );\r
1700         PERF_DECLARE( RecvTcp );\r
1701         PERF_DECLARE( RecvUdp );\r
1702         PERF_DECLARE( RecvDhcp );\r
1703         PERF_DECLARE( RecvArp );\r
1704 \r
1705         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1706 \r
1707         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1708         {\r
1709                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1710                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1711                 recv_cnt++;\r
1712 \r
1713                 if( p_wc->status != IB_WCS_SUCCESS )\r
1714                 {\r
1715                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1716                         {\r
1717                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1718                                         ("Failed completion %s\n",\r
1719                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1720                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1721                         }\r
1722                         else\r
1723                         {\r
1724                                 IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
1725                                         ("Flushed completion %s\n",\r
1726                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1727                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1728                         }\r
1729                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1730                         /* Dereference the port object on behalf of the failed receive. */\r
1731                         cl_obj_deref( &p_port->obj );\r
1732                         continue;\r
1733                 }\r
1734 \r
1735                 len = p_wc->length;\r
1736 \r
1737                 if( len < (sizeof(ib_grh_t) + sizeof(ipoib_hdr_t)) )\r
1738                 {\r
1739                         IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1740                                 ("Received ETH packet < min size\n") );\r
1741                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1742                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1743                         cl_obj_deref( &p_port->obj );\r
1744                         continue;\r
1745                 }\r
1746 \r
1747                 /* Successful completion.  Get the receive information. */\r
1748                 cl_perf_start( GetRecvEndpts );\r
1749                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1750                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1751 \r
1752 #if IPOIB_INLINE_RECV\r
1753                 p_ipoib = &p_desc->buf.ib.pkt;\r
1754                 p_eth = &p_desc->buf.eth.pkt;\r
1755 #else   /* IPOIB_INLINE_RECV */\r
1756                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1757                 p_eth = &p_desc->p_buf->eth.pkt;\r
1758 #endif  /*IPOIB_INLINE_RECV */\r
1759 \r
1760                 switch( p_ipoib->hdr.type )\r
1761                 {\r
1762                 case ETH_PROT_TYPE_IP:\r
1763                         if( len <\r
1764                                 (sizeof(ib_grh_t) + sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1765                         {\r
1766                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1767                                         ("Received IP packet < min size\n") );\r
1768                                 status = IB_INVALID_SETTING;\r
1769                                 break;\r
1770                         }\r
1771 \r
1772                         if( p_ipoib->type.ip.hdr.offset ||\r
1773                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1774                         {\r
1775                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1776                                 cl_perf_start( RecvTcp );\r
1777                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1778                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1779                                 break;\r
1780                         }\r
1781 \r
1782                         /* First packet of a UDP transfer. */\r
1783                         if( len < (sizeof(ib_grh_t) + sizeof(ipoib_hdr_t) +\r
1784                                 sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1785                         {\r
1786                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1787                                         ("Received UDP packet < min size\n") );\r
1788                                 status = IB_INVALID_SETTING;\r
1789                                 break;\r
1790                         }\r
1791 \r
1792                         /* Check if DHCP conversion is required. */\r
1793                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1794                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1795                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1796                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1797                         {\r
1798                                 if( len < (sizeof(ib_grh_t) + sizeof(ipoib_hdr_t) +\r
1799                                         sizeof(ip_hdr_t) + sizeof(udp_hdr_t) + DHCP_MIN_SIZE ))\r
1800                                 {\r
1801                                         IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1802                                                 ("Received DHCP < min size\n") );\r
1803                                         status = IB_INVALID_SETTING;\r
1804                                         break;\r
1805                                 }\r
1806                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1807                                 cl_perf_start( RecvDhcp );\r
1808                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1809                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1810                         }\r
1811                         else\r
1812                         {\r
1813                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1814                                 cl_perf_start( RecvUdp );\r
1815                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1816                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1817                         }\r
1818                         break;\r
1819 \r
1820                 case ETH_PROT_TYPE_ARP:\r
1821                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1822                         {\r
1823                                 IPOIB_TRACE( IPOIB_DBG_ERROR,\r
1824                                         ("Received ARP < min size\n") );\r
1825                                 status = IB_INVALID_SETTING;\r
1826                                 break;\r
1827                         }\r
1828                         cl_perf_start( RecvArp );\r
1829                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1830                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1831                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1832                         break;\r
1833 \r
1834                 default:\r
1835                         /* Unfiltered.  Setup the ethernet header and report. */\r
1836                         cl_perf_start( RecvGen );\r
1837                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1838                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1839                 }\r
1840 \r
1841                 if( status != IB_SUCCESS )\r
1842                 {\r
1843                         /* Update stats. */\r
1844                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1845                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1846                         /* Dereference the port object on behalf of the failed receive. */\r
1847                         cl_obj_deref( &p_port->obj );\r
1848                 }\r
1849                 else\r
1850                 {\r
1851                         p_desc->len =\r
1852                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1853                         if( p_dst )\r
1854                         {\r
1855                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1856                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1857                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1858                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1859                                 {\r
1860                                         p_desc->type = PKT_TYPE_BCAST;\r
1861                                 }\r
1862                                 else\r
1863                                 {\r
1864                                         p_desc->type = PKT_TYPE_MCAST;\r
1865                                 }\r
1866                         }\r
1867                         else\r
1868                         {\r
1869                                 p_desc->type = PKT_TYPE_UCAST;\r
1870                         }\r
1871                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1872                 }\r
1873         }\r
1874 \r
1875         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1876         return recv_cnt;\r
1877 }\r
1878 \r
1879 \r
1880 static ib_api_status_t\r
1881 __recv_gen(\r
1882         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1883                 OUT                     eth_pkt_t* const                        p_eth,\r
1884         IN                              ipoib_endpt_t* const            p_src,\r
1885         IN                              ipoib_endpt_t* const            p_dst )\r
1886 {\r
1887         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1888 \r
1889         if( !p_src || !p_dst )\r
1890         {\r
1891                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1892                         ("Received packet with no matching endpoints.\n") );\r
1893                 return IB_NOT_DONE;\r
1894         }\r
1895 \r
1896         /*\r
1897          * Fill in the ethernet header.  Note that doing so will overwrite\r
1898          * the IPoIB header, so start by moving the information from the IPoIB\r
1899          * header.\r
1900          */\r
1901         p_eth->hdr.type = p_ipoib->hdr.type;\r
1902         p_eth->hdr.src = p_src->mac;\r
1903         p_eth->hdr.dst = p_dst->mac;\r
1904 \r
1905         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1906         return IB_SUCCESS;\r
1907 }\r
1908 \r
1909 \r
1910 static ib_api_status_t\r
1911 __recv_dhcp(\r
1912         IN                              ipoib_port_t* const                     p_port,\r
1913         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1914                 OUT                     eth_pkt_t* const                        p_eth,\r
1915         IN                              ipoib_endpt_t* const            p_src,\r
1916         IN                              ipoib_endpt_t* const            p_dst )\r
1917 {\r
1918         ib_api_status_t         status;\r
1919         dhcp_pkt_t                      *p_dhcp;\r
1920         uint8_t                         *p_option;\r
1921         uint8_t                         *p_cid = NULL;\r
1922         ib_gid_t                        gid;\r
1923         uint8_t                         msg = 0;\r
1924 \r
1925         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1926 \r
1927         /* Create the ethernet header. */\r
1928         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1929         if( status != IB_SUCCESS )\r
1930         {\r
1931                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1932                         ("__recv_gen returned %s.\n", \r
1933                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1934                 return status;\r
1935         }\r
1936 \r
1937         /* Fixup the payload. */\r
1938         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
1939         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
1940         {\r
1941                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("Invalid DHCP op code.\n") );\r
1942                 return IB_INVALID_SETTING;\r
1943         }\r
1944         /* find a better way to check for echo packets ? */\r
1945         if ( p_port->p_adapter->mac.addr[0] == p_eth->hdr.src.addr[0] &&\r
1946                  p_port->p_adapter->mac.addr[1] == p_eth->hdr.src.addr[1] &&\r
1947                  p_port->p_adapter->mac.addr[2] == p_eth->hdr.src.addr[2] &&\r
1948                  p_port->p_adapter->mac.addr[3] == p_eth->hdr.src.addr[3] &&\r
1949                  p_port->p_adapter->mac.addr[4] == p_eth->hdr.src.addr[4] &&\r
1950                  p_port->p_adapter->mac.addr[5] == p_eth->hdr.src.addr[5] )\r
1951         {\r
1952                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("Local echo dhcp msg.\n") );\r
1953                 return IB_INVALID_PARAMETER;\r
1954         }\r
1955 \r
1956         /*\r
1957          * Find the client identifier option, making sure to skip\r
1958          * the "magic cookie".\r
1959          */\r
1960         p_option = &p_dhcp->options[0];\r
1961         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
1962         {\r
1963                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
1964                         ("DHCP cookie corrupted.\n") );\r
1965                 return IB_INVALID_PARAMETER;\r
1966         }\r
1967 \r
1968         p_option = &p_dhcp->options[4];\r
1969         while( *p_option != DHCP_OPT_END )\r
1970         {\r
1971                 switch( *p_option )\r
1972                 {\r
1973                 case DHCP_OPT_PAD:\r
1974                         p_option++;\r
1975                         break;\r
1976 \r
1977                 case DHCP_OPT_MSG:\r
1978                         msg = p_option[2];\r
1979                         p_option += 3;\r
1980                         break;\r
1981 \r
1982                 case DHCP_OPT_CLIENT_ID:\r
1983                         p_cid = p_option;\r
1984                         /* Fall through. */\r
1985 \r
1986                 default:\r
1987                         /*\r
1988                          * All other options have a length byte following the option code.\r
1989                          * Offset by the length to get to the next option.\r
1990                          */\r
1991                         p_option += (p_option[1] + 2);\r
1992                 }\r
1993         }\r
1994 \r
1995         switch( msg )\r
1996         {\r
1997         /* message from client */\r
1998         case DHCPDISCOVER:\r
1999         case DHCPREQUEST:\r
2000         case DHCPDECLINE:\r
2001         case DHCPRELEASE:\r
2002         case DHCPINFORM:\r
2003                 if( !p_cid )\r
2004                 {\r
2005                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2006                                 ("Failed to find required Client-identifier option.\n") );\r
2007                         return IB_INVALID_SETTING;\r
2008                 }\r
2009                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2010                 {\r
2011                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2012                                 ("Invalid hardware address type.\n") );\r
2013                         return IB_INVALID_SETTING;\r
2014                 }\r
2015                 break;\r
2016         /* message from DHCP server */\r
2017         case DHCPOFFER:\r
2018         case DHCPACK:\r
2019         case DHCPNAK:\r
2020                 break;\r
2021 \r
2022         default:\r
2023                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2024                         ("Invalide message type.\n") );\r
2025                 return IB_INVALID_PARAMETER;\r
2026         }\r
2027         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2028         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2029         p_dhcp->hlen = HW_ADDR_LEN;\r
2030 \r
2031         if( p_cid ) /* from client */\r
2032         {\r
2033                 /* Validate that the length and type of the option is as required. */\r
2034                 if( p_cid[1] != 21 )\r
2035                 {\r
2036                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2037                                 ("Client-identifier length not 21 as required.\n") );\r
2038                         return IB_INVALID_SETTING;\r
2039                 }\r
2040                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2041                 {\r
2042                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2043                                 ("Client-identifier type is wrong.\n") );\r
2044                         return IB_INVALID_SETTING;\r
2045                 }\r
2046                 /*\r
2047                  * Copy the GID value from the option so that we can make aligned\r
2048                  * accesses to the contents.\r
2049                  * Recover CID to standard type.\r
2050                  */\r
2051                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2052                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2053                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2054                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2055                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2056         }\r
2057         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2058         return status;\r
2059 }\r
2060 \r
2061 \r
2062 static ib_api_status_t\r
2063 __recv_arp(\r
2064         IN                              ipoib_port_t* const                     p_port,\r
2065         IN                              ib_wc_t* const                          p_wc,\r
2066         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2067                 OUT                     eth_pkt_t* const                        p_eth,\r
2068         IN                              ipoib_endpt_t** const           pp_src,\r
2069         IN                              ipoib_endpt_t* const            p_dst )\r
2070 {\r
2071         ib_api_status_t                 status;\r
2072         arp_pkt_t                               *p_arp;\r
2073         const ipoib_arp_pkt_t   *p_ib_arp;\r
2074         ib_gid_t                                gid;\r
2075         mac_addr_t                              mac;\r
2076         ipoib_hw_addr_t                 null_hw = {0};\r
2077 \r
2078         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2079 \r
2080         if( !p_dst )\r
2081         {\r
2082                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2083                         ("Unknown destination endpoint\n") );\r
2084                 return IB_INVALID_SETTING;\r
2085         }\r
2086 \r
2087         p_ib_arp = &p_ipoib->type.arp;\r
2088         p_arp = &p_eth->type.arp;\r
2089 \r
2090         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2091         {\r
2092                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2093                         ("ARP hardware type is not IB\n") );\r
2094                 return IB_INVALID_SETTING;\r
2095         }\r
2096 \r
2097         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2098         {\r
2099                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2100                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2101                 return IB_INVALID_SETTING;\r
2102         }\r
2103 \r
2104         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2105         {\r
2106                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP protocal type not IP\n") );\r
2107                 return IB_INVALID_SETTING;\r
2108         }\r
2109 \r
2110         /*\r
2111          * If we don't have a source, lookup the endpoint specified in the payload.\r
2112          */\r
2113         if( !*pp_src )\r
2114                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2115 \r
2116         /*\r
2117          * If the endpoint exists for the GID, make sure\r
2118          * the dlid and qpn match the arp.\r
2119          */\r
2120         if( *pp_src )\r
2121         {\r
2122                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2123                         sizeof(ib_gid_t) ) )\r
2124                 {\r
2125                         /*\r
2126                          * GIDs for the endpoint are different.  The ARP must\r
2127                          * have been proxied.  Dereference it.\r
2128                          */\r
2129                         *pp_src = NULL;\r
2130                 }\r
2131                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2132                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2133                 {\r
2134                         /* Out of date!  Destroy the endpoint and replace it. */\r
2135                         __endpt_mgr_remove( p_port, *pp_src );\r
2136                         *pp_src = NULL;\r
2137                 }\r
2138         }\r
2139 \r
2140         /* Do we need to create an endpoint for this GID? */\r
2141         if( !*pp_src )\r
2142         {\r
2143                 /* Copy the src GID to allow aligned access */\r
2144                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2145                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2146                 if( status != IB_SUCCESS )\r
2147                 {\r
2148                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2149                                 ("ipoib_mac_from_guid returned %s\n",\r
2150                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2151                         return status;\r
2152                 }\r
2153                 /*\r
2154                  * Create the endpoint.  Note that the LID is left blank and will be\r
2155                  * resolved by a path query as needed.  This is done because the\r
2156                  * remote LID/GID from the work completion may not be the original\r
2157                  * initiator.\r
2158                  */\r
2159                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2160                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2161                 if( !*pp_src )\r
2162                 {\r
2163                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2164                                 ("ipoib_endpt_create returned %s\n",\r
2165                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2166                         return status;\r
2167                 }\r
2168 \r
2169                 cl_obj_lock( &p_port->obj );\r
2170                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2171                 cl_obj_unlock( &p_port->obj );\r
2172         }\r
2173 \r
2174         CL_ASSERT( !cl_memcmp(\r
2175                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2176         CL_ASSERT( (*pp_src)->qpn ==\r
2177                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2178         /* Now swizzle the data. */\r
2179         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2180         p_arp->hw_size = sizeof(mac_addr_t);\r
2181         p_arp->src_hw = (*pp_src)->mac;\r
2182         p_arp->src_ip = p_ib_arp->src_ip;\r
2183 \r
2184         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2185         {\r
2186 \r
2187                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2188                 {\r
2189                         /*\r
2190                          * We received bcast ARP packet that means\r
2191                          * remote port lets everyone know it was changed IP/MAC\r
2192                          * or just activated\r
2193                          */\r
2194                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2195                         {\r
2196                                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP: is not ARP MCAST\n") );\r
2197                                 return IB_INVALID_SETTING;\r
2198                         }\r
2199 \r
2200                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2201                         p_dst->mac = p_port->p_local_endpt->mac;\r
2202                         /*\r
2203                          * we don't care what receiver ip addr is,\r
2204                          * as long as OS' ARP table is global  ???\r
2205                          */\r
2206                         p_arp->dst_ip = (net32_t)0;\r
2207 \r
2208                 }\r
2209                 else /* we've got reply to our ARP request */\r
2210                 {\r
2211                         p_arp->dst_hw = p_dst->mac;\r
2212                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2213                         CL_ASSERT( p_dst->qpn == \r
2214                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2215                 }\r
2216         }\r
2217         else /* we got ARP reqeust */\r
2218         {\r
2219                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2220                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2221         }\r
2222 \r
2223         /*\r
2224          * Create the ethernet header.  Note that this is done last so that\r
2225          * we have a chance to create a new endpoint.\r
2226          */\r
2227         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2228         if( status != IB_SUCCESS )\r
2229         {\r
2230                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2231                         ("__recv_gen returned %s.\n", \r
2232                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2233                 return status;\r
2234         }\r
2235 \r
2236         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2237         return IB_SUCCESS;\r
2238 }\r
2239 \r
2240 \r
2241 static ib_api_status_t\r
2242 __recv_mgr_prepare_pkt(\r
2243         IN                              ipoib_port_t* const                     p_port,\r
2244         IN                              ipoib_recv_desc_t* const        p_desc,\r
2245                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2246 {\r
2247         NDIS_STATUS                                                     status;\r
2248         uint32_t                                                        pkt_filter;\r
2249         ip_stat_sel_t                                           type;\r
2250         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2251         PERF_DECLARE( GetNdisPkt );\r
2252 \r
2253         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2254 \r
2255         pkt_filter = p_port->p_adapter->packet_filter;\r
2256         /* Check the packet filter. */\r
2257         switch( p_desc->type )\r
2258         {\r
2259         default:\r
2260         case PKT_TYPE_UCAST:\r
2261                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2262                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2263                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2264                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2265                 {\r
2266                         /* OK to report. */\r
2267                         type = IP_STAT_UCAST_BYTES;\r
2268                         status = NDIS_STATUS_SUCCESS;\r
2269                 }\r
2270                 else\r
2271                 {\r
2272                         type = IP_STAT_DROPPED;\r
2273                         status = NDIS_STATUS_FAILURE;\r
2274                 }\r
2275                 break;\r
2276         case PKT_TYPE_BCAST:\r
2277                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2278                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2279                 {\r
2280                         /* OK to report. */\r
2281                         type = IP_STAT_BCAST_BYTES;\r
2282                         status = NDIS_STATUS_SUCCESS;\r
2283                 }\r
2284                 else\r
2285                 {\r
2286                         type = IP_STAT_DROPPED;\r
2287                         status = NDIS_STATUS_FAILURE;\r
2288                 }\r
2289                 break;\r
2290         case PKT_TYPE_MCAST:\r
2291                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2292                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2293                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2294                 {\r
2295                         /* OK to report. */\r
2296                         type = IP_STAT_MCAST_BYTES;\r
2297                         status = NDIS_STATUS_SUCCESS;\r
2298                 }\r
2299                 else\r
2300                 {\r
2301                         type = IP_STAT_DROPPED;\r
2302                         status = NDIS_STATUS_FAILURE;\r
2303                 }\r
2304                 break;\r
2305         }\r
2306 \r
2307         if( status != NDIS_STATUS_SUCCESS )\r
2308         {\r
2309                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2310                 /* Return the receive descriptor to the pool. */\r
2311                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2312                 IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
2313                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2314                 /*\r
2315                  * Return IB_NOT_DONE since the packet has been completed,\r
2316                  * but has not consumed an array entry.\r
2317                  */\r
2318                 return IB_NOT_DONE;\r
2319         }\r
2320 \r
2321         cl_perf_start( GetNdisPkt );\r
2322         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2323         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2324         if( !*pp_packet )\r
2325         {\r
2326                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2327                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2328                 return IB_INSUFFICIENT_RESOURCES;\r
2329         }\r
2330 \r
2331         /* Flag the checksums as having been calculated. */\r
2332         chksum.Value = 0;\r
2333         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2334         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2335         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2336         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2337                 (void*)(uintn_t)chksum.Value;\r
2338 \r
2339         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2340 \r
2341         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2342         return IB_SUCCESS;\r
2343 }\r
2344 \r
2345 \r
2346 static uint32_t\r
2347 __recv_mgr_build_pkt_array(\r
2348         IN                              ipoib_port_t* const                     p_port,\r
2349                 OUT                     cl_qlist_t* const                       p_done_list )\r
2350 {\r
2351         cl_list_item_t                  *p_item;\r
2352         ipoib_recv_desc_t               *p_desc;\r
2353         uint32_t                                i = 0;\r
2354         ib_api_status_t                 status;\r
2355         PERF_DECLARE( PreparePkt );\r
2356 \r
2357         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2358 \r
2359         cl_spinlock_acquire( &p_port->recv_lock );\r
2360 \r
2361         /* Move any existing receives to the head to preserve ordering. */\r
2362         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2363         p_item = cl_qlist_remove_head( p_done_list );\r
2364         while( p_item != cl_qlist_end( p_done_list ) )\r
2365         {\r
2366                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2367 \r
2368                 cl_perf_start( PreparePkt );\r
2369                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2370                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2371                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2372                 if( status == IB_SUCCESS )\r
2373                 {\r
2374                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2375                         i++;\r
2376                 }\r
2377                 else if( status != IB_NOT_DONE )\r
2378                 {\r
2379                         IPOIB_TRACE( IPOIB_DBG_RECV,\r
2380                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2381                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2382                         /* Put all completed receives on the port's done list. */\r
2383                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2384                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2385                         break;\r
2386                 }\r
2387 \r
2388                 p_item = cl_qlist_remove_head( p_done_list );\r
2389         }\r
2390 \r
2391         cl_spinlock_release( &p_port->recv_lock );\r
2392 \r
2393         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2394         return i;\r
2395 }\r
2396 \r
2397 \r
2398 \r
2399 \r
2400 /******************************************************************************\r
2401 *\r
2402 * Send manager implementation.\r
2403 *\r
2404 ******************************************************************************/\r
2405 static void\r
2406 __send_mgr_construct(\r
2407         IN                              ipoib_port_t* const                     p_port )\r
2408 {\r
2409         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2410         p_port->send_mgr.depth = 0;\r
2411         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2412         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2413 }\r
2414 \r
2415 \r
2416 static void\r
2417 __send_mgr_destroy(\r
2418         IN                              ipoib_port_t* const                     p_port )\r
2419 {\r
2420         cl_list_item_t  *p_item;\r
2421         NDIS_PACKET             *p_packet;\r
2422 \r
2423         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2424 \r
2425         /* Complete any pending packets. */\r
2426         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2427                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2428                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2429         {\r
2430                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2431                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2432                         NDIS_STATUS_REQUEST_ABORTED );\r
2433         }\r
2434 \r
2435         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2436 }\r
2437 \r
2438 \r
2439 static NDIS_STATUS\r
2440 __send_mgr_filter(\r
2441         IN                              ipoib_port_t* const                     p_port,\r
2442         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2443         IN                              NDIS_BUFFER* const                      p_buf,\r
2444         IN                              size_t                                          buf_len,\r
2445         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2446 {\r
2447         NDIS_STATUS             status;\r
2448 \r
2449         PERF_DECLARE( FilterIp );\r
2450         PERF_DECLARE( FilterArp );\r
2451         PERF_DECLARE( SendGen );\r
2452 \r
2453         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2454 \r
2455         /*\r
2456          * We already checked the ethernet header length, so we know it's safe\r
2457          * to decrement the buf_len without underflowing.\r
2458          */\r
2459         buf_len -= sizeof(eth_hdr_t);\r
2460 \r
2461         switch( p_eth_hdr->type )\r
2462         {\r
2463         case ETH_PROT_TYPE_IP:\r
2464                 cl_perf_start( FilterIp );\r
2465                 status = __send_mgr_filter_ip(\r
2466                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2467                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2468                 break;\r
2469 \r
2470         case ETH_PROT_TYPE_ARP:\r
2471                 cl_perf_start( FilterArp );\r
2472                 status = __send_mgr_filter_arp(\r
2473                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2474                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2475                 break;\r
2476 \r
2477         default:\r
2478                 /*\r
2479                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2480                  * Just send the payload and hope for the best.\r
2481                  */\r
2482                 cl_perf_start( SendGen );\r
2483                 status = __send_gen( p_port, p_desc );\r
2484                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2485                 break;\r
2486         }\r
2487 \r
2488         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2489         return status;\r
2490 }\r
2491 \r
2492 \r
2493 static NDIS_STATUS\r
2494 __send_copy(\r
2495         IN                              ipoib_port_t* const                     p_port,\r
2496         IN                              ipoib_send_desc_t* const        p_desc )\r
2497 {\r
2498         NDIS_PACKET                             *p_packet;\r
2499         NDIS_BUFFER                             *p_buf;\r
2500         NDIS_STATUS                             status;\r
2501         UINT                                    tot_len, bytes_copied = 0;\r
2502 \r
2503         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2504 \r
2505         p_desc->p_buf = \r
2506                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2507         if( !p_desc->p_buf )\r
2508         {\r
2509                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2510                         ("Failed to allocate buffer for packet copy.\n") );\r
2511                 return NDIS_STATUS_RESOURCES;\r
2512         }\r
2513 \r
2514         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2515         if( status != NDIS_STATUS_SUCCESS )\r
2516         {\r
2517                 IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
2518                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2519                 return status;\r
2520         }\r
2521 \r
2522         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2523                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2524         if( status != NDIS_STATUS_SUCCESS )\r
2525         {\r
2526                 NdisFreePacket( p_packet );\r
2527                 IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
2528                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2529                 return status;\r
2530         }\r
2531 \r
2532         NdisChainBufferAtFront( p_packet, p_buf );\r
2533 \r
2534         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2535 \r
2536         /* Setup the work request. */\r
2537         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2538                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2539         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2540         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2541         p_desc->wr.num_ds = 2;\r
2542 \r
2543         /* Copy the packet. */\r
2544         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2545                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2546                 NormalPagePriority );\r
2547 \r
2548         /* Free our temp packet now that the data is copied. */\r
2549         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2550         NdisFreeBuffer( p_buf );\r
2551         NdisFreePacket( p_packet );\r
2552 \r
2553         if( bytes_copied != tot_len )\r
2554         {\r
2555                 /* Something went wrong.  Drop the packet. */\r
2556                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2557                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2558                         bytes_copied, tot_len) );\r
2559                 return NDIS_STATUS_RESOURCES;\r
2560         }\r
2561 \r
2562         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2563         return NDIS_STATUS_SUCCESS;\r
2564 }\r
2565 \r
2566 \r
2567 #if !IPOIB_USE_DMA\r
2568 /* Send using the MDL's page information rather than the SGL. */\r
2569 static ib_api_status_t\r
2570 __send_gen(\r
2571         IN                              ipoib_port_t* const                     p_port,\r
2572         IN                              ipoib_send_desc_t* const        p_desc )\r
2573 {\r
2574         uint32_t                                i, j = 1;\r
2575         ULONG                                   offset;\r
2576         MDL                                             *p_mdl;\r
2577         UINT                                    num_pages, tot_len;\r
2578         ULONG                                   buf_len;\r
2579         PPFN_NUMBER                             page_array;\r
2580         boolean_t                               hdr_done = FALSE;\r
2581         ib_api_status_t                 status;\r
2582 \r
2583         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2584 \r
2585         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2586                 &tot_len );\r
2587 \r
2588         if( !p_mdl )\r
2589         {\r
2590                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2591                         ("No buffers associated with packet.\n") );\r
2592                 return IB_ERROR;\r
2593         }\r
2594 \r
2595         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2596         if( num_pages >= MAX_SEND_SGE )\r
2597         {\r
2598                 IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
2599                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2600                 status = __send_copy( p_port, p_desc );\r
2601                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2602                 return status;\r
2603         }\r
2604 \r
2605         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2606         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2607         /*\r
2608          * Assume that the ethernet header is always fully contained\r
2609          * in the first page of the first MDL.  This makes for much\r
2610          * simpler code.\r
2611          */\r
2612         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2613         CL_ASSERT( offset <= PAGE_SIZE );\r
2614 \r
2615         while( tot_len )\r
2616         {\r
2617                 buf_len = MmGetMdlByteCount( p_mdl );\r
2618                 page_array = MmGetMdlPfnArray( p_mdl );\r
2619                 CL_ASSERT( page_array );\r
2620                 i = 0;\r
2621                 if( !hdr_done )\r
2622                 {\r
2623                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2624                         /* Skip the ethernet header. */\r
2625                         buf_len -= sizeof(eth_hdr_t);\r
2626                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2627                         if( buf_len )\r
2628                         {\r
2629                                 /* The ethernet header is a subset of this MDL. */\r
2630                                 CL_ASSERT( i == 0 );\r
2631                                 if( offset < PAGE_SIZE )\r
2632                                 {\r
2633                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2634                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2635                                         /* Add the byte offset since we're on the 1st page. */\r
2636                                         p_desc->local_ds[j].vaddr += offset;\r
2637                                         if( offset + buf_len > PAGE_SIZE )\r
2638                                         {\r
2639                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2640                                                 buf_len -= p_desc->local_ds[j].length;\r
2641                                         }\r
2642                                         else\r
2643                                         {\r
2644                                                 p_desc->local_ds[j].length = buf_len;\r
2645                                                 buf_len = 0;\r
2646                                         }\r
2647                                         /* This data segment is done.  Move to the next. */\r
2648                                         j++;\r
2649                                 }\r
2650                                 /* This page is done.  Move to the next. */\r
2651                                 i++;\r
2652                         }\r
2653                         /* Done handling the ethernet header. */\r
2654                         hdr_done = TRUE;\r
2655                 }\r
2656 \r
2657                 /* Finish this MDL */\r
2658                 while( buf_len )\r
2659                 {\r
2660                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2661                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2662                         /* Add the first page's offset if we're on the first page. */\r
2663                         if( i == 0 )\r
2664                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2665 \r
2666                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2667                         {\r
2668                                 /* Buffers spans pages. */\r
2669                                 p_desc->local_ds[j].length =\r
2670                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2671                                 buf_len -= p_desc->local_ds[j].length;\r
2672                                 /* This page is done.  Move to the next. */\r
2673                                 i++;\r
2674                         }\r
2675                         else\r
2676                         {\r
2677                                 /* Last page of the buffer. */\r
2678                                 p_desc->local_ds[j].length = buf_len;\r
2679                                 buf_len = 0;\r
2680                         }\r
2681                         /* This data segment is done.  Move to the next. */\r
2682                         j++;\r
2683                 }\r
2684 \r
2685                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2686                 if( !tot_len )\r
2687                         break;\r
2688 \r
2689                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2690                 if( !p_mdl )\r
2691                 {\r
2692                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2693                                 ("Failed to get next buffer.\n") );\r
2694                         return IB_ERROR;\r
2695                 }\r
2696         }\r
2697 \r
2698         /* Set the number of data segments. */\r
2699         p_desc->wr.num_ds = j;\r
2700 \r
2701         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2702         return IB_SUCCESS;\r
2703 }\r
2704 \r
2705 #else\r
2706 \r
2707 static NDIS_STATUS\r
2708 __send_gen(\r
2709         IN                              ipoib_port_t* const                     p_port,\r
2710         IN                              ipoib_send_desc_t* const        p_desc )\r
2711 {\r
2712         ib_api_status_t                 status;\r
2713         SCATTER_GATHER_LIST             *p_sgl;\r
2714         uint32_t                                i, j = 1;\r
2715         uint32_t                                offset = sizeof(eth_hdr_t);\r
2716         PERF_DECLARE( SendCopy );\r
2717 \r
2718         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2719 \r
2720         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2721                 ScatterGatherListPacketInfo );\r
2722         if( !p_sgl )\r
2723         {\r
2724                 ASSERT( p_sgl );\r
2725                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2726                         ("Failed to get SGL from packet.\n") );\r
2727                 return NDIS_STATUS_FAILURE;\r
2728         }\r
2729 \r
2730         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2731         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2732                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2733                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2734                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2735         {\r
2736                 IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
2737                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2738                 cl_perf_start( SendCopy );\r
2739                 status = __send_copy( p_port, p_desc );\r
2740                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2741                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2742                 return status;\r
2743         }\r
2744 \r
2745         /*\r
2746          * Skip the ethernet header.  It is either the first element,\r
2747          * or part of it.\r
2748          */\r
2749         i = 0;\r
2750         while( offset )\r
2751         {\r
2752                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2753                 {\r
2754                         offset -= p_sgl->Elements[i++].Length;\r
2755                 }\r
2756                 else\r
2757                 {\r
2758                         p_desc->local_ds[j].vaddr =\r
2759                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2760                         p_desc->local_ds[j].length =\r
2761                                 p_sgl->Elements[i].Length - offset;\r
2762                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2763                         i++;\r
2764                         j++;\r
2765                         break;\r
2766                 }\r
2767         }\r
2768         /* Now fill in the rest of the local data segments. */\r
2769         while( i < p_sgl->NumberOfElements )\r
2770         {\r
2771                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2772                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2773                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2774                 i++;\r
2775                 j++;\r
2776         }\r
2777 \r
2778         /* Set the number of data segments. */\r
2779         p_desc->wr.num_ds = j;\r
2780 \r
2781         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2782         return NDIS_STATUS_SUCCESS;\r
2783 }\r
2784 #endif\r
2785 \r
2786 \r
2787 static NDIS_STATUS\r
2788 __send_mgr_filter_ip(\r
2789         IN                              ipoib_port_t* const                     p_port,\r
2790         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2791         IN                              NDIS_BUFFER*                            p_buf,\r
2792         IN                              size_t                                          buf_len,\r
2793         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2794 {\r
2795         NDIS_STATUS             status;\r
2796         ip_hdr_t                *p_ip_hdr;\r
2797 \r
2798         PERF_DECLARE( QueryIp );\r
2799         PERF_DECLARE( SendTcp );\r
2800         PERF_DECLARE( FilterUdp );\r
2801 \r
2802         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2803 \r
2804         if( !buf_len )\r
2805         {\r
2806                 cl_perf_start( QueryIp );\r
2807                 NdisGetNextBuffer( p_buf, &p_buf );\r
2808                 if( !p_buf )\r
2809                 {\r
2810                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2811                                 ("Failed to get IP header buffer.\n") );\r
2812                         return NDIS_STATUS_FAILURE;\r
2813                 }\r
2814                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2815                 if( !p_ip_hdr )\r
2816                 {\r
2817                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2818                                 ("Failed to query IP header buffer.\n") );\r
2819                         return NDIS_STATUS_FAILURE;\r
2820                 }\r
2821                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2822         }\r
2823         else\r
2824         {\r
2825                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2826         }\r
2827         if( buf_len < sizeof(ip_hdr_t) )\r
2828         {\r
2829                 /* This buffer is done for.  Get the next buffer. */\r
2830                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2831                         ("Buffer too small for IP packet.\n") );\r
2832                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2833         }\r
2834 \r
2835         if( p_ip_hdr->offset ||\r
2836                 p_ip_hdr->prot != IP_PROT_UDP )\r
2837         {\r
2838                 /* Not a UDP packet. */\r
2839                 cl_perf_start( SendTcp );\r
2840                 status = __send_gen( p_port, p_desc );\r
2841                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2842                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2843                 return status;\r
2844         }\r
2845 \r
2846         buf_len -= sizeof(ip_hdr_t);\r
2847 \r
2848         cl_perf_start( FilterUdp );\r
2849         status = __send_mgr_filter_udp(\r
2850                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2851         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2852 \r
2853         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2854         return status;\r
2855 }\r
2856 \r
2857 \r
2858 static NDIS_STATUS\r
2859 __send_mgr_filter_udp(\r
2860         IN                              ipoib_port_t* const                     p_port,\r
2861         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2862         IN                              NDIS_BUFFER*                            p_buf,\r
2863         IN                              size_t                                          buf_len,\r
2864         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2865 {\r
2866         ib_api_status_t         status;\r
2867         udp_hdr_t                       *p_udp_hdr;\r
2868         PERF_DECLARE( QueryUdp );\r
2869         PERF_DECLARE( SendUdp );\r
2870         PERF_DECLARE( FilterDhcp );\r
2871 \r
2872         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2873 \r
2874         if( !buf_len )\r
2875         {\r
2876                 cl_perf_start( QueryUdp );\r
2877                 NdisGetNextBuffer( p_buf, &p_buf );\r
2878                 if( !p_buf )\r
2879                 {\r
2880                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2881                                 ("Failed to get UDP header buffer.\n") );\r
2882                         return NDIS_STATUS_FAILURE;\r
2883                 }\r
2884                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2885                 if( !p_udp_hdr )\r
2886                 {\r
2887                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2888                                 ("Failed to query UDP header buffer.\n") );\r
2889                         return NDIS_STATUS_FAILURE;\r
2890                 }\r
2891                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
2892         }\r
2893         else\r
2894         {\r
2895                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
2896         }\r
2897         /* Get the UDP header and check the destination port numbers. */\r
2898         if( buf_len < sizeof(udp_hdr_t) )\r
2899         {\r
2900                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2901                         ("Buffer not large enough for UDP packet.\n") );\r
2902                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2903         }\r
2904 \r
2905         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
2906                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r
2907                 (p_udp_hdr->src_port != DHCP_PORT_SERVER ||\r
2908                 p_udp_hdr->dst_port != DHCP_PORT_CLIENT) )\r
2909         {\r
2910                 /* Not a DHCP packet. */\r
2911                 cl_perf_start( SendUdp );\r
2912                 status = __send_gen( p_port, p_desc );\r
2913                 cl_perf_stop( &p_port->p_adapter->perf, SendUdp );\r
2914                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2915                 return status;\r
2916         }\r
2917 \r
2918         buf_len -= sizeof(udp_hdr_t);\r
2919 \r
2920         /* Allocate our scratch buffer. */\r
2921         p_desc->p_buf = (send_buf_t*)\r
2922                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2923         if( !p_desc->p_buf )\r
2924         {\r
2925                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2926                         ("Failed to query DHCP packet buffer.\n") );\r
2927                 return NDIS_STATUS_RESOURCES;\r
2928         }\r
2929         /* Copy the IP and UDP headers. */\r
2930         cl_memcpy( &p_desc->p_buf->ip.hdr, p_ip_hdr , sizeof(ip_hdr_t) );\r
2931         cl_memcpy(\r
2932                 &p_desc->p_buf->ip.prot.udp.hdr, p_udp_hdr, sizeof(udp_hdr_t) );\r
2933 \r
2934         cl_perf_start( FilterDhcp );\r
2935         status = __send_mgr_filter_dhcp(\r
2936                 p_port, p_udp_hdr, p_buf, buf_len, p_desc );\r
2937         cl_perf_stop( &p_port->p_adapter->perf, FilterDhcp );\r
2938 \r
2939         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2940         return status;\r
2941 }\r
2942 \r
2943 \r
2944 static NDIS_STATUS\r
2945 __send_mgr_filter_dhcp(\r
2946         IN                              ipoib_port_t* const                     p_port,\r
2947         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
2948         IN                              NDIS_BUFFER*                            p_buf,\r
2949         IN                              size_t                                          buf_len,\r
2950         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2951 {\r
2952         dhcp_pkt_t                      *p_dhcp;\r
2953         dhcp_pkt_t                      *p_ib_dhcp;\r
2954         uint8_t                         *p_option, *p_cid = NULL;\r
2955         uint8_t                         msg = 0;\r
2956         size_t                          len;\r
2957         ib_gid_t                        gid;\r
2958 \r
2959         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2960 \r
2961         if( !buf_len )\r
2962         {\r
2963                 NdisGetNextBuffer( p_buf, &p_buf );\r
2964                 if( !p_buf )\r
2965                 {\r
2966                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2967                                 ("Failed to get DHCP buffer.\n") );\r
2968                         return NDIS_STATUS_FAILURE;\r
2969                 }\r
2970                 NdisQueryBufferSafe( p_buf, &p_dhcp, &buf_len, NormalPagePriority );\r
2971                 if( !p_dhcp )\r
2972                 {\r
2973                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2974                                 ("Failed to query DHCP buffer.\n") );\r
2975                         return NDIS_STATUS_FAILURE;\r
2976                 }\r
2977         }\r
2978         else\r
2979         {\r
2980                 p_dhcp = (dhcp_pkt_t*)(p_udp_hdr + 1);\r
2981         }\r
2982 \r
2983         if( buf_len < DHCP_MIN_SIZE )\r
2984         {\r
2985                 IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
2986                         ("Buffer not large enough for DHCP packet.\n") );\r
2987                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2988         }\r
2989 \r
2990         p_ib_dhcp = &p_desc->p_buf->ip.prot.udp.dhcp;\r
2991         cl_memcpy( p_ib_dhcp, p_dhcp, buf_len );\r
2992 \r
2993         /* Now scan through the options looking for the client identifier. */\r
2994         p_option = &p_ib_dhcp->options[4];\r
2995         while( *p_option != DHCP_OPT_END )\r
2996         {\r
2997                 switch( *p_option )\r
2998                 {\r
2999                 case DHCP_OPT_PAD:\r
3000                         p_option++;\r
3001                         break;\r
3002 \r
3003                 case DHCP_OPT_MSG:\r
3004                         msg = p_option[2];\r
3005                         p_option += 3;\r
3006                         break;\r
3007 \r
3008                 case DHCP_OPT_CLIENT_ID:\r
3009                         p_cid = p_option;\r
3010                         /* Fall through. */\r
3011 \r
3012                 default:\r
3013                         /*\r
3014                          * All other options have a length byte following the option code.\r
3015                          * Offset by the length to get to the next option.\r
3016                          */\r
3017                         p_option += (p_option[1] + 2);\r
3018                 }\r
3019         }\r
3020 \r
3021         switch( msg )\r
3022         {\r
3023         /* Client messages */\r
3024         case DHCPDISCOVER:\r
3025         case DHCPREQUEST:\r
3026                         p_ib_dhcp->flags |= DHCP_FLAGS_BROADCAST;\r
3027                 /* Fall through */\r
3028         case DHCPDECLINE:\r
3029         case DHCPRELEASE:\r
3030         case DHCPINFORM:\r
3031                 /* Fix up the client identifier option */\r
3032                 if( p_cid )\r
3033                 {\r
3034                         /* do we need to replace it ?  len eq ETH MAC sz 'and' MAC is mine */\r
3035                         if( p_cid[1] == HW_ADDR_LEN+1 && !cl_memcmp( &p_cid[3], &p_port->p_adapter->mac.addr, HW_ADDR_LEN  ) )\r
3036                         {\r
3037                                  /* Make sure there's room to extend it.  23 is the size of\r
3038                                  * the CID option for IPoIB.\r
3039                                  */\r
3040                                 if( buf_len + 23 - p_cid[1] > sizeof(dhcp_pkt_t) )\r
3041                                 {\r
3042                                         IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
3043                                                 ("Can't convert CID to IPoIB format.\n") );\r