[ipoib] Always connect to mcast groups as a full member. (mlnx: 2845)
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
index 62a2fb2..1e0f946 100644 (file)
@@ -1,5 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
 #include "ipoib_port.h"\r
 #include "ipoib_adapter.h"\r
 #include "ipoib_debug.h"\r
-\r
-\r
-/*\r
- * PR 102492 relates to the SA not accepting wildcard values for MCMemberRecord\r
- * Sets used to create/join Multicast groups.  Defining this keyword causes all\r
- * member record fields to be hard coded to well known values.\r
- */\r
-//#define PR_102492\r
-\r
-/*\r
- * PR 102801 relates to the SA ignoring the MGID when doing a GET_TABLE\r
- * request that specifies an MGID.  The ESM returns every McMemberRecord\r
- * instead.\r
- */\r
-//#define PR_102801\r
+#if defined(EVENT_TRACING)\r
+#ifdef offsetof\r
+#undef offsetof\r
+#endif\r
+#include "ipoib_port.tmh"\r
+#endif\r
 \r
 \r
 /* Amount of physical memory to register. */\r
@@ -212,8 +205,7 @@ __recv_mgr_destroy(
 /* Posts receive buffers to the receive queue. */\r
 static ib_api_status_t\r
 __recv_mgr_repost(\r
-       IN                              ipoib_port_t* const                     p_port,\r
-       IN                              uint32_t                                        num_wrs );\r
+       IN                              ipoib_port_t* const                     p_port );\r
 \r
 static void\r
 __recv_cb(\r
@@ -228,7 +220,7 @@ __recv_get_endpts(
                OUT                     ipoib_endpt_t** const           pp_src,\r
                OUT                     ipoib_endpt_t** const           pp_dst );\r
 \r
-static uint32_t\r
+static int32_t\r
 __recv_mgr_filter(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              ib_wc_t* const                          p_done_wc_list,\r
@@ -268,7 +260,9 @@ __recv_mgr_prepare_pkt(
 static uint32_t\r
 __recv_mgr_build_pkt_array(\r
        IN                              ipoib_port_t* const                     p_port,\r
-               OUT                     cl_qlist_t* const                       p_done_list );\r
+       IN                              int32_t                                         shortage,\r
+               OUT                     cl_qlist_t* const                       p_done_list,\r
+               OUT                     int32_t* const                          p_discarded );\r
 \r
 /******************************************************************************\r
 *\r
@@ -370,6 +364,10 @@ __endpt_mgr_remove(
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              ipoib_endpt_t* const            p_endpt );\r
 \r
+static void\r
+__endpt_mgr_reset_all(\r
+       IN                              ipoib_port_t* const                     p_port );\r
+\r
 static inline NDIS_STATUS\r
 __endpt_mgr_ref(\r
        IN                              ipoib_port_t* const                     p_port,\r
@@ -393,13 +391,13 @@ __endpt_mgr_get_by_lid(
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   net16_t                                         lid );\r
 \r
-static inline void\r
+static inline ib_api_status_t\r
 __endpt_mgr_insert_locked(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt );\r
 \r
-static inline void\r
+static inline ib_api_status_t\r
 __endpt_mgr_insert(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
@@ -420,16 +418,16 @@ __endpt_mgr_add_bcast(
 * MCast operations.\r
 *\r
 ******************************************************************************/\r
-static void\r
-__port_get_mcast(\r
+static ib_api_status_t\r
+__port_get_bcast(\r
        IN                              ipoib_port_t* const                     p_port );\r
 \r
-static void\r
+static ib_api_status_t\r
 __port_join_bcast(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              ib_member_rec_t* const          p_member_rec );\r
 \r
-static void\r
+static ib_api_status_t\r
 __port_create_bcast(\r
        IN                              ipoib_port_t* const                     p_port );\r
 \r
@@ -452,6 +450,10 @@ static void
 __mcast_cb(\r
        IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
 \r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context );\r
+\r
 \r
 static intn_t\r
 __gid_cmp(\r
@@ -462,6 +464,33 @@ __gid_cmp(
 }\r
 \r
 \r
+inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
+{\r
+       cl_obj_ref( &p_port->obj );\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
+#else\r
+       UNREFERENCED_PARAMETER(type);\r
+#endif\r
+}\r
+\r
+\r
+inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
+{\r
+       cl_obj_deref( &p_port->obj );\r
+\r
+#if DBG\r
+       cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
+#else\r
+       UNREFERENCED_PARAMETER(type);\r
+#endif\r
+}\r
+\r
+\r
 /******************************************************************************\r
 *\r
 * Implementation\r
@@ -484,7 +513,7 @@ ipoib_create_port(
                (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
        if( !p_port )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
                        sizeof(ipoib_port_t)) );\r
                return IB_INSUFFICIENT_MEMORY;\r
@@ -499,7 +528,7 @@ ipoib_create_port(
        status = __port_init( p_port, p_adapter, p_pnp_rec );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ipoib_port_init returned %s.\n", \r
                        p_adapter->p_ifc->get_err_str( status )) );\r
                __port_cleanup( &p_port->obj );\r
@@ -548,6 +577,9 @@ __port_construct(
 \r
        __endpt_mgr_construct( p_port );\r
 \r
+       KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
+       KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
+       \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -569,16 +601,16 @@ __port_init(
        cl_status = cl_spinlock_init( &p_port->send_lock );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                       ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_spinlock_init returned %#x\n", cl_status) );\r
                return IB_ERROR;\r
        }\r
 \r
        cl_status = cl_spinlock_init( &p_port->recv_lock );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                       ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_spinlock_init returned %#x\n", cl_status) );\r
                return IB_ERROR;\r
        }\r
 \r
@@ -586,7 +618,7 @@ __port_init(
        status = __ib_mgr_init( p_port );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__ib_mgr_init returned %s\n", \r
                        p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -596,7 +628,7 @@ __port_init(
        status = __buf_mgr_init( p_port );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__buf_mgr_init returned %s\n", \r
                        p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -606,7 +638,7 @@ __port_init(
        status = __recv_mgr_init( p_port );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__recv_mgr_init returned %s\n", \r
                        p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -616,7 +648,7 @@ __port_init(
        status = __endpt_mgr_init( p_port );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__endpt_mgr_init returned %s\n", \r
                        p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -625,14 +657,34 @@ __port_init(
        /* We only ever destroy from the PnP callback thread. */\r
        cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
                __port_destroying, __port_cleanup, __port_free );\r
+\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_init] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
+       if( cl_status != CL_SUCCESS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_obj_init returned %#x\n", cl_status) );\r
+               return IB_ERROR;\r
+       }\r
+\r
+       cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                       ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
+               cl_obj_destroy( &p_port->obj );\r
                return IB_ERROR;\r
        }\r
 \r
-       cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_init] );\r
+       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
+#endif\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return IB_SUCCESS;\r
@@ -653,6 +705,10 @@ __port_destroying(
 \r
        ipoib_port_down( p_port );\r
 \r
+       __endpt_mgr_remove_all( p_port );\r
+\r
+       ipoib_port_resume( p_port );\r
+\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -738,6 +794,8 @@ __ib_mgr_init(
        uint64_t                        vaddr;\r
        net32_t                         rkey;\r
        ib_qp_attr_t            qp_attr;\r
+       ib_ca_attr_t            *ca_attr;\r
+       uint32_t                        ca_size;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
@@ -749,12 +807,58 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_OPEN_CA, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_open_ca returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
 \r
+       /* Query the CA for Pkey table */\r
+       status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, NULL, &ca_size);\r
+       if(status != IB_INSUFFICIENT_MEMORY)\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ib_query_ca failed\n"));\r
+                       return status;\r
+       }\r
+\r
+       ca_attr = (ib_ca_attr_t*)cl_zalloc(ca_size);\r
+       if      (!ca_attr)\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_zalloc can't allocate %d\n",ca_size));\r
+               return IB_INSUFFICIENT_RESOURCES;\r
+       }\r
+\r
+       status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, ca_attr,&ca_size);     \r
+       if( status != IB_SUCCESS )\r
+       {\r
+               cl_free(ca_attr);\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ib_query_ca returned %s\n", \r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
+       if( ca_attr->p_port_attr->link_state == IB_LINK_ACTIVE)\r
+       {\r
+               uint16_t index;\r
+               CL_ASSERT(ca_attr->p_port_attr->p_pkey_table[0] == IB_DEFAULT_PKEY);\r
+               for(index = 0; index < ca_attr->p_port_attr->num_pkeys; index++)\r
+               {\r
+                       if(p_port->p_adapter->guids.port_guid.pkey == ca_attr->p_port_attr->p_pkey_table[index])\r
+                               break;\r
+               }\r
+               if(index >= ca_attr->p_port_attr->num_pkeys)\r
+               {\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("Pkey table is invalid, index not found\n"));\r
+                   return IB_NOT_FOUND;\r
+               }\r
+               p_port->pkey_index = index;\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_IB,\r
+                       ("for PKEY = 0x%04X got index = %d\n",p_port->p_adapter->guids.port_guid.pkey,index));\r
+       }\r
+       cl_free(ca_attr);\r
        /* Allocate the PD. */\r
        status = p_port->p_adapter->p_ifc->alloc_pd(\r
                p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
@@ -762,7 +866,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_ALLOC_PD, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_alloc_pd returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -780,7 +884,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_create_cq returned %s.\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -797,7 +901,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_create_cq returned %s.\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -821,7 +925,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_CREATE_QP, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_create_qp returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -833,7 +937,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_QUERY_QP, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_query_qp returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -857,7 +961,7 @@ __ib_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_REG_PHYS, 1, status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_reg_phys returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -898,7 +1002,7 @@ static void
 __buf_mgr_construct(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
 \r
@@ -906,12 +1010,12 @@ __buf_mgr_construct(
        p_port->buf_mgr.h_buffer_pool = NULL;\r
 \r
        ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
-               NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
+               NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
 \r
        p_port->buf_mgr.h_send_pkt_pool = NULL;\r
        p_port->buf_mgr.h_send_buf_pool = NULL;\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
 \r
@@ -923,7 +1027,7 @@ __buf_mgr_init(
        NDIS_STATUS             ndis_status;\r
        ipoib_params_t  *p_params;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_ENTER(IPOIB_DBG_INIT );\r
 \r
        CL_ASSERT( p_port );\r
        CL_ASSERT( p_port->p_adapter );\r
@@ -942,9 +1046,9 @@ __buf_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                       ("cl_qpool_init for recvs returned %s\n",\r
-                       cl_status_text[cl_status]) );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_qpool_init for recvs returned %#x\n",\r
+                       cl_status) );\r
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
 \r
@@ -955,7 +1059,7 @@ __buf_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
@@ -966,7 +1070,7 @@ __buf_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
@@ -978,7 +1082,7 @@ __buf_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
@@ -989,12 +1093,12 @@ __buf_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return IB_SUCCESS;\r
 }\r
 \r
@@ -1003,7 +1107,7 @@ static void
 __buf_mgr_destroy(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_ENTER(IPOIB_DBG_INIT );\r
 \r
        CL_ASSERT( p_port );\r
 \r
@@ -1025,7 +1129,7 @@ __buf_mgr_destroy(
        /* Free the lookaside list of scratch buffers. */\r
        ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_INIT );\r
+       IPOIB_EXIT(  IPOIB_DBG_INIT );\r
 }\r
 \r
 \r
@@ -1039,7 +1143,7 @@ __recv_ctor(
        ipoib_port_t            *p_port;\r
        uint32_t                        ds0_len;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
+       IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
 \r
        CL_ASSERT( p_object );\r
        CL_ASSERT( context );\r
@@ -1083,7 +1187,7 @@ __recv_ctor(
        p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
        if( !p_desc->p_buf )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to allocate receive buffer.\n") );\r
                return CL_INSUFFICIENT_MEMORY;\r
        }\r
@@ -1100,7 +1204,7 @@ __recv_ctor(
 \r
        *pp_pool_item = &p_desc->item;\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
+       IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
        return CL_SUCCESS;\r
 }\r
 \r
@@ -1113,7 +1217,7 @@ __recv_dtor(
 {\r
        ipoib_recv_desc_t       *p_desc;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
+       IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
 \r
        UNUSED_PARAM( context );\r
 \r
@@ -1122,7 +1226,7 @@ __recv_dtor(
        if( p_desc->p_buf )\r
                cl_free( p_desc->p_buf );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_ALLOC );\r
+       IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
 }\r
 #endif\r
 \r
@@ -1132,12 +1236,12 @@ __buf_mgr_get_recv(
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        ipoib_recv_desc_t       *p_desc;\r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       IPOIB_ENTER( IPOIB_DBG_RECV );\r
        p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
        /* Reference the port object for the send. */\r
        if( p_desc )\r
        {\r
-               cl_obj_ref( &p_port->obj );\r
+               ipoib_port_ref( p_port, ref_get_recv );\r
                CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
 #if IPOIB_INLINE_RECV\r
                CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
@@ -1150,7 +1254,7 @@ __buf_mgr_get_recv(
 #endif /* IPOIB_INLINE_RECV */\r
                CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
        }\r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       IPOIB_EXIT( IPOIB_DBG_RECV );\r
        return p_desc;\r
 }\r
 \r
@@ -1163,7 +1267,7 @@ __buf_mgr_put_recv(
 {\r
        NDIS_BUFFER             *p_buf;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       IPOIB_ENTER(IPOIB_DBG_RECV );\r
 \r
        if( p_packet )\r
        {\r
@@ -1181,9 +1285,8 @@ __buf_mgr_put_recv(
        /*\r
         * Dereference the port object since the receive is no longer outstanding.\r
         */\r
-       cl_obj_deref( &p_port->obj );\r
-\r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       ipoib_port_deref( p_port, ref_get_recv );\r
+       IPOIB_EXIT(  IPOIB_DBG_RECV );\r
 }\r
 \r
 \r
@@ -1192,9 +1295,9 @@ __buf_mgr_put_recv_list(
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              cl_qlist_t* const                       p_list )\r
 {\r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
        cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
 }\r
 \r
 \r
@@ -1207,13 +1310,13 @@ __buf_mgr_get_ndis_pkt(
        NDIS_PACKET                             *p_packet;\r
        NDIS_BUFFER                             *p_buffer;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       IPOIB_ENTER(  IPOIB_DBG_RECV );\r
 \r
        NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
                p_port->buf_mgr.h_packet_pool );\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
                return NULL;\r
        }\r
@@ -1229,7 +1332,7 @@ __buf_mgr_get_ndis_pkt(
 #endif /* IPOIB_INLINE_RECV */\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
                NdisDprFreePacketNonInterlocked( p_packet );\r
                return NULL;\r
@@ -1237,9 +1340,8 @@ __buf_mgr_get_ndis_pkt(
 \r
        NdisChainBufferAtFront( p_packet, p_buffer );\r
        NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
-       NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_BUF | IPOIB_DBG_RECV );\r
+       IPOIB_EXIT(  IPOIB_DBG_RECV );\r
        return p_packet;\r
 }\r
 \r
@@ -1275,7 +1377,7 @@ __recv_mgr_init(
        {\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
@@ -1301,11 +1403,15 @@ __recv_mgr_destroy(
 }\r
 \r
 \r
-/* Posts receive buffers to the receive queue. */\r
-static ib_api_status_t\r
+/*\r
+ * Posts receive buffers to the receive queue and returns the number\r
+ * of receives needed to bring the RQ to its low water mark.  Note\r
+ * that the value is signed, and can go negative.  All tests must\r
+ * be for > 0.\r
+ */\r
+static int32_t\r
 __recv_mgr_repost(\r
-       IN                              ipoib_port_t* const                     p_port,\r
-       IN                              uint32_t                                        num_wrs )\r
+       IN                              ipoib_port_t* const                     p_port )\r
 {\r
        ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
        ib_api_status_t         status;\r
@@ -1316,35 +1422,27 @@ __recv_mgr_repost(
        IPOIB_ENTER( IPOIB_DBG_RECV );\r
 \r
        CL_ASSERT( p_port );\r
-       CL_ASSERT( num_wrs );\r
        cl_obj_lock( &p_port->obj );\r
-       if( p_port->state == IB_QPS_ERROR )\r
+       if( p_port->state != IB_QPS_RTS )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
                        ("Port in invalid state.  Not reposting.\n") );\r
-               return IB_SUCCESS;\r
+               return 0;\r
        }\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_repost );\r
        cl_obj_unlock( &p_port->obj );\r
 \r
-       while( num_wrs-- )\r
+       while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
        {\r
-               if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )\r
-               {\r
-                       IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
-                               ("Receive queue already full\n") );\r
-                       break;\r
-               }\r
-\r
                /* Pull receives out of the pool and chain them up. */\r
                cl_perf_start( GetRecv );\r
                p_next = __buf_mgr_get_recv( p_port );\r
                cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
                if( !p_next )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
-                               ("Out of receive descriptors!\n") );\r
+                       IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+                               ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
                        break;\r
                }\r
 \r
@@ -1360,45 +1458,36 @@ __recv_mgr_repost(
 \r
                p_head = p_next;\r
 \r
-               cl_atomic_inc( &p_port->recv_mgr.depth );\r
+               p_port->recv_mgr.depth++;\r
        }\r
 \r
-       if( !p_head )\r
+       if( p_head )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_EXIT( IPOIB_DBG_RECV );\r
-               return IB_SUCCESS;\r
-       }\r
+               cl_perf_start( PostRecv );\r
+               status = p_port->p_adapter->p_ifc->post_recv(\r
+                       p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
+               cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
 \r
-       cl_perf_start( PostRecv );\r
-       status = p_port->p_adapter->p_ifc->post_recv(\r
-               p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
-       cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
-\r
-       /*\r
-        * If we failed, fix up the work completion list and return those\r
-        * buffers to the pool\r
-        */\r
-       if( status != IB_SUCCESS )\r
-       {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
-                       ("ip_post_recv returned %s\n", \r
-                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               /* return the descriptors to the pool */\r
-               while( p_failed )\r
+               if( status != IB_SUCCESS )\r
                {\r
-                       p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
-                       p_failed = p_failed->p_next;\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("ip_post_recv returned %s\n", \r
+                               p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                       /* return the descriptors to the pool */\r
+                       while( p_failed )\r
+                       {\r
+                               p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
+                               p_failed = p_failed->p_next;\r
 \r
-                       __buf_mgr_put_recv( p_port, p_head, NULL );\r
-                       cl_atomic_dec( &p_port->recv_mgr.depth );\r
+                               __buf_mgr_put_recv( p_port, p_head, NULL );\r
+                               p_port->recv_mgr.depth--;\r
+                       }\r
                }\r
        }\r
 \r
-       cl_obj_deref( &p_port->obj );\r
-\r
+       ipoib_port_deref( p_port, ref_repost );\r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
-       return status;\r
+       return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
 }\r
 \r
 \r
@@ -1411,6 +1500,7 @@ ipoib_return_packet(
        ipoib_port_t            *p_port;\r
        ipoib_recv_desc_t       *p_desc;\r
        ib_api_status_t         status = IB_NOT_DONE;\r
+       int32_t                         shortage;\r
        PERF_DECLARE( ReturnPacket );\r
        PERF_DECLARE( ReturnPutRecv );\r
        PERF_DECLARE( ReturnRepostRecv );\r
@@ -1424,30 +1514,25 @@ ipoib_return_packet(
 \r
        cl_perf_start( ReturnPacket );\r
 \r
-       /* Get the manager and descriptor from the packet. */\r
+       /* Get the port and descriptor from the packet. */\r
        p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
        p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
 \r
        cl_spinlock_acquire( &p_port->recv_lock );\r
+\r
        cl_perf_start( ReturnPutRecv );\r
        __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
        cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
 \r
        /* Repost buffers. */\r
        cl_perf_start( ReturnRepostRecv );\r
-       __recv_mgr_repost( p_port, 0xFFFF );\r
+       shortage = __recv_mgr_repost( p_port );\r
        cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
 \r
-       /* Complete any additional receives waiting for a packet. */\r
-       p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
-       do\r
+       for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
+               p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
+               p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
        {\r
-               if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
-               {\r
-                       cl_spinlock_release( &p_port->recv_lock );\r
-                       break;\r
-               }\r
-\r
                p_desc = (ipoib_recv_desc_t*)p_item;\r
 \r
                cl_perf_start( ReturnPreparePkt );\r
@@ -1455,27 +1540,41 @@ ipoib_return_packet(
                cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
                if( status == IB_SUCCESS )\r
                {\r
+                       if( shortage > 0 )\r
+                               NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
+                       else\r
+                               NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
+\r
                        cl_spinlock_release( &p_port->recv_lock );\r
                        cl_perf_start( ReturnNdisIndicate );\r
                        NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
                                &p_packet, 1 );\r
                        cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
+                       cl_spinlock_acquire( &p_port->recv_lock );\r
+\r
+                       if( shortage > 0 )\r
+                       {\r
+                               cl_perf_start( ReturnPutRecv );\r
+                               __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
+                               cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
+\r
+                               /* Repost buffers. */\r
+                               cl_perf_start( ReturnRepostRecv );\r
+                               shortage = __recv_mgr_repost( p_port );\r
+                               cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
+                       }\r
                }\r
                else if( status != IB_NOT_DONE )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_RECV,\r
+                       IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
                                ("__recv_mgr_prepare_pkt returned %s\n",\r
                                p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                        /* Return the item to the head of the list. */\r
                        cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
-                       cl_spinlock_release( &p_port->recv_lock );\r
-               }\r
-               else\r
-               {\r
-                       p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
+                       break;\r
                }\r
-\r
-       } while( status == IB_NOT_DONE );\r
+       }\r
+       cl_spinlock_release( &p_port->recv_lock );\r
        cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
@@ -1490,7 +1589,7 @@ __recv_cb(
        ipoib_port_t            *p_port;\r
        ib_api_status_t         status;\r
        ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
-       uint32_t                        pkt_cnt, recv_cnt = 0;\r
+       int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
        cl_qlist_t                      done_list, bad_list;\r
        size_t                          i;\r
        PERF_DECLARE( RecvCompBundle );\r
@@ -1516,8 +1615,7 @@ __recv_cb(
        cl_qlist_init( &done_list );\r
        cl_qlist_init( &bad_list );\r
 \r
-       cl_obj_ref( &p_port->obj );\r
-\r
+       ipoib_port_ref( p_port, ref_recv_cb );\r
        for( i = 0; i < MAX_RECV_WC; i++ )\r
                wc[i].p_next = &wc[i + 1];\r
        wc[MAX_RECV_WC - 1].p_next = NULL;\r
@@ -1551,35 +1649,78 @@ __recv_cb(
        /* We're done looking at the endpoint map, release the reference. */\r
        cl_atomic_dec( &p_port->endpt_rdr );\r
 \r
+       cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
+\r
+       cl_spinlock_acquire( &p_port->recv_lock );\r
+\r
        /* Update our posted depth. */\r
-       cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
+       p_port->recv_mgr.depth -= recv_cnt;\r
 \r
-       cl_perf_start( BuildPktArray );\r
-       /* Notify NDIS of any and all possible receive buffers. */\r
-       pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
-       cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
+       /* Return any discarded receives to the pool */\r
+       cl_perf_start( PutRecvList );\r
+       __buf_mgr_put_recv_list( p_port, &bad_list );\r
+       cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
 \r
-       /* Only indicate receives if we actually had any. */\r
-       if( pkt_cnt )\r
+       do\r
        {\r
+               /* Repost ASAP so we don't starve the RQ. */\r
+               cl_perf_start( RepostRecv );\r
+               shortage = __recv_mgr_repost( p_port );\r
+               cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
+\r
+               cl_perf_start( BuildPktArray );\r
+               /* Notify NDIS of any and all possible receive buffers. */\r
+               pkt_cnt = __recv_mgr_build_pkt_array(\r
+                       p_port, shortage, &done_list, &discarded );\r
+               cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
+\r
+               /* Only indicate receives if we actually had any. */\r
+               if( discarded && shortage > 0 )\r
+               {\r
+                       /* We may have thrown away packets, and have a shortage */\r
+                       cl_perf_start( RepostRecv );\r
+                       __recv_mgr_repost( p_port );\r
+                       cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
+               }\r
+\r
+               if( !pkt_cnt )\r
+                       break;\r
+\r
+               cl_spinlock_release( &p_port->recv_lock );\r
+\r
                cl_perf_start( RecvNdisIndicate );\r
                NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
                        p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
                cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
-       }\r
 \r
-       cl_spinlock_acquire( &p_port->recv_lock );\r
+               /*\r
+                * Cap the number of receives to put back to what we just indicated\r
+                * with NDIS_STATUS_RESOURCES.\r
+                */\r
+               if( shortage > 0 )\r
+               {\r
+                       if( pkt_cnt < shortage )\r
+                               shortage = pkt_cnt;\r
 \r
-       /* Return any discarded receives to the pool */\r
-       cl_perf_start( PutRecvList );\r
-       __buf_mgr_put_recv_list( p_port, &bad_list );\r
-       cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
+                       /* Return all but the last packet to the pool. */\r
+                       cl_spinlock_acquire( &p_port->recv_lock );\r
+                       while( shortage-- > 1 )\r
+                       {\r
+                               __buf_mgr_put_recv( p_port,\r
+                                       IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
+                                       p_port->recv_mgr.recv_pkt_array[shortage] );\r
+                       }\r
+                       cl_spinlock_release( &p_port->recv_lock );\r
 \r
-       /* Repost receives. */\r
-       cl_perf_start( RepostRecv );\r
-       __recv_mgr_repost( p_port, recv_cnt );\r
-       cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
+                       /*\r
+                        * Return the last packet as if NDIS returned it, so that we repost\r
+                        * and report any other pending receives.\r
+                        */\r
+                       ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
+               }\r
+               cl_spinlock_acquire( &p_port->recv_lock );\r
 \r
+       } while( pkt_cnt );\r
        cl_spinlock_release( &p_port->recv_lock );\r
 \r
        /*\r
@@ -1593,10 +1734,9 @@ __recv_cb(
        cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
        CL_ASSERT( status == IB_SUCCESS );\r
 \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_recv_cb );\r
 \r
        cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
-       cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
 }\r
@@ -1660,7 +1800,7 @@ __recv_get_endpts(
 #endif /* IPOIB_INLINE_RECV */\r
                        if( status != IB_SUCCESS )\r
                        {\r
-                               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("ipoib_mac_from_guid returned %s\n",\r
                                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                                return;\r
@@ -1672,17 +1812,24 @@ __recv_get_endpts(
 #else  /* IPOIB_INLINE_RECV */\r
                        *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
 #endif /* IPOIB_INLINE_RECV */\r
-                               p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
+                               0, p_wc->recv.ud.remote_qp );\r
                        if( !*pp_src )\r
                        {\r
-                               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                                       ("ipoib_endpt_create returned %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("ipoib_endpt_create failed\n") );\r
                                return;\r
                        }\r
                        cl_perf_start( EndptInsert );\r
                        cl_obj_lock( &p_port->obj );\r
-                       __endpt_mgr_insert( p_port, mac, *pp_src );\r
+                       status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
+                       if( status != IB_SUCCESS )\r
+                       {\r
+                               cl_obj_unlock( &p_port->obj );\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("__endpt_mgr_insert returned %s\n",\r
+                                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                               return;\r
+                       }\r
                        cl_obj_unlock( &p_port->obj );\r
                        cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
                }\r
@@ -1700,22 +1847,37 @@ __recv_get_endpts(
                CL_ASSERT( *pp_dst );\r
        }\r
 \r
-       IPOIB_TRACE( (IPOIB_DBG_RECV | IPOIB_DBG_INFO),\r
-               ("Recv:\n"\r
-               "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
-               "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
-               (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
-               (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
-               (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
-               (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
-               (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
-               (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
+       if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
+               (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
+       {\r
+               /* Update the QPN for the endpoint. */\r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+                       ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
+                       (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
+                       (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
+                       (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
+               (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
+       }\r
+\r
+       if( *pp_src && *pp_dst )\r
+       {\r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+                       ("Recv:\n"\r
+                       "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
+                       "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
+                       (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
+                       (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
+                       (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
+                       (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
+                       (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
+                       (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
+       }\r
 \r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
 }\r
 \r
 \r
-static uint32_t\r
+static int32_t\r
 __recv_mgr_filter(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              ib_wc_t* const                          p_done_wc_list,\r
@@ -1728,7 +1890,8 @@ __recv_mgr_filter(
        eth_pkt_t                               *p_eth;\r
        ipoib_endpt_t                   *p_src, *p_dst;\r
        ib_api_status_t                 status;\r
-       uint32_t                                len, recv_cnt = 0;\r
+       uint32_t                                len;\r
+       int32_t                                 recv_cnt = 0;\r
        PERF_DECLARE( GetRecvEndpts );\r
        PERF_DECLARE( RecvGen );\r
        PERF_DECLARE( RecvTcp );\r
@@ -1740,7 +1903,7 @@ __recv_mgr_filter(
 \r
        for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
        {\r
-               CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
+               CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
                p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
                recv_cnt++;\r
 \r
@@ -1748,21 +1911,22 @@ __recv_mgr_filter(
                {\r
                        if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
                        {\r
-                               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
-                                       ("Failed completion %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
+                               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("Failed completion %s  (vendor specific %#x)\n",\r
+                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
+                                       (int)p_wc->vendor_specific) );\r
                                ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        }\r
                        else\r
                        {\r
-                               IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,\r
+                               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
                                        ("Flushed completion %s\n",\r
                                        p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
                                ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
                        }\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
                        /* Dereference the port object on behalf of the failed receive. */\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_failed_recv_wc );\r
                        continue;\r
                }\r
 \r
@@ -1770,15 +1934,27 @@ __recv_mgr_filter(
 \r
                if( len < sizeof(ipoib_hdr_t) )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Received ETH packet < min size\n") );\r
                        ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_recv_inv_len );\r
                        continue;\r
                }\r
 \r
+               if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
+               {\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("Received ETH packet > payload MTU (%d)\n",\r
+                               p_port->p_adapter->params.payload_mtu) );\r
+                       ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
+                       cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
+                       ipoib_port_deref( p_port, ref_recv_inv_len );\r
+                       continue;\r
+                       \r
+               }\r
                /* Successful completion.  Get the receive information. */\r
+               p_desc->ndis_csum.Value = (ULONG) p_wc->csum_ok;\r
                cl_perf_start( GetRecvEndpts );\r
                __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
                cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
@@ -1791,18 +1967,21 @@ __recv_mgr_filter(
                p_eth = &p_desc->p_buf->eth.pkt;\r
 #endif /*IPOIB_INLINE_RECV */\r
 \r
-               /* Don't report loopback traffic - we requested SW loopback. */\r
-               if( !cl_memcmp( &p_port->p_adapter->mac, &p_eth->hdr.src,\r
-                       sizeof(p_port->p_adapter->mac) ) )\r
+               if( p_src )\r
                {\r
-                       /*\r
-                        * "This is not the packet you're looking for" - don't update\r
-                        * receive statistics, the packet never happened.\r
-                        */\r
-                       cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
-                       /* Dereference the port object on behalf of the failed receive. */\r
-                       cl_obj_deref( &p_port->obj );\r
-                       continue;\r
+                       /* Don't report loopback traffic - we requested SW loopback. */\r
+                       if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
+                               &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
+                       {\r
+                               /*\r
+                                * "This is not the packet you're looking for" - don't update\r
+                                * receive statistics, the packet never happened.\r
+                                */\r
+                               cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
+                               /* Dereference the port object on behalf of the failed recv. */\r
+                               ipoib_port_deref( p_port, ref_recv_loopback );\r
+                               continue;\r
+                       }\r
                }\r
 \r
                switch( p_ipoib->hdr.type )\r
@@ -1810,7 +1989,7 @@ __recv_mgr_filter(
                case ETH_PROT_TYPE_IP:\r
                        if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
                        {\r
-                               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("Received IP packet < min size\n") );\r
                                status = IB_INVALID_SETTING;\r
                                break;\r
@@ -1830,7 +2009,7 @@ __recv_mgr_filter(
                        if( len <\r
                                (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
                        {\r
-                               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("Received UDP packet < min size\n") );\r
                                status = IB_INVALID_SETTING;\r
                                break;\r
@@ -1845,11 +2024,16 @@ __recv_mgr_filter(
                                if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
                                        sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
                                {\r
-                                       IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                                ("Received DHCP < min size\n") );\r
                                        status = IB_INVALID_SETTING;\r
                                        break;\r
                                }\r
+                               if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
+                                       // If there are IP options in this message, we are in trouble in any case\r
+                                       status = IB_INVALID_SETTING;\r
+                                       break;                                  \r
+                               }\r
                                /* UDP packet with BOOTP ports in src/dst port numbers. */\r
                                cl_perf_start( RecvDhcp );\r
                                status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
@@ -1867,7 +2051,7 @@ __recv_mgr_filter(
                case ETH_PROT_TYPE_ARP:\r
                        if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
                        {\r
-                               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("Received ARP < min size\n") );\r
                                status = IB_INVALID_SETTING;\r
                                break;\r
@@ -1891,7 +2075,7 @@ __recv_mgr_filter(
                        ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
                        /* Dereference the port object on behalf of the failed receive. */\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_recv_filter );\r
                }\r
                else\r
                {\r
@@ -1935,7 +2119,7 @@ __recv_gen(
 \r
        if( !p_src || !p_dst )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Received packet with no matching endpoints.\n") );\r
                return IB_NOT_DONE;\r
        }\r
@@ -1977,7 +2161,7 @@ __recv_dhcp(
        status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__recv_gen returned %s.\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -1987,7 +2171,8 @@ __recv_dhcp(
        p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
        if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("Invalid DHCP op code.\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("Invalid DHCP op code.\n") );\r
                return IB_INVALID_SETTING;\r
        }\r
 \r
@@ -1998,13 +2183,13 @@ __recv_dhcp(
        p_option = &p_dhcp->options[0];\r
        if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("DHCP cookie corrupted.\n") );\r
                return IB_INVALID_PARAMETER;\r
        }\r
 \r
        p_option = &p_dhcp->options[4];\r
-       while( *p_option != DHCP_OPT_END )\r
+       while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
        {\r
                switch( *p_option )\r
                {\r
@@ -2040,13 +2225,13 @@ __recv_dhcp(
        case DHCPINFORM:\r
                if( !p_cid )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to find required Client-identifier option.\n") );\r
                        return IB_INVALID_SETTING;\r
                }\r
                if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Invalid hardware address type.\n") );\r
                        return IB_INVALID_SETTING;\r
                }\r
@@ -2058,7 +2243,7 @@ __recv_dhcp(
                break;\r
 \r
        default:\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Invalide message type.\n") );\r
                return IB_INVALID_PARAMETER;\r
        }\r
@@ -2071,13 +2256,13 @@ __recv_dhcp(
                /* Validate that the length and type of the option is as required. */\r
                if( p_cid[1] != 21 )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Client-identifier length not 21 as required.\n") );\r
                        return IB_INVALID_SETTING;\r
                }\r
                if( p_cid[2] != DHCP_HW_TYPE_IB )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Client-identifier type is wrong.\n") );\r
                        return IB_INVALID_SETTING;\r
                }\r
@@ -2117,7 +2302,7 @@ __recv_arp(
 \r
        if( !p_dst )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Unknown destination endpoint\n") );\r
                return IB_INVALID_SETTING;\r
        }\r
@@ -2127,21 +2312,22 @@ __recv_arp(
 \r
        if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ARP hardware type is not IB\n") );\r
                return IB_INVALID_SETTING;\r
        }\r
 \r
        if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
                return IB_INVALID_SETTING;\r
        }\r
 \r
        if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP protocal type not IP\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ARP protocal type not IP\n") );\r
                return IB_INVALID_SETTING;\r
        }\r
 \r
@@ -2166,8 +2352,26 @@ __recv_arp(
                         */\r
                        *pp_src = NULL;\r
                }\r
-               else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
-                       (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
+               else if( (*pp_src)->dlid &&\r
+                       (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
+               {\r
+                       /* Out of date!  Destroy the endpoint and replace it. */\r
+                       __endpt_mgr_remove( p_port, *pp_src );\r
+                       *pp_src = NULL;\r
+               }\r
+               else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
+               {\r
+                       if( (*pp_src)->qpn !=\r
+                               (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
+                               p_wc->recv.ud.remote_qp !=\r
+                               (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
+                       {\r
+                               /* Out of date!  Destroy the endpoint and replace it. */\r
+                               __endpt_mgr_remove( p_port, *pp_src );\r
+                               *pp_src = NULL;\r
+                       }\r
+               }\r
+               else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
                {\r
                        /* Out of date!  Destroy the endpoint and replace it. */\r
                        __endpt_mgr_remove( p_port, *pp_src );\r
@@ -2183,7 +2387,7 @@ __recv_arp(
                status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
                if( status != IB_SUCCESS )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("ipoib_mac_from_guid returned %s\n",\r
                                p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                        return status;\r
@@ -2196,22 +2400,32 @@ __recv_arp(
                 */\r
                *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
                        0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
+\r
                if( !*pp_src )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
-                               ("ipoib_endpt_create returned %s\n",\r
-                               p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("ipoib_endpt_create failed\n") );\r
                        return status;\r
                }\r
 \r
                cl_obj_lock( &p_port->obj );\r
-               __endpt_mgr_insert( p_port, mac, *pp_src );\r
+               status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
+               if( status != IB_SUCCESS )\r
+               {\r
+                       cl_obj_unlock( &p_port->obj );\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("__endpt_mgr_insert return %s \n",\r
+                               p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                       return status;\r
+               }\r
+\r
                cl_obj_unlock( &p_port->obj );\r
        }\r
 \r
        CL_ASSERT( !cl_memcmp(\r
                &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
-       CL_ASSERT( (*pp_src)->qpn ==\r
+       CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
+               (*pp_src)->qpn ==\r
                (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
        /* Now swizzle the data. */\r
        p_arp->hw_type = ARP_HW_TYPE_ETH;\r
@@ -2221,17 +2435,21 @@ __recv_arp(
 \r
        if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
        {\r
-\r
-               if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
+               if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
                {\r
                        /*\r
                         * We received bcast ARP packet that means\r
                         * remote port lets everyone know it was changed IP/MAC\r
                         * or just activated\r
                         */\r
-                       if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
+\r
+                       /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
+\r
+                       if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
+                                !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
                        {\r
-                               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ARP: is not ARP MCAST\n") );\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("ARP: is not ARP MCAST\n") );\r
                                return IB_INVALID_SETTING;\r
                        }\r
 \r
@@ -2242,7 +2460,6 @@ __recv_arp(
                         * as long as OS' ARP table is global  ???\r
                         */\r
                        p_arp->dst_ip = (net32_t)0;\r
-\r
                }\r
                else /* we've got reply to our ARP request */\r
                {\r
@@ -2265,7 +2482,7 @@ __recv_arp(
        status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__recv_gen returned %s.\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -2285,7 +2502,6 @@ __recv_mgr_prepare_pkt(
        NDIS_STATUS                                                     status;\r
        uint32_t                                                        pkt_filter;\r
        ip_stat_sel_t                                           type;\r
-       NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
        PERF_DECLARE( GetNdisPkt );\r
 \r
        IPOIB_ENTER( IPOIB_DBG_RECV );\r
@@ -2347,7 +2563,7 @@ __recv_mgr_prepare_pkt(
                ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
                /* Return the receive descriptor to the pool. */\r
                __buf_mgr_put_recv( p_port, p_desc, NULL );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_RECV,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
                        ("Packet filter doesn't match receive.  Dropping.\n") );\r
                /*\r
                 * Return IB_NOT_DONE since the packet has been completed,\r
@@ -2361,19 +2577,14 @@ __recv_mgr_prepare_pkt(
        cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
        if( !*pp_packet )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__buf_mgr_get_ndis_pkt failed\n") );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
 \r
-       /* Flag the checksums as having been calculated. */\r
-       chksum.Value = 0;\r
-       chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
-       chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
-       chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
-       NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
-               (void*)(uintn_t)chksum.Value;\r
-\r
+       /* Get the checksums directly from packet information. */\r
+       NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
+               (PVOID) (uintn_t) (p_desc->ndis_csum.Value);\r
        ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
@@ -2384,7 +2595,9 @@ __recv_mgr_prepare_pkt(
 static uint32_t\r
 __recv_mgr_build_pkt_array(\r
        IN                              ipoib_port_t* const                     p_port,\r
-               OUT                     cl_qlist_t* const                       p_done_list )\r
+       IN                              int32_t                                         shortage,\r
+               OUT                     cl_qlist_t* const                       p_done_list,\r
+               OUT                     int32_t* const                          p_discarded )\r
 {\r
        cl_list_item_t                  *p_item;\r
        ipoib_recv_desc_t               *p_desc;\r
@@ -2394,7 +2607,7 @@ __recv_mgr_build_pkt_array(
 \r
        IPOIB_ENTER( IPOIB_DBG_RECV );\r
 \r
-       cl_spinlock_acquire( &p_port->recv_lock );\r
+       *p_discarded = 0;\r
 \r
        /* Move any existing receives to the head to preserve ordering. */\r
        cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
@@ -2410,11 +2623,25 @@ __recv_mgr_build_pkt_array(
                if( status == IB_SUCCESS )\r
                {\r
                        CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
+                       if( shortage-- > 0 )\r
+                       {\r
+                               NDIS_SET_PACKET_STATUS(\r
+                                       p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
+                       }\r
+                       else\r
+                       {\r
+                               NDIS_SET_PACKET_STATUS(\r
+                                       p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
+                       }\r
                        i++;\r
                }\r
-               else if( status != IB_NOT_DONE )\r
+               else if( status == IB_NOT_DONE )\r
+               {\r
+                       (*p_discarded)++;\r
+               }\r
+               else\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_RECV,\r
+                       IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
                                ("__recv_mgr_prepare_pkt returned %s\n",\r
                                p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                        /* Put all completed receives on the port's done list. */\r
@@ -2426,8 +2653,6 @@ __recv_mgr_build_pkt_array(
                p_item = cl_qlist_remove_head( p_done_list );\r
        }\r
 \r
-       cl_spinlock_release( &p_port->recv_lock );\r
-\r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
        return i;\r
 }\r
@@ -2451,15 +2676,14 @@ __send_mgr_construct(
 }\r
 \r
 \r
-static void\r
-__send_mgr_destroy(\r
+static void \r
+__pending_list_destroy(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        cl_list_item_t  *p_item;\r
        NDIS_PACKET             *p_packet;\r
-\r
-       IPOIB_ENTER( IPOIB_DBG_SEND );\r
-\r
+       \r
+       cl_spinlock_acquire( &p_port->send_lock );\r
        /* Complete any pending packets. */\r
        for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
                p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
@@ -2467,8 +2691,17 @@ __send_mgr_destroy(
        {\r
                p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
                NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
-                       NDIS_STATUS_REQUEST_ABORTED );\r
+                       NDIS_STATUS_RESET_IN_PROGRESS );\r
        }\r
+       cl_spinlock_release( &p_port->send_lock );\r
+}\r
+\r
+static void\r
+__send_mgr_destroy(\r
+       IN                              ipoib_port_t* const                     p_port )\r
+{\r
+       IPOIB_ENTER( IPOIB_DBG_SEND );\r
+       __pending_list_destroy(p_port);\r
 \r
        IPOIB_EXIT( IPOIB_DBG_SEND );\r
 }\r
@@ -2544,7 +2777,7 @@ __send_copy(
                ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
        if( !p_desc->p_buf )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to allocate buffer for packet copy.\n") );\r
                return NDIS_STATUS_RESOURCES;\r
        }\r
@@ -2552,17 +2785,17 @@ __send_copy(
        NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                        ("Failed to allocate NDIS_PACKET for copy.\n") );\r
                return status;\r
        }\r
 \r
        NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
-               p_desc->p_buf, XFER_BLOCK_SIZE );\r
+               p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
                NdisFreePacket( p_packet );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                        ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
                return status;\r
        }\r
@@ -2591,7 +2824,7 @@ __send_copy(
        if( bytes_copied != tot_len )\r
        {\r
                /* Something went wrong.  Drop the packet. */\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to copy full packet: %d of %d bytes copied.\n",\r
                        bytes_copied, tot_len) );\r
                return NDIS_STATUS_RESOURCES;\r
@@ -2625,7 +2858,7 @@ __send_gen(
 \r
        if( !p_mdl )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("No buffers associated with packet.\n") );\r
                return IB_ERROR;\r
        }\r
@@ -2633,7 +2866,7 @@ __send_gen(
        /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
        if( num_pages >= MAX_SEND_SGE )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                        ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
                status = __send_copy( p_port, p_desc );\r
                IPOIB_EXIT( IPOIB_DBG_SEND );\r
@@ -2641,7 +2874,7 @@ __send_gen(
        }\r
 \r
        CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
-       CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
+       CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
        /*\r
         * Assume that the ethernet header is always fully contained\r
         * in the first page of the first MDL.  This makes for much\r
@@ -2661,7 +2894,7 @@ __send_gen(
                        CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
                        /* Skip the ethernet header. */\r
                        buf_len -= sizeof(eth_hdr_t);\r
-                       CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
+                       CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
                        if( buf_len )\r
                        {\r
                                /* The ethernet header is a subset of this MDL. */\r
@@ -2727,7 +2960,7 @@ __send_gen(
                NdisGetNextBuffer( p_mdl, &p_mdl );\r
                if( !p_mdl )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get next buffer.\n") );\r
                        return IB_ERROR;\r
                }\r
@@ -2760,7 +2993,7 @@ __send_gen(
        if( !p_sgl )\r
        {\r
                ASSERT( p_sgl );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to get SGL from packet.\n") );\r
                return NDIS_STATUS_FAILURE;\r
        }\r
@@ -2771,7 +3004,7 @@ __send_gen(
                ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
                p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_SEND | IPOIB_DBG_INFO,\r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                        ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
                cl_perf_start( SendCopy );\r
                status = __send_copy( p_port, p_desc );\r
@@ -2845,14 +3078,14 @@ __send_mgr_filter_ip(
                NdisGetNextBuffer( p_buf, &p_buf );\r
                if( !p_buf )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get IP header buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
                NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
                if( !p_ip_hdr )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to query IP header buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
@@ -2865,7 +3098,7 @@ __send_mgr_filter_ip(
        if( buf_len < sizeof(ip_hdr_t) )\r
        {\r
                /* This buffer is done for.  Get the next buffer. */\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Buffer too small for IP packet.\n") );\r
                return NDIS_STATUS_BUFFER_TOO_SHORT;\r
        }\r
@@ -2915,14 +3148,14 @@ __send_mgr_filter_udp(
                NdisGetNextBuffer( p_buf, &p_buf );\r
                if( !p_buf )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get UDP header buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
                NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
                if( !p_udp_hdr )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to query UDP header buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
@@ -2935,7 +3168,7 @@ __send_mgr_filter_udp(
        /* Get the UDP header and check the destination port numbers. */\r
        if( buf_len < sizeof(udp_hdr_t) )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Buffer not large enough for UDP packet.\n") );\r
                return NDIS_STATUS_BUFFER_TOO_SHORT;\r
        }\r
@@ -2960,7 +3193,7 @@ __send_mgr_filter_udp(
                ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
        if( !p_desc->p_buf )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to query DHCP packet buffer.\n") );\r
                return NDIS_STATUS_RESOURCES;\r
        }\r
@@ -2979,6 +3212,19 @@ __send_mgr_filter_udp(
 }\r
 \r
 \r
+unsigned short ipchksum(unsigned short *ip, int len)\r
+{\r
+    unsigned long sum = 0;\r
+\r
+    len >>= 1;\r
+    while (len--) {\r
+        sum += *(ip++);\r
+        if (sum > 0xFFFF)\r
+            sum -= 0xFFFF;\r
+    }\r
+    return (unsigned short)((~sum) & 0x0000FFFF);\r
+}\r
+\r
 static NDIS_STATUS\r
 __send_mgr_filter_dhcp(\r
        IN                              ipoib_port_t* const                     p_port,\r
@@ -3001,14 +3247,14 @@ __send_mgr_filter_dhcp(
                NdisGetNextBuffer( p_buf, &p_buf );\r
                if( !p_buf )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get DHCP buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
                NdisQueryBufferSafe( p_buf, &p_dhcp, &buf_len, NormalPagePriority );\r
                if( !p_dhcp )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to query DHCP buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
@@ -3020,7 +3266,7 @@ __send_mgr_filter_dhcp(
 \r
        if( buf_len < DHCP_MIN_SIZE )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Buffer not large enough for DHCP packet.\n") );\r
                return NDIS_STATUS_BUFFER_TOO_SHORT;\r
        }\r
@@ -3030,7 +3276,7 @@ __send_mgr_filter_dhcp(
 \r
        /* Now scan through the options looking for the client identifier. */\r
        p_option = &p_ib_dhcp->options[4];\r
-       while( *p_option != DHCP_OPT_END )\r
+       while( *p_option != DHCP_OPT_END && p_option < &p_ib_dhcp->options[312] )\r
        {\r
                switch( *p_option )\r
                {\r
@@ -3070,21 +3316,22 @@ __send_mgr_filter_dhcp(
                if( p_cid )\r
                {\r
                        /* do we need to replace it ?  len eq ETH MAC sz 'and' MAC is mine */\r
-                       if( p_cid[1] == HW_ADDR_LEN+1 && !cl_memcmp( &p_cid[3], &p_port->p_adapter->mac.addr, HW_ADDR_LEN  ) )\r
+                       if( p_cid[1] == HW_ADDR_LEN+1 && !cl_memcmp( &p_cid[3],\r
+                               &p_port->p_adapter->params.conf_mac.addr, HW_ADDR_LEN ) )\r
                        {\r
-                                /* Make sure there's room to extend it.  23 is the size of\r
+                               /* Make sure there's room to extend it.  23 is the size of\r
                                 * the CID option for IPoIB.\r
                                 */\r
                                if( buf_len + 23 - p_cid[1] > sizeof(dhcp_pkt_t) )\r
                                {\r
-                                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                                ("Can't convert CID to IPoIB format.\n") );\r
-                                       return IB_INSUFFICIENT_MEMORY;\r
+                                       return NDIS_STATUS_RESOURCES;\r
                                }\r
                                /* Move the existing options down, and add a new CID option */\r
                                len = p_option - ( p_cid + p_cid[1] + 2 );\r
                                p_option = p_cid + p_cid[1] + 2;\r
-                               cl_memcpy ( p_cid, p_option, len );\r
+                               RtlMoveMemory( p_cid, p_option, len );\r
                                \r
                                p_cid += len;\r
                                p_cid[0] = DHCP_OPT_CLIENT_ID;\r
@@ -3104,7 +3351,7 @@ __send_mgr_filter_dhcp(
                         */\r
                        if( buf_len + 23 > sizeof(dhcp_pkt_t) )\r
                        {\r
-                               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("Can't convert CID to IPoIB format.\n") );\r
                                return NDIS_STATUS_RESOURCES;\r
                        }\r
@@ -3114,17 +3361,26 @@ __send_mgr_filter_dhcp(
                        p_option[0] = DHCP_OPT_END;\r
                        p_cid[0] = DHCP_OPT_CLIENT_ID;\r
                        p_cid[1] = 21;\r
+                       p_cid[2] = DHCP_HW_TYPE_IB;\r
                }\r
 \r
                CL_ASSERT( p_cid[1] == 21 );\r
                p_cid[23]= DHCP_OPT_END;\r
-               ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid );\r
+               ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid.guid );\r
                cl_memcpy( &p_cid[7], &gid, sizeof(ib_gid_t) );\r
-               cl_memcpy( &p_cid[3], &p_port->ib_mgr.qpn, sizeof(p_port->ib_mgr.qpn) );\r
-               /* Clear the hardware address. */\r
+               cl_memcpy( &p_cid[3], &p_port->ib_mgr.qpn, sizeof(p_port->ib_mgr.qpn) );                \r
                p_ib_dhcp->htype = DHCP_HW_TYPE_IB;\r
-               p_ib_dhcp->hlen = 0;\r
-               cl_memclr( p_ib_dhcp->chaddr, sizeof(p_ib_dhcp->chaddr) );\r
+\r
+               /* update lengths to include any change we made */\r
+               p_desc->p_buf->ip.hdr.length = cl_ntoh16( sizeof(ip_hdr_t) + sizeof(udp_hdr_t) + sizeof(dhcp_pkt_t) );\r
+               p_desc->p_buf->ip.prot.udp.hdr.length = cl_ntoh16( sizeof(udp_hdr_t) + sizeof(dhcp_pkt_t) );\r
+\r
+               /* update crc in ip header */\r
+               if( !p_port->p_adapter->params.send_chksum_offload )\r
+               {\r
+                       p_desc->p_buf->ip.hdr.chksum = 0;\r
+                       p_desc->p_buf->ip.hdr.chksum = ipchksum((unsigned short*) &p_desc->p_buf->ip.hdr, sizeof(ip_hdr_t));\r
+               }\r
                break;\r
 \r
        /* Server messages. */\r
@@ -3135,7 +3391,7 @@ __send_mgr_filter_dhcp(
                break;\r
 \r
        default:\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Invalide message type.\n") );\r
                return NDIS_STATUS_INVALID_DATA;\r
        }\r
@@ -3170,14 +3426,14 @@ __send_mgr_filter_arp(
                NdisGetNextBuffer( p_buf, &p_buf );\r
                if( !p_buf )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get ARP buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
                NdisQueryBufferSafe( p_buf, &p_arp, &buf_len, NormalPagePriority );\r
                if( !p_arp )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed to get query ARP buffer.\n") );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
@@ -3190,14 +3446,14 @@ __send_mgr_filter_arp(
        /* Single buffer ARP packet. */\r
        if( buf_len < sizeof(arp_pkt_t) )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Buffer too short for ARP.\n") );\r
                return NDIS_STATUS_BUFFER_TOO_SHORT;\r
        }\r
 \r
        if( p_arp->prot_type != ETH_PROT_TYPE_IP )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Unsupported protocol type.\n") );\r
                return NDIS_STATUS_INVALID_DATA;\r
        }\r
@@ -3207,7 +3463,7 @@ __send_mgr_filter_arp(
                ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
        if( !p_desc->p_buf )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to query ARP packet buffer.\n") );\r
                return NDIS_STATUS_RESOURCES;\r
        }\r
@@ -3221,7 +3477,7 @@ __send_mgr_filter_arp(
        p_ib_arp->op = p_arp->op;\r
        p_ib_arp->src_hw.flags_qpn = p_port->ib_mgr.qpn;\r
        ib_gid_set_default( &p_ib_arp->src_hw.gid,\r
-               p_port->p_adapter->guids.port_guid );\r
+               p_port->p_adapter->guids.port_guid.guid );\r
        p_ib_arp->src_ip = p_arp->src_ip;\r
        if( cl_memcmp( &p_arp->dst_hw, &null_hw, sizeof(mac_addr_t) ) )\r
        {\r
@@ -3230,7 +3486,7 @@ __send_mgr_filter_arp(
                        &p_ib_arp->dst_hw.gid, &p_ib_arp->dst_hw.flags_qpn );\r
                if( status != NDIS_STATUS_SUCCESS )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Failed lookup of destination HW address\n") );\r
                        return status;\r
                }\r
@@ -3256,9 +3512,9 @@ __send_mgr_get_eth_hdr(
        IN                              NDIS_PACKET* const                      p_packet,\r
                OUT                     NDIS_BUFFER** const                     pp_buf,\r
                OUT                     eth_hdr_t** const                       pp_eth_hdr,\r
-               OUT                     size_t*                                         p_buf_len )\r
+               OUT                     UINT*                                           p_buf_len )\r
 {\r
-       size_t                          tot_len;\r
+       UINT                            tot_len;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_SEND );\r
 \r
@@ -3268,20 +3524,20 @@ __send_mgr_get_eth_hdr(
        if( !*pp_eth_hdr )\r
        {\r
                /* Failed to get first buffer. */\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("NdisMGetFirstBufferSafe failed.\n") );\r
                return NDIS_STATUS_FAILURE;\r
        }\r
 \r
        if( *p_buf_len < sizeof(eth_hdr_t) )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("First buffer in packet smaller than eth_hdr_t: %d.\n",\r
                        *p_buf_len) );\r
                return NDIS_STATUS_BUFFER_TOO_SHORT;\r
        }\r
 \r
-       IPOIB_TRACE_EXIT( (IPOIB_DBG_SEND | IPOIB_DBG_INFO),\r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                ("Ethernet header:\n"\r
                "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
                "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
@@ -3313,7 +3569,7 @@ __send_mgr_queue(
        /* Check the send queue and pend the request if not empty. */\r
        if( cl_qlist_count( &p_port->send_mgr.pending_list ) )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                        ("Pending list not empty.\n") );\r
                return NDIS_STATUS_PENDING;\r
        }\r
@@ -3321,7 +3577,7 @@ __send_mgr_queue(
        /* Check the send queue and pend the request if not empty. */\r
        if( p_port->send_mgr.depth == p_port->p_adapter->params.sq_depth )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                        ("No available WQEs.\n") );\r
                return NDIS_STATUS_PENDING;\r
        }\r
@@ -3331,13 +3587,12 @@ __send_mgr_queue(
        cl_perf_stop( &p_port->p_adapter->perf, GetEndpt );\r
 \r
        if( status == NDIS_STATUS_NO_ROUTE_TO_DESTINATION &&\r
-               p_eth_hdr->dst.addr[0] == 0x01 &&\r
-               p_eth_hdr->dst.addr[1] == 0x00 &&\r
-               p_eth_hdr->dst.addr[2] == 0x5E )\r
+               ETH_IS_MULTICAST( p_eth_hdr->dst.addr ) )\r
        {\r
-               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst ) == IB_SUCCESS )\r
+               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst, \r
+                       IB_MC_REC_STATE_FULL_MEMBER) == IB_SUCCESS )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_ENDPT,\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                                ("Multicast Mac - trying to join.\n") );\r
                        return NDIS_STATUS_PENDING;\r
                }\r
@@ -3358,6 +3613,8 @@ __build_send_desc(
 {\r
        NDIS_STATUS                     status;\r
        int32_t                         hdr_idx;\r
+       PNDIS_PACKET_EXTENSION                          PktExt;\r
+       PNDIS_TCP_IP_CHECKSUM_PACKET_INFO       pChecksumPktInfo; //NDIS 5.1\r
 \r
        PERF_DECLARE( SendMgrFilter );\r
 \r
@@ -3370,7 +3627,7 @@ __build_send_desc(
        cl_perf_stop( &p_port->p_adapter->perf, SendMgrFilter );\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__send_mgr_filter returned 0x%08X.\n", status) );\r
                return status;\r
        }\r
@@ -3392,12 +3649,29 @@ __build_send_desc(
        p_desc->wr.wr_id = (uintn_t)p_desc->p_pkt;\r
        p_desc->wr.wr_type = WR_SEND;\r
        p_desc->wr.send_opt = IB_SEND_OPT_SIGNALED;\r
-       p_desc->wr.ds_array = p_desc->local_ds;\r
-\r
-       p_desc->wr.dgrm.ud.remote_qp = p_desc->p_endpt->qpn;\r
-       p_desc->wr.dgrm.ud.remote_qkey = p_port->ib_mgr.bcast_rec.qkey;\r
+       \r
+       PktExt = NDIS_PACKET_EXTENSION_FROM_PACKET(p_desc->p_pkt);\r
+       pChecksumPktInfo = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&PktExt->NdisPacketInfo[TcpIpChecksumPacketInfo];\r
+       if(p_port->p_adapter->params.send_chksum_offload & \r
+               (pChecksumPktInfo->Transmit.NdisPacketChecksumV4 || pChecksumPktInfo->Transmit.NdisPacketChecksumV6))\r
+       {\r
+               // Set transimition checksum offloading \r
+               if (pChecksumPktInfo->Transmit.NdisPacketIpChecksum) \r
+               {\r
+                       p_desc->wr.send_opt |= IB_SEND_OPT_TX_IP_CSUM;\r
+               }\r
+               if(pChecksumPktInfo->Transmit.NdisPacketTcpChecksum  ) \r
+               {\r
+                       p_desc->wr.send_opt |= IB_SEND_OPT_TX_TCP_UDP_CSUM;\r
+               }\r
+       }\r
+       \r
+       p_desc->wr.ds_array = p_desc->local_ds;\r
+\r
+       p_desc->wr.dgrm.ud.remote_qp = p_desc->p_endpt->qpn;\r
+       p_desc->wr.dgrm.ud.remote_qkey = p_port->ib_mgr.bcast_rec.qkey;\r
        p_desc->wr.dgrm.ud.h_av = p_desc->p_endpt->h_av;\r
-       p_desc->wr.dgrm.ud.pkey_index = 0;\r
+       p_desc->wr.dgrm.ud.pkey_index = p_port->pkey_index;\r
        p_desc->wr.dgrm.ud.rsvd = NULL;\r
 \r
        /* Store context in our reserved area of the packet. */\r
@@ -3448,7 +3722,7 @@ ipoib_port_send(
        uint32_t                        i;\r
        eth_hdr_t                       *p_eth_hdr;\r
        NDIS_BUFFER                     *p_buf;\r
-       size_t                          buf_len;\r
+       UINT                            buf_len;\r
 \r
        PERF_DECLARE( GetEthHdr );\r
        PERF_DECLARE( BuildSendDesc );\r
@@ -3459,6 +3733,27 @@ ipoib_port_send(
 \r
        IPOIB_ENTER( IPOIB_DBG_SEND );\r
 \r
+\r
+       cl_obj_lock( &p_port->obj );\r
+       if( p_port->state != IB_QPS_RTS )\r
+       {\r
+               cl_obj_unlock( &p_port->obj );\r
+               for( i = 0; i < num_packets; ++i )\r
+               {\r
+                       ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
+                       /* Complete the packet. */\r
+                       NdisMSendComplete( p_port->p_adapter->h_adapter,\r
+                               p_packet_array[i], NDIS_STATUS_ADAPTER_NOT_READY );\r
+                       \r
+               }\r
+\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
+                       ("Invalid state - Aborting.\n") );\r
+               return;\r
+       }\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       \r
        cl_spinlock_acquire( &p_port->send_lock );\r
        for( i = 0; i < num_packets; i++ )\r
        {\r
@@ -3524,7 +3819,7 @@ ipoib_port_send(
                cl_perf_stop( &p_port->p_adapter->perf, PostSend );\r
                if( ib_status != IB_SUCCESS )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("ib_post_send returned %s\n", \r
                                p_port->p_adapter->p_ifc->get_err_str( ib_status )) );\r
                        cl_perf_start( ProcessFailedSends );\r
@@ -3553,7 +3848,7 @@ ipoib_port_resume(
        ipoib_send_desc_t       desc;\r
        eth_hdr_t                       *p_eth_hdr;\r
        NDIS_BUFFER                     *p_buf;\r
-       size_t                          buf_len;\r
+       UINT                            buf_len;\r
 \r
        PERF_DECLARE( GetEndpt );\r
        PERF_DECLARE( BuildSendDesc );\r
@@ -3562,6 +3857,17 @@ ipoib_port_resume(
 \r
        IPOIB_ENTER( IPOIB_DBG_SEND );\r
 \r
+\r
+       cl_obj_lock( &p_port->obj );\r
+       if( p_port->state != IB_QPS_RTS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
+                       ("Invalid state - Aborting.\n") );\r
+               cl_obj_unlock( &p_port->obj );\r
+               return;\r
+       }\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
        cl_spinlock_acquire( &p_port->send_lock );\r
 \r
        for( p_item = cl_qlist_head( &p_port->send_mgr.pending_list );\r
@@ -3571,7 +3877,7 @@ ipoib_port_resume(
                /* Check the send queue and pend the request if not empty. */\r
                if( p_port->send_mgr.depth == p_port->p_adapter->params.sq_depth )\r
                {\r
-                       IPOIB_TRACE_EXIT( IPOIB_DBG_SEND | IPOIB_DBG_WARN,\r
+                       IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                                ("No available WQEs.\n") );\r
                        break;\r
                }\r
@@ -3605,14 +3911,12 @@ ipoib_port_resume(
                {\r
                        ASSERT( status == NDIS_STATUS_NO_ROUTE_TO_DESTINATION );\r
 \r
-                       if( p_eth_hdr->dst.addr[0] == 0x01 &&\r
-                               p_eth_hdr->dst.addr[1] == 0x00 &&\r
-                               p_eth_hdr->dst.addr[2] == 0x5E )\r
+                       if( ETH_IS_MULTICAST( p_eth_hdr->dst.addr ) )\r
                        {\r
-                               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst ) ==\r
-                                       IB_SUCCESS )\r
+                               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst,\r
+                                       IB_MC_REC_STATE_FULL_MEMBER) == IB_SUCCESS )\r
                                {\r
-                                       IPOIB_TRACE_EXIT( IPOIB_DBG_ENDPT,\r
+                                       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                                                ("Multicast Mac - trying to join.\n") );\r
                                        cl_qlist_insert_head( &p_port->send_mgr.pending_list,\r
                                                IPOIB_LIST_ITEM_FROM_PACKET( desc.p_pkt ) );\r
@@ -3647,7 +3951,7 @@ ipoib_port_resume(
                cl_perf_stop( &p_port->p_adapter->perf, PostSend );\r
                if( ib_status != IB_SUCCESS )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("ib_post_send returned %s\n", \r
                                p_port->p_adapter->p_ifc->get_err_str( ib_status )) );\r
                        cl_perf_start( ProcessFailedSends );\r
@@ -3701,7 +4005,7 @@ __send_cb(
 \r
        p_port = (ipoib_port_t*)cq_context;\r
 \r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_send_cb );\r
 \r
        for( i = 0; i < MAX_SEND_WC; i++ )\r
                wc[i].p_next = &wc[i + 1];\r
@@ -3718,7 +4022,7 @@ __send_cb(
                while( p_wc )\r
                {\r
                        cl_perf_start( SendComp );\r
-                       CL_ASSERT( p_wc->wc_type == IB_WC_SEND );\r
+                       CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_SEND );\r
                        p_packet = (NDIS_PACKET*)(uintn_t)p_wc->wr_id;\r
                        CL_ASSERT( p_packet );\r
                        CL_ASSERT( IPOIB_PORT_FROM_PACKET( p_packet ) == p_port );\r
@@ -3753,7 +4057,7 @@ __send_cb(
                                break;\r
 \r
                        case IB_WCS_WR_FLUSHED_ERR:\r
-                               IPOIB_TRACE( IPOIB_DBG_RECV | IPOIB_DBG_INFO,\r
+                               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                                        ("Flushed send completion.\n") );\r
                                ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
                                NdisMSendComplete( p_port->p_adapter->h_adapter,\r
@@ -3761,8 +4065,10 @@ __send_cb(
                                break;\r
 \r
                        default:\r
-                               IPOIB_TRACE( IPOIB_DBG_ERROR, ("Send failed with %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
+                               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("Send failed with %s (vendor specific %#x)\n",\r
+                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
+                                       (int)p_wc->vendor_specific) );\r
                                ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                                NdisMSendComplete( p_port->p_adapter->h_adapter,\r
                                        p_packet, NDIS_STATUS_FAILURE );\r
@@ -3799,7 +4105,7 @@ __send_cb(
        ipoib_port_resume( p_port );\r
        cl_perf_stop( &p_port->p_adapter->perf, PortResume );\r
        \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_send_cb );\r
 \r
        cl_perf_stop( &p_port->p_adapter->perf, SendCb );\r
        cl_perf_update_ctr( &p_port->p_adapter->perf, SendCompBundle );\r
@@ -3852,33 +4158,119 @@ __endpt_mgr_destroy(
 static void\r
 __endpt_mgr_remove_all(\r
        IN                              ipoib_port_t* const                     p_port )\r
+{\r
+       IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
+       \r
+       cl_obj_lock( &p_port->obj );\r
+       /* Wait for all readers to complete. */\r
+       while( p_port->endpt_rdr )\r
+               ;\r
+       /*\r
+        * We don't need to initiate destruction - this is called only\r
+        * from the __port_destroying function, and destruction cascades\r
+        * to all child objects.  Just clear all the maps.\r
+        */\r
+       cl_qmap_remove_all( &p_port->endpt_mgr.mac_endpts );\r
+       cl_qmap_remove_all( &p_port->endpt_mgr.lid_endpts );\r
+       cl_fmap_remove_all( &p_port->endpt_mgr.gid_endpts );\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+}\r
+\r
+\r
+static void\r
+__endpt_mgr_reset_all(\r
+       IN                              ipoib_port_t* const                     p_port )\r
 {\r
        cl_map_item_t   *p_item;\r
-       ipoib_endpt_t   *p_endpt;\r
+       ipoib_endpt_t           *p_endpt;\r
+       cl_qlist_t                      mc_list;\r
+       uint32_t                        local_exist = 0;\r
+\r
 \r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
+\r
+       cl_qlist_init( &mc_list );\r
        \r
        cl_obj_lock( &p_port->obj );\r
        /* Wait for all readers to complete. */\r
        while( p_port->endpt_rdr )\r
                ;\r
+\r
+       if( p_port->p_local_endpt )\r
+       {\r
+               cl_qmap_remove_item( &p_port->endpt_mgr.mac_endpts,\r
+                       &p_port->p_local_endpt->mac_item );\r
+               cl_fmap_remove_item( &p_port->endpt_mgr.gid_endpts,\r
+                       &p_port->p_local_endpt->gid_item );\r
+               cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,\r
+                       &p_port->p_local_endpt->lid_item );\r
+               \r
+               cl_qlist_insert_head(\r
+                       &mc_list, &p_port->p_local_endpt->mac_item.pool_item.list_item );\r
+               local_exist = 1;\r
+\r
+               p_port->p_local_endpt = NULL;\r
+       }\r
+\r
        p_item = cl_qmap_head( &p_port->endpt_mgr.mac_endpts );\r
        while( p_item != cl_qmap_end( &p_port->endpt_mgr.mac_endpts ) )\r
        {\r
                p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, mac_item );\r
-               cl_qmap_remove_item( &p_port->endpt_mgr.mac_endpts, p_item );\r
-               cl_obj_unlock( &p_port->obj );\r
-               cl_obj_destroy( &p_endpt->obj );\r
-               cl_obj_lock( &p_port->obj );\r
-               /* Wait for all readers to complete. */\r
-               while( p_port->endpt_rdr )\r
-                       ;\r
-               p_item = cl_qmap_head( &p_port->endpt_mgr.mac_endpts );\r
+               p_item = cl_qmap_next( p_item );\r
+               if( p_endpt->h_mcast )\r
+               {\r
+                       /*\r
+                        * We destroy MC endpoints since they will get recreated\r
+                        * when the port comes back up and we rejoin the MC groups.\r
+                        */\r
+                       cl_qmap_remove_item( &p_port->endpt_mgr.mac_endpts,\r
+                               &p_endpt->mac_item );\r
+                       cl_fmap_remove_item( &p_port->endpt_mgr.gid_endpts,\r
+                               &p_endpt->gid_item );\r
+\r
+                       cl_qlist_insert_tail(\r
+                               &mc_list, &p_endpt->mac_item.pool_item.list_item );\r
+               }\r
+               else if( p_endpt->h_av )\r
+               {\r
+                       /* Destroy the AV for all other endpoints. */\r
+                       p_port->p_adapter->p_ifc->destroy_av( p_endpt->h_av );\r
+                       p_endpt->h_av = NULL;\r
+               }\r
+               \r
+               if( p_endpt->dlid )\r
+               {\r
+                       cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,\r
+                               &p_endpt->lid_item );\r
+                       p_endpt->dlid = 0;\r
+               }\r
+               \r
        }\r
-       cl_qmap_remove_all( &p_port->endpt_mgr.lid_endpts );\r
-       cl_fmap_remove_all( &p_port->endpt_mgr.gid_endpts );\r
+\r
        cl_obj_unlock( &p_port->obj );\r
 \r
+\r
+       if(cl_qlist_count( &mc_list ) - local_exist)\r
+       {\r
+               p_port->mcast_cnt =  (uint32_t)cl_qlist_count( &mc_list ) - local_exist;\r
+       }\r
+       else\r
+       {\r
+               p_port->mcast_cnt = 0;\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }       \r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt - local_exist));\r
+\r
+       /* Destroy all multicast endpoints now that we have released the lock. */\r
+       while( cl_qlist_count( &mc_list ) )\r
+       {\r
+               cl_obj_destroy( &PARENT_STRUCT( cl_qlist_remove_head( &mc_list ),\r
+                       ipoib_endpt_t, mac_item.pool_item.list_item )->obj );\r
+       }\r
+\r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
 }\r
 \r
@@ -3929,7 +4321,7 @@ ipoib_mac_to_gid(
        IN              const   mac_addr_t                                      mac,\r
                OUT                     ib_gid_t*                                       p_gid )\r
 {\r
-       ipoib_endpt_t* p_endpt;\r
+       ipoib_endpt_t*  p_endpt;\r
        cl_map_item_t   *p_item;\r
        uint64_t                key = 0;\r
 \r
@@ -3943,7 +4335,8 @@ ipoib_mac_to_gid(
        if( p_item == cl_qmap_end( &p_port->endpt_mgr.mac_endpts ) )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ENDPT, ("Failed endpoint lookup.\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("Failed endpoint lookup.\n") );\r
                return STATUS_INVALID_PARAMETER;\r
        }\r
 \r
@@ -3971,12 +4364,21 @@ __endpt_mgr_ref(
 \r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
+       if( !cl_memcmp( &mac, &p_port->p_adapter->params.conf_mac, sizeof(mac) ) )\r
+       {\r
+               /* Discard loopback traffic. */\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ENDPT,\r
+                       ("Discarding loopback traffic\n") );\r
+               IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+               return NDIS_STATUS_NO_ROUTE_TO_DESTINATION;\r
+       }\r
+\r
        key = 0;\r
        cl_memcpy( &key, &mac, sizeof(mac_addr_t) );\r
 \r
        cl_obj_lock( &p_port->obj );\r
 \r
-       IPOIB_TRACE( (IPOIB_DBG_ENDPT | IPOIB_DBG_INFO),\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
                ("Look for :\t  MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
                mac.addr[0], mac.addr[1], mac.addr[2],\r
                mac.addr[3], mac.addr[4], mac.addr[5]) );\r
@@ -3985,7 +4387,8 @@ __endpt_mgr_ref(
        if( p_item == cl_qmap_end( &p_port->endpt_mgr.mac_endpts ) )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ENDPT, ("Failed endpoint lookup.\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
+                       ("Failed endpoint lookup.\n") );\r
                return NDIS_STATUS_NO_ROUTE_TO_DESTINATION;\r
        }\r
 \r
@@ -4027,7 +4430,8 @@ __endpt_mgr_get_gid_qpn(
        if( p_item == cl_qmap_end( &p_port->endpt_mgr.mac_endpts ) )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ENDPT, ("Failed endpoint lookup.\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
+                       ("Failed endpoint lookup.\n") );\r
                return NDIS_STATUS_FAILURE;\r
        }\r
 \r
@@ -4085,15 +4489,17 @@ __endpt_mgr_get_by_lid(
 }\r
 \r
 \r
-inline void\r
+inline ib_api_status_t\r
 __endpt_mgr_insert_locked(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt )\r
 {\r
+       ib_api_status_t status;\r
+\r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
-       IPOIB_TRACE( (IPOIB_DBG_ENDPT | IPOIB_DBG_INFO),\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
                ("insert  :\t  MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
                mac.addr[0], mac.addr[1], mac.addr[2],\r
                mac.addr[3], mac.addr[4], mac.addr[5]) );\r
@@ -4106,21 +4512,22 @@ __endpt_mgr_insert_locked(
        }\r
        /* __endpt_mgr_insert expects *one* reference to be held when being called. */\r
        cl_atomic_inc( &p_port->endpt_rdr );\r
-       __endpt_mgr_insert( p_port, mac, p_endpt );\r
+       status= __endpt_mgr_insert( p_port, mac, p_endpt );\r
        cl_atomic_dec( &p_port->endpt_rdr );\r
        cl_obj_unlock( &p_port->obj );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+       return status;\r
 }\r
 \r
 \r
-void\r
+inline ib_api_status_t\r
 __endpt_mgr_insert(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt )\r
 {\r
        uint64_t                key;\r
+       cl_status_t             cl_status;\r
        cl_map_item_t   *p_qitem;\r
        cl_fmap_item_t  *p_fitem;\r
 \r
@@ -4131,9 +4538,21 @@ __endpt_mgr_insert(
                ;\r
 \r
        /* Link the endpoint to the port. */\r
-       cl_obj_insert_rel_parent_locked(\r
+       cl_status = cl_obj_insert_rel_parent_locked(\r
                &p_endpt->rel, &p_port->obj, &p_endpt->obj );\r
 \r
+       if( cl_status != CL_SUCCESS )\r
+       {\r
+               cl_obj_destroy( &p_endpt->obj );\r
+               return IB_INVALID_STATE;\r
+       }\r
+\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_endpt_track] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref  type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
        p_endpt->mac = mac;\r
        key = 0;\r
        cl_memcpy( &key, &mac, sizeof(mac_addr_t) );\r
@@ -4151,6 +4570,7 @@ __endpt_mgr_insert(
        }\r
 \r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+       return IB_SUCCESS;\r
 }\r
 \r
 \r
@@ -4175,7 +4595,7 @@ __endpt_mgr_add_bcast(
                p_mcast_rec->p_member_rec->mlid, CL_HTON32(0x00FFFFFF) );\r
        if( !p_endpt )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ipoib_endpt_create failed.\n") );\r
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
@@ -4185,18 +4605,18 @@ __endpt_mgr_add_bcast(
                p_port->port_num, p_mcast_rec );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ipoib_create_mcast_endpt returned %s\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
 \r
-       /* Add the broadcast endpoint to the enpoint map. */\r
+       /* Add the broadcast endpoint to the endpoint map. */\r
        cl_memset( &bcast_mac, 0xFF, sizeof(bcast_mac) );\r
-       __endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );\r
+       status = __endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
-       return IB_SUCCESS;\r
+       return status;\r
 }\r
 \r
 \r
@@ -4242,6 +4662,12 @@ ipoib_port_remove_endpt(
 \r
                cl_obj_unlock( &p_port->obj );\r
                cl_obj_destroy( &p_endpt->obj );\r
+#if DBG\r
+               cl_atomic_dec( &p_port->ref[ref_endpt_track] );\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+                       ("ref type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
        }\r
        else\r
        {\r
@@ -4322,11 +4748,14 @@ ipoib_port_up(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       /* Wait for all receives to get flushed. */\r
-       while( p_port->recv_mgr.depth )\r
+       /* Wait for all work requests to get flushed. */\r
+       while( p_port->recv_mgr.depth || p_port->send_mgr.depth )\r
                cl_thread_suspend( 0 );\r
 \r
-       p_port->state = IB_QPS_RTS;\r
+       cl_obj_lock( &p_port->obj );\r
+       p_port->state = IB_QPS_INIT;\r
+       KeResetEvent( &p_port->sa_event );\r
+       cl_obj_unlock( &p_port->obj );\r
 \r
        info.method = IB_MAD_METHOD_GET;\r
        info.attr_id = IB_MAD_ATTR_PORTINFO_RECORD;\r
@@ -4341,22 +4770,23 @@ ipoib_port_up(
        cl_memclr( &query, sizeof(ib_query_req_t) );\r
        query.query_type = IB_QUERY_USER_DEFINED;\r
        query.p_query_input = &info;\r
-       query.port_guid = p_port->p_adapter->guids.port_guid;\r
+       query.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        query.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        query.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
        query.query_context = p_port;\r
        query.pfn_query_cb = __port_info_cb;\r
 \r
        /* reference the object for the multicast query. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_port_up );\r
 \r
-       status =\r
-               p_port->p_adapter->p_ifc->query( p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
+       status = p_port->p_adapter->p_ifc->query(\r
+               p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
        if( status != IB_SUCCESS )\r
        {\r
-               p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
+               ipoib_set_inactive( p_port->p_adapter );\r
+               ipoib_port_deref( p_port, ref_port_up );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_query returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return;\r
@@ -4378,13 +4808,13 @@ __endpt_mgr_add_local(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid );\r
+       ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid.guid );\r
        p_endpt = ipoib_endpt_create(\r
                &gid, p_port_info->base_lid, p_port->ib_mgr.qpn );\r
        if( !p_endpt )\r
        {\r
                p_port->p_adapter->hung = TRUE;\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Failed to create local endpt\n") );\r
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
@@ -4395,23 +4825,33 @@ __endpt_mgr_add_local(
        av_attr.dlid = p_port_info->base_lid;\r
        av_attr.static_rate = ib_port_info_compute_rate( p_port_info );\r
        av_attr.path_bits = 0;\r
-       status =\r
-               p_port->p_adapter->p_ifc->create_av( p_port->ib_mgr.h_pd, &av_attr, &p_endpt->h_av );\r
+       status = p_port->p_adapter->p_ifc->create_av(\r
+               p_port->ib_mgr.h_pd, &av_attr, &p_endpt->h_av );\r
        if( status != IB_SUCCESS )\r
        {\r
                cl_obj_destroy( &p_endpt->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_create_av for local endpoint returned %s\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
 \r
-       __endpt_mgr_insert_locked( p_port, p_port->p_adapter->mac, p_endpt );\r
+       /* __endpt_mgr_insert expects *one* reference to be held. */\r
+       cl_atomic_inc( &p_port->endpt_rdr );\r
+       status = __endpt_mgr_insert( p_port, p_port->p_adapter->params.conf_mac, p_endpt );\r
+       cl_atomic_dec( &p_port->endpt_rdr );\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("__endpt_mgr_insert for local endpoint returned %s\n",\r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
 \r
        p_port->p_local_endpt = p_endpt;\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
-       return IB_SUCCESS;\r
+       return status;\r
 }\r
 \r
 \r
@@ -4425,13 +4865,20 @@ __port_info_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_query_rec->query_context;\r
+       p_port = (ipoib_port_t*)p_query_rec->query_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        p_port->ib_mgr.h_query = NULL;\r
-       cl_obj_unlock( &p_port->obj );\r
 \r
-       switch( p_query_rec->status )\r
+       if( p_port->state != IB_QPS_INIT )\r
+       {\r
+               status = IB_CANCELED;\r
+               goto done;\r
+       }\r
+\r
+       status = p_query_rec->status;\r
+\r
+       switch( status )\r
        {\r
        case IB_SUCCESS:\r
                /* Note that the we report the rate from the port info. */\r
@@ -4441,7 +4888,7 @@ __port_info_cb(
                status = __endpt_mgr_add_local( p_port, &p_port_rec->port_info );\r
                if( status == IB_SUCCESS )\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_INFO,\r
+                       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                                ("Received port info: link width = %d.\n",\r
                                p_port_rec->port_info.link_width_active) );\r
 \r
@@ -4452,38 +4899,54 @@ __port_info_cb(
                                p_port_rec->port_info.link_width_active,\r
                                ib_port_info_get_link_speed_active( &p_port_rec->port_info ) );\r
 \r
-                       __port_get_mcast( p_port );\r
+                       status = __port_get_bcast( p_port );\r
                }\r
                else\r
                {\r
-                       IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("__endpt_mgr_add_local returned %s\n",\r
                                p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                }\r
                break;\r
 \r
        case IB_CANCELED:\r
-               IPOIB_TRACE( IPOIB_DBG_INIT,\r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                        ("Instance destroying - Aborting.\n") );\r
                break;\r
 \r
        case IB_TIMEOUT:\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_PORT_INFO_TIMEOUT, 0 );\r
-               IPOIB_TRACE( IPOIB_DBG_INFO, ("Port info query timed out.\n") );\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Port info query timed out.\n") );\r
                break;\r
 \r
        case IB_REMOTE_ERROR:\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_PORT_INFO_REJECT, 0 );\r
-               IPOIB_TRACE( IPOIB_DBG_INFO, ("Port info query rejected by SA.\n") );\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Port info query rejected by SA.\n") );\r
                break;\r
 \r
        default:\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_QUERY_PORT_INFO, 1, p_query_rec->status );\r
                /* Hopefully we'll get an SM change event that will restart things. */\r
-               IPOIB_TRACE( IPOIB_DBG_INFO, ("Port info query failed.\n") );\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Port info query failed.\n") );\r
+       }\r
+\r
+done:\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               if( status != IB_CANCELED )\r
+               {\r
+                       ipoib_set_inactive( p_port->p_adapter );\r
+                       __endpt_mgr_reset_all( p_port );\r
+               }\r
+               KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
        }\r
 \r
        /* Return the response MAD to AL. */\r
@@ -4491,14 +4954,14 @@ __port_info_cb(
                p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
 \r
        /* Release the reference taken when issuing the port info query. */\r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_port_info_cb );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
 \r
-static void\r
-__port_get_mcast(\r
+static ib_api_status_t\r
+__port_get_bcast(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        ib_api_status_t         status;\r
@@ -4506,9 +4969,9 @@ __port_get_mcast(
        ib_user_query_t         info;\r
        ib_member_rec_t         member_rec;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       info.method = IB_MAD_METHOD_GET;\r
+       info.method = IB_MAD_METHOD_GETTABLE;\r
        info.attr_id = IB_MAD_ATTR_MCMEMBER_RECORD;\r
        info.attr_size = sizeof(ib_member_rec_t);\r
        info.comp_mask = IB_MCR_COMPMASK_MGID;\r
@@ -4518,30 +4981,33 @@ __port_get_mcast(
        cl_memclr( &member_rec, sizeof(ib_member_rec_t) );\r
        member_rec.mgid = bcast_mgid_template;\r
 \r
+    member_rec.mgid.raw[4] = (uint8_t) (p_port->p_adapter->guids.port_guid.pkey >> 8) ;\r
+       member_rec.mgid.raw[5] = (uint8_t) p_port->p_adapter->guids.port_guid.pkey;\r
+       member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
        cl_memclr( &query, sizeof(ib_query_req_t) );\r
        query.query_type = IB_QUERY_USER_DEFINED;\r
        query.p_query_input = &info;\r
-       query.port_guid = p_port->p_adapter->guids.port_guid;\r
+       query.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        query.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        query.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
        query.query_context = p_port;\r
        query.pfn_query_cb = __bcast_get_cb;\r
 \r
        /* reference the object for the multicast query. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_get_bcast );\r
 \r
-       status =\r
-               p_port->p_adapter->p_ifc->query( p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
+       status = p_port->p_adapter->p_ifc->query(\r
+               p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               ipoib_port_deref( p_port, ref_get_bcast );\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_query returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               return;\r
        }\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
+       return status;\r
 }\r
 \r
 \r
@@ -4552,66 +5018,62 @@ __bcast_get_cb(
 {\r
        ipoib_port_t            *p_port;\r
        ib_member_rec_t         *p_mc_req;\r
-#ifdef PR_102801\r
-       uint32_t                        i;\r
-#endif\r
+       ib_api_status_t         status;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_query_rec->query_context;\r
+       p_port = (ipoib_port_t*)p_query_rec->query_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        p_port->ib_mgr.h_query = NULL;\r
-       cl_obj_unlock( &p_port->obj );\r
+       if( p_port->state != IB_QPS_INIT )\r
+       {\r
+               status = IB_CANCELED;\r
+               goto done;\r
+       }\r
+\r
+       status = p_query_rec->status;\r
 \r
-       switch( p_query_rec->status )\r
+       switch( status )\r
        {\r
        case IB_SUCCESS:\r
-#ifdef PR_102801\r
-               p_mc_req = NULL;\r
-               /* Loop through the records until we find one with the proper MGID. */\r
-               for( i = 0; i < p_query_rec->result_cnt; i++ )\r
+               if( p_query_rec->result_cnt )\r
                {\r
                        p_mc_req = (ib_member_rec_t*)\r
-                               ib_get_query_result( p_query_rec->p_result_mad, i );\r
-                       if( cl_memcmp(\r
-                               &p_mc_req->mgid, &bcast_mgid_template, sizeof(ib_gid_t) ) == 0 )\r
-                       {\r
-                               break;\r
-                       }\r
-               }\r
-               /*\r
-                * If we didn't get any records with the desired MGID,\r
-                * create the group.\r
-                */\r
-               if( i == p_query_rec->result_cnt )\r
-               {\r
-                       __port_create_bcast( p_port );\r
+                               ib_get_query_result( p_query_rec->p_result_mad, 0 );\r
+\r
+                       /* Join the broadcast group. */\r
+                       status = __port_join_bcast( p_port, p_mc_req );\r
                        break;\r
                }\r
-#else\r
-               p_mc_req = (ib_member_rec_t*)\r
-                       ib_get_query_result( p_query_rec->p_result_mad, 0 );\r
-#endif\r
-               /* Join the broadcast group. */\r
-               __port_join_bcast( p_port, p_mc_req );\r
-               break;\r
+               /* Fall through. */\r
 \r
        case IB_REMOTE_ERROR:\r
-               /*\r
-                * SA failed the query.  Broadcast group doesn't exist, so create it.\r
-                */\r
-               __port_create_bcast( p_port );\r
+               /* SA failed the query.  Broadcast group doesn't exist, create it. */\r
+               status = __port_create_bcast( p_port );\r
                break;\r
 \r
        case IB_CANCELED:\r
-               IPOIB_TRACE( IPOIB_DBG_INIT, ("Instance destroying - Aborting.\n") );\r
-       \r
+               IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Instance destroying - Aborting.\n") );\r
+               break;\r
+\r
        default:\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_BCAST_GET, 1, p_query_rec->status );\r
-               /* Flag the adapter as hung. */\r
-               p_port->p_adapter->hung = TRUE;\r
+       }\r
+\r
+done:\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               if( status != IB_CANCELED )\r
+               {\r
+                       ipoib_set_inactive( p_port->p_adapter );\r
+                       __endpt_mgr_reset_all( p_port );\r
+               }\r
+               KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
        }\r
 \r
        /* Return the response MAD to AL. */\r
@@ -4619,13 +5081,13 @@ __bcast_get_cb(
                p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
 \r
        /* Release the reference taken when issuing the member record query. */\r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_bcast_get_cb );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
 \r
-static void\r
+static ib_api_status_t\r
 __port_join_bcast(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              ib_member_rec_t* const          p_member_rec )\r
@@ -4633,22 +5095,23 @@ __port_join_bcast(
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        /* Check that the rate is realizable for our port. */\r
-       if( p_port->ib_mgr.rate < (p_member_rec->rate & 0x3F) )\r
+       if( p_port->ib_mgr.rate < (p_member_rec->rate & 0x3F) &&\r
+               (g_ipoib.bypass_check_bcast_rate == 0))\r
        {\r
                /*\r
                 * The MC group rate is higher than our port's rate.  Log an error\r
                 * and stop.  A port transition will drive the retry.\r
                 */\r
-               IPOIB_TRACE( IPOIB_DBG_WARN,\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,\r
                        ("Unrealizable join due to rate mismatch.\n") );\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_BCAST_RATE, 2,\r
                        (uint32_t)(p_member_rec->rate & 0x3F),\r
                        (uint32_t)p_port->ib_mgr.rate );\r
-               return;\r
+               return IB_ERROR;\r
        }\r
 \r
        /* Join the broadcast group. */\r
@@ -4658,48 +5121,49 @@ __port_join_bcast(
        /* We specify our port GID for the join operation. */\r
        mcast_req.member_rec.port_gid.unicast.prefix = IB_DEFAULT_SUBNET_PREFIX;\r
        mcast_req.member_rec.port_gid.unicast.interface_id =\r
-               p_port->p_adapter->guids.port_guid;\r
+               p_port->p_adapter->guids.port_guid.guid;\r
 \r
        mcast_req.mcast_context = p_port;\r
        mcast_req.pfn_mcast_cb = __bcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
 \r
        if( ib_member_get_state( mcast_req.member_rec.scope_state ) !=\r
                IB_MC_REC_STATE_FULL_MEMBER )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_WARN,\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,\r
                        ("Incorrect MC member rec join state in query response.\n") );\r
                ib_member_set_state( &mcast_req.member_rec.scope_state,\r
                        IB_MC_REC_STATE_FULL_MEMBER );\r
        }\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_bcast );\r
 \r
-       status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );\r
+       status = p_port->p_adapter->p_ifc->join_mcast(\r
+               p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               ipoib_port_deref( p_port, ref_bcast_join_failed );\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               return;\r
        }\r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
+       return status;\r
 }\r
 \r
 \r
-static void\r
+static ib_api_status_t\r
 __port_create_bcast(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        /* Join the broadcast group. */\r
        cl_memclr( &mcast_req, sizeof(mcast_req) );\r
@@ -4711,27 +5175,21 @@ __port_create_bcast(
         * We specify the MGID since we don't want the SA to generate it for us.\r
         */\r
        mcast_req.member_rec.mgid = bcast_mgid_template;\r
+       mcast_req.member_rec.mgid.raw[4] = (uint8_t) (p_port->p_adapter->guids.port_guid.pkey >> 8); \r
+       mcast_req.member_rec.mgid.raw[5] = (uint8_t) p_port->p_adapter->guids.port_guid.pkey;\r
        ib_gid_set_default( &mcast_req.member_rec.port_gid,\r
-               p_port->p_adapter->guids.port_guid );\r
+               p_port->p_adapter->guids.port_guid.guid );\r
        /*\r
         * IPOIB spec requires that the QKEY have the MSb set so that the QKEY\r
         * from the QP is used rather than the QKEY in the send WR.\r
         */\r
-#ifdef PR_102492\r
-       mcast_req.member_rec.qkey = CL_HTON32(0x80000B1B);\r
-#else\r
        mcast_req.member_rec.qkey =\r
                (uint32_t)(uintn_t)p_port | IB_QP_PRIVILEGED_Q_KEY;\r
-#endif\r
        mcast_req.member_rec.mtu =\r
-               (IB_PATH_SELECTOR_EXACTLY << 6) | IB_MTU_2048;\r
+               (IB_PATH_SELECTOR_EXACTLY << 6) | IB_MTU_LEN_2048;\r
 \r
-       mcast_req.member_rec.pkey = IB_DEFAULT_PKEY;\r
+       mcast_req.member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
 \r
-#ifdef PR_102492\r
-       mcast_req.member_rec.rate =\r
-               (IB_PATH_SELECTOR_LARGEST << 6);// | IB_PATH_RECORD_RATE_10_GBS;\r
-#endif\r
        mcast_req.member_rec.sl_flow_hop = ib_member_set_sl_flow_hop( 0, 0, 0 );\r
        mcast_req.member_rec.scope_state =\r
                ib_member_set_scope_state( 2, IB_MC_REC_STATE_FULL_MEMBER );\r
@@ -4740,21 +5198,22 @@ __port_create_bcast(
        mcast_req.pfn_mcast_cb = __bcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_bcast );\r
 \r
        status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               ipoib_port_deref( p_port, ref_bcast_create_failed );\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
        }\r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
+       return status;\r
 }\r
 \r
 \r
@@ -4767,7 +5226,13 @@ ipoib_port_down(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       /* Mark our state.  This causes all callbacks to abort. */\r
+       /*\r
+        * Mark our state.  This causes all callbacks to abort.\r
+        * Note that we hold the receive lock so that we synchronize\r
+        * with reposting.  We must take the receive lock before the\r
+        * object lock since that is the order taken when reposting.\r
+        */\r
+       cl_spinlock_acquire( &p_port->recv_lock );\r
        cl_obj_lock( &p_port->obj );\r
        p_port->state = IB_QPS_ERROR;\r
 \r
@@ -4776,10 +5241,15 @@ ipoib_port_down(
 \r
        if( p_port->ib_mgr.h_query )\r
        {\r
-               p_port->p_adapter->p_ifc->cancel_query( p_port->p_adapter->h_al, p_port->ib_mgr.h_query );\r
+               p_port->p_adapter->p_ifc->cancel_query(\r
+                       p_port->p_adapter->h_al, p_port->ib_mgr.h_query );\r
                p_port->ib_mgr.h_query = NULL;\r
        }\r
        cl_obj_unlock( &p_port->obj );\r
+       cl_spinlock_release( &p_port->recv_lock );\r
+\r
+       KeWaitForSingleObject(\r
+               &p_port->sa_event, Executive, KernelMode, FALSE, NULL );\r
 \r
        /*\r
         * Put the QP in the error state.  This removes the need to\r
@@ -4791,15 +5261,26 @@ ipoib_port_down(
        status = p_port->p_adapter->p_ifc->modify_qp( p_port->ib_mgr.h_qp, &qp_mod );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_modify_qp to error state returned %s.\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                p_port->p_adapter->hung = TRUE;\r
                return;\r
        }\r
 \r
-       /* Initiate cleanup of all endpoints */\r
-       __endpt_mgr_remove_all( p_port );\r
+       KeResetEvent(&p_port->leave_mcast_event);\r
+\r
+       /* Reset all endpoints so we don't flush our ARP cache. */\r
+       __endpt_mgr_reset_all( p_port );\r
+\r
+       KeWaitForSingleObject(\r
+               &p_port->leave_mcast_event, Executive, KernelMode, FALSE, NULL );\r
+\r
+       __pending_list_destroy(p_port);\r
+       \r
+       cl_obj_lock( &p_port->p_adapter->obj );\r
+       ipoib_dereg_addrs( p_port->p_adapter );\r
+       cl_obj_unlock( &p_port->p_adapter->obj );\r
        \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
@@ -4814,25 +5295,33 @@ __bcast_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_mcast_rec->mcast_context;\r
+       p_port = (ipoib_port_t*)p_mcast_rec->mcast_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
-       if( p_port->state == IB_QPS_ERROR )\r
+       if( p_port->state != IB_QPS_INIT )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_INIT,\r
-                       ("Instance destroying - Aborting.\n") );\r
+               if( p_mcast_rec->status == IB_SUCCESS )\r
+\r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
+               KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
+               ipoib_port_deref( p_port, ref_bcast_inv_state );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Invalid state - Aborting.\n") );\r
                return;\r
        }\r
-       cl_obj_unlock( &p_port->obj );\r
 \r
-       if( p_mcast_rec->status != IB_SUCCESS )\r
+       status = p_mcast_rec->status;\r
+\r
+       if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Multicast join for broadcast group returned %s.\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( p_mcast_rec->status )) );\r
-               if( p_mcast_rec->status == IB_REMOTE_ERROR )\r
+               if( status == IB_REMOTE_ERROR )\r
                {\r
                        /*\r
                         * Either:\r
@@ -4846,57 +5335,76 @@ __bcast_cb(
                         * over with the Get.\r
                         */\r
                        /* TODO: Assert is a place holder.  Can we ever get here if the\r
-                       state isn't IB_PNP_PORT_ADD or PORT_DOWN? */\r
+                       state isn't IB_PNP_PORT_ADD or PORT_DOWN or PORT_INIT? */\r
                        CL_ASSERT( p_port->p_adapter->state == IB_PNP_PORT_ADD ||\r
-                               p_port->p_adapter->state == IB_PNP_PORT_DOWN );\r
-                       __port_get_mcast( p_port );\r
+                               p_port->p_adapter->state == IB_PNP_PORT_DOWN ||\r
+                               p_port->p_adapter->state == IB_PNP_PORT_INIT );\r
+                       status = __port_get_bcast( p_port );\r
                }\r
                else\r
                {\r
                        NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                                EVENT_IPOIB_BCAST_JOIN, 1, p_mcast_rec->status );\r
-                       /* Flag the adapter as hung. */\r
-                       p_port->p_adapter->hung = TRUE;\r
                }\r
 \r
-               cl_obj_deref( &p_port->obj );\r
+               cl_obj_unlock( &p_port->obj );\r
+               if( status != IB_SUCCESS )\r
+               {\r
+                       ipoib_set_inactive( p_port->p_adapter );\r
+                       __endpt_mgr_reset_all( p_port );\r
+                       KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
+               }\r
+               ipoib_port_deref( p_port, ref_bcast_req_failed );\r
                IPOIB_EXIT( IPOIB_DBG_INIT );\r
                return;\r
        }\r
+       cl_obj_unlock( &p_port->obj );\r
 \r
        status = __endpt_mgr_add_bcast( p_port, p_mcast_rec );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__endpt_mgr_add_bcast returned %s\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
                CL_ASSERT( status == IB_SUCCESS );\r
-               /* Flag the adapter as hung. */\r
-               p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_EXIT( IPOIB_DBG_INIT );\r
-               return;\r
+               goto err;\r
        }\r
 \r
        /* Get the QP ready for action. */\r
        status = __ib_mgr_activate( p_port );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__ib_mgr_activate returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+\r
+err:\r
                /* Flag the adapter as hung. */\r
                p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
+               KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
+               ipoib_port_deref( p_port, ref_bcast_error );\r
                IPOIB_EXIT( IPOIB_DBG_INIT );\r
                return;\r
        }\r
 \r
+       cl_obj_lock( &p_port->obj );\r
+       /* Only change the state if we're still in INIT. */\r
+       if( p_port->state == IB_QPS_INIT )\r
+               p_port->state = IB_QPS_RTS;\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       /* Prepost receives. */\r
+       cl_spinlock_acquire( &p_port->recv_lock );\r
+       __recv_mgr_repost( p_port );\r
+       cl_spinlock_release( &p_port->recv_lock );\r
+\r
        /* Notify the adapter that we now have an active connection. */\r
        ipoib_set_active( p_port->p_adapter );\r
 \r
-       cl_obj_deref( &p_port->obj );\r
+       KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
+       ipoib_port_deref( p_port, ref_join_bcast );\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -4907,9 +5415,7 @@ __qp_event(
 {\r
        UNUSED_PARAM( p_event_rec );\r
        CL_ASSERT( p_event_rec->context );\r
-       /* Place holder for proper error handler. */\r
-       cl_msg_out( "Async QP event: %d\n", p_event_rec->code );\r
-       CL_ASSERT( p_event_rec->code == IB_AE_UNKNOWN );\r
+       ((ipoib_port_t*)p_event_rec->context)->p_adapter->hung = TRUE;\r
 }\r
 \r
 \r
@@ -4919,8 +5425,7 @@ __cq_event(
 {\r
        UNUSED_PARAM( p_event_rec );\r
        CL_ASSERT( p_event_rec->context );\r
-       /* Place holder for proper error handler. */\r
-       CL_ASSERT( p_event_rec->code == IB_AE_UNKNOWN );\r
+       ((ipoib_port_t*)p_event_rec->context)->p_adapter->hung = TRUE;\r
 }\r
 \r
 \r
@@ -4942,20 +5447,20 @@ __ib_mgr_activate(
        status = p_port->p_adapter->p_ifc->modify_qp( p_port->ib_mgr.h_qp, &qp_mod );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_modify_qp returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
 \r
        /* Move the QP to RTS. */\r
-       dgrm_info.port_guid = p_port->p_adapter->guids.port_guid;\r
+       dgrm_info.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        dgrm_info.qkey = p_port->ib_mgr.bcast_rec.qkey;\r
-       dgrm_info.pkey_index = 0;\r
+       dgrm_info.pkey_index = p_port->pkey_index;\r
        status = p_port->p_adapter->p_ifc->init_dgrm_svc( p_port->ib_mgr.h_qp, &dgrm_info );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_init_dgrm_svc returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -4965,7 +5470,7 @@ __ib_mgr_activate(
        status = p_port->p_adapter->p_ifc->rearm_cq( p_port->ib_mgr.h_recv_cq, FALSE );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_rearm_cq for recv returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
@@ -4973,17 +5478,12 @@ __ib_mgr_activate(
        status = p_port->p_adapter->p_ifc->rearm_cq( p_port->ib_mgr.h_send_cq, FALSE );\r
        if( status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_rearm_cq for send returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
 \r
-       /* Prepost receives. */\r
-       cl_spinlock_acquire( &p_port->recv_lock );\r
-       __recv_mgr_repost( p_port, p_port->p_adapter->params.rq_depth );\r
-       cl_spinlock_release( &p_port->recv_lock );\r
-\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return IB_SUCCESS;\r
 }\r
@@ -4993,7 +5493,8 @@ __ib_mgr_activate(
 ib_api_status_t\r
 ipoib_port_join_mcast(\r
        IN                              ipoib_port_t* const                     p_port,\r
-       IN              const   mac_addr_t                                      mac )\r
+       IN              const   mac_addr_t                              mac,\r
+       IN              const   uint8_t                                 state)\r
 {\r
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
@@ -5026,6 +5527,7 @@ ipoib_port_join_mcast(
        mcast_req.member_rec = p_port->ib_mgr.bcast_rec;\r
        /* Clear fields that aren't specified in the join */\r
        mcast_req.member_rec.mlid = 0;\r
+       ib_member_set_state( &mcast_req.member_rec.scope_state,state);\r
 \r
        if( mac.addr[0] == 1 && mac.addr[1] == 0 && mac.addr[2] == 0x5E )\r
        {\r
@@ -5043,9 +5545,9 @@ ipoib_port_join_mcast(
        else\r
        {\r
                /* Handle non IP mutlicast MAC addresses. */\r
-               /* Update the signature. */\r
-               mcast_req.member_rec.mgid.raw[2] = 0x06;\r
-               mcast_req.member_rec.mgid.raw[3] = 0x6A;\r
+               /* Update the signature to use the lower 2 bytes of the OpenIB OUI. */\r
+               mcast_req.member_rec.mgid.raw[2] = 0x14;\r
+               mcast_req.member_rec.mgid.raw[3] = 0x05;\r
                /* Now copy the MAC address into the last 6 bytes of the GID. */\r
                cl_memcpy( &mcast_req.member_rec.mgid.raw[10], mac.addr, 6 );\r
        }\r
@@ -5054,9 +5556,9 @@ ipoib_port_join_mcast(
        mcast_req.pfn_mcast_cb = __mcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
-\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
+       mcast_req.member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
        /*\r
         * Create the endpoint and insert it in the port.  Since we don't wait for\r
         * the mcast SA operations to complete before returning from the multicast\r
@@ -5069,20 +5571,28 @@ ipoib_port_join_mcast(
                &mcast_req.member_rec.mgid, 0, CL_HTON32(0x00FFFFFF) );\r
        if( !p_endpt )\r
        {\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, ("ipoib_endpt_create failed.\n") );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ipoib_endpt_create failed.\n") );\r
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
 \r
-       __endpt_mgr_insert_locked( p_port, mac, p_endpt );\r
+       status = __endpt_mgr_insert_locked( p_port, mac, p_endpt );\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("__endpt_mgr_insert_locked returned %s\n", \r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_mcast );\r
 \r
        status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               ipoib_port_deref( p_port, ref_mcast_join_failed );\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
        }\r
@@ -5104,27 +5614,33 @@ __mcast_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_MCAST );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_mcast_rec->mcast_context;\r
+       p_port = (ipoib_port_t*)p_mcast_rec->mcast_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
-       if( p_port->state == IB_QPS_ERROR )\r
+       if( p_port->state != IB_QPS_RTS )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               cl_obj_deref( &p_port->obj );\r
-               IPOIB_TRACE_EXIT( IPOIB_DBG_INIT,\r
-                       ("Instance destroying - Aborting.\n") );\r
+               if( p_mcast_rec->status == IB_SUCCESS )\r
+\r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
+               ipoib_port_deref( p_port, ref_mcast_inv_state );\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Invalid state - Aborting.\n") );\r
                return;\r
        }\r
        cl_obj_unlock( &p_port->obj );\r
 \r
        if( p_mcast_rec->status != IB_SUCCESS )\r
        {\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Multicast join request failed with status %s.\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( p_mcast_rec->status )) );\r
                /* Flag the adapter as hung. */\r
-               p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
+               p_port->p_adapter->hung =TRUE;\r
+               ipoib_port_deref( p_port, ref_mcast_req_failed );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5140,10 +5656,12 @@ __mcast_cb(
                 * is not an error.\r
                 */\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE( IPOIB_DBG_MCAST,\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
                        ("Failed to find endpoint for update.\n") );\r
-               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
-               cl_obj_deref( &p_port->obj );\r
+\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               ipoib_port_deref( p_port, ref_mcast_no_endpt );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5157,12 +5675,12 @@ __mcast_cb(
        if( status != IB_SUCCESS )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
-               IPOIB_TRACE( IPOIB_DBG_ERROR,\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_MCAST,\r
                        ("ipoib_endpt_set_mcast returned %s.\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                /* Flag the adapter as hung. */\r
                p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_mcast_av_failed );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5182,7 +5700,57 @@ __mcast_cb(
        /* Try to send all pending sends. */\r
        ipoib_port_resume( p_port );\r
 \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_join_mcast );\r
+\r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r
+void\r
+ipoib_leave_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t*)context;\r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_VERBOSE, IPOIB_DBG_MCAST,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt));\r
+       \r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       cl_atomic_dec( &p_port->mcast_cnt);\r
+       \r
+       if(0 == p_port->mcast_cnt)\r
+       {\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }\r
+       \r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leave mcast callback deref ipoib_port \n") );\r
+       \r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r
+\r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
 \r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t*)context;\r
+\r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leave mcast callback deref ipoib_port \n") );\r
+       \r
        IPOIB_EXIT( IPOIB_DBG_MCAST );\r
 }\r
+\r
+\r
+\r
+\r