[ipoib] Always connect to mcast groups as a full member. (mlnx: 2845)
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
index 37b62df..1e0f946 100644 (file)
@@ -1,5 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -389,13 +391,13 @@ __endpt_mgr_get_by_lid(
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   net16_t                                         lid );\r
 \r
-static inline void\r
+static inline ib_api_status_t\r
 __endpt_mgr_insert_locked(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt );\r
 \r
-static inline void\r
+static inline ib_api_status_t\r
 __endpt_mgr_insert(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
@@ -448,6 +450,10 @@ static void
 __mcast_cb(\r
        IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
 \r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context );\r
+\r
 \r
 static intn_t\r
 __gid_cmp(\r
@@ -458,6 +464,33 @@ __gid_cmp(
 }\r
 \r
 \r
+inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
+{\r
+       cl_obj_ref( &p_port->obj );\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
+#else\r
+       UNREFERENCED_PARAMETER(type);\r
+#endif\r
+}\r
+\r
+\r
+inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
+{\r
+       cl_obj_deref( &p_port->obj );\r
+\r
+#if DBG\r
+       cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
+#else\r
+       UNREFERENCED_PARAMETER(type);\r
+#endif\r
+}\r
+\r
+\r
 /******************************************************************************\r
 *\r
 * Implementation\r
@@ -545,7 +578,8 @@ __port_construct(
        __endpt_mgr_construct( p_port );\r
 \r
        KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
-\r
+       KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
+       \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -568,7 +602,7 @@ __port_init(
        if( cl_status != CL_SUCCESS )\r
        {\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
+                       ("cl_spinlock_init returned %#x\n", cl_status) );\r
                return IB_ERROR;\r
        }\r
 \r
@@ -576,7 +610,7 @@ __port_init(
        if( cl_status != CL_SUCCESS )\r
        {\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
+                       ("cl_spinlock_init returned %#x\n", cl_status) );\r
                return IB_ERROR;\r
        }\r
 \r
@@ -623,14 +657,34 @@ __port_init(
        /* We only ever destroy from the PnP callback thread. */\r
        cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
                __port_destroying, __port_cleanup, __port_free );\r
+\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_init] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
+       if( cl_status != CL_SUCCESS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_obj_init returned %#x\n", cl_status) );\r
+               return IB_ERROR;\r
+       }\r
+\r
+       cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
+                       ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
+               cl_obj_destroy( &p_port->obj );\r
                return IB_ERROR;\r
        }\r
 \r
-       cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_init] );\r
+       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
+               ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
+#endif\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return IB_SUCCESS;\r
@@ -653,6 +707,8 @@ __port_destroying(
 \r
        __endpt_mgr_remove_all( p_port );\r
 \r
+       ipoib_port_resume( p_port );\r
+\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -738,6 +794,8 @@ __ib_mgr_init(
        uint64_t                        vaddr;\r
        net32_t                         rkey;\r
        ib_qp_attr_t            qp_attr;\r
+       ib_ca_attr_t            *ca_attr;\r
+       uint32_t                        ca_size;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
@@ -755,6 +813,52 @@ __ib_mgr_init(
                return status;\r
        }\r
 \r
+       /* Query the CA for Pkey table */\r
+       status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, NULL, &ca_size);\r
+       if(status != IB_INSUFFICIENT_MEMORY)\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ib_query_ca failed\n"));\r
+                       return status;\r
+       }\r
+\r
+       ca_attr = (ib_ca_attr_t*)cl_zalloc(ca_size);\r
+       if      (!ca_attr)\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("cl_zalloc can't allocate %d\n",ca_size));\r
+               return IB_INSUFFICIENT_RESOURCES;\r
+       }\r
+\r
+       status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, ca_attr,&ca_size);     \r
+       if( status != IB_SUCCESS )\r
+       {\r
+               cl_free(ca_attr);\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("ib_query_ca returned %s\n", \r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
+       if( ca_attr->p_port_attr->link_state == IB_LINK_ACTIVE)\r
+       {\r
+               uint16_t index;\r
+               CL_ASSERT(ca_attr->p_port_attr->p_pkey_table[0] == IB_DEFAULT_PKEY);\r
+               for(index = 0; index < ca_attr->p_port_attr->num_pkeys; index++)\r
+               {\r
+                       if(p_port->p_adapter->guids.port_guid.pkey == ca_attr->p_port_attr->p_pkey_table[index])\r
+                               break;\r
+               }\r
+               if(index >= ca_attr->p_port_attr->num_pkeys)\r
+               {\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("Pkey table is invalid, index not found\n"));\r
+                   return IB_NOT_FOUND;\r
+               }\r
+               p_port->pkey_index = index;\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_IB,\r
+                       ("for PKEY = 0x%04X got index = %d\n",p_port->p_adapter->guids.port_guid.pkey,index));\r
+       }\r
+       cl_free(ca_attr);\r
        /* Allocate the PD. */\r
        status = p_port->p_adapter->p_ifc->alloc_pd(\r
                p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
@@ -906,7 +1010,7 @@ __buf_mgr_construct(
        p_port->buf_mgr.h_buffer_pool = NULL;\r
 \r
        ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
-               NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
+               NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
 \r
        p_port->buf_mgr.h_send_pkt_pool = NULL;\r
        p_port->buf_mgr.h_send_buf_pool = NULL;\r
@@ -943,8 +1047,8 @@ __buf_mgr_init(
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("cl_qpool_init for recvs returned %s\n",\r
-                       cl_status_text[cl_status]) );\r
+                       ("cl_qpool_init for recvs returned %#x\n",\r
+                       cl_status) );\r
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
 \r
@@ -1137,7 +1241,7 @@ __buf_mgr_get_recv(
        /* Reference the port object for the send. */\r
        if( p_desc )\r
        {\r
-               cl_obj_ref( &p_port->obj );\r
+               ipoib_port_ref( p_port, ref_get_recv );\r
                CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
 #if IPOIB_INLINE_RECV\r
                CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
@@ -1181,8 +1285,7 @@ __buf_mgr_put_recv(
        /*\r
         * Dereference the port object since the receive is no longer outstanding.\r
         */\r
-       cl_obj_deref( &p_port->obj );\r
-\r
+       ipoib_port_deref( p_port, ref_get_recv );\r
        IPOIB_EXIT(  IPOIB_DBG_RECV );\r
 }\r
 \r
@@ -1192,9 +1295,9 @@ __buf_mgr_put_recv_list(
        IN                              ipoib_port_t* const                     p_port,\r
        IN                              cl_qlist_t* const                       p_list )\r
 {\r
-       IPOIB_ENTER(  IPOIB_DBG_RECV );\r
+       //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
        cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
-       IPOIB_EXIT(  IPOIB_DBG_RECV );\r
+       //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
 }\r
 \r
 \r
@@ -1327,7 +1430,7 @@ __recv_mgr_repost(
                        ("Port in invalid state.  Not reposting.\n") );\r
                return 0;\r
        }\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_repost );\r
        cl_obj_unlock( &p_port->obj );\r
 \r
        while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
@@ -1339,7 +1442,7 @@ __recv_mgr_repost(
                if( !p_next )\r
                {\r
                        IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
-                               ("Out of receive descriptors!\n") );\r
+                               ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
                        break;\r
                }\r
 \r
@@ -1382,8 +1485,7 @@ __recv_mgr_repost(
                }\r
        }\r
 \r
-       cl_obj_deref( &p_port->obj );\r
-\r
+       ipoib_port_deref( p_port, ref_repost );\r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
        return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
 }\r
@@ -1513,8 +1615,7 @@ __recv_cb(
        cl_qlist_init( &done_list );\r
        cl_qlist_init( &bad_list );\r
 \r
-       cl_obj_ref( &p_port->obj );\r
-\r
+       ipoib_port_ref( p_port, ref_recv_cb );\r
        for( i = 0; i < MAX_RECV_WC; i++ )\r
                wc[i].p_next = &wc[i + 1];\r
        wc[MAX_RECV_WC - 1].p_next = NULL;\r
@@ -1633,7 +1734,7 @@ __recv_cb(
        cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
        CL_ASSERT( status == IB_SUCCESS );\r
 \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_recv_cb );\r
 \r
        cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
 \r
@@ -1711,17 +1812,24 @@ __recv_get_endpts(
 #else  /* IPOIB_INLINE_RECV */\r
                        *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
 #endif /* IPOIB_INLINE_RECV */\r
-                               p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
+                               0, p_wc->recv.ud.remote_qp );\r
                        if( !*pp_src )\r
                        {\r
                                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                                       ("ipoib_endpt_create returned %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                                       ("ipoib_endpt_create failed\n") );\r
                                return;\r
                        }\r
                        cl_perf_start( EndptInsert );\r
                        cl_obj_lock( &p_port->obj );\r
-                       __endpt_mgr_insert( p_port, mac, *pp_src );\r
+                       status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
+                       if( status != IB_SUCCESS )\r
+                       {\r
+                               cl_obj_unlock( &p_port->obj );\r
+                               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                                       ("__endpt_mgr_insert returned %s\n",\r
+                                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                               return;\r
+                       }\r
                        cl_obj_unlock( &p_port->obj );\r
                        cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
                }\r
@@ -1739,7 +1847,8 @@ __recv_get_endpts(
                CL_ASSERT( *pp_dst );\r
        }\r
 \r
-       if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
+       if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
+               (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
        {\r
                /* Update the QPN for the endpoint. */\r
                IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
@@ -1794,7 +1903,7 @@ __recv_mgr_filter(
 \r
        for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
        {\r
-               CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
+               CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
                p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
                recv_cnt++;\r
 \r
@@ -1803,8 +1912,9 @@ __recv_mgr_filter(
                        if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
                        {\r
                                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                                       ("Failed completion %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
+                                       ("Failed completion %s  (vendor specific %#x)\n",\r
+                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
+                                       (int)p_wc->vendor_specific) );\r
                                ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        }\r
                        else\r
@@ -1816,7 +1926,7 @@ __recv_mgr_filter(
                        }\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
                        /* Dereference the port object on behalf of the failed receive. */\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_failed_recv_wc );\r
                        continue;\r
                }\r
 \r
@@ -1828,11 +1938,23 @@ __recv_mgr_filter(
                                ("Received ETH packet < min size\n") );\r
                        ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_recv_inv_len );\r
                        continue;\r
                }\r
 \r
+               if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
+               {\r
+                       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("Received ETH packet > payload MTU (%d)\n",\r
+                               p_port->p_adapter->params.payload_mtu) );\r
+                       ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
+                       cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
+                       ipoib_port_deref( p_port, ref_recv_inv_len );\r
+                       continue;\r
+                       \r
+               }\r
                /* Successful completion.  Get the receive information. */\r
+               p_desc->ndis_csum.Value = (ULONG) p_wc->csum_ok;\r
                cl_perf_start( GetRecvEndpts );\r
                __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
                cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
@@ -1857,7 +1979,7 @@ __recv_mgr_filter(
                                 */\r
                                cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
                                /* Dereference the port object on behalf of the failed recv. */\r
-                               cl_obj_deref( &p_port->obj );\r
+                               ipoib_port_deref( p_port, ref_recv_loopback );\r
                                continue;\r
                        }\r
                }\r
@@ -1907,6 +2029,11 @@ __recv_mgr_filter(
                                        status = IB_INVALID_SETTING;\r
                                        break;\r
                                }\r
+                               if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
+                                       // If there are IP options in this message, we are in trouble in any case\r
+                                       status = IB_INVALID_SETTING;\r
+                                       break;                                  \r
+                               }\r
                                /* UDP packet with BOOTP ports in src/dst port numbers. */\r
                                cl_perf_start( RecvDhcp );\r
                                status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
@@ -1948,7 +2075,7 @@ __recv_mgr_filter(
                        ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                        cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
                        /* Dereference the port object on behalf of the failed receive. */\r
-                       cl_obj_deref( &p_port->obj );\r
+                       ipoib_port_deref( p_port, ref_recv_filter );\r
                }\r
                else\r
                {\r
@@ -2062,7 +2189,7 @@ __recv_dhcp(
        }\r
 \r
        p_option = &p_dhcp->options[4];\r
-       while( *p_option != DHCP_OPT_END )\r
+       while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
        {\r
                switch( *p_option )\r
                {\r
@@ -2225,8 +2352,26 @@ __recv_arp(
                         */\r
                        *pp_src = NULL;\r
                }\r
-               else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
-                       (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
+               else if( (*pp_src)->dlid &&\r
+                       (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
+               {\r
+                       /* Out of date!  Destroy the endpoint and replace it. */\r
+                       __endpt_mgr_remove( p_port, *pp_src );\r
+                       *pp_src = NULL;\r
+               }\r
+               else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
+               {\r
+                       if( (*pp_src)->qpn !=\r
+                               (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
+                               p_wc->recv.ud.remote_qp !=\r
+                               (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
+                       {\r
+                               /* Out of date!  Destroy the endpoint and replace it. */\r
+                               __endpt_mgr_remove( p_port, *pp_src );\r
+                               *pp_src = NULL;\r
+                       }\r
+               }\r
+               else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
                {\r
                        /* Out of date!  Destroy the endpoint and replace it. */\r
                        __endpt_mgr_remove( p_port, *pp_src );\r
@@ -2255,22 +2400,32 @@ __recv_arp(
                 */\r
                *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
                        0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
+\r
                if( !*pp_src )\r
                {\r
                        IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                               ("ipoib_endpt_create returned %s\n",\r
-                               p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                               ("ipoib_endpt_create failed\n") );\r
                        return status;\r
                }\r
 \r
                cl_obj_lock( &p_port->obj );\r
-               __endpt_mgr_insert( p_port, mac, *pp_src );\r
+               status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
+               if( status != IB_SUCCESS )\r
+               {\r
+                       cl_obj_unlock( &p_port->obj );\r
+                       IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                               ("__endpt_mgr_insert return %s \n",\r
+                               p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+                       return status;\r
+               }\r
+\r
                cl_obj_unlock( &p_port->obj );\r
        }\r
 \r
        CL_ASSERT( !cl_memcmp(\r
                &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
-       CL_ASSERT( (*pp_src)->qpn ==\r
+       CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
+               (*pp_src)->qpn ==\r
                (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
        /* Now swizzle the data. */\r
        p_arp->hw_type = ARP_HW_TYPE_ETH;\r
@@ -2280,15 +2435,18 @@ __recv_arp(
 \r
        if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
        {\r
-\r
-               if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
+               if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
                {\r
                        /*\r
                         * We received bcast ARP packet that means\r
                         * remote port lets everyone know it was changed IP/MAC\r
                         * or just activated\r
                         */\r
-                       if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
+\r
+                       /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
+\r
+                       if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
+                                !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
                        {\r
                                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                        ("ARP: is not ARP MCAST\n") );\r
@@ -2302,7 +2460,6 @@ __recv_arp(
                         * as long as OS' ARP table is global  ???\r
                         */\r
                        p_arp->dst_ip = (net32_t)0;\r
-\r
                }\r
                else /* we've got reply to our ARP request */\r
                {\r
@@ -2345,7 +2502,6 @@ __recv_mgr_prepare_pkt(
        NDIS_STATUS                                                     status;\r
        uint32_t                                                        pkt_filter;\r
        ip_stat_sel_t                                           type;\r
-       NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
        PERF_DECLARE( GetNdisPkt );\r
 \r
        IPOIB_ENTER( IPOIB_DBG_RECV );\r
@@ -2426,14 +2582,9 @@ __recv_mgr_prepare_pkt(
                return IB_INSUFFICIENT_RESOURCES;\r
        }\r
 \r
-       /* Flag the checksums as having been calculated. */\r
-       chksum.Value = 0;\r
-       chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
-       chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
-       chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
-       NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
-               (void*)(uintn_t)chksum.Value;\r
-\r
+       /* Get the checksums directly from packet information. */\r
+       NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
+               (PVOID) (uintn_t) (p_desc->ndis_csum.Value);\r
        ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_RECV );\r
@@ -2525,15 +2676,14 @@ __send_mgr_construct(
 }\r
 \r
 \r
-static void\r
-__send_mgr_destroy(\r
+static void \r
+__pending_list_destroy(\r
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        cl_list_item_t  *p_item;\r
        NDIS_PACKET             *p_packet;\r
-\r
-       IPOIB_ENTER( IPOIB_DBG_SEND );\r
-\r
+       \r
+       cl_spinlock_acquire( &p_port->send_lock );\r
        /* Complete any pending packets. */\r
        for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
                p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
@@ -2543,6 +2693,15 @@ __send_mgr_destroy(
                NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
                        NDIS_STATUS_RESET_IN_PROGRESS );\r
        }\r
+       cl_spinlock_release( &p_port->send_lock );\r
+}\r
+\r
+static void\r
+__send_mgr_destroy(\r
+       IN                              ipoib_port_t* const                     p_port )\r
+{\r
+       IPOIB_ENTER( IPOIB_DBG_SEND );\r
+       __pending_list_destroy(p_port);\r
 \r
        IPOIB_EXIT( IPOIB_DBG_SEND );\r
 }\r
@@ -2632,7 +2791,7 @@ __send_copy(
        }\r
 \r
        NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
-               p_desc->p_buf, XFER_BLOCK_SIZE );\r
+               p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
        if( status != NDIS_STATUS_SUCCESS )\r
        {\r
                NdisFreePacket( p_packet );\r
@@ -2715,7 +2874,7 @@ __send_gen(
        }\r
 \r
        CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
-       CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
+       CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
        /*\r
         * Assume that the ethernet header is always fully contained\r
         * in the first page of the first MDL.  This makes for much\r
@@ -2735,7 +2894,7 @@ __send_gen(
                        CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
                        /* Skip the ethernet header. */\r
                        buf_len -= sizeof(eth_hdr_t);\r
-                       CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
+                       CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
                        if( buf_len )\r
                        {\r
                                /* The ethernet header is a subset of this MDL. */\r
@@ -3053,6 +3212,19 @@ __send_mgr_filter_udp(
 }\r
 \r
 \r
+unsigned short ipchksum(unsigned short *ip, int len)\r
+{\r
+    unsigned long sum = 0;\r
+\r
+    len >>= 1;\r
+    while (len--) {\r
+        sum += *(ip++);\r
+        if (sum > 0xFFFF)\r
+            sum -= 0xFFFF;\r
+    }\r
+    return (unsigned short)((~sum) & 0x0000FFFF);\r
+}\r
+\r
 static NDIS_STATUS\r
 __send_mgr_filter_dhcp(\r
        IN                              ipoib_port_t* const                     p_port,\r
@@ -3104,7 +3276,7 @@ __send_mgr_filter_dhcp(
 \r
        /* Now scan through the options looking for the client identifier. */\r
        p_option = &p_ib_dhcp->options[4];\r
-       while( *p_option != DHCP_OPT_END )\r
+       while( *p_option != DHCP_OPT_END && p_option < &p_ib_dhcp->options[312] )\r
        {\r
                switch( *p_option )\r
                {\r
@@ -3154,7 +3326,7 @@ __send_mgr_filter_dhcp(
                                {\r
                                        IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                                ("Can't convert CID to IPoIB format.\n") );\r
-                                       return IB_INSUFFICIENT_MEMORY;\r
+                                       return NDIS_STATUS_RESOURCES;\r
                                }\r
                                /* Move the existing options down, and add a new CID option */\r
                                len = p_option - ( p_cid + p_cid[1] + 2 );\r
@@ -3189,17 +3361,26 @@ __send_mgr_filter_dhcp(
                        p_option[0] = DHCP_OPT_END;\r
                        p_cid[0] = DHCP_OPT_CLIENT_ID;\r
                        p_cid[1] = 21;\r
+                       p_cid[2] = DHCP_HW_TYPE_IB;\r
                }\r
 \r
                CL_ASSERT( p_cid[1] == 21 );\r
                p_cid[23]= DHCP_OPT_END;\r
-               ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid );\r
+               ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid.guid );\r
                cl_memcpy( &p_cid[7], &gid, sizeof(ib_gid_t) );\r
-               cl_memcpy( &p_cid[3], &p_port->ib_mgr.qpn, sizeof(p_port->ib_mgr.qpn) );\r
-               /* Clear the hardware address. */\r
+               cl_memcpy( &p_cid[3], &p_port->ib_mgr.qpn, sizeof(p_port->ib_mgr.qpn) );                \r
                p_ib_dhcp->htype = DHCP_HW_TYPE_IB;\r
-               p_ib_dhcp->hlen = 0;\r
-               cl_memclr( p_ib_dhcp->chaddr, sizeof(p_ib_dhcp->chaddr) );\r
+\r
+               /* update lengths to include any change we made */\r
+               p_desc->p_buf->ip.hdr.length = cl_ntoh16( sizeof(ip_hdr_t) + sizeof(udp_hdr_t) + sizeof(dhcp_pkt_t) );\r
+               p_desc->p_buf->ip.prot.udp.hdr.length = cl_ntoh16( sizeof(udp_hdr_t) + sizeof(dhcp_pkt_t) );\r
+\r
+               /* update crc in ip header */\r
+               if( !p_port->p_adapter->params.send_chksum_offload )\r
+               {\r
+                       p_desc->p_buf->ip.hdr.chksum = 0;\r
+                       p_desc->p_buf->ip.hdr.chksum = ipchksum((unsigned short*) &p_desc->p_buf->ip.hdr, sizeof(ip_hdr_t));\r
+               }\r
                break;\r
 \r
        /* Server messages. */\r
@@ -3296,7 +3477,7 @@ __send_mgr_filter_arp(
        p_ib_arp->op = p_arp->op;\r
        p_ib_arp->src_hw.flags_qpn = p_port->ib_mgr.qpn;\r
        ib_gid_set_default( &p_ib_arp->src_hw.gid,\r
-               p_port->p_adapter->guids.port_guid );\r
+               p_port->p_adapter->guids.port_guid.guid );\r
        p_ib_arp->src_ip = p_arp->src_ip;\r
        if( cl_memcmp( &p_arp->dst_hw, &null_hw, sizeof(mac_addr_t) ) )\r
        {\r
@@ -3408,7 +3589,8 @@ __send_mgr_queue(
        if( status == NDIS_STATUS_NO_ROUTE_TO_DESTINATION &&\r
                ETH_IS_MULTICAST( p_eth_hdr->dst.addr ) )\r
        {\r
-               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst ) == IB_SUCCESS )\r
+               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst, \r
+                       IB_MC_REC_STATE_FULL_MEMBER) == IB_SUCCESS )\r
                {\r
                        IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                                ("Multicast Mac - trying to join.\n") );\r
@@ -3431,6 +3613,8 @@ __build_send_desc(
 {\r
        NDIS_STATUS                     status;\r
        int32_t                         hdr_idx;\r
+       PNDIS_PACKET_EXTENSION                          PktExt;\r
+       PNDIS_TCP_IP_CHECKSUM_PACKET_INFO       pChecksumPktInfo; //NDIS 5.1\r
 \r
        PERF_DECLARE( SendMgrFilter );\r
 \r
@@ -3465,12 +3649,29 @@ __build_send_desc(
        p_desc->wr.wr_id = (uintn_t)p_desc->p_pkt;\r
        p_desc->wr.wr_type = WR_SEND;\r
        p_desc->wr.send_opt = IB_SEND_OPT_SIGNALED;\r
+       \r
+       PktExt = NDIS_PACKET_EXTENSION_FROM_PACKET(p_desc->p_pkt);\r
+       pChecksumPktInfo = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&PktExt->NdisPacketInfo[TcpIpChecksumPacketInfo];\r
+       if(p_port->p_adapter->params.send_chksum_offload & \r
+               (pChecksumPktInfo->Transmit.NdisPacketChecksumV4 || pChecksumPktInfo->Transmit.NdisPacketChecksumV6))\r
+       {\r
+               // Set transimition checksum offloading \r
+               if (pChecksumPktInfo->Transmit.NdisPacketIpChecksum) \r
+               {\r
+                       p_desc->wr.send_opt |= IB_SEND_OPT_TX_IP_CSUM;\r
+               }\r
+               if(pChecksumPktInfo->Transmit.NdisPacketTcpChecksum  ) \r
+               {\r
+                       p_desc->wr.send_opt |= IB_SEND_OPT_TX_TCP_UDP_CSUM;\r
+               }\r
+       }\r
+       \r
        p_desc->wr.ds_array = p_desc->local_ds;\r
 \r
        p_desc->wr.dgrm.ud.remote_qp = p_desc->p_endpt->qpn;\r
        p_desc->wr.dgrm.ud.remote_qkey = p_port->ib_mgr.bcast_rec.qkey;\r
        p_desc->wr.dgrm.ud.h_av = p_desc->p_endpt->h_av;\r
-       p_desc->wr.dgrm.ud.pkey_index = 0;\r
+       p_desc->wr.dgrm.ud.pkey_index = p_port->pkey_index;\r
        p_desc->wr.dgrm.ud.rsvd = NULL;\r
 \r
        /* Store context in our reserved area of the packet. */\r
@@ -3532,6 +3733,27 @@ ipoib_port_send(
 \r
        IPOIB_ENTER( IPOIB_DBG_SEND );\r
 \r
+\r
+       cl_obj_lock( &p_port->obj );\r
+       if( p_port->state != IB_QPS_RTS )\r
+       {\r
+               cl_obj_unlock( &p_port->obj );\r
+               for( i = 0; i < num_packets; ++i )\r
+               {\r
+                       ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
+                       /* Complete the packet. */\r
+                       NdisMSendComplete( p_port->p_adapter->h_adapter,\r
+                               p_packet_array[i], NDIS_STATUS_ADAPTER_NOT_READY );\r
+                       \r
+               }\r
+\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
+                       ("Invalid state - Aborting.\n") );\r
+               return;\r
+       }\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
+       \r
        cl_spinlock_acquire( &p_port->send_lock );\r
        for( i = 0; i < num_packets; i++ )\r
        {\r
@@ -3635,6 +3857,17 @@ ipoib_port_resume(
 \r
        IPOIB_ENTER( IPOIB_DBG_SEND );\r
 \r
+\r
+       cl_obj_lock( &p_port->obj );\r
+       if( p_port->state != IB_QPS_RTS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
+                       ("Invalid state - Aborting.\n") );\r
+               cl_obj_unlock( &p_port->obj );\r
+               return;\r
+       }\r
+       cl_obj_unlock( &p_port->obj );\r
+\r
        cl_spinlock_acquire( &p_port->send_lock );\r
 \r
        for( p_item = cl_qlist_head( &p_port->send_mgr.pending_list );\r
@@ -3644,7 +3877,7 @@ ipoib_port_resume(
                /* Check the send queue and pend the request if not empty. */\r
                if( p_port->send_mgr.depth == p_port->p_adapter->params.sq_depth )\r
                {\r
-                       IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
+                       IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
                                ("No available WQEs.\n") );\r
                        break;\r
                }\r
@@ -3678,12 +3911,10 @@ ipoib_port_resume(
                {\r
                        ASSERT( status == NDIS_STATUS_NO_ROUTE_TO_DESTINATION );\r
 \r
-                       if( p_eth_hdr->dst.addr[0] == 0x01 &&\r
-                               p_eth_hdr->dst.addr[1] == 0x00 &&\r
-                               p_eth_hdr->dst.addr[2] == 0x5E )\r
+                       if( ETH_IS_MULTICAST( p_eth_hdr->dst.addr ) )\r
                        {\r
-                               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst ) ==\r
-                                       IB_SUCCESS )\r
+                               if( ipoib_port_join_mcast( p_port, p_eth_hdr->dst,\r
+                                       IB_MC_REC_STATE_FULL_MEMBER) == IB_SUCCESS )\r
                                {\r
                                        IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
                                                ("Multicast Mac - trying to join.\n") );\r
@@ -3774,7 +4005,7 @@ __send_cb(
 \r
        p_port = (ipoib_port_t*)cq_context;\r
 \r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_send_cb );\r
 \r
        for( i = 0; i < MAX_SEND_WC; i++ )\r
                wc[i].p_next = &wc[i + 1];\r
@@ -3791,7 +4022,7 @@ __send_cb(
                while( p_wc )\r
                {\r
                        cl_perf_start( SendComp );\r
-                       CL_ASSERT( p_wc->wc_type == IB_WC_SEND );\r
+                       CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_SEND );\r
                        p_packet = (NDIS_PACKET*)(uintn_t)p_wc->wr_id;\r
                        CL_ASSERT( p_packet );\r
                        CL_ASSERT( IPOIB_PORT_FROM_PACKET( p_packet ) == p_port );\r
@@ -3835,8 +4066,9 @@ __send_cb(
 \r
                        default:\r
                                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                                       ("Send failed with %s\n",\r
-                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
+                                       ("Send failed with %s (vendor specific %#x)\n",\r
+                                       p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
+                                       (int)p_wc->vendor_specific) );\r
                                ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
                                NdisMSendComplete( p_port->p_adapter->h_adapter,\r
                                        p_packet, NDIS_STATUS_FAILURE );\r
@@ -3873,7 +4105,7 @@ __send_cb(
        ipoib_port_resume( p_port );\r
        cl_perf_stop( &p_port->p_adapter->perf, PortResume );\r
        \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_send_cb );\r
 \r
        cl_perf_stop( &p_port->p_adapter->perf, SendCb );\r
        cl_perf_update_ctr( &p_port->p_adapter->perf, SendCompBundle );\r
@@ -3952,8 +4184,10 @@ __endpt_mgr_reset_all(
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        cl_map_item_t   *p_item;\r
-       ipoib_endpt_t   *p_endpt;\r
-       cl_qlist_t              mc_list;\r
+       ipoib_endpt_t           *p_endpt;\r
+       cl_qlist_t                      mc_list;\r
+       uint32_t                        local_exist = 0;\r
+\r
 \r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
@@ -3972,9 +4206,10 @@ __endpt_mgr_reset_all(
                        &p_port->p_local_endpt->gid_item );\r
                cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,\r
                        &p_port->p_local_endpt->lid_item );\r
-\r
+               \r
                cl_qlist_insert_head(\r
                        &mc_list, &p_port->p_local_endpt->mac_item.pool_item.list_item );\r
+               local_exist = 1;\r
 \r
                p_port->p_local_endpt = NULL;\r
        }\r
@@ -3994,11 +4229,7 @@ __endpt_mgr_reset_all(
                                &p_endpt->mac_item );\r
                        cl_fmap_remove_item( &p_port->endpt_mgr.gid_endpts,\r
                                &p_endpt->gid_item );\r
-                       if( p_endpt->dlid )\r
-                       {\r
-                               cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,\r
-                                       &p_endpt->lid_item );\r
-                       }\r
+\r
                        cl_qlist_insert_tail(\r
                                &mc_list, &p_endpt->mac_item.pool_item.list_item );\r
                }\r
@@ -4008,9 +4239,31 @@ __endpt_mgr_reset_all(
                        p_port->p_adapter->p_ifc->destroy_av( p_endpt->h_av );\r
                        p_endpt->h_av = NULL;\r
                }\r
+               \r
+               if( p_endpt->dlid )\r
+               {\r
+                       cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,\r
+                               &p_endpt->lid_item );\r
+                       p_endpt->dlid = 0;\r
+               }\r
+               \r
        }\r
+\r
        cl_obj_unlock( &p_port->obj );\r
 \r
+\r
+       if(cl_qlist_count( &mc_list ) - local_exist)\r
+       {\r
+               p_port->mcast_cnt =  (uint32_t)cl_qlist_count( &mc_list ) - local_exist;\r
+       }\r
+       else\r
+       {\r
+               p_port->mcast_cnt = 0;\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }       \r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt - local_exist));\r
+\r
        /* Destroy all multicast endpoints now that we have released the lock. */\r
        while( cl_qlist_count( &mc_list ) )\r
        {\r
@@ -4236,12 +4489,14 @@ __endpt_mgr_get_by_lid(
 }\r
 \r
 \r
-inline void\r
+inline ib_api_status_t\r
 __endpt_mgr_insert_locked(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt )\r
 {\r
+       ib_api_status_t status;\r
+\r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
        IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
@@ -4257,21 +4512,22 @@ __endpt_mgr_insert_locked(
        }\r
        /* __endpt_mgr_insert expects *one* reference to be held when being called. */\r
        cl_atomic_inc( &p_port->endpt_rdr );\r
-       __endpt_mgr_insert( p_port, mac, p_endpt );\r
+       status= __endpt_mgr_insert( p_port, mac, p_endpt );\r
        cl_atomic_dec( &p_port->endpt_rdr );\r
        cl_obj_unlock( &p_port->obj );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+       return status;\r
 }\r
 \r
 \r
-void\r
+inline ib_api_status_t\r
 __endpt_mgr_insert(\r
        IN                              ipoib_port_t* const                     p_port,\r
        IN              const   mac_addr_t                                      mac,\r
        IN                              ipoib_endpt_t* const            p_endpt )\r
 {\r
        uint64_t                key;\r
+       cl_status_t             cl_status;\r
        cl_map_item_t   *p_qitem;\r
        cl_fmap_item_t  *p_fitem;\r
 \r
@@ -4282,9 +4538,21 @@ __endpt_mgr_insert(
                ;\r
 \r
        /* Link the endpoint to the port. */\r
-       cl_obj_insert_rel_parent_locked(\r
+       cl_status = cl_obj_insert_rel_parent_locked(\r
                &p_endpt->rel, &p_port->obj, &p_endpt->obj );\r
 \r
+       if( cl_status != CL_SUCCESS )\r
+       {\r
+               cl_obj_destroy( &p_endpt->obj );\r
+               return IB_INVALID_STATE;\r
+       }\r
+\r
+#if DBG\r
+       cl_atomic_inc( &p_port->ref[ref_endpt_track] );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+               ("ref  type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
        p_endpt->mac = mac;\r
        key = 0;\r
        cl_memcpy( &key, &mac, sizeof(mac_addr_t) );\r
@@ -4302,6 +4570,7 @@ __endpt_mgr_insert(
        }\r
 \r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+       return IB_SUCCESS;\r
 }\r
 \r
 \r
@@ -4344,10 +4613,10 @@ __endpt_mgr_add_bcast(
 \r
        /* Add the broadcast endpoint to the endpoint map. */\r
        cl_memset( &bcast_mac, 0xFF, sizeof(bcast_mac) );\r
-       __endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );\r
+       status = __endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
-       return IB_SUCCESS;\r
+       return status;\r
 }\r
 \r
 \r
@@ -4393,6 +4662,12 @@ ipoib_port_remove_endpt(
 \r
                cl_obj_unlock( &p_port->obj );\r
                cl_obj_destroy( &p_endpt->obj );\r
+#if DBG\r
+               cl_atomic_dec( &p_port->ref[ref_endpt_track] );\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
+                       ("ref type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );\r
+#endif\r
+\r
        }\r
        else\r
        {\r
@@ -4495,14 +4770,14 @@ ipoib_port_up(
        cl_memclr( &query, sizeof(ib_query_req_t) );\r
        query.query_type = IB_QUERY_USER_DEFINED;\r
        query.p_query_input = &info;\r
-       query.port_guid = p_port->p_adapter->guids.port_guid;\r
+       query.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        query.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        query.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
        query.query_context = p_port;\r
        query.pfn_query_cb = __port_info_cb;\r
 \r
        /* reference the object for the multicast query. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_port_up );\r
 \r
        status = p_port->p_adapter->p_ifc->query(\r
                p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
@@ -4510,7 +4785,7 @@ ipoib_port_up(
        {\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
                ipoib_set_inactive( p_port->p_adapter );\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_port_up );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_query returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
@@ -4533,7 +4808,7 @@ __endpt_mgr_add_local(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid );\r
+       ib_gid_set_default( &gid, p_port->p_adapter->guids.port_guid.guid );\r
        p_endpt = ipoib_endpt_create(\r
                &gid, p_port_info->base_lid, p_port->ib_mgr.qpn );\r
        if( !p_endpt )\r
@@ -4563,13 +4838,20 @@ __endpt_mgr_add_local(
 \r
        /* __endpt_mgr_insert expects *one* reference to be held. */\r
        cl_atomic_inc( &p_port->endpt_rdr );\r
-       __endpt_mgr_insert( p_port, p_port->p_adapter->params.conf_mac, p_endpt );\r
+       status = __endpt_mgr_insert( p_port, p_port->p_adapter->params.conf_mac, p_endpt );\r
        cl_atomic_dec( &p_port->endpt_rdr );\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("__endpt_mgr_insert for local endpoint returned %s\n",\r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
 \r
        p_port->p_local_endpt = p_endpt;\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
-       return IB_SUCCESS;\r
+       return status;\r
 }\r
 \r
 \r
@@ -4583,7 +4865,7 @@ __port_info_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_query_rec->query_context;\r
+       p_port = (ipoib_port_t*)p_query_rec->query_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        p_port->ib_mgr.h_query = NULL;\r
@@ -4660,8 +4942,10 @@ done:
        if( status != IB_SUCCESS )\r
        {\r
                if( status != IB_CANCELED )\r
+               {\r
                        ipoib_set_inactive( p_port->p_adapter );\r
-\r
+                       __endpt_mgr_reset_all( p_port );\r
+               }\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
        }\r
 \r
@@ -4670,7 +4954,7 @@ done:
                p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
 \r
        /* Release the reference taken when issuing the port info query. */\r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_port_info_cb );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
@@ -4685,7 +4969,7 @@ __port_get_bcast(
        ib_user_query_t         info;\r
        ib_member_rec_t         member_rec;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        info.method = IB_MAD_METHOD_GETTABLE;\r
        info.attr_id = IB_MAD_ATTR_MCMEMBER_RECORD;\r
@@ -4697,29 +4981,32 @@ __port_get_bcast(
        cl_memclr( &member_rec, sizeof(ib_member_rec_t) );\r
        member_rec.mgid = bcast_mgid_template;\r
 \r
+    member_rec.mgid.raw[4] = (uint8_t) (p_port->p_adapter->guids.port_guid.pkey >> 8) ;\r
+       member_rec.mgid.raw[5] = (uint8_t) p_port->p_adapter->guids.port_guid.pkey;\r
+       member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
        cl_memclr( &query, sizeof(ib_query_req_t) );\r
        query.query_type = IB_QUERY_USER_DEFINED;\r
        query.p_query_input = &info;\r
-       query.port_guid = p_port->p_adapter->guids.port_guid;\r
+       query.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        query.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        query.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
        query.query_context = p_port;\r
        query.pfn_query_cb = __bcast_get_cb;\r
 \r
        /* reference the object for the multicast query. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_get_bcast );\r
 \r
        status = p_port->p_adapter->p_ifc->query(\r
                p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_get_bcast );\r
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_query returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
        }\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return status;\r
 }\r
 \r
@@ -4733,9 +5020,9 @@ __bcast_get_cb(
        ib_member_rec_t         *p_mc_req;\r
        ib_api_status_t         status;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_query_rec->query_context;\r
+       p_port = (ipoib_port_t*)p_query_rec->query_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        p_port->ib_mgr.h_query = NULL;\r
@@ -4782,8 +5069,10 @@ done:
        if( status != IB_SUCCESS )\r
        {\r
                if( status != IB_CANCELED )\r
+               {\r
                        ipoib_set_inactive( p_port->p_adapter );\r
-\r
+                       __endpt_mgr_reset_all( p_port );\r
+               }\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
        }\r
 \r
@@ -4792,9 +5081,9 @@ done:
                p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
 \r
        /* Release the reference taken when issuing the member record query. */\r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_bcast_get_cb );\r
 \r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
 \r
@@ -4806,16 +5095,17 @@ __port_join_bcast(
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        /* Check that the rate is realizable for our port. */\r
-       if( p_port->ib_mgr.rate < (p_member_rec->rate & 0x3F) )\r
+       if( p_port->ib_mgr.rate < (p_member_rec->rate & 0x3F) &&\r
+               (g_ipoib.bypass_check_bcast_rate == 0))\r
        {\r
                /*\r
                 * The MC group rate is higher than our port's rate.  Log an error\r
                 * and stop.  A port transition will drive the retry.\r
                 */\r
-               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_MCAST,\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,\r
                        ("Unrealizable join due to rate mismatch.\n") );\r
                NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
                        EVENT_IPOIB_BCAST_RATE, 2,\r
@@ -4831,37 +5121,37 @@ __port_join_bcast(
        /* We specify our port GID for the join operation. */\r
        mcast_req.member_rec.port_gid.unicast.prefix = IB_DEFAULT_SUBNET_PREFIX;\r
        mcast_req.member_rec.port_gid.unicast.interface_id =\r
-               p_port->p_adapter->guids.port_guid;\r
+               p_port->p_adapter->guids.port_guid.guid;\r
 \r
        mcast_req.mcast_context = p_port;\r
        mcast_req.pfn_mcast_cb = __bcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
 \r
        if( ib_member_get_state( mcast_req.member_rec.scope_state ) !=\r
                IB_MC_REC_STATE_FULL_MEMBER )\r
        {\r
-               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_MCAST,\r
+               IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,\r
                        ("Incorrect MC member rec join state in query response.\n") );\r
                ib_member_set_state( &mcast_req.member_rec.scope_state,\r
                        IB_MC_REC_STATE_FULL_MEMBER );\r
        }\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_bcast );\r
 \r
        status = p_port->p_adapter->p_ifc->join_mcast(\r
                p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_bcast_join_failed );\r
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
        }\r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return status;\r
 }\r
 \r
@@ -4873,7 +5163,7 @@ __port_create_bcast(
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        /* Join the broadcast group. */\r
        cl_memclr( &mcast_req, sizeof(mcast_req) );\r
@@ -4885,8 +5175,10 @@ __port_create_bcast(
         * We specify the MGID since we don't want the SA to generate it for us.\r
         */\r
        mcast_req.member_rec.mgid = bcast_mgid_template;\r
+       mcast_req.member_rec.mgid.raw[4] = (uint8_t) (p_port->p_adapter->guids.port_guid.pkey >> 8); \r
+       mcast_req.member_rec.mgid.raw[5] = (uint8_t) p_port->p_adapter->guids.port_guid.pkey;\r
        ib_gid_set_default( &mcast_req.member_rec.port_gid,\r
-               p_port->p_adapter->guids.port_guid );\r
+               p_port->p_adapter->guids.port_guid.guid );\r
        /*\r
         * IPOIB spec requires that the QKEY have the MSb set so that the QKEY\r
         * from the QP is used rather than the QKEY in the send WR.\r
@@ -4894,9 +5186,9 @@ __port_create_bcast(
        mcast_req.member_rec.qkey =\r
                (uint32_t)(uintn_t)p_port | IB_QP_PRIVILEGED_Q_KEY;\r
        mcast_req.member_rec.mtu =\r
-               (IB_PATH_SELECTOR_EXACTLY << 6) | IB_MTU_2048;\r
+               (IB_PATH_SELECTOR_EXACTLY << 6) | IB_MTU_LEN_2048;\r
 \r
-       mcast_req.member_rec.pkey = IB_DEFAULT_PKEY;\r
+       mcast_req.member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
 \r
        mcast_req.member_rec.sl_flow_hop = ib_member_set_sl_flow_hop( 0, 0, 0 );\r
        mcast_req.member_rec.scope_state =\r
@@ -4906,21 +5198,21 @@ __port_create_bcast(
        mcast_req.pfn_mcast_cb = __bcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_bcast );\r
 \r
        status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_bcast_create_failed );\r
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
        }\r
-       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+       IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return status;\r
 }\r
 \r
@@ -4976,9 +5268,16 @@ ipoib_port_down(
                return;\r
        }\r
 \r
+       KeResetEvent(&p_port->leave_mcast_event);\r
+\r
        /* Reset all endpoints so we don't flush our ARP cache. */\r
        __endpt_mgr_reset_all( p_port );\r
 \r
+       KeWaitForSingleObject(\r
+               &p_port->leave_mcast_event, Executive, KernelMode, FALSE, NULL );\r
+\r
+       __pending_list_destroy(p_port);\r
+       \r
        cl_obj_lock( &p_port->p_adapter->obj );\r
        ipoib_dereg_addrs( p_port->p_adapter );\r
        cl_obj_unlock( &p_port->p_adapter->obj );\r
@@ -4996,17 +5295,20 @@ __bcast_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_mcast_rec->mcast_context;\r
+       p_port = (ipoib_port_t*)p_mcast_rec->mcast_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        if( p_port->state != IB_QPS_INIT )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
                if( p_mcast_rec->status == IB_SUCCESS )\r
-                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
 \r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_bcast_inv_state );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                        ("Invalid state - Aborting.\n") );\r
                return;\r
@@ -5049,9 +5351,10 @@ __bcast_cb(
                if( status != IB_SUCCESS )\r
                {\r
                        ipoib_set_inactive( p_port->p_adapter );\r
+                       __endpt_mgr_reset_all( p_port );\r
                        KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
                }\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_bcast_req_failed );\r
                IPOIB_EXIT( IPOIB_DBG_INIT );\r
                return;\r
        }\r
@@ -5063,7 +5366,8 @@ __bcast_cb(
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__endpt_mgr_add_bcast returned %s\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
                CL_ASSERT( status == IB_SUCCESS );\r
                goto err;\r
        }\r
@@ -5080,7 +5384,7 @@ err:
                /* Flag the adapter as hung. */\r
                p_port->p_adapter->hung = TRUE;\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_bcast_error );\r
                IPOIB_EXIT( IPOIB_DBG_INIT );\r
                return;\r
        }\r
@@ -5100,7 +5404,7 @@ err:
        ipoib_set_active( p_port->p_adapter );\r
 \r
        KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_join_bcast );\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -5111,7 +5415,7 @@ __qp_event(
 {\r
        UNUSED_PARAM( p_event_rec );\r
        CL_ASSERT( p_event_rec->context );\r
-       ((ipoib_port_t* __ptr64)p_event_rec->context)->p_adapter->hung = TRUE;\r
+       ((ipoib_port_t*)p_event_rec->context)->p_adapter->hung = TRUE;\r
 }\r
 \r
 \r
@@ -5121,7 +5425,7 @@ __cq_event(
 {\r
        UNUSED_PARAM( p_event_rec );\r
        CL_ASSERT( p_event_rec->context );\r
-       ((ipoib_port_t* __ptr64)p_event_rec->context)->p_adapter->hung = TRUE;\r
+       ((ipoib_port_t*)p_event_rec->context)->p_adapter->hung = TRUE;\r
 }\r
 \r
 \r
@@ -5150,9 +5454,9 @@ __ib_mgr_activate(
        }\r
 \r
        /* Move the QP to RTS. */\r
-       dgrm_info.port_guid = p_port->p_adapter->guids.port_guid;\r
+       dgrm_info.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
        dgrm_info.qkey = p_port->ib_mgr.bcast_rec.qkey;\r
-       dgrm_info.pkey_index = 0;\r
+       dgrm_info.pkey_index = p_port->pkey_index;\r
        status = p_port->p_adapter->p_ifc->init_dgrm_svc( p_port->ib_mgr.h_qp, &dgrm_info );\r
        if( status != IB_SUCCESS )\r
        {\r
@@ -5189,7 +5493,8 @@ __ib_mgr_activate(
 ib_api_status_t\r
 ipoib_port_join_mcast(\r
        IN                              ipoib_port_t* const                     p_port,\r
-       IN              const   mac_addr_t                                      mac )\r
+       IN              const   mac_addr_t                              mac,\r
+       IN              const   uint8_t                                 state)\r
 {\r
        ib_api_status_t         status;\r
        ib_mcast_req_t          mcast_req;\r
@@ -5222,6 +5527,7 @@ ipoib_port_join_mcast(
        mcast_req.member_rec = p_port->ib_mgr.bcast_rec;\r
        /* Clear fields that aren't specified in the join */\r
        mcast_req.member_rec.mlid = 0;\r
+       ib_member_set_state( &mcast_req.member_rec.scope_state,state);\r
 \r
        if( mac.addr[0] == 1 && mac.addr[1] == 0 && mac.addr[2] == 0x5E )\r
        {\r
@@ -5250,9 +5556,9 @@ ipoib_port_join_mcast(
        mcast_req.pfn_mcast_cb = __mcast_cb;\r
        mcast_req.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
        mcast_req.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-       mcast_req.port_guid = p_port->p_adapter->guids.port_guid;\r
-       mcast_req.pkey_index = 0;\r
-\r
+       mcast_req.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
+       mcast_req.pkey_index = p_port->pkey_index;\r
+       mcast_req.member_rec.pkey = p_port->p_adapter->guids.port_guid.pkey;\r
        /*\r
         * Create the endpoint and insert it in the port.  Since we don't wait for\r
         * the mcast SA operations to complete before returning from the multicast\r
@@ -5270,15 +5576,22 @@ ipoib_port_join_mcast(
                return IB_INSUFFICIENT_MEMORY;\r
        }\r
 \r
-       __endpt_mgr_insert_locked( p_port, mac, p_endpt );\r
+       status = __endpt_mgr_insert_locked( p_port, mac, p_endpt );\r
+       if( status != IB_SUCCESS )\r
+       {\r
+               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("__endpt_mgr_insert_locked returned %s\n", \r
+                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+               return status;\r
+       }\r
 \r
        /* reference the object for the multicast join request. */\r
-       cl_obj_ref( &p_port->obj );\r
+       ipoib_port_ref( p_port, ref_join_mcast );\r
 \r
        status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );\r
        if( status != IB_SUCCESS )\r
        {\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_mcast_join_failed );\r
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_join_mcast returned %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
@@ -5301,16 +5614,19 @@ __mcast_cb(
 \r
        IPOIB_ENTER( IPOIB_DBG_MCAST );\r
 \r
-       p_port = (ipoib_port_t* __ptr64)p_mcast_rec->mcast_context;\r
+       p_port = (ipoib_port_t*)p_mcast_rec->mcast_context;\r
 \r
        cl_obj_lock( &p_port->obj );\r
        if( p_port->state != IB_QPS_RTS )\r
        {\r
                cl_obj_unlock( &p_port->obj );\r
                if( p_mcast_rec->status == IB_SUCCESS )\r
-                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
 \r
-               cl_obj_deref( &p_port->obj );\r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
+               ipoib_port_deref( p_port, ref_mcast_inv_state );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                        ("Invalid state - Aborting.\n") );\r
                return;\r
@@ -5322,8 +5638,9 @@ __mcast_cb(
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("Multicast join request failed with status %s.\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( p_mcast_rec->status )) );\r
-               ipoib_set_inactive( p_port->p_adapter );\r
-               cl_obj_deref( &p_port->obj );\r
+               /* Flag the adapter as hung. */\r
+               p_port->p_adapter->hung =TRUE;\r
+               ipoib_port_deref( p_port, ref_mcast_req_failed );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5341,8 +5658,10 @@ __mcast_cb(
                cl_obj_unlock( &p_port->obj );\r
                IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
                        ("Failed to find endpoint for update.\n") );\r
-               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
-               cl_obj_deref( &p_port->obj );\r
+\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               ipoib_port_deref( p_port, ref_mcast_no_endpt );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5361,7 +5680,7 @@ __mcast_cb(
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
                /* Flag the adapter as hung. */\r
                p_port->p_adapter->hung = TRUE;\r
-               cl_obj_deref( &p_port->obj );\r
+               ipoib_port_deref( p_port, ref_mcast_av_failed );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
        }\r
@@ -5381,7 +5700,57 @@ __mcast_cb(
        /* Try to send all pending sends. */\r
        ipoib_port_resume( p_port );\r
 \r
-       cl_obj_deref( &p_port->obj );\r
+       ipoib_port_deref( p_port, ref_join_mcast );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_MCAST );\r
 }\r
+\r
+\r
+void\r
+ipoib_leave_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t*)context;\r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_VERBOSE, IPOIB_DBG_MCAST,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt));\r
+       \r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       cl_atomic_dec( &p_port->mcast_cnt);\r
+       \r
+       if(0 == p_port->mcast_cnt)\r
+       {\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }\r
+       \r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leave mcast callback deref ipoib_port \n") );\r
+       \r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r
+\r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t*)context;\r
+\r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leave mcast callback deref ipoib_port \n") );\r
+       \r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r
+\r
+\r