[IPOIB, IBBUS] g_stat: first patch of several, adding and populating global g_stat...
authorleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Tue, 17 Nov 2009 14:12:21 +0000 (14:12 +0000)
committerleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Tue, 17 Nov 2009 14:12:21 +0000 (14:12 +0000)
For debug purposes.
Usable in both checked and free version.
Usage: open Watch Window in Debugger and add ipoib!g_stat or ibbus!g_stat.

An example of a real problem: MLX4 driver is stuck on unload.

It can happen when IBAL is stuck on resource reclamation.
The reclamation is being done in one of IBAL threads, which are started on IBAL's start up.
So you'd like to look into this thread.
But where it is ?
You can find it by printing ALL the threads of System process and looking for ibbus in their stack traces.
It can take minutes.

After this patch you can do that in 3 seconds this way:
1. open Watch Window in WinDbg and add ibbus!g_stat.
2. open
ibbus!g_stat->Drv->Gp_async_obj_mgr->Thread_pool->P_thread[0]->Osd->P_thread

This field contains the address of the thread, you wanted.

git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2580 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

17 files changed:
core/al/al_init.c
core/al/al_mgr.h
core/al/kernel/SOURCES
core/bus/kernel/SOURCES
core/bus/kernel/bus_driver.c
core/bus/kernel/bus_driver.h
core/bus/kernel/bus_pnp.c
core/bus/kernel/bus_stat.c [new file with mode: 0644]
core/bus/kernel/bus_stat.h [new file with mode: 0644]
core/complib/cl_threadpool.c
core/complib/kernel/cl_thread.c
inc/complib/cl_threadpool.h
ulp/ipoib/kernel/SOURCES
ulp/ipoib/kernel/ipoib_adapter.h
ulp/ipoib/kernel/ipoib_driver.c
ulp/ipoib/kernel/ipoib_stat.c [new file with mode: 0644]
ulp/ipoib/kernel/ipoib_stat.h [new file with mode: 0644]

index 2b2aca5..b57fc9d 100644 (file)
@@ -88,7 +88,7 @@ al_initialize( void )
                return ib_convert_cl_status( cl_status );\r
        }\r
 #endif\r
-       cl_status = cl_async_proc_init( gp_async_proc_mgr, 1, "Althread" );\r
+       cl_status = cl_async_proc_init( gp_async_proc_mgr, 1, "AL_MISC" );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
                AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,\r
index 583374c..684c6cb 100644 (file)
@@ -74,6 +74,9 @@ cl_async_proc_t               *gp_async_pnp_mgr;
 al_mgr_t                       *gp_al_mgr;\r
 ib_al_handle_t         gh_al;\r
 ib_pool_handle_t       gh_mad_pool;\r
+void                           *gp_tmp;\r
+cl_obj_mgr_t           *gp_obj_mgr;\r
+\r
 \r
 \r
 ib_api_status_t\r
index 23bfebc..f4a2cc7 100644 (file)
@@ -56,7 +56,7 @@ SOURCES= ibal.rc                      \
        ..\ib_common.c                  \\r
        ..\ib_statustext.c\r
 \r
-INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel;..\..\bus\kernel\$O;\r
+INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel;..\..\bus\kernel;..\..\bus\kernel\$O;\r
 \r
 C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS \\r
        -DEXPORT_AL_SYMBOLS\r
index 09d90f2..42bfa26 100644 (file)
@@ -14,7 +14,8 @@ SOURCES= ibbus.rc             \
        bus_driver.c            \\r
        bus_pnp.c                       \\r
        bus_port_mgr.c          \\r
-       bus_iou_mgr.c\r
+       bus_iou_mgr.c           \\r
+       bus_stat.c\r
 \r
 INCLUDES=..\..\..\inc;..\..\..\inc\kernel;..\..\al;..\..\al\kernel;..\..\bus\kernel\$O;\r
 \r
index bee7eb5..c190c93 100644 (file)
@@ -1045,6 +1045,8 @@ DriverEntry(
 \r
        /* Store the driver object pointer in the global parameters. */\r
        bus_globals.p_driver_obj = p_driver_obj;\r
+       bus_st_init();\r
+       g_stat.drv.p_globals = &bus_globals;\r
 \r
        /* Get the registry values. */\r
        status = __read_registry( p_registry_path );\r
index ff20ee0..368b6f9 100644 (file)
@@ -47,6 +47,7 @@
 #include "bus_iou_mgr.h"\r
 #include "al_dev.h"\r
 #include <rdma/verbs.h>\r
+#include "bus_stat.h"\r
 \r
 /* Safe string functions. */\r
 #if WINVER == 0x500\r
@@ -135,6 +136,8 @@ typedef struct _bus_fdo_ext
        PIO_WORKITEM                    p_po_work_item;\r
        /* current PnP state */\r
        PNP_DEVICE_STATE                pnp_state; /* state for PnP Manager */\r
+       /* statistics */\r
+       PBUS_ST_DEVICE                  p_stat;\r
 \r
 }      bus_fdo_ext_t;\r
 \r
index 7634e93..e549a85 100644 (file)
@@ -465,6 +465,11 @@ fdo_start(
        }\r
        p_ext->device_power_state = PowerDeviceD0;\r
 \r
+       p_ext->p_stat = bus_st_dev_add();\r
+       if ( p_ext->p_stat ) \r
+               p_ext->p_stat->p_fdo = p_ext;\r
+       \r
+\r
        lock_control_event();\r
        if ( !gp_async_proc_mgr ) {\r
                /* Initialize AL */\r
@@ -479,6 +484,12 @@ fdo_start(
                }\r
                AL_init_here = TRUE;\r
                BUS_TRACE( BUS_DBG_PNP, ("AL initialized\n"));\r
+               /* for debug */\r
+               g_stat.drv.gp_async_obj_mgr = gp_async_obj_mgr;\r
+               g_stat.drv.gp_async_proc_mgr = gp_async_proc_mgr;\r
+               g_stat.drv.gp_async_pnp_mgr = gp_async_pnp_mgr;\r
+               g_stat.drv.gp_al_mgr = gp_al_mgr;\r
+               g_stat.drv.gp_obj_mgr = gp_obj_mgr;\r
        }\r
        unlock_control_event();\r
 \r
@@ -623,11 +634,13 @@ fdo_release_resources(
        NTSTATUS                status;\r
        bus_filter_t    *p_bfi;\r
        int                             ic;\r
+       PBUS_ST_DEVICE p_stat;\r
 \r
        BUS_ENTER( BUS_DBG_PNP );\r
 \r
        p_ext = p_dev_obj->DeviceExtension;\r
        ic = get_bfi_count();\r
+       p_stat = p_ext->p_stat;\r
 \r
        p_bfi = p_ext->bus_filter;\r
        CL_ASSERT( p_bfi );\r
@@ -665,6 +678,7 @@ fdo_release_resources(
        /* if not last Buf Filter Instance, then exit, otherwise cleanup/shutdown */\r
        if ( ic > 0 ) {\r
                BUS_TRACE_EXIT( BUS_DBG_PNP, ("%d remaining BusFilters\n", ic ));\r
+               bus_st_dev_rmv( p_stat );\r
                return;\r
        }\r
 \r
@@ -707,6 +721,8 @@ fdo_release_resources(
        }\r
 #endif \r
 \r
+       bus_st_dev_rmv( p_stat );\r
+\r
        BUS_TRACE_EXIT( BUS_DBG_PNP, \r
                ("=====> IBBUS: fdo_release_resources exited \n") );\r
 }\r
diff --git a/core/bus/kernel/bus_stat.c b/core/bus/kernel/bus_stat.c
new file mode 100644 (file)
index 0000000..19579f3
--- /dev/null
@@ -0,0 +1,53 @@
+/*++
+
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.
+
+Module Name:
+       bus_stat.h
+
+Abstract:
+       Statistics Collector header file
+
+Revision History:
+
+Notes:
+
+--*/
+
+#include "al_debug.h"
+#include "bus_stat.h"
+
+#if defined (EVENT_TRACING)
+#ifdef offsetof
+#undef offsetof
+#endif
+#include "bus_stat.tmh"
+#endif 
+
+BUS_ST_STAT g_stat;
+
+void bus_st_dev_rmv( PBUS_ST_DEVICE p_stat )
+{
+       if ( p_stat )
+               p_stat->valid = FALSE;
+}
+
+PBUS_ST_DEVICE bus_st_dev_add()
+{
+       int i;
+
+       for ( i = 0; i < BUS_ST_MAX_DEVICES; ++i ) {
+               if ( g_stat.dev[i].valid == FALSE ) {
+                       g_stat.dev[i].valid = TRUE;
+                       return &g_stat.dev[i];
+               }
+       }
+
+       return NULL;
+}
+
+void bus_st_init()
+{
+       memset( &g_stat, 0, sizeof(g_stat) );
+}
+
diff --git a/core/bus/kernel/bus_stat.h b/core/bus/kernel/bus_stat.h
new file mode 100644 (file)
index 0000000..b793177
--- /dev/null
@@ -0,0 +1,84 @@
+/*++\r
+\r
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.\r
+\r
+Module Name:\r
+       bus_stat.h\r
+\r
+Abstract:\r
+       Statistics Collector header file\r
+\r
+Revision History:\r
+\r
+Notes:\r
+\r
+--*/\r
+\r
+#pragma once\r
+\r
+#include <ntddk.h>\r
+#include "al_mgr.h"\r
+//\r
+// restrictions\r
+//\r
+\r
+#define BUS_ST_MAX_DEVICES                     8\r
+\r
+//\r
+// enums\r
+// \r
+\r
+//\r
+// structures\r
+//\r
+\r
+// device\r
+\r
+typedef struct _bus_fdo_ext    bus_fdo_ext_t;\r
+typedef struct _bus_globals            bus_globals_t;\r
+typedef struct _al_mgr                 al_mgr_t;\r
+\r
+\r
+\r
+typedef struct _BUS_ST_DEVICE\r
+{\r
+       boolean_t                       valid;\r
+       bus_fdo_ext_t           *p_fdo;\r
+       \r
+} BUS_ST_DEVICE, *PBUS_ST_DEVICE;\r
+\r
+// driver\r
+typedef struct _BUS_ST_DRIVER\r
+{\r
+       bus_globals_t           *p_globals;\r
+       cl_obj_mgr_t            *gp_obj_mgr;\r
+       cl_async_proc_t         *gp_async_proc_mgr;\r
+       cl_async_proc_t         *gp_async_pnp_mgr;\r
+       cl_async_proc_t         *gp_async_obj_mgr;\r
+       al_mgr_t                        *gp_al_mgr;\r
+       \r
+} BUS_ST_DRIVER, *PBUS_ST_DRIVER;\r
+\r
+// driver stack\r
+\r
+typedef struct _BUS_ST_STAT\r
+{\r
+       BUS_ST_DRIVER           drv;\r
+       BUS_ST_DEVICE           dev[BUS_ST_MAX_DEVICES];\r
+       \r
+} BUS_ST_STAT, *PBUS_ST_STAT;\r
+\r
+extern BUS_ST_STAT g_stat;\r
+\r
+//\r
+// functions \r
+//\r
+\r
+void bus_st_dev_rmv( PBUS_ST_DEVICE p_stat );\r
+\r
+PBUS_ST_DEVICE bus_st_dev_add();\r
+\r
+void bus_st_init();\r
+\r
+\r
+\r
index cba8f2c..d45ca3f 100644 (file)
@@ -169,6 +169,10 @@ cl_thread_pool_init(
                 * will signal all the threads.\r
                 */\r
                cl_atomic_inc( &p_thread_pool->running_count );\r
+\r
+               /* for debug only */\r
+               if ( i < sizeof(p_thread_pool->p_thread)/sizeof(cl_thread_t*) )\r
+                       p_thread_pool->p_thread[i] = p_thread;\r
        }\r
        p_thread_pool->state = CL_INITIALIZED;\r
        return( CL_SUCCESS );\r
index f68eb16..f9ebd42 100644 (file)
@@ -29,7 +29,6 @@
  * $Id$\r
  */\r
 \r
-\r
 #include "complib/cl_thread.h"\r
 \r
 \r
@@ -72,12 +71,11 @@ cl_thread_init(
        CL_ASSERT( p_thread && pfn_callback );\r
        CL_ASSERT( KeGetCurrentIrql() == PASSIVE_LEVEL );\r
 \r
-       UNUSED_PARAM( name );\r
-\r
        cl_thread_construct( p_thread );\r
 \r
        p_thread->pfn_callback = pfn_callback;\r
        p_thread->context = context;\r
+       strncpy( &p_thread->name[0], name, sizeof(p_thread->name) );\r
 \r
        /* Create a new thread, storing both the handle and thread id. */\r
        InitializeObjectAttributes( &attr, NULL, OBJ_KERNEL_HANDLE, NULL, NULL );\r
@@ -135,4 +133,4 @@ cl_is_current_thread(
        IN      const cl_thread_t* const        p_thread )\r
 {\r
        return( p_thread->osd.p_thread == KeGetCurrentThread() );\r
-}
\ No newline at end of file
+}\r
index b4c0089..d84fe1c 100644 (file)
@@ -98,6 +98,8 @@ typedef struct _cl_thread_pool
        boolean_t                                       exit;\r
        cl_state_t                                      state;\r
        atomic32_t                                      running_count;\r
+       // for debug\r
+       cl_thread_t                                     *p_thread[8];\r
 \r
 } cl_thread_pool_t;\r
 /*\r
index ffd58fe..12e93b4 100644 (file)
@@ -24,7 +24,8 @@ SOURCES=      ipoib_log.mc \
                ipoib_endpoint.c \\r
                ipoib_port.c \\r
                ipoib_ibat.c \\r
-               ipoib_xfr_mgr.c\r
+               ipoib_xfr_mgr.c \\r
+               ipoib_stat.c\r
 \r
 INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel;\r
 \r
index f6600a2..492863e 100644 (file)
@@ -46,6 +46,7 @@
 #include <complib/cl_vector.h>\r
 #include <ip_packet.h>\r
 #include "ip_stats.h"\r
+#include "ipoib_stat.h"\r
 \r
 \r
 /*\r
@@ -194,6 +195,7 @@ typedef struct _ipoib_adapter
 \r
        cl_perf_t                               perf;\r
        ib_al_ifc_t                             *p_ifc;\r
+       PIPOIB_ST_DEVICE                p_stat;\r
 \r
 }      ipoib_adapter_t;\r
 /*\r
index 5bd0f36..3b11aba 100644 (file)
@@ -354,6 +354,8 @@ DriverEntry(
        \r
        KeInitializeSpinLock( &g_ipoib.lock );\r
        cl_qlist_init( &g_ipoib.adapter_list );\r
+       ipoib_st_init();\r
+       g_stat.drv.obj = p_drv_obj;\r
 \r
        NdisMInitializeWrapper(\r
                &g_ipoib.h_ndis_wrapper, p_drv_obj, p_registry_path, NULL );\r
@@ -785,6 +787,10 @@ ipoib_initialize(
                return NDIS_STATUS_FAILURE;\r
        }\r
 \r
+       p_adapter->p_stat = ipoib_st_dev_add();\r
+       if ( p_adapter->p_stat ) \r
+               p_adapter->p_stat->p_adapter = p_adapter;\r
+       \r
        ipoib_ref_ibat();\r
 \r
        IPOIB_PRINT_EXIT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT, \r
@@ -804,6 +810,7 @@ ipoib_halt(
        IN                              NDIS_HANDLE                                     adapter_context )\r
 {\r
        ipoib_adapter_t *p_adapter;\r
+       PIPOIB_ST_DEVICE p_stat;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
@@ -811,6 +818,7 @@ ipoib_halt(
 \r
        CL_ASSERT( adapter_context );\r
        p_adapter = (ipoib_adapter_t*)adapter_context;\r
+       p_stat = p_adapter->p_stat;\r
 \r
        IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                        ("Port %016I64x (CA %016I64x port %d) halting\n",\r
@@ -818,6 +826,7 @@ ipoib_halt(
                        p_adapter->guids.port_num) );\r
 \r
        ipoib_destroy_adapter( p_adapter );\r
+       ipoib_st_dev_rmv( p_stat );\r
 \r
        IPOIB_PRINT_EXIT(TRACE_LEVEL_WARNING, IPOIB_DBG_INIT, \r
                ("=====> ipoib_halt exited\n"));\r
@@ -1926,7 +1935,10 @@ ipoib_pnp_notify(
                cl_obj_unlock( &p_adapter->obj );\r
 \r
                ipoib_resume_oids( p_adapter );\r
+               p_adapter->p_stat->n_pnp_irps++;\r
        }\r
+       else \r
+               p_adapter->p_stat->n_power_irps++;\r
 \r
        IPOIB_PRINT_EXIT(TRACE_LEVEL_WARNING, IPOIB_DBG_PNP, \r
                ("=====> ipoib_pnp_notify exited, PnP event %d\n", pnp_event));\r
diff --git a/ulp/ipoib/kernel/ipoib_stat.c b/ulp/ipoib/kernel/ipoib_stat.c
new file mode 100644 (file)
index 0000000..29675eb
--- /dev/null
@@ -0,0 +1,37 @@
+#include "ipoib_adapter.h"
+#include "ipoib_debug.h"
+
+#if defined (EVENT_TRACING)
+#ifdef offsetof
+#undef offsetof
+#endif
+#include "ipoib_stat.tmh"
+#endif 
+
+IPOIB_ST_STAT g_stat;
+
+void ipoib_st_dev_rmv( PIPOIB_ST_DEVICE p_stat )
+{
+       if ( p_stat )
+               p_stat->valid = FALSE;
+}
+
+PIPOIB_ST_DEVICE ipoib_st_dev_add()
+{
+       int i;
+
+       for ( i = 0; i < IPOIB_ST_MAX_DEVICES; ++i ) {
+               if ( g_stat.dev[i].valid == FALSE ) {
+                       g_stat.dev[i].valid = TRUE;
+                       return &g_stat.dev[i];
+               }
+       }
+
+       return NULL;
+}
+
+void ipoib_st_init()
+{
+       memset( &g_stat, 0, sizeof(g_stat) );
+}
+
diff --git a/ulp/ipoib/kernel/ipoib_stat.h b/ulp/ipoib/kernel/ipoib_stat.h
new file mode 100644 (file)
index 0000000..1613a7d
--- /dev/null
@@ -0,0 +1,76 @@
+/*++\r
+\r
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.\r
+\r
+Module Name:\r
+       ipoib_stat.h\r
+\r
+Abstract:\r
+       Statistics Collector header file\r
+\r
+Revision History:\r
+\r
+Notes:\r
+\r
+--*/\r
+\r
+#pragma once\r
+\r
+#include <ntddk.h>\r
+\r
+//\r
+// restrictions\r
+//\r
+\r
+#define IPOIB_ST_MAX_DEVICES                   8\r
+\r
+//\r
+// enums\r
+// \r
+\r
+//\r
+// structures\r
+//\r
+\r
+// device\r
+typedef struct _ipoib_adapter ipoib_adapter_t;\r
+\r
+typedef struct _IPOIB_ST_DEVICE\r
+{\r
+       boolean_t                       valid;\r
+       ipoib_adapter_t         *p_adapter;\r
+       int                                     n_power_irps;   // NdisDevicePnPEventPowerProfileChanged \r
+       int                                     n_pnp_irps;             // NdisDevicePnPEventSurpriseRemoved \r
+       \r
+} IPOIB_ST_DEVICE, *PIPOIB_ST_DEVICE;\r
+\r
+// driver\r
+typedef struct _IPOIB_ST_DRIVER\r
+{\r
+       PDRIVER_OBJECT          obj;\r
+       \r
+} IPOIB_ST_DRIVER, *PIPOIB_ST_DRIVER;\r
+\r
+// driver stack\r
+\r
+typedef struct _IPOIB_ST_STAT\r
+{\r
+       IPOIB_ST_DRIVER         drv;\r
+       IPOIB_ST_DEVICE         dev[IPOIB_ST_MAX_DEVICES];\r
+       \r
+} IPOIB_ST_STAT, *PIPOIB_ST_STAT;\r
+\r
+extern IPOIB_ST_STAT g_stat;\r
+\r
+//\r
+// functions \r
+//\r
+\r
+void ipoib_st_dev_rmv( PIPOIB_ST_DEVICE p_stat );\r
+\r
+PIPOIB_ST_DEVICE ipoib_st_dev_add();\r
+\r
+void ipoib_st_init();\r
+\r
+\r
+\r