[TOOLS] add support to ConnectX HCAs
authorleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 14 Apr 2008 07:59:31 +0000 (07:59 +0000)
committerleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 14 Apr 2008 07:59:31 +0000 (07:59 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@1055 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

tools/mtcr/user/mtcr.c

index 77d58b4..4b2eeea 100644 (file)
 #define USB_DEV_NAME    "mtusb-1"\r
 #define CLEAR(st)               memset(&(st), 0, sizeof(st))\r
 \r
-\r
+// HERMON ORDERING WA:\r
+#define HERMON_WA_BASE 0xf0384 // SEM BASE ADDR. SEM 0xf0380 is reserved for external tools usage.\r
+#define HERMON_WA_SIZE 3       // Size in entries\r
 \r
 #define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL"\r
+\r
+/* \r
+ * DLL global variables \r
+ */\r
+#pragma data_seg(".sdata")\r
+\r
+/* Process / thread count */\r
+DWORD g_Slots[HERMON_WA_SIZE]; /* 3 slots */\r
+\r
 #ifdef DBG\r
 ULONG g_DebugLevel = DEBUG_LEVEL_MID;\r
 #else\r
 ULONG g_DebugLevel = DEBUG_LEVEL_LOW;\r
 #endif\r
 \r
+#pragma data_seg()\r
+\r
+\r
 //-----------------------------------------------------\r
 \r
 #define MAX_HCA_NUM 16\r
 \r
+// flags in below structure\r
+#define FIX_ORDERING_BUG       1\r
 \r
 typedef struct mfile_ibal_t {\r
     mfile            s;\r
     ib_al_handle_t   h_al;      \r
     ib_ca_handle_t   h_ca;\r
     map_crspace      cr_map;\r
+\r
+    int                  bugs;\r
+    int                  slot_num;\r
+    u_int32_t    hermon_wa_slot; /* apply hermon cr write workaround */\r
+    int                  hermon_wa_last_op_write;\r
+    u_int64_t    hermon_wa_max_retries;\r
+    u_int64_t    hermon_wa_num_of_writes;\r
+    u_int64_t    hermon_wa_num_of_retry_writes;\r
 } mfile_ibal;\r
 \r
 \r
+// Return slot_num or -1 when no free slots \r
+static int __get_slot()\r
+{\r
+       int i, prev_val;\r
+\r
+       for (i = 0; i < HERMON_WA_SIZE; ++i) {\r
+               prev_val = InterlockedCompareExchange(\r
+                       (LONG volatile* )&g_Slots[i], 0, 1 );\r
+               if ( prev_val ) \r
+                       return i;\r
+       }\r
+       return -1;\r
+}\r
+\r
+static void __put_slot(int slot_num)\r
+{\r
+       int prev_val;\r
+       prev_val = InterlockedExchange(\r
+               (LONG volatile* )&g_Slots[slot_num], 1 );\r
+       if ( !prev_val ) \r
+               DPRINT1(("Wrong semaphore %d value \n", slot_num));\r
+       \r
+}\r
+\r
 BOOL APIENTRY DllMain( HANDLE hModule,\r
                        DWORD  ul_reason_for_call,\r
                        LPVOID lpReserved\r
                                          )\r
 {\r
+    int i;\r
     char* pszDbgLevel;\r
     switch (ul_reason_for_call)\r
     {\r
@@ -59,6 +108,9 @@ BOOL APIENTRY DllMain( HANDLE hModule,
         if (pszDbgLevel) {\r
             g_DebugLevel = atol(pszDbgLevel);\r
         }\r
+               // set slot data\r
+               for (i = 0; i < HERMON_WA_SIZE; ++i)\r
+                       g_Slots[i] = 1;\r
 \r
         break;\r
         \r
@@ -85,13 +137,20 @@ BOOL APIENTRY DllMain( HANDLE hModule,
 #define SINAI_4X_CONF_DEV_ID                            24205\r
 #define SINAI_8X_DEV_ID                                 25204\r
 #define SINAI_8X_CONF_DEV_ID                            25205\r
+#define HERMON_SDR_DEV_ID                               25408\r
+#define HERMON_DDR_DEV_ID                               25418\r
+#define HERMON_QDR_DEV_ID                               25428\r
+#define HERMON_DDR_PCI5_DEV_ID                          26418\r
+#define HERMON_QDR_PCI5_DEV_ID                          26428\r
+#define HERMON_CONF_DEV_ID                              401\r
 \r
 #define IS_CONF_DEV(dev_id)     \\r
-        ((dev_id == TAVOR_CONF_DEV_ID)    || \\r
-         (dev_id == ARBEL_TM_CONF_DEV_ID) || \\r
-         (dev_id == ARBEL_CONF_DEV_ID)    || \\r
-         (dev_id == SINAI_4X_CONF_DEV_ID) || \\r
-         (dev_id == SINAI_8X_CONF_DEV_ID))\r
+               ((dev_id == TAVOR_CONF_DEV_ID)    || \\r
+               (dev_id == ARBEL_TM_CONF_DEV_ID) || \\r
+               (dev_id == ARBEL_CONF_DEV_ID)    || \\r
+               (dev_id == SINAI_4X_CONF_DEV_ID) || \\r
+               (dev_id == SINAI_8X_CONF_DEV_ID) || \\r
+               (dev_id == HERMON_CONF_DEV_ID))\r
         \r
 #define MAX_DEV_NAME            32\r
 typedef struct {\r
@@ -115,7 +174,15 @@ static DEVICE_DB_T db[] = {
         {       SINAI_4X_DEV_ID,        "InfiniHost_III_Lx",    MDEVS_TAVOR     },\r
         {       SINAI_4X_CONF_DEV_ID,   "InfiniHostBd",         MDEVS_TAVOR_CR  },\r
         {       SINAI_8X_DEV_ID,        "InfiniHost_III_Lx",    MDEVS_TAVOR     },\r
-        {       SINAI_8X_CONF_DEV_ID,   "InfiniHostBd",         MDEVS_TAVOR_CR  },\r
+        {       SINAI_8X_CONF_DEV_ID,  "InfiniHostBd",                 MDEVS_TAVOR_CR  },\r
+\r
+                       {       HERMON_SDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR  },\r
+                       {       HERMON_DDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR  },\r
+                       {       HERMON_QDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR  },\r
+                       {       HERMON_DDR_PCI5_DEV_ID, "ConnectX",             MDEVS_TAVOR_CR  },\r
+                       {       HERMON_QDR_PCI5_DEV_ID, "ConnectX",             MDEVS_TAVOR_CR  },\r
+                       \r
+                       {       HERMON_CONF_DEV_ID, "ConnectXBd",               MDEVS_TAVOR_CR  },\r
 };\r
 #define DEVICE_DB_SIZE  (sizeof(db) / sizeof(DEVICE_DB_T))\r
 \r
@@ -123,6 +190,14 @@ Mdevs dmasks[] = { MDEVS_TAVOR_CR, MDEVS_TAVOR_CR, MDEVS_TAVOR_UAR, MDEVS_TAVOR_
 char *dsuffix[] = { "conf", "_cr", "_uar", "_ddr"};\r
 #define MASKS_SIZE      (sizeof(dmasks) / sizeof(Mdevs))\r
 \r
+static int is_hermon(USHORT dev_id)\r
+{\r
+       return (dev_id == HERMON_SDR_DEV_ID) ||\r
+               (dev_id == HERMON_DDR_DEV_ID) ||\r
+               (dev_id == HERMON_QDR_DEV_ID) ||\r
+               (dev_id == HERMON_DDR_PCI5_DEV_ID) ||\r
+               (dev_id == HERMON_QDR_PCI5_DEV_ID);\r
+}\r
 \r
 // Return: < 0 - Error.   > 0 - Numbern of characters written (including last '\0')\r
 int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, int name_len, int *cnt)\r
@@ -160,7 +235,8 @@ int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name,
     }\r
     \r
     // names generation\r
-    for (j=0; j<MASKS_SIZE; j++) {\r\r
+    for (j=0; j<MASKS_SIZE; j++) {\r
+\r
         if ((db[i].mask & mask) & dmasks[j]) {\r
             char l_name[MAX_DEV_NAME]; int len; \r
             // create one name\r
@@ -430,6 +506,68 @@ int get_and_inc_dev_idx(u_int32_t* dev_idx_by_type, USHORT dev_id) {
 }\r
 \r
 \r
+\r
+//\r
+// Hermon PCI ordering bug work around (in kernel):\r
+//\r
+// Toggle bit is done using a HW semaphore. 3 slots (sems) are allocated for MST tools and\r
+// are dynamically allocated by the mst_pci.o module. 1 slot is reserved for external tools usage.\r
+// This means to more than 3 tools that use the PCI dev can run simultaniously.\r
+// \r
+// Open:\r
+//   If the wa is enabled ("MTCR_HERMON_WA" env variable does NOT exist or exist and equals 1):\r
+//   and HERMON-A0 dev id is read:\r
+//   get a vacant slot - ioctl( , PCI_HERMON_WA ,);\r
+//   read the sem in the slot (to lock it).\r
+//\r
+//\r
+// Flush (hermon_wa_write_confirm_sem):\r
+//   If enabled on first read after write:\r
+//   write 0 to sem.\r
+//   read sem untlill sem = 0;\r
+//\r
+// Close/release\r
+//   Zero the slot.\r
+//\r
+\r
+int hermon_wa_write_confirm_sem(mfile* mf, u_int32_t addr) {\r
+    u_int64_t max_retries = 0x1000000;\r
+    u_int64_t retries = 0;\r
+    mfile_ibal *mfi = (mfile_ibal *)mf;\r
+\r
+    mfi->hermon_wa_num_of_writes++;\r
+\r
+    // clear sem\r
+    *((u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)) = 0;\r
+\r
+    while (retries < max_retries) {\r
+         u_int32_t slot_value ;\r
+\r
+         slot_value = __be32_to_cpu(*((volatile u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)));\r
+         if (slot_value == 0) {\r
+\r
+             // Good - The written value was read\r
+             if (retries) {\r
+                 // NOT REALLY NEEDED - just to collect statistics\r
+\r
+                 // printf("-D- Hermon WA addr %06x took %d retries\n", addr, retries);\r
+                 mfi->hermon_wa_num_of_retry_writes++;\r
+                 if (retries > mfi->hermon_wa_max_retries) {\r
+                     mfi->hermon_wa_max_retries = retries;\r
+                 }\r
+             }\r
+             return 4;\r
+         } else {\r
+             // sem is still locked from previous read - read passed the write\r
+             retries++;\r
+         }\r
+     }\r
+\r
+     DPRINT1(("-D- Hermon WA addr %06x failed after %ld retries\n", addr, retries));\r
+     return 0;\r
+}  \r
+\r
+\r
 //\r
 //\r
 // List devices in their MST compatible names.\r
@@ -628,7 +766,7 @@ MTCR_API mfile *mopend(const char *name, DType dtype)
             }\r
         } else {\r
                     \r
-            int bar_num;\r
+            int bar_num, slot_num;\r
 \r
             // Type of file \r
             mf->s.tp = MST_PCI;\r
@@ -648,8 +786,36 @@ MTCR_API mfile *mopend(const char *name, DType dtype)
             }\r
 \r
             mf->s.ptr = (void*)(ULONG_PTR)mf->cr_map.va;\r
-       }\r
-           \r
+\r
+                       // check whether we need to perform the workaround\r
+                       if (is_hermon(dev_id)) {\r
+                               char* hermon_wa_env = getenv("MTCR_HERMON_WA");\r
+                               // if there is no env variable or it = "0" - we perform the workaround\r
+                               if (hermon_wa_env == NULL || strcmp(hermon_wa_env, "0")) {\r
+                                       // perform the workaround only for A0 revision\r
+                                       if (access_type == MDEVS_TAVOR_CR) {\r
+                                               u_int32_t rev = 0;\r
+                                               mread4((mfile*)mf, 0xf0014, &rev);\r
+                                               if (rev == 0xa00190)  { // TODO: Should I ignore REV here ?\r
+                                                       slot_num = __get_slot();\r
+                                                       if (slot_num < 0) // no free slot \r
+                                                               goto ErrExit;\r
+                                                       mf->bugs |= FIX_ORDERING_BUG;\r
+                                                       mf->slot_num = slot_num;\r
+                                               }\r
+                                       }\r
+                               }\r
+                       }\r
+                       \r
+                       // prepare for the workaround\r
+                       if (mf->bugs & FIX_ORDERING_BUG) {      // Hermon WA setup\r
+                               DPRINT1(("-D- Hermon WA setup\n"));\r
+                               mf->hermon_wa_slot = HERMON_WA_BASE + 4*slot_num;\r
+                               hermon_wa_write_confirm_sem((mfile*)mf, mf->hermon_wa_slot);\r
+                       }\r
+\r
+               }\r
+                          \r
     } else if (dev_id == DEVASYS_DEV_ID) {\r
         // Type of file \r
         h = usb_open();\r
@@ -701,6 +867,16 @@ MTCR_API int mclose(mfile *mf)
         if (mf->tp == MST_PCICONF) {\r
             ibal_access(mfi->h_ca, 0x0, &stub, 4, FW_CLOSE_IF );\r
         } else if (mf->tp = MST_PCI) {\r
+\r
+            if (mfi->bugs & FIX_ORDERING_BUG) {\r
+                       __put_slot( mfi->slot_num );\r
+                    DPRINT1(("-D- Hermon WA stats:\n"));\r
+                    DPRINT1(("-D- : num of write flushes: %8ld\n", mfi->hermon_wa_num_of_writes));\r
+                    DPRINT1(("-D- : num of retry flushes: %8ld\n", mfi->hermon_wa_num_of_retry_writes));\r
+                    DPRINT1(("-D- : max_retries:          %8ld\n", mfi->hermon_wa_max_retries));\r
+            }\r
+\r
+\r
             if (mfi->cr_map.size) {\r
                 \r
                 if (ibal_access(mfi->h_ca, 0x0, NULL, 0, FW_UNMAP_CRSPACE ) != IB_SUCCESS) {\r
@@ -1029,3 +1205,4 @@ unsigned char mset_i2c_slave(mfile *mf, unsigned char new_i2c_slave)
     return ret;\r
 }\r
 \r
+\r