[HW] Remove FUNC_PTR64.
authorleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 13 Jul 2008 10:24:27 +0000 (10:24 +0000)
committerleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 13 Jul 2008 10:24:27 +0000 (10:24 +0000)
Signed-off-by: Fab Tillier <ftillier@microsoft.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@1379 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

25 files changed:
hw/mlx4/kernel/hca/data.h
hw/mlx4/kernel/hca/fw.c
hw/mlx4/user/hca/cq.c
hw/mlx4/user/hca/mlx4.h
hw/mlx4/user/hca/qp.c
hw/mlx4/user/hca/srq.c
hw/mthca/kernel/hca_data.h
hw/mthca/kernel/hca_direct.c
hw/mthca/kernel/hca_driver.c
hw/mthca/kernel/hca_mcast.c
hw/mthca/kernel/hca_memory.c
hw/mthca/kernel/hca_verbs.c
hw/mthca/kernel/ib_verbs.h
hw/mthca/kernel/mt_verbs.c
hw/mthca/kernel/mthca_provider.c
hw/mthca/user/mlnx_ual_av.c
hw/mthca/user/mlnx_ual_ca.c
hw/mthca/user/mlnx_ual_cq.c
hw/mthca/user/mlnx_ual_mcast.c
hw/mthca/user/mlnx_ual_mrw.c
hw/mthca/user/mlnx_ual_osbypass.c
hw/mthca/user/mlnx_ual_pd.c
hw/mthca/user/mlnx_ual_qp.c
hw/mthca/user/mlnx_ual_srq.c
inc/iba/ib_types.h

index 9816a0c..6f8e79e 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -221,8 +222,8 @@ mlnx_local_mad (
 \r
 ib_api_status_t\r
 fw_access_ctrl(\r
-       IN              const   void* FUNC_PTR64                                context,\r
-       IN              const   void* FUNC_PTR64 * const                handle_array    OPTIONAL,\r
+       IN              const   void*                                           context,\r
+       IN              const   void** const                            handle_array    OPTIONAL,\r
        IN                              uint32_t                                        num_handles,\r
        IN                              ib_ci_op_t* const                       p_ci_op,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf              OPTIONAL);\r
index 470b323..f242644 100644 (file)
-/*
- * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.
- * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 
- *
- * This software is available to you under the OpenIB.org BSD license
- * below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * $Id: al.c 1611 2006-08-20 14:48:55Z sleybo $
- */
-
-#include "precomp.h"
-
-#if defined(EVENT_TRACING)
-#ifdef offsetof
-#undef offsetof
-#endif
-#include "fw.tmh"
-#endif
-
-
-/***********************************
-Firmware Update definitions
-***********************************/
-#define PCI_CONF_ADDR  (0x00000058)
-#define PCI_CONF_DATA  (0x0000005c)
-#define FLASH_OFFSET   (0x000f01a4)
-#define READ_BIT               (1<<29)
-#define WRITE_BIT              (2<<29)
-#define ADDR_MSK               (0x0007ffff)
-#define CMD_MASK               (0xe0000000)
-#define BANK_SHIFT             (19)
-#define BANK_MASK              (0xfff80000)
-#define MAX_FLASH_SIZE (0x80000) // 512K
-
-#define SEMAP63                                (0xf03fc)
-#define GPIO_DIR_L                     (0xf008c)
-#define GPIO_POL_L                     (0xf0094)
-#define GPIO_MOD_L                     (0xf009c)
-#define GPIO_DAT_L                     (0xf0084)
-#define GPIO_DATACLEAR_L       (0xf00d4)
-#define GPIO_DATASET_L         (0xf00dc)
-
-#define CPUMODE                                (0xf0150)
-#define CPUMODE_MSK                    (0xc0000000UL)
-#define CPUMODE_SHIFT          (30)
-
-/* Definitions intended to become shared with UM. Later... */
-#define FW_READ                        0x00
-#define FW_WRITE               0x01
-#define FW_READ_CMD            0x08
-#define FW_WRITE_CMD   0x09
-#define FW_OPEN_IF             0xe7
-#define FW_CLOSE_IF            0x7e
-
-#define FW_SIGNATURE           (0x5a445a44)
-#define FW_SECT_SIZE           (0x10000)
-
-typedef struct Primary_Sector{
-       uint32_t fi_addr;
-       uint32_t fi_size;
-       uint32_t signature;
-       uint32_t fw_reserved[5];
-       uint32_t vsd[56];
-       uint32_t branch_to;
-       uint32_t crc016;
-} primary_sector_t;
-
-static uint32_t old_dir;
-static uint32_t old_pol;
-static uint32_t old_mod;
-static uint32_t old_dat;
-
-
-static NTSTATUS
-fw_access_pciconf (
-               IN              BUS_INTERFACE_STANDARD          *p_BusInterface,
-               IN              ULONG                                           op_flag,
-               IN              PVOID                                           p_buffer,
-               IN              ULONG                                           offset,
-               IN              ULONG POINTER_ALIGNMENT         length )
-{
-
-       ULONG                           bytes;  
-       NTSTATUS                        status = STATUS_SUCCESS;
-
-       PAGED_CODE();
-
-       if( !p_buffer )
-               return STATUS_INVALID_PARAMETER;
-
-       if (p_BusInterface)
-       {
-
-               bytes = p_BusInterface->SetBusData(
-                                               p_BusInterface->Context,
-                                               PCI_WHICHSPACE_CONFIG,
-                                               (PVOID)&offset,
-                                               PCI_CONF_ADDR,
-                                               sizeof(ULONG) );
-
-               if( op_flag == 0 )
-               {
-                       if ( bytes )
-                               bytes = p_BusInterface->GetBusData(
-                                                       p_BusInterface->Context,
-                                                       PCI_WHICHSPACE_CONFIG,
-                                                       p_buffer,
-                                                       PCI_CONF_DATA,
-                                                       length );
-                       if ( !bytes )
-                               status = STATUS_NOT_SUPPORTED;
-               }
-
-               else
-               {
-                       if ( bytes )
-                               bytes = p_BusInterface->SetBusData(
-                                                       p_BusInterface->Context,
-                                                       PCI_WHICHSPACE_CONFIG,
-                                                       p_buffer,
-                                                       PCI_CONF_DATA,
-                                                       length);
-
-                       if ( !bytes )
-                               status = STATUS_NOT_SUPPORTED;
-               }
-       }
-       return status;
-}
-
-
-static NTSTATUS
-__map_crspace(
-       IN              struct ib_ucontext *            p_uctx,
-       IN              PVOID                                           p_buf,
-       IN              ULONG                                           buf_size
-       )
-{
-       NTSTATUS status;
-       PMDL p_mdl;
-       PVOID ua, ka;
-       ULONG sz;
-       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;
-       map_crspace *p_res = (map_crspace *)p_buf;
-       struct pci_dev *p_pdev = p_fdo->bus_ib_ifc.pdev;
-
-       HCA_ENTER( HCA_DBG_PNP );
-
-       // sanity checks
-       if ( buf_size < sizeof *p_res || !p_buf ) {
-               status = STATUS_INVALID_PARAMETER;
-               goto err_invalid_params;
-       }
-
-       // map memory
-       sz =(ULONG)p_pdev->bar[HCA_BAR_TYPE_HCR].size;
-       if (!p_pdev->bar[HCA_BAR_TYPE_HCR].virt) {
-               PHYSICAL_ADDRESS pa;
-               pa.QuadPart = p_pdev->bar[HCA_BAR_TYPE_HCR].phys;
-               ka = MmMapIoSpace( pa, sz, MmNonCached ); 
-               if ( ka == NULL) {
-                       HCA_PRINT(TRACE_LEVEL_ERROR  , HCA_DBG_SHIM,
-                               ("No kernel mapping of CR space.\n") );
-                       status = STATUS_INSUFFICIENT_RESOURCES;
-                       goto err_map_to_kernel;
-               }
-               p_pdev->bar[HCA_BAR_TYPE_HCR].virt = ka;
-       }
-       ka = p_pdev->bar[HCA_BAR_TYPE_HCR].virt;
-
-       // prepare for mapping to user space 
-       p_mdl = IoAllocateMdl( ka, sz, FALSE,FALSE,NULL);
-       if (p_mdl == NULL) {
-               HCA_PRINT(TRACE_LEVEL_ERROR  , HCA_DBG_SHIM, 
-                       ("IoAllocateMdl failed.\n") );
-               status = STATUS_INSUFFICIENT_RESOURCES;
-               goto err_alloc_mdl;
-       }
-
-       // fill MDL
-       MmBuildMdlForNonPagedPool(p_mdl);
-       
-       // map the buffer into user space 
-       __try
-       {
-               ua = MmMapLockedPagesSpecifyCache( p_mdl, UserMode, MmNonCached,
-                       NULL, FALSE, NormalPagePriority );
-       }
-       __except(EXCEPTION_EXECUTE_HANDLER)
-       {
-               HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM,
-                       ("MmMapLockedPagesSpecifyCache failed.\n") );
-               status = STATUS_INSUFFICIENT_RESOURCES;
-               goto err_map_to_user;
-       }
-       
-       // fill the results
-       p_res->va = (uint64_t)(ULONG_PTR)ua;
-       p_res->size = sz;
-
-       // resource tracking
-       p_uctx->x.p_mdl = p_mdl;
-       p_uctx->x.va = ua;
-
-#if 0  
-       HCA_PRINT(TRACE_LEVEL_INFORMATION, HCA_DBG_SHIM,
-               ("MTHCA: __map_crspace succeeded with .ka %I64x, size %I64x va %I64x, size %x, pa %I64x \n",
-               p_pdev->bar[HCA_BAR_TYPE_HCR].virt, p_pdev->bar[HCA_BAR_TYPE_HCR].size, 
-               p_res->va, p_res->size, p_pdev->bar[HCA_BAR_TYPE_HCR].phys ));
-#endif
-       status = STATUS_SUCCESS;
-       goto out;
-
-err_map_to_user:
-       IoFreeMdl( p_mdl );
-err_alloc_mdl:
-err_map_to_kernel:
-err_invalid_params:    
-out:   
-       HCA_EXIT( HCA_DBG_PNP );
-       return status;
-}
-
-
-static void
-__unmap_crspace(
-       IN              struct ib_ucontext *                    p_uctx
-       )
-{
-       HCA_ENTER( HCA_DBG_PNP );
-
-       if (p_uctx->x.va && p_uctx->x.p_mdl) {
-               MmUnmapLockedPages(p_uctx->x.va, p_uctx->x.p_mdl);
-               IoFreeMdl( p_uctx->x.p_mdl );
-               p_uctx->x.va = p_uctx->x.p_mdl = NULL;
-               //NB: the unmap of IO space is being done in __UnmapHcaMemoryResources
-       }
-
-       HCA_EXIT( HCA_DBG_PNP );
-}
-
-static void
-__open_fw_access(
-       IN                              struct ib_ucontext*                     p_uctx,
-       IN                              PBUS_INTERFACE_STANDARD         p_bus_interface )
-{
-       if( !p_uctx->x.fw_if_open )
-       {
-               p_bus_interface->InterfaceReference( p_bus_interface->Context );
-               p_uctx->x.fw_if_open = TRUE;
-       }
-}
-
-static void 
-__close_fw_access(
-       IN              struct ib_ucontext *    p_uctx,
-       IN              PBUS_INTERFACE_STANDARD p_bus_interface
-       )
-{
-       if (p_uctx->x.fw_if_open ) {
-               p_bus_interface->InterfaceDereference((PVOID)p_bus_interface->Context);
-               p_uctx->x.fw_if_open = FALSE;
-       }
-}
-
-
-void
-unmap_crspace_for_all( struct ib_ucontext *p_uctx )
-{
-       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;
-       PBUS_INTERFACE_STANDARD    p_bus_interface = &p_fdo->bus_pci_ifc;
-
-       HCA_ENTER( HCA_DBG_PNP );
-
-       mutex_lock( &p_uctx->x.mutex );
-       __unmap_crspace( p_uctx);
-       __close_fw_access(p_uctx, p_bus_interface);
-       mutex_unlock( &p_uctx->x.mutex );
-
-       HCA_EXIT( HCA_DBG_PNP );
-}
-
-static NTSTATUS
-fw_flash_write_data (
-               IN              BUS_INTERFACE_STANDARD                  *p_BusInterface,
-               IN              PVOID                                                   p_buffer,
-               IN              ULONG                                                   offset,
-               IN              ULONG POINTER_ALIGNMENT                 length )
-{
-       NTSTATUS                status;
-       uint32_t                cnt = 0;
-       uint32_t                lcl_data;
-
-       if (!length)
-               return IB_INVALID_PARAMETER;
-       
-       lcl_data = (*((uint32_t*)p_buffer) << 24);
-
-       status = fw_access_pciconf(p_BusInterface, FW_WRITE , &lcl_data, FLASH_OFFSET+4, length );
-       if ( status != STATUS_SUCCESS )
-               return status;
-       lcl_data = ( WRITE_BIT | (offset & ADDR_MSK));
-               
-       status = fw_access_pciconf(p_BusInterface, FW_WRITE , &lcl_data, FLASH_OFFSET, 4 );
-       if ( status != STATUS_SUCCESS )
-       return status;
-
-       lcl_data = 0;
-       
-       do
-       {
-               if (++cnt > 5000)
-               {
-                       return STATUS_DEVICE_NOT_READY;
-               }
-
-               status = fw_access_pciconf(p_BusInterface, FW_READ , &lcl_data, FLASH_OFFSET, 4 );
-               if ( status != STATUS_SUCCESS )
-               return status;
-
-       } while(lcl_data & CMD_MASK);
-
-       return status;
-}
-
-
-static NTSTATUS
-fw_flash_read_data (
-               IN              BUS_INTERFACE_STANDARD                  *p_BusInterface,
-               IN              PVOID                                                   p_buffer,
-               IN              ULONG                                                   offset,
-               IN              ULONG POINTER_ALIGNMENT                 length )
-{
-       NTSTATUS        status = STATUS_SUCCESS;
-       uint32_t        cnt = 0;
-       uint32_t        lcl_data = ( READ_BIT | (offset & ADDR_MSK));
-
-       if (!length)
-               return IB_INVALID_PARAMETER;
-       
-       status = fw_access_pciconf(p_BusInterface, FW_WRITE, &lcl_data, FLASH_OFFSET, 4 );
-       if ( status != STATUS_SUCCESS )
-               return status;
-
-       lcl_data = 0;
-       do
-       {
-               // Timeout checks
-               if (++cnt > 5000 )
-               {
-                       return STATUS_DEVICE_NOT_READY;
-       }
-
-               status = fw_access_pciconf(p_BusInterface, FW_READ, &lcl_data, FLASH_OFFSET, 4 );
-       
-               if ( status != STATUS_SUCCESS )
-                       return status;
-
-       } while(lcl_data & CMD_MASK);
-
-       status = fw_access_pciconf(p_BusInterface, FW_READ, p_buffer, FLASH_OFFSET+4, length );
-       return status;
-}
-
-ib_api_status_t
-fw_access_ctrl(
-       IN              const   ib_ca_handle_t                          h_ca,
-       IN              const   void* FUNC_PTR64* const         handle_array    OPTIONAL,
-       IN                              uint32_t                                        num_handles,
-       IN                              ib_ci_op_t* const                       p_ci_op,
-       IN      OUT                     ci_umv_buf_t                            *p_umv_buf )
-{
-       NTSTATUS                                        status = STATUS_SUCCESS;
-       PVOID                                           p_data;
-       ULONG                                           offset;
-       ULONG POINTER_ALIGNMENT         length;
-       struct ib_ucontext *p_uctx = (struct ib_ucontext *)h_ca;
-       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;
-       PBUS_INTERFACE_STANDARD         p_bus_interface = &p_fdo->bus_pci_ifc;
-
-       UNREFERENCED_PARAMETER(handle_array);
-       UNREFERENCED_PARAMETER(num_handles);
-
-       if( !p_umv_buf )
-               return IB_UNSUPPORTED;
-
-       if ( !p_ci_op )
-               return IB_INVALID_PARAMETER;
-
-       length = p_ci_op->buf_size;
-       offset = p_ci_op->buf_info;
-       p_data = p_ci_op->p_buf;
-
-       mutex_lock( &p_uctx->x.mutex );
-
-       switch ( p_ci_op->command )
-       {
-       case FW_MAP_CRSPACE:
-               status = __map_crspace(p_uctx, p_data, length);
-               break;
-               
-       case FW_UNMAP_CRSPACE:
-               __unmap_crspace(p_uctx);
-               break;
-                               
-       case FW_OPEN_IF: // open BusInterface
-               if (p_fdo->bus_pci_ifc_taken)
-                       __open_fw_access( p_uctx, p_bus_interface );
-               break;
-
-       case FW_READ: // read data from flash
-               if ( p_uctx->x.fw_if_open )
-                       status = fw_flash_read_data(p_bus_interface, p_data, offset, length);
-               break;
-
-       case FW_WRITE: // write data to flash
-               if ( p_uctx->x.fw_if_open )
-                       status = fw_flash_write_data(p_bus_interface, p_data, offset, length);
-               break;
-
-       case FW_READ_CMD:
-               if ( p_uctx->x.fw_if_open )
-                       status = fw_access_pciconf(p_bus_interface, 0 , p_data, offset, 4);
-               break;
-
-       case FW_WRITE_CMD:
-               if ( p_uctx->x.fw_if_open )
-                       status = fw_access_pciconf(p_bus_interface, 1 , p_data, offset, 4);
-               break;
-
-       case FW_CLOSE_IF: // close BusInterface
-               __close_fw_access(p_uctx, p_bus_interface);
-               break;
-
-       default:
-               status = STATUS_INVALID_DEVICE_REQUEST;
-       }
-
-       if ( status != STATUS_SUCCESS ) {
-               __close_fw_access(p_uctx, p_bus_interface);
-               HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_INIT, 
-                       ("fw_access_ctrl failed, ntstatus: %08x.\n", status));
-       }
-
-       mutex_unlock( &p_uctx->x.mutex );
-
-       switch( status ) {
-               case STATUS_SUCCESS:                                    return IB_SUCCESS;
-               case STATUS_INVALID_DEVICE_REQUEST:     return IB_UNSUPPORTED;
-               case STATUS_INSUFFICIENT_RESOURCES:     return IB_INSUFFICIENT_RESOURCES;
-               default:                                                                        return IB_ERROR;
-       }
-}
-
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id: al.c 1611 2006-08-20 14:48:55Z sleybo $\r
+ */\r
+\r
+#include "precomp.h"\r
+\r
+#if defined(EVENT_TRACING)\r
+#ifdef offsetof\r
+#undef offsetof\r
+#endif\r
+#include "fw.tmh"\r
+#endif\r
+\r
+\r
+/***********************************\r
+Firmware Update definitions\r
+***********************************/\r
+#define PCI_CONF_ADDR  (0x00000058)\r
+#define PCI_CONF_DATA  (0x0000005c)\r
+#define FLASH_OFFSET   (0x000f01a4)\r
+#define READ_BIT               (1<<29)\r
+#define WRITE_BIT              (2<<29)\r
+#define ADDR_MSK               (0x0007ffff)\r
+#define CMD_MASK               (0xe0000000)\r
+#define BANK_SHIFT             (19)\r
+#define BANK_MASK              (0xfff80000)\r
+#define MAX_FLASH_SIZE (0x80000) // 512K\r
+\r
+#define SEMAP63                                (0xf03fc)\r
+#define GPIO_DIR_L                     (0xf008c)\r
+#define GPIO_POL_L                     (0xf0094)\r
+#define GPIO_MOD_L                     (0xf009c)\r
+#define GPIO_DAT_L                     (0xf0084)\r
+#define GPIO_DATACLEAR_L       (0xf00d4)\r
+#define GPIO_DATASET_L         (0xf00dc)\r
+\r
+#define CPUMODE                                (0xf0150)\r
+#define CPUMODE_MSK                    (0xc0000000UL)\r
+#define CPUMODE_SHIFT          (30)\r
+\r
+/* Definitions intended to become shared with UM. Later... */\r
+#define FW_READ                        0x00\r
+#define FW_WRITE               0x01\r
+#define FW_READ_CMD            0x08\r
+#define FW_WRITE_CMD   0x09\r
+#define FW_OPEN_IF             0xe7\r
+#define FW_CLOSE_IF            0x7e\r
+\r
+#define FW_SIGNATURE           (0x5a445a44)\r
+#define FW_SECT_SIZE           (0x10000)\r
+\r
+typedef struct Primary_Sector{\r
+       uint32_t fi_addr;\r
+       uint32_t fi_size;\r
+       uint32_t signature;\r
+       uint32_t fw_reserved[5];\r
+       uint32_t vsd[56];\r
+       uint32_t branch_to;\r
+       uint32_t crc016;\r
+} primary_sector_t;\r
+\r
+static uint32_t old_dir;\r
+static uint32_t old_pol;\r
+static uint32_t old_mod;\r
+static uint32_t old_dat;\r
+\r
+\r
+static NTSTATUS\r
+fw_access_pciconf (\r
+               IN              BUS_INTERFACE_STANDARD          *p_BusInterface,\r
+               IN              ULONG                                           op_flag,\r
+               IN              PVOID                                           p_buffer,\r
+               IN              ULONG                                           offset,\r
+               IN              ULONG POINTER_ALIGNMENT         length )\r
+{\r
+\r
+       ULONG                           bytes;  \r
+       NTSTATUS                        status = STATUS_SUCCESS;\r
+\r
+       PAGED_CODE();\r
+\r
+       if( !p_buffer )\r
+               return STATUS_INVALID_PARAMETER;\r
+\r
+       if (p_BusInterface)\r
+       {\r
+\r
+               bytes = p_BusInterface->SetBusData(\r
+                                               p_BusInterface->Context,\r
+                                               PCI_WHICHSPACE_CONFIG,\r
+                                               (PVOID)&offset,\r
+                                               PCI_CONF_ADDR,\r
+                                               sizeof(ULONG) );\r
+\r
+               if( op_flag == 0 )\r
+               {\r
+                       if ( bytes )\r
+                               bytes = p_BusInterface->GetBusData(\r
+                                                       p_BusInterface->Context,\r
+                                                       PCI_WHICHSPACE_CONFIG,\r
+                                                       p_buffer,\r
+                                                       PCI_CONF_DATA,\r
+                                                       length );\r
+                       if ( !bytes )\r
+                               status = STATUS_NOT_SUPPORTED;\r
+               }\r
+\r
+               else\r
+               {\r
+                       if ( bytes )\r
+                               bytes = p_BusInterface->SetBusData(\r
+                                                       p_BusInterface->Context,\r
+                                                       PCI_WHICHSPACE_CONFIG,\r
+                                                       p_buffer,\r
+                                                       PCI_CONF_DATA,\r
+                                                       length);\r
+\r
+                       if ( !bytes )\r
+                               status = STATUS_NOT_SUPPORTED;\r
+               }\r
+       }\r
+       return status;\r
+}\r
+\r
+\r
+static NTSTATUS\r
+__map_crspace(\r
+       IN              struct ib_ucontext *            p_uctx,\r
+       IN              PVOID                                           p_buf,\r
+       IN              ULONG                                           buf_size\r
+       )\r
+{\r
+       NTSTATUS status;\r
+       PMDL p_mdl;\r
+       PVOID ua, ka;\r
+       ULONG sz;\r
+       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;\r
+       map_crspace *p_res = (map_crspace *)p_buf;\r
+       struct pci_dev *p_pdev = p_fdo->bus_ib_ifc.pdev;\r
+\r
+       HCA_ENTER( HCA_DBG_PNP );\r
+\r
+       // sanity checks\r
+       if ( buf_size < sizeof *p_res || !p_buf ) {\r
+               status = STATUS_INVALID_PARAMETER;\r
+               goto err_invalid_params;\r
+       }\r
+\r
+       // map memory\r
+       sz =(ULONG)p_pdev->bar[HCA_BAR_TYPE_HCR].size;\r
+       if (!p_pdev->bar[HCA_BAR_TYPE_HCR].virt) {\r
+               PHYSICAL_ADDRESS pa;\r
+               pa.QuadPart = p_pdev->bar[HCA_BAR_TYPE_HCR].phys;\r
+               ka = MmMapIoSpace( pa, sz, MmNonCached ); \r
+               if ( ka == NULL) {\r
+                       HCA_PRINT(TRACE_LEVEL_ERROR  , HCA_DBG_SHIM,\r
+                               ("No kernel mapping of CR space.\n") );\r
+                       status = STATUS_INSUFFICIENT_RESOURCES;\r
+                       goto err_map_to_kernel;\r
+               }\r
+               p_pdev->bar[HCA_BAR_TYPE_HCR].virt = ka;\r
+       }\r
+       ka = p_pdev->bar[HCA_BAR_TYPE_HCR].virt;\r
+\r
+       // prepare for mapping to user space \r
+       p_mdl = IoAllocateMdl( ka, sz, FALSE,FALSE,NULL);\r
+       if (p_mdl == NULL) {\r
+               HCA_PRINT(TRACE_LEVEL_ERROR  , HCA_DBG_SHIM, \r
+                       ("IoAllocateMdl failed.\n") );\r
+               status = STATUS_INSUFFICIENT_RESOURCES;\r
+               goto err_alloc_mdl;\r
+       }\r
+\r
+       // fill MDL\r
+       MmBuildMdlForNonPagedPool(p_mdl);\r
+       \r
+       // map the buffer into user space \r
+       __try\r
+       {\r
+               ua = MmMapLockedPagesSpecifyCache( p_mdl, UserMode, MmNonCached,\r
+                       NULL, FALSE, NormalPagePriority );\r
+       }\r
+       __except(EXCEPTION_EXECUTE_HANDLER)\r
+       {\r
+               HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM,\r
+                       ("MmMapLockedPagesSpecifyCache failed.\n") );\r
+               status = STATUS_INSUFFICIENT_RESOURCES;\r
+               goto err_map_to_user;\r
+       }\r
+       \r
+       // fill the results\r
+       p_res->va = (uint64_t)(ULONG_PTR)ua;\r
+       p_res->size = sz;\r
+\r
+       // resource tracking\r
+       p_uctx->x.p_mdl = p_mdl;\r
+       p_uctx->x.va = ua;\r
+\r
+#if 0  \r
+       HCA_PRINT(TRACE_LEVEL_INFORMATION, HCA_DBG_SHIM,\r
+               ("MTHCA: __map_crspace succeeded with .ka %I64x, size %I64x va %I64x, size %x, pa %I64x \n",\r
+               p_pdev->bar[HCA_BAR_TYPE_HCR].virt, p_pdev->bar[HCA_BAR_TYPE_HCR].size, \r
+               p_res->va, p_res->size, p_pdev->bar[HCA_BAR_TYPE_HCR].phys ));\r
+#endif\r
+       status = STATUS_SUCCESS;\r
+       goto out;\r
+\r
+err_map_to_user:\r
+       IoFreeMdl( p_mdl );\r
+err_alloc_mdl:\r
+err_map_to_kernel:\r
+err_invalid_params:    \r
+out:   \r
+       HCA_EXIT( HCA_DBG_PNP );\r
+       return status;\r
+}\r
+\r
+\r
+static void\r
+__unmap_crspace(\r
+       IN              struct ib_ucontext *                    p_uctx\r
+       )\r
+{\r
+       HCA_ENTER( HCA_DBG_PNP );\r
+\r
+       if (p_uctx->x.va && p_uctx->x.p_mdl) {\r
+               MmUnmapLockedPages(p_uctx->x.va, p_uctx->x.p_mdl);\r
+               IoFreeMdl( p_uctx->x.p_mdl );\r
+               p_uctx->x.va = p_uctx->x.p_mdl = NULL;\r
+               //NB: the unmap of IO space is being done in __UnmapHcaMemoryResources\r
+       }\r
+\r
+       HCA_EXIT( HCA_DBG_PNP );\r
+}\r
+\r
+static void\r
+__open_fw_access(\r
+       IN                              struct ib_ucontext*                     p_uctx,\r
+       IN                              PBUS_INTERFACE_STANDARD         p_bus_interface )\r
+{\r
+       if( !p_uctx->x.fw_if_open )\r
+       {\r
+               p_bus_interface->InterfaceReference( p_bus_interface->Context );\r
+               p_uctx->x.fw_if_open = TRUE;\r
+       }\r
+}\r
+\r
+static void \r
+__close_fw_access(\r
+       IN              struct ib_ucontext *    p_uctx,\r
+       IN              PBUS_INTERFACE_STANDARD p_bus_interface\r
+       )\r
+{\r
+       if (p_uctx->x.fw_if_open ) {\r
+               p_bus_interface->InterfaceDereference((PVOID)p_bus_interface->Context);\r
+               p_uctx->x.fw_if_open = FALSE;\r
+       }\r
+}\r
+\r
+\r
+void\r
+unmap_crspace_for_all( struct ib_ucontext *p_uctx )\r
+{\r
+       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;\r
+       PBUS_INTERFACE_STANDARD    p_bus_interface = &p_fdo->bus_pci_ifc;\r
+\r
+       HCA_ENTER( HCA_DBG_PNP );\r
+\r
+       mutex_lock( &p_uctx->x.mutex );\r
+       __unmap_crspace( p_uctx);\r
+       __close_fw_access(p_uctx, p_bus_interface);\r
+       mutex_unlock( &p_uctx->x.mutex );\r
+\r
+       HCA_EXIT( HCA_DBG_PNP );\r
+}\r
+\r
+static NTSTATUS\r
+fw_flash_write_data (\r
+               IN              BUS_INTERFACE_STANDARD                  *p_BusInterface,\r
+               IN              PVOID                                                   p_buffer,\r
+               IN              ULONG                                                   offset,\r
+               IN              ULONG POINTER_ALIGNMENT                 length )\r
+{\r
+       NTSTATUS                status;\r
+       uint32_t                cnt = 0;\r
+       uint32_t                lcl_data;\r
+\r
+       if (!length)\r
+               return IB_INVALID_PARAMETER;\r
+       \r
+       lcl_data = (*((uint32_t*)p_buffer) << 24);\r
+\r
+       status = fw_access_pciconf(p_BusInterface, FW_WRITE , &lcl_data, FLASH_OFFSET+4, length );\r
+       if ( status != STATUS_SUCCESS )\r
+               return status;\r
+       lcl_data = ( WRITE_BIT | (offset & ADDR_MSK));\r
+               \r
+       status = fw_access_pciconf(p_BusInterface, FW_WRITE , &lcl_data, FLASH_OFFSET, 4 );\r
+       if ( status != STATUS_SUCCESS )\r
+       return status;\r
+\r
+       lcl_data = 0;\r
+       \r
+       do\r
+       {\r
+               if (++cnt > 5000)\r
+               {\r
+                       return STATUS_DEVICE_NOT_READY;\r
+               }\r
+\r
+               status = fw_access_pciconf(p_BusInterface, FW_READ , &lcl_data, FLASH_OFFSET, 4 );\r
+               if ( status != STATUS_SUCCESS )\r
+               return status;\r
+\r
+       } while(lcl_data & CMD_MASK);\r
+\r
+       return status;\r
+}\r
+\r
+\r
+static NTSTATUS\r
+fw_flash_read_data (\r
+               IN              BUS_INTERFACE_STANDARD                  *p_BusInterface,\r
+               IN              PVOID                                                   p_buffer,\r
+               IN              ULONG                                                   offset,\r
+               IN              ULONG POINTER_ALIGNMENT                 length )\r
+{\r
+       NTSTATUS        status = STATUS_SUCCESS;\r
+       uint32_t        cnt = 0;\r
+       uint32_t        lcl_data = ( READ_BIT | (offset & ADDR_MSK));\r
+\r
+       if (!length)\r
+               return IB_INVALID_PARAMETER;\r
+       \r
+       status = fw_access_pciconf(p_BusInterface, FW_WRITE, &lcl_data, FLASH_OFFSET, 4 );\r
+       if ( status != STATUS_SUCCESS )\r
+               return status;\r
+\r
+       lcl_data = 0;\r
+       do\r
+       {\r
+               // Timeout checks\r
+               if (++cnt > 5000 )\r
+               {\r
+                       return STATUS_DEVICE_NOT_READY;\r
+       }\r
+\r
+               status = fw_access_pciconf(p_BusInterface, FW_READ, &lcl_data, FLASH_OFFSET, 4 );\r
+       \r
+               if ( status != STATUS_SUCCESS )\r
+                       return status;\r
+\r
+       } while(lcl_data & CMD_MASK);\r
+\r
+       status = fw_access_pciconf(p_BusInterface, FW_READ, p_buffer, FLASH_OFFSET+4, length );\r
+       return status;\r
+}\r
+\r
+ib_api_status_t\r
+fw_access_ctrl(\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
+       IN              const   void** const                            handle_array    OPTIONAL,\r
+       IN                              uint32_t                                        num_handles,\r
+       IN                              ib_ci_op_t* const                       p_ci_op,\r
+       IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
+{\r
+       NTSTATUS                                        status = STATUS_SUCCESS;\r
+       PVOID                                           p_data;\r
+       ULONG                                           offset;\r
+       ULONG POINTER_ALIGNMENT         length;\r
+       struct ib_ucontext *p_uctx = (struct ib_ucontext *)h_ca;\r
+       PFDO_DEVICE_DATA p_fdo = p_uctx->device->x.p_fdo;\r
+       PBUS_INTERFACE_STANDARD         p_bus_interface = &p_fdo->bus_pci_ifc;\r
+\r
+       UNREFERENCED_PARAMETER(handle_array);\r
+       UNREFERENCED_PARAMETER(num_handles);\r
+\r
+       if( !p_umv_buf )\r
+               return IB_UNSUPPORTED;\r
+\r
+       if ( !p_ci_op )\r
+               return IB_INVALID_PARAMETER;\r
+\r
+       length = p_ci_op->buf_size;\r
+       offset = p_ci_op->buf_info;\r
+       p_data = p_ci_op->p_buf;\r
+\r
+       mutex_lock( &p_uctx->x.mutex );\r
+\r
+       switch ( p_ci_op->command )\r
+       {\r
+       case FW_MAP_CRSPACE:\r
+               status = __map_crspace(p_uctx, p_data, length);\r
+               break;\r
+               \r
+       case FW_UNMAP_CRSPACE:\r
+               __unmap_crspace(p_uctx);\r
+               break;\r
+                               \r
+       case FW_OPEN_IF: // open BusInterface\r
+               if (p_fdo->bus_pci_ifc_taken)\r
+                       __open_fw_access( p_uctx, p_bus_interface );\r
+               break;\r
+\r
+       case FW_READ: // read data from flash\r
+               if ( p_uctx->x.fw_if_open )\r
+                       status = fw_flash_read_data(p_bus_interface, p_data, offset, length);\r
+               break;\r
+\r
+       case FW_WRITE: // write data to flash\r
+               if ( p_uctx->x.fw_if_open )\r
+                       status = fw_flash_write_data(p_bus_interface, p_data, offset, length);\r
+               break;\r
+\r
+       case FW_READ_CMD:\r
+               if ( p_uctx->x.fw_if_open )\r
+                       status = fw_access_pciconf(p_bus_interface, 0 , p_data, offset, 4);\r
+               break;\r
+\r
+       case FW_WRITE_CMD:\r
+               if ( p_uctx->x.fw_if_open )\r
+                       status = fw_access_pciconf(p_bus_interface, 1 , p_data, offset, 4);\r
+               break;\r
+\r
+       case FW_CLOSE_IF: // close BusInterface\r
+               __close_fw_access(p_uctx, p_bus_interface);\r
+               break;\r
+\r
+       default:\r
+               status = STATUS_INVALID_DEVICE_REQUEST;\r
+       }\r
+\r
+       if ( status != STATUS_SUCCESS ) {\r
+               __close_fw_access(p_uctx, p_bus_interface);\r
+               HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_INIT, \r
+                       ("fw_access_ctrl failed, ntstatus: %08x.\n", status));\r
+       }\r
+\r
+       mutex_unlock( &p_uctx->x.mutex );\r
+\r
+       switch( status ) {\r
+               case STATUS_SUCCESS:                                    return IB_SUCCESS;\r
+               case STATUS_INVALID_DEVICE_REQUEST:     return IB_UNSUPPORTED;\r
+               case STATUS_INSUFFICIENT_RESOURCES:     return IB_INSUFFICIENT_RESOURCES;\r
+               default:                                                                        return IB_ERROR;\r
+       }\r
+}\r
+\r
index eb085f7..7c8b9f8 100644 (file)
-/*
- * Copyright (c) 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
- * Copyright (c) 2006, 2007 Cisco Systems.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "mlx4.h"
-#include "doorbell.h"
-#include "mlx4_debug.h"
-
-#if defined(EVENT_TRACING)
-#include "cq.tmh"
-#endif
-
-enum {
-       MLX4_CQ_DOORBELL                        = 0x20
-};
-
-enum {
-       CQ_OK                                   =  0,
-       CQ_EMPTY                                = -1,
-       CQ_POLL_ERR                             = -2
-};
-
-#define MLX4_CQ_DB_REQ_NOT_SOL                 (1 << 24)
-#define MLX4_CQ_DB_REQ_NOT                     (2 << 24)
-
-enum {
-       MLX4_CQE_OWNER_MASK                     = 0x80,
-       MLX4_CQE_IS_SEND_MASK                   = 0x40,
-       MLX4_CQE_OPCODE_MASK                    = 0x1f
-};
-
-enum {
-       MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR              = 0x01,
-       MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR               = 0x02,
-       MLX4_CQE_SYNDROME_LOCAL_PROT_ERR                = 0x04,
-       MLX4_CQE_SYNDROME_WR_FLUSH_ERR                  = 0x05,
-       MLX4_CQE_SYNDROME_MW_BIND_ERR                   = 0x06,
-       MLX4_CQE_SYNDROME_BAD_RESP_ERR                  = 0x10,
-       MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR              = 0x11,
-       MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR          = 0x12,
-       MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR             = 0x13,
-       MLX4_CQE_SYNDROME_REMOTE_OP_ERR                 = 0x14,
-       MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR       = 0x15,
-       MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR             = 0x16,
-       MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR            = 0x22,
-};
-
-struct mlx4_cqe {
-       uint32_t        my_qpn;
-       uint32_t        immed_rss_invalid;
-       uint32_t        g_mlpath_rqpn;
-       uint8_t         sl;
-       uint8_t         reserved1;
-       uint16_t        rlid;
-       uint32_t        reserved2;
-       uint32_t        byte_cnt;
-       uint16_t        wqe_index;
-       uint16_t        checksum;
-       uint8_t         reserved3[3];
-       uint8_t         owner_sr_opcode;
-};
-
-struct mlx4_err_cqe {
-       uint32_t        my_qpn;
-       uint32_t        reserved1[5];
-       uint16_t        wqe_index;
-       uint8_t         vendor_err;
-       uint8_t         syndrome;
-       uint8_t         reserved2[3];
-       uint8_t         owner_sr_opcode;
-};
-
-static struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry)
-{
-       return (struct mlx4_cqe *)(cq->buf.buf + entry * MLX4_CQ_ENTRY_SIZE);
-}
-
-static void *get_sw_cqe(struct mlx4_cq *cq, int n)
-{
-       struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
-
-       return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
-               !!(n & (cq->ibv_cq.cqe + 1))) ? NULL : cqe;
-}
-
-static struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq)
-{
-       return get_sw_cqe(cq, cq->cons_index);
-}
-
-static void update_cons_index(struct mlx4_cq *cq)
-{
-       *cq->set_ci_db = htonl(cq->cons_index & 0xffffff);
-}
-
-static void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, ib_wc_t *wc)
-{
-       if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR)
-               printf(PFX "local QP operation err "
-                      "(QPN %06x, WQE index %x, vendor syndrome %02x, "
-                      "opcode = %02x)\n",
-                      htonl(cqe->my_qpn), htonl(cqe->wqe_index),
-                      cqe->vendor_err,
-                      cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
-
-       switch (cqe->syndrome) {
-       case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
-               wc->status = IB_WCS_LOCAL_LEN_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
-               wc->status = IB_WCS_LOCAL_OP_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
-               wc->status = IB_WCS_LOCAL_PROTECTION_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
-               wc->status = IB_WCS_WR_FLUSHED_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_MW_BIND_ERR:
-               wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
-               wc->status = IB_WCS_BAD_RESP_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
-               wc->status = IB_WCS_LOCAL_ACCESS_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
-               wc->status = IB_WCS_REM_INVALID_REQ_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
-               wc->status = IB_WCS_REM_ACCESS_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
-               wc->status = IB_WCS_REM_OP_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
-               wc->status = IB_WCS_TIMEOUT_RETRY_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
-               wc->status = IB_WCS_RNR_RETRY_ERR;
-               break;
-       case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
-               wc->status = IB_WCS_REM_ABORT_ERR;
-               break;
-       }
-
-       wc->vendor_specific = cqe->vendor_err;
-}
-
-static int mlx4_poll_one(struct mlx4_cq *cq, struct mlx4_qp **cur_qp, ib_wc_t *wc)
-{
-       struct mlx4_wq *wq;
-       struct mlx4_cqe *cqe;
-       struct mlx4_srq *srq = NULL;
-       uint32_t qpn;
-       uint16_t wqe_index;
-       int is_error;
-       int is_send;
-#ifdef XRC_SUPPORT
-       int is_xrc_recv = 0;
-#endif
-
-       cqe = next_cqe_sw(cq);
-       if (!cqe)
-               return CQ_EMPTY;
-
-       ++cq->cons_index;
-
-       /*
-        * Make sure we read CQ entry contents after we've checked the
-        * ownership bit.
-        */
-       rmb();
-
-       qpn = ntohl(cqe->my_qpn);
-
-       is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
-       is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
-               MLX4_CQE_OPCODE_ERROR;
-
-#ifdef XRC_SUPPORT
-       if (qpn & MLX4_XRC_QPN_BIT && !is_send) {
-               uint32_t srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff;
-               /*
-               * We do not have to take the XRC SRQ table lock here,
-               * because CQs will be locked while XRC SRQs are removed
-               * from the table.
-               */
-               srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn);
-               if (!srq)
-                       return CQ_POLL_ERR;
-               is_xrc_recv = 1;
-       } else 
-#endif
-       if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) {
-               struct mlx4_qp *tmp_qp;
-               /*
-               * We do not have to take the QP table lock here,
-               * because CQs will be locked while QPs are removed
-               * from the table.
-               */
-               tmp_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context), qpn & 0xffffff);
-                       if (!tmp_qp) {
-                                       MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_CQ, (
-                                               "cqe_qpn %#x, wr_id %#I64x, ix %d, cons_index %d, asked_qpn %#x \n", 
-                                               qpn, wc->wr_id, ntohs(cqe->wqe_index), cq->cons_index - 1,
-                                               (*cur_qp) ? (*cur_qp)->ibv_qp.qp_num : 0 )); 
-                       return CQ_POLL_ERR;
-                       }
-                       *cur_qp = tmp_qp;
-               }
-
-       if (is_send) {
-               wq = &(*cur_qp)->sq;
-               wqe_index = ntohs(cqe->wqe_index);
-               wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);
-               wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
-               ++wq->tail;
-       } else 
-#ifdef XRC_SUPPORT
-       if (is_xrc_recv) {
-               wqe_index = htons(cqe->wqe_index);
-               wc->wr_id = srq->wrid[wqe_index];
-               mlx4_free_srq_wqe(srq, wqe_index);
-       } else 
-#endif 
-       if ((*cur_qp)->ibv_qp.srq) {
-               srq = to_msrq((*cur_qp)->ibv_qp.srq);
-               wqe_index = htons(cqe->wqe_index);
-               wc->wr_id = srq->wrid[wqe_index];
-               mlx4_free_srq_wqe(srq, wqe_index);
-       } else {
-               wq = &(*cur_qp)->rq;
-               wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
-               ++wq->tail;
-       }
-
-       if (is_send) {
-               wc->recv.ud.recv_opt = 0;
-               switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
-               case MLX4_OPCODE_RDMA_WRITE_IMM:
-                       wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;
-               case MLX4_OPCODE_RDMA_WRITE:
-                       wc->wc_type   = IB_WC_RDMA_WRITE;
-                       break;
-               case MLX4_OPCODE_SEND_IMM:
-                       wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;
-               case MLX4_OPCODE_SEND:
-                       wc->wc_type   = IB_WC_SEND;
-                       break;
-               case MLX4_OPCODE_RDMA_READ:
-                       wc->wc_type   = IB_WC_RDMA_READ;
-                       wc->length      = ntohl(cqe->byte_cnt);
-                       break;
-               case MLX4_OPCODE_ATOMIC_CS:
-                       wc->wc_type   = IB_WC_COMPARE_SWAP;
-                       wc->length      = 8;
-                       break;
-               case MLX4_OPCODE_ATOMIC_FA:
-                       wc->wc_type   = IB_WC_FETCH_ADD;
-                       wc->length      = 8;
-                       break;
-               case MLX4_OPCODE_BIND_MW:
-                       wc->wc_type   = IB_WC_MW_BIND;
-                       break;
-               default:
-                       /* assume it's a send completion */
-                       wc->wc_type   = IB_WC_SEND;
-                       break;
-               }
-       } else {
-               wc->length = ntohl(cqe->byte_cnt);
-
-               switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
-               case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
-                       wc->wc_type   = IB_WC_RECV;
-                       wc->recv.ud.recv_opt  = IB_RECV_OPT_IMMEDIATE;
-                       wc->recv.ud.immediate_data = cqe->immed_rss_invalid;
-                       break;
-               case MLX4_RECV_OPCODE_SEND:
-                       wc->wc_type   = IB_WC_RECV;
-                       wc->recv.ud.recv_opt = 0;
-                       break;
-               case MLX4_RECV_OPCODE_SEND_IMM:
-                       wc->wc_type   = IB_WC_RECV;
-                       wc->recv.ud.recv_opt  = IB_RECV_OPT_IMMEDIATE;
-                       wc->recv.ud.immediate_data = cqe->immed_rss_invalid;
-                       break;
-               default:
-                       /* assume it's a recv completion */
-                       wc->recv.ud.recv_opt  = 0;
-                       wc->wc_type = IB_WC_RECV;
-                       break;
-               }
-
-               wc->recv.ud.remote_lid  = cqe->rlid;
-               wc->recv.ud.remote_sl           = cqe->sl >> 4;
-               wc->recv.ud.remote_qp   = cqe->g_mlpath_rqpn & 0xffffff00;
-               wc->recv.ud.path_bits           = (uint8_t)(cqe->g_mlpath_rqpn & 0x7f);
-               wc->recv.ud.recv_opt            |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;
-               wc->recv.ud.pkey_index  = (uint16_t)(ntohl(cqe->immed_rss_invalid) & 0x7f);
-       }
-
-       if (is_error) 
-               mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
-       else
-               wc->status = IB_WCS_SUCCESS;
-
-       MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_CQ, ("qpn %#x, wr_id %#I64x, ix %d, cons_index %d, is_error %d \n", 
-               qpn, wc->wr_id, ntohs(cqe->wqe_index), cq->cons_index - 1, is_error )); 
-
-       return CQ_OK;
-}
-
-ib_api_status_t
-mlx4_poll_cq(
-       IN              const   void* FUNC_PTR64                                                h_cq,
-       IN      OUT                     ib_wc_t**       const                   pp_free_wclist,
-               OUT                     ib_wc_t**       const                   pp_done_wclist)
-{
-       struct mlx4_cq *cq = to_mcq((struct ibv_cq *)/*Ptr64ToPtr(*/ h_cq /*)*/);
-       struct mlx4_qp *qp = NULL;
-       ib_wc_t *wc_p, **next_pp;
-       int npolled = 0;
-       int err = CQ_OK;
-       ib_api_status_t status = IB_SUCCESS;
-
-       pthread_spin_lock(&cq->lock);
-       // loop through CQ
-       next_pp = pp_done_wclist;
-       wc_p = *pp_free_wclist;
-       while( wc_p ) {
-               err = mlx4_poll_one(cq, &qp, wc_p);
-               if (err != CQ_OK)
-                       break;
-
-               // prepare for the next step
-               *next_pp = wc_p;
-               next_pp = &wc_p->p_next;
-               wc_p = wc_p->p_next;
-               ++npolled;
-       }
-
-       // prepare the results
-       *pp_free_wclist = wc_p;         /* Set the head of the free list. */
-       *next_pp = NULL;                                /* Clear the tail of the done list. */
-
-       if (npolled)
-               update_cons_index(cq);
-
-       pthread_spin_unlock(&cq->lock);
-
-       if (err == CQ_POLL_ERR)
-               status = IB_ERROR;
-       else if (err == CQ_EMPTY && npolled == 0 )
-               status = IB_NOT_FOUND;
-       
-       return status;
-}
-
-ib_api_status_t
-mlx4_arm_cq (
-       IN              const   void* FUNC_PTR64                                h_cq,
-       IN              const   boolean_t                                       solicited)
-{
-       struct ibv_cq *ibvcq = (struct ibv_cq *)/*Ptr64ToPtr(*/ h_cq /*)*/;
-       struct mlx4_cq *cq = to_mcq(ibvcq);
-       uint32_t doorbell[2];
-       uint32_t sn;
-       uint32_t ci;
-       uint32_t cmd;
-
-       sn  = cq->arm_sn & 3;
-       ci  = cq->cons_index & 0xffffff;
-       cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
-
-       *cq->arm_db = htonl(sn << 28 | cmd | ci);
-
-       /*
-        * Make sure that the doorbell record in host memory is
-        * written before ringing the doorbell via PCI MMIO.
-        */
-       wmb();
-
-       doorbell[0] = htonl(sn << 28 | cmd | cq->cqn);
-       doorbell[1] = htonl(ci);
-
-       mlx4_write64(doorbell, to_mctx(ibvcq->context), MLX4_CQ_DOORBELL);
-
-       return IB_SUCCESS;
-}
-
-#if 0
-// this function could be called in Windows
-// we do it in kernel
-void mlx4_cq_event(struct ibv_cq *cq)
-{
-       to_mcq(cq)->arm_sn++;
-}
-#endif
-
-void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
-{
-       struct mlx4_cqe *cqe, *dest;
-       uint32_t prod_index;
-       uint8_t owner_bit;
-       int nfreed = 0;
-#ifdef XRC_SUPPORT
-       int is_xrc_srq = 0;
-
-       if (srq && srq->ibv_srq.xrc_cq)
-               is_xrc_srq = 1;
-#endif 
-
-       pthread_spin_lock(&cq->lock);
-
-       /*
-        * First we need to find the current producer index, so we
-        * know where to start cleaning from.  It doesn't matter if HW
-        * adds new entries after this loop -- the QP we're worried
-        * about is already in RESET, so the new entries won't come
-        * from our QP and therefore don't need to be checked.
-        */
-       for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
-               if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
-                       break;
-
-       /*
-        * Now sweep backwards through the CQ, removing CQ entries
-        * that match our QP by copying older entries on top of them.
-        */
-       while ((int) --prod_index - (int) cq->cons_index >= 0) {
-               cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
-#ifdef XRC_SUPPORT
-               if (is_xrc_srq &&
-                   (ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&
-                   !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
-                       mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
-                       ++nfreed;
-               } else 
-#endif         
-               if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
-                       if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
-                               mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
-                       ++nfreed;
-               } else if (nfreed) {
-                       dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
-                       owner_bit =  (uint8_t)(dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK);
-                       memcpy(dest, cqe, sizeof *cqe);
-                       dest->owner_sr_opcode = (uint8_t)(owner_bit |
-                               (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK));
-               }
-       }
-
-       if (nfreed) {
-               cq->cons_index += nfreed;
-               /*
-                * Make sure update of buffer contents is done before
-                * updating consumer index.
-                */
-               wmb();
-               update_cons_index(cq);
-       }
-
-       pthread_spin_unlock(&cq->lock);
-}
-
-void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe)
-{
-       UNREFERENCED_PARAMETER(cq);
-       UNREFERENCED_PARAMETER(buf);
-       UNREFERENCED_PARAMETER(old_cqe);
-}
+/*\r
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.\r
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.\r
+ * Copyright (c) 2006, 2007 Cisco Systems.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses.  You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include "mlx4.h"\r
+#include "doorbell.h"\r
+#include "mlx4_debug.h"\r
+\r
+#if defined(EVENT_TRACING)\r
+#include "cq.tmh"\r
+#endif\r
+\r
+enum {\r
+       MLX4_CQ_DOORBELL                        = 0x20\r
+};\r
+\r
+enum {\r
+       CQ_OK                                   =  0,\r
+       CQ_EMPTY                                = -1,\r
+       CQ_POLL_ERR                             = -2\r
+};\r
+\r
+#define MLX4_CQ_DB_REQ_NOT_SOL                 (1 << 24)\r
+#define MLX4_CQ_DB_REQ_NOT                     (2 << 24)\r
+\r
+enum {\r
+       MLX4_CQE_OWNER_MASK                     = 0x80,\r
+       MLX4_CQE_IS_SEND_MASK                   = 0x40,\r
+       MLX4_CQE_OPCODE_MASK                    = 0x1f\r
+};\r
+\r
+enum {\r
+       MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR              = 0x01,\r
+       MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR               = 0x02,\r
+       MLX4_CQE_SYNDROME_LOCAL_PROT_ERR                = 0x04,\r
+       MLX4_CQE_SYNDROME_WR_FLUSH_ERR                  = 0x05,\r
+       MLX4_CQE_SYNDROME_MW_BIND_ERR                   = 0x06,\r
+       MLX4_CQE_SYNDROME_BAD_RESP_ERR                  = 0x10,\r
+       MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR              = 0x11,\r
+       MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR          = 0x12,\r
+       MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR             = 0x13,\r
+       MLX4_CQE_SYNDROME_REMOTE_OP_ERR                 = 0x14,\r
+       MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR       = 0x15,\r
+       MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR             = 0x16,\r
+       MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR            = 0x22,\r
+};\r
+\r
+struct mlx4_cqe {\r
+       uint32_t        my_qpn;\r
+       uint32_t        immed_rss_invalid;\r
+       uint32_t        g_mlpath_rqpn;\r
+       uint8_t         sl;\r
+       uint8_t         reserved1;\r
+       uint16_t        rlid;\r
+       uint32_t        reserved2;\r
+       uint32_t        byte_cnt;\r
+       uint16_t        wqe_index;\r
+       uint16_t        checksum;\r
+       uint8_t         reserved3[3];\r
+       uint8_t         owner_sr_opcode;\r
+};\r
+\r
+struct mlx4_err_cqe {\r
+       uint32_t        my_qpn;\r
+       uint32_t        reserved1[5];\r
+       uint16_t        wqe_index;\r
+       uint8_t         vendor_err;\r
+       uint8_t         syndrome;\r
+       uint8_t         reserved2[3];\r
+       uint8_t         owner_sr_opcode;\r
+};\r
+\r
+static struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry)\r
+{\r
+       return (struct mlx4_cqe *)(cq->buf.buf + entry * MLX4_CQ_ENTRY_SIZE);\r
+}\r
+\r
+static void *get_sw_cqe(struct mlx4_cq *cq, int n)\r
+{\r
+       struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);\r
+\r
+       return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^\r
+               !!(n & (cq->ibv_cq.cqe + 1))) ? NULL : cqe;\r
+}\r
+\r
+static struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq)\r
+{\r
+       return get_sw_cqe(cq, cq->cons_index);\r
+}\r
+\r
+static void update_cons_index(struct mlx4_cq *cq)\r
+{\r
+       *cq->set_ci_db = htonl(cq->cons_index & 0xffffff);\r
+}\r
+\r
+static void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, ib_wc_t *wc)\r
+{\r
+       if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR)\r
+               printf(PFX "local QP operation err "\r
+                      "(QPN %06x, WQE index %x, vendor syndrome %02x, "\r
+                      "opcode = %02x)\n",\r
+                      htonl(cqe->my_qpn), htonl(cqe->wqe_index),\r
+                      cqe->vendor_err,\r
+                      cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
+\r
+       switch (cqe->syndrome) {\r
+       case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:\r
+               wc->status = IB_WCS_LOCAL_LEN_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:\r
+               wc->status = IB_WCS_LOCAL_OP_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:\r
+               wc->status = IB_WCS_LOCAL_PROTECTION_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:\r
+               wc->status = IB_WCS_WR_FLUSHED_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_MW_BIND_ERR:\r
+               wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_BAD_RESP_ERR:\r
+               wc->status = IB_WCS_BAD_RESP_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:\r
+               wc->status = IB_WCS_LOCAL_ACCESS_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:\r
+               wc->status = IB_WCS_REM_INVALID_REQ_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:\r
+               wc->status = IB_WCS_REM_ACCESS_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:\r
+               wc->status = IB_WCS_REM_OP_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:\r
+               wc->status = IB_WCS_TIMEOUT_RETRY_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:\r
+               wc->status = IB_WCS_RNR_RETRY_ERR;\r
+               break;\r
+       case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:\r
+               wc->status = IB_WCS_REM_ABORT_ERR;\r
+               break;\r
+       }\r
+\r
+       wc->vendor_specific = cqe->vendor_err;\r
+}\r
+\r
+static int mlx4_poll_one(struct mlx4_cq *cq, struct mlx4_qp **cur_qp, ib_wc_t *wc)\r
+{\r
+       struct mlx4_wq *wq;\r
+       struct mlx4_cqe *cqe;\r
+       struct mlx4_srq *srq = NULL;\r
+       uint32_t qpn;\r
+       uint16_t wqe_index;\r
+       int is_error;\r
+       int is_send;\r
+#ifdef XRC_SUPPORT\r
+       int is_xrc_recv = 0;\r
+#endif\r
+\r
+       cqe = next_cqe_sw(cq);\r
+       if (!cqe)\r
+               return CQ_EMPTY;\r
+\r
+       ++cq->cons_index;\r
+\r
+       /*\r
+        * Make sure we read CQ entry contents after we've checked the\r
+        * ownership bit.\r
+        */\r
+       rmb();\r
+\r
+       qpn = ntohl(cqe->my_qpn);\r
+\r
+       is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;\r
+       is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==\r
+               MLX4_CQE_OPCODE_ERROR;\r
+\r
+#ifdef XRC_SUPPORT\r
+       if (qpn & MLX4_XRC_QPN_BIT && !is_send) {\r
+               uint32_t srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff;\r
+               /*\r
+               * We do not have to take the XRC SRQ table lock here,\r
+               * because CQs will be locked while XRC SRQs are removed\r
+               * from the table.\r
+               */\r
+               srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn);\r
+               if (!srq)\r
+                       return CQ_POLL_ERR;\r
+               is_xrc_recv = 1;\r
+       } else \r
+#endif\r
+       if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) {\r
+               struct mlx4_qp *tmp_qp;\r
+               /*\r
+               * We do not have to take the QP table lock here,\r
+               * because CQs will be locked while QPs are removed\r
+               * from the table.\r
+               */\r
+               tmp_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context), qpn & 0xffffff);\r
+                       if (!tmp_qp) {\r
+                                       MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_CQ, (\r
+                                               "cqe_qpn %#x, wr_id %#I64x, ix %d, cons_index %d, asked_qpn %#x \n", \r
+                                               qpn, wc->wr_id, ntohs(cqe->wqe_index), cq->cons_index - 1,\r
+                                               (*cur_qp) ? (*cur_qp)->ibv_qp.qp_num : 0 )); \r
+                       return CQ_POLL_ERR;\r
+                       }\r
+                       *cur_qp = tmp_qp;\r
+               }\r
+\r
+       if (is_send) {\r
+               wq = &(*cur_qp)->sq;\r
+               wqe_index = ntohs(cqe->wqe_index);\r
+               wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);\r
+               wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
+               ++wq->tail;\r
+       } else \r
+#ifdef XRC_SUPPORT\r
+       if (is_xrc_recv) {\r
+               wqe_index = htons(cqe->wqe_index);\r
+               wc->wr_id = srq->wrid[wqe_index];\r
+               mlx4_free_srq_wqe(srq, wqe_index);\r
+       } else \r
+#endif \r
+       if ((*cur_qp)->ibv_qp.srq) {\r
+               srq = to_msrq((*cur_qp)->ibv_qp.srq);\r
+               wqe_index = htons(cqe->wqe_index);\r
+               wc->wr_id = srq->wrid[wqe_index];\r
+               mlx4_free_srq_wqe(srq, wqe_index);\r
+       } else {\r
+               wq = &(*cur_qp)->rq;\r
+               wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
+               ++wq->tail;\r
+       }\r
+\r
+       if (is_send) {\r
+               wc->recv.ud.recv_opt = 0;\r
+               switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
+               case MLX4_OPCODE_RDMA_WRITE_IMM:\r
+                       wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
+               case MLX4_OPCODE_RDMA_WRITE:\r
+                       wc->wc_type   = IB_WC_RDMA_WRITE;\r
+                       break;\r
+               case MLX4_OPCODE_SEND_IMM:\r
+                       wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
+               case MLX4_OPCODE_SEND:\r
+                       wc->wc_type   = IB_WC_SEND;\r
+                       break;\r
+               case MLX4_OPCODE_RDMA_READ:\r
+                       wc->wc_type   = IB_WC_RDMA_READ;\r
+                       wc->length      = ntohl(cqe->byte_cnt);\r
+                       break;\r
+               case MLX4_OPCODE_ATOMIC_CS:\r
+                       wc->wc_type   = IB_WC_COMPARE_SWAP;\r
+                       wc->length      = 8;\r
+                       break;\r
+               case MLX4_OPCODE_ATOMIC_FA:\r
+                       wc->wc_type   = IB_WC_FETCH_ADD;\r
+                       wc->length      = 8;\r
+                       break;\r
+               case MLX4_OPCODE_BIND_MW:\r
+                       wc->wc_type   = IB_WC_MW_BIND;\r
+                       break;\r
+               default:\r
+                       /* assume it's a send completion */\r
+                       wc->wc_type   = IB_WC_SEND;\r
+                       break;\r
+               }\r
+       } else {\r
+               wc->length = ntohl(cqe->byte_cnt);\r
+\r
+               switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
+               case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:\r
+                       wc->wc_type   = IB_WC_RECV;\r
+                       wc->recv.ud.recv_opt  = IB_RECV_OPT_IMMEDIATE;\r
+                       wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
+                       break;\r
+               case MLX4_RECV_OPCODE_SEND:\r
+                       wc->wc_type   = IB_WC_RECV;\r
+                       wc->recv.ud.recv_opt = 0;\r
+                       break;\r
+               case MLX4_RECV_OPCODE_SEND_IMM:\r
+                       wc->wc_type   = IB_WC_RECV;\r
+                       wc->recv.ud.recv_opt  = IB_RECV_OPT_IMMEDIATE;\r
+                       wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
+                       break;\r
+               default:\r
+                       /* assume it's a recv completion */\r
+                       wc->recv.ud.recv_opt  = 0;\r
+                       wc->wc_type = IB_WC_RECV;\r
+                       break;\r
+               }\r
+\r
+               wc->recv.ud.remote_lid  = cqe->rlid;\r
+               wc->recv.ud.remote_sl           = cqe->sl >> 4;\r
+               wc->recv.ud.remote_qp   = cqe->g_mlpath_rqpn & 0xffffff00;\r
+               wc->recv.ud.path_bits           = (uint8_t)(cqe->g_mlpath_rqpn & 0x7f);\r
+               wc->recv.ud.recv_opt            |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;\r
+               wc->recv.ud.pkey_index  = (uint16_t)(ntohl(cqe->immed_rss_invalid) & 0x7f);\r
+       }\r
+\r
+       if (is_error) \r
+               mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);\r
+       else\r
+               wc->status = IB_WCS_SUCCESS;\r
+\r
+       MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_CQ, ("qpn %#x, wr_id %#I64x, ix %d, cons_index %d, is_error %d \n", \r
+               qpn, wc->wr_id, ntohs(cqe->wqe_index), cq->cons_index - 1, is_error )); \r
+\r
+       return CQ_OK;\r
+}\r
+\r
+ib_api_status_t\r
+mlx4_poll_cq(\r
+       IN              const   void*                                           h_cq,\r
+       IN      OUT                     ib_wc_t**       const                   pp_free_wclist,\r
+               OUT                     ib_wc_t**       const                   pp_done_wclist)\r
+{\r
+       struct mlx4_cq *cq = to_mcq((struct ibv_cq *)/*Ptr64ToPtr(*/ h_cq /*)*/);\r
+       struct mlx4_qp *qp = NULL;\r
+       ib_wc_t *wc_p, **next_pp;\r
+       int npolled = 0;\r
+       int err = CQ_OK;\r
+       ib_api_status_t status = IB_SUCCESS;\r
+\r
+       pthread_spin_lock(&cq->lock);\r
\r
+       // loop through CQ\r
+       next_pp = pp_done_wclist;\r
+       wc_p = *pp_free_wclist;\r
+       while( wc_p ) {\r
+               err = mlx4_poll_one(cq, &qp, wc_p);\r
+               if (err != CQ_OK)\r
+                       break;\r
+\r
+               // prepare for the next step\r
+               *next_pp = wc_p;\r
+               next_pp = &wc_p->p_next;\r
+               wc_p = wc_p->p_next;\r
+               ++npolled;\r
+       }\r
+\r
+       // prepare the results\r
+       *pp_free_wclist = wc_p;         /* Set the head of the free list. */\r
+       *next_pp = NULL;                                /* Clear the tail of the done list. */\r
+\r
+       if (npolled)\r
+               update_cons_index(cq);\r
+\r
+       pthread_spin_unlock(&cq->lock);\r
+\r
+       if (err == CQ_POLL_ERR)\r
+               status = IB_ERROR;\r
+       else if (err == CQ_EMPTY && npolled == 0 )\r
+               status = IB_NOT_FOUND;\r
+       \r
+       return status;\r
+}\r
+\r
+ib_api_status_t\r
+mlx4_arm_cq (\r
+       IN              const   void*                                           h_cq,\r
+       IN              const   boolean_t                                       solicited)\r
+{\r
+       struct ibv_cq *ibvcq = (struct ibv_cq *)/*Ptr64ToPtr(*/ h_cq /*)*/;\r
+       struct mlx4_cq *cq = to_mcq(ibvcq);\r
+       uint32_t doorbell[2];\r
+       uint32_t sn;\r
+       uint32_t ci;\r
+       uint32_t cmd;\r
+\r
+       sn  = cq->arm_sn & 3;\r
+       ci  = cq->cons_index & 0xffffff;\r
+       cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;\r
+\r
+       *cq->arm_db = htonl(sn << 28 | cmd | ci);\r
+\r
+       /*\r
+        * Make sure that the doorbell record in host memory is\r
+        * written before ringing the doorbell via PCI MMIO.\r
+        */\r
+       wmb();\r
+\r
+       doorbell[0] = htonl(sn << 28 | cmd | cq->cqn);\r
+       doorbell[1] = htonl(ci);\r
+\r
+       mlx4_write64(doorbell, to_mctx(ibvcq->context), MLX4_CQ_DOORBELL);\r
+\r
+       return IB_SUCCESS;\r
+}\r
+\r
+#if 0\r
+// this function could be called in Windows\r
+// we do it in kernel\r
+void mlx4_cq_event(struct ibv_cq *cq)\r
+{\r
+       to_mcq(cq)->arm_sn++;\r
+}\r
+#endif\r
+\r
+void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)\r
+{\r
+       struct mlx4_cqe *cqe, *dest;\r
+       uint32_t prod_index;\r
+       uint8_t owner_bit;\r
+       int nfreed = 0;\r
+#ifdef XRC_SUPPORT\r
+       int is_xrc_srq = 0;\r
+\r
+       if (srq && srq->ibv_srq.xrc_cq)\r
+               is_xrc_srq = 1;\r
+#endif \r
+\r
+       pthread_spin_lock(&cq->lock);\r
+\r
+       /*\r
+        * First we need to find the current producer index, so we\r
+        * know where to start cleaning from.  It doesn't matter if HW\r
+        * adds new entries after this loop -- the QP we're worried\r
+        * about is already in RESET, so the new entries won't come\r
+        * from our QP and therefore don't need to be checked.\r
+        */\r
+       for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)\r
+               if (prod_index == cq->cons_index + cq->ibv_cq.cqe)\r
+                       break;\r
+\r
+       /*\r
+        * Now sweep backwards through the CQ, removing CQ entries\r
+        * that match our QP by copying older entries on top of them.\r
+        */\r
+       while ((int) --prod_index - (int) cq->cons_index >= 0) {\r
+               cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);\r
+#ifdef XRC_SUPPORT\r
+               if (is_xrc_srq &&\r
+                   (ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&\r
+                   !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {\r
+                       mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));\r
+                       ++nfreed;\r
+               } else \r
+#endif         \r
+               if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {\r
+                       if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))\r
+                               mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));\r
+                       ++nfreed;\r
+               } else if (nfreed) {\r
+                       dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);\r
+                       owner_bit =  (uint8_t)(dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK);\r
+                       memcpy(dest, cqe, sizeof *cqe);\r
+                       dest->owner_sr_opcode = (uint8_t)(owner_bit |\r
+                               (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK));\r
+               }\r
+       }\r
+\r
+       if (nfreed) {\r
+               cq->cons_index += nfreed;\r
+               /*\r
+                * Make sure update of buffer contents is done before\r
+                * updating consumer index.\r
+                */\r
+               wmb();\r
+               update_cons_index(cq);\r
+       }\r
+\r
+       pthread_spin_unlock(&cq->lock);\r
+}\r
+\r
+void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe)\r
+{\r
+       UNREFERENCED_PARAMETER(cq);\r
+       UNREFERENCED_PARAMETER(buf);\r
+       UNREFERENCED_PARAMETER(old_cqe);\r
+}\r
index eaa7832..9ba5dcc 100644 (file)
-/*
- * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef MLX4_H
-#define MLX4_H
-
-
-#include <iba\ib_types.h>
-#include <iba\ib_uvp.h>
-
-#include "verbs.h"
-#include "mx_abi.h"
-
-#include "l2w.h"
-
-#define PFX            "mlx4: "
-
-#ifndef max
-#define max(a,b) \
-       ({ typeof (a) _a = (a); \
-          typeof (b) _b = (b); \
-          _a > _b ? _a : _b; })
-#endif
-
-#ifndef min
-#define min(a,b) \
-       ({ typeof (a) _a = (a); \
-          typeof (b) _b = (b); \
-          _a < _b ? _a : _b; })
-#endif
-
-enum {
-       MLX4_CQ_ENTRY_SIZE              = 0x20
-};
-
-enum {
-       MLX4_STAT_RATE_OFFSET           = 5
-};
-
-enum {
-       MLX4_QP_TABLE_BITS              = 8,
-       MLX4_QP_TABLE_SIZE              = 1 << MLX4_QP_TABLE_BITS,
-       MLX4_QP_TABLE_MASK              = MLX4_QP_TABLE_SIZE - 1
-};
-
-#ifdef XRC_SUPPORT
-enum {
-       MLX4_XRC_SRQ_TABLE_BITS         = 8,
-       MLX4_XRC_SRQ_TABLE_SIZE         = 1 << MLX4_XRC_SRQ_TABLE_BITS,
-       MLX4_XRC_SRQ_TABLE_MASK         = MLX4_XRC_SRQ_TABLE_SIZE - 1
-};
-
-enum {
-       MLX4_XRC_QPN_BIT                = (1 << 23)
-};
-#endif
-
-enum mlx4_db_type {
-       MLX4_DB_TYPE_CQ,
-       MLX4_DB_TYPE_RQ,
-       MLX4_NUM_DB_TYPE
-};
-
-enum mlx4_opcode_type {
-       MLX4_OPCODE_NOP                 = 0x00,
-       MLX4_OPCODE_SEND_INVAL          = 0x01,
-       MLX4_OPCODE_RDMA_WRITE          = 0x08,
-       MLX4_OPCODE_RDMA_WRITE_IMM      = 0x09,
-       MLX4_OPCODE_SEND                = 0x0a,
-       MLX4_OPCODE_SEND_IMM            = 0x0b,
-       MLX4_OPCODE_LSO                 = 0x0e,
-       MLX4_OPCODE_RDMA_READ           = 0x10,
-       MLX4_OPCODE_ATOMIC_CS           = 0x11,
-       MLX4_OPCODE_ATOMIC_FA           = 0x12,
-       MLX4_OPCODE_ATOMIC_MASK_CS      = 0x14,
-       MLX4_OPCODE_ATOMIC_MASK_FA      = 0x15,
-       MLX4_OPCODE_BIND_MW             = 0x18,
-       MLX4_OPCODE_FMR                 = 0x19,
-       MLX4_OPCODE_LOCAL_INVAL         = 0x1b,
-       MLX4_OPCODE_CONFIG_CMD          = 0x1f,
-
-       MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00,
-       MLX4_RECV_OPCODE_SEND           = 0x01,
-       MLX4_RECV_OPCODE_SEND_IMM       = 0x02,
-       MLX4_RECV_OPCODE_SEND_INVAL     = 0x03,
-
-       MLX4_CQE_OPCODE_ERROR           = 0x1e,
-       MLX4_CQE_OPCODE_RESIZE          = 0x16,
-
-       MLX4_OPCODE_INVALID                     = 0xff
-};
-
-struct mlx4_db_page;
-
-struct mlx4_context {
-       struct ibv_context              ibv_ctx;
-
-       uint8_t                        *uar;
-       pthread_spinlock_t              uar_lock;
-
-       uint8_t                        *bf_page;
-       int                             bf_buf_size;
-       int                             bf_offset;
-       pthread_spinlock_t              bf_lock;
-
-       struct {
-               struct mlx4_qp        **table;
-               int                     refcnt;
-       }                               qp_table[MLX4_QP_TABLE_SIZE];
-       pthread_mutex_t                 qp_table_mutex;
-       int                             num_qps;
-       int                             qp_table_shift;
-       int                             qp_table_mask;
-       int                             max_qp_wr;
-       int                             max_sge;
-       int                             max_cqe;
-
-#ifdef XRC_SUPPORT
-       struct {
-               struct mlx4_srq       **table;
-               int                     refcnt;
-       }                               xrc_srq_table[MLX4_XRC_SRQ_TABLE_SIZE];
-       pthread_mutex_t                 xrc_srq_table_mutex;
-       int                             num_xrc_srqs;
-       int                             xrc_srq_table_shift;
-       int                             xrc_srq_table_mask;
-#endif
-
-       struct mlx4_db_page            *db_list[MLX4_NUM_DB_TYPE];
-       pthread_mutex_t                 db_list_mutex;
-};
-
-struct mlx4_buf {
-       uint8_t                        *buf;
-       int                                     length;
-};
-
-struct mlx4_pd {
-       struct ibv_pd                   ibv_pd;
-       uint32_t                        pdn;
-};
-
-struct mlx4_cq {
-       struct ibv_cq                   ibv_cq;
-       struct mlx4_buf                 buf;
-       pthread_spinlock_t              lock;
-       uint32_t                        cqn;
-       uint32_t                        cons_index;
-       uint32_t                       *set_ci_db;
-       uint32_t                       *arm_db;
-       int                             arm_sn;
-};
-
-struct mlx4_srq {
-       struct ibv_srq                  ibv_srq;
-       struct mlx4_buf                 buf;
-       pthread_spinlock_t              lock;
-       uint64_t                       *wrid;
-       uint32_t                        srqn;
-       int                             max;
-       int                             max_gs;
-       int                             wqe_shift;
-       int                             head;
-       int                             tail;
-       uint32_t                       *db;
-       uint16_t                        counter;
-};
-
-struct mlx4_wq {
-       uint64_t                       *wrid;
-       pthread_spinlock_t              lock;
-       int                             wqe_cnt;
-       int                             max_post;
-       unsigned                        head;
-       unsigned                        tail;
-       int                             max_gs;
-       int                             wqe_shift;
-       int                             offset;
-};
-
-struct mlx4_qp {
-       struct ibv_qp                   ibv_qp;
-       struct mlx4_buf                 buf;
-       int                             max_inline_data;
-       int                             buf_size;
-
-       uint32_t                        doorbell_qpn;
-       uint32_t                        sq_signal_bits;
-       int                             sq_spare_wqes;
-       struct mlx4_wq                  sq;
-
-       uint32_t                       *db;
-       struct mlx4_wq                  rq;
-};
-
-struct mlx4_av {
-       uint32_t                        port_pd;
-       uint8_t                         reserved1;
-       uint8_t                         g_slid;
-       uint16_t                        dlid;
-       uint8_t                         reserved2;
-       uint8_t                         gid_index;
-       uint8_t                         stat_rate;
-       uint8_t                         hop_limit;
-       uint32_t                        sl_tclass_flowlabel;
-       uint8_t                         dgid[16];
-};
-
-struct mlx4_ah {
-       struct ibv_ah                   ibv_ah;
-       struct mlx4_av                  av;
-};
-
-#ifdef XRC_SUPPORT
-struct mlx4_xrc_domain {
-       struct ibv_xrc_domain           ibv_xrcd;
-       uint32_t                        xrcdn;
-};
-#endif
-
-static inline unsigned long align(unsigned long val, unsigned long align)
-{
-       return (val + align - 1) & ~(align - 1);
-}
-
-#define to_mxxx(xxx, type)                                             \
-       ((struct mlx4_##type *)                                 \
-        ((uint8_t *) ib##xxx - offsetof(struct mlx4_##type, ibv_##xxx)))
-
-static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)
-{
-       return to_mxxx(ctx, context);
-}
-
-static inline struct mlx4_pd *to_mpd(struct ibv_pd *ibpd)
-{
-       return to_mxxx(pd, pd);
-}
-
-static inline struct mlx4_cq *to_mcq(struct ibv_cq *ibcq)
-{
-       return to_mxxx(cq, cq);
-}
-
-static inline struct mlx4_srq *to_msrq(struct ibv_srq *ibsrq)
-{
-       return to_mxxx(srq, srq);
-}
-
-static inline struct mlx4_qp *to_mqp(struct ibv_qp *ibqp)
-{
-       return to_mxxx(qp, qp);
-}
-
-static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)
-{
-       return to_mxxx(ah, ah);
-}
-
-#ifdef XRC_SUPPORT
-static inline struct mlx4_xrc_domain *to_mxrcd(struct ibv_xrc_domain *ibxrcd)
-{
-       return to_mxxx(xrcd, xrc_domain);
-}
-#endif
-
-struct ibv_context * mlx4_alloc_context();
-struct ibv_context * mlx4_fill_context(struct ibv_context *ctx,
-                                                                               struct ibv_get_context_resp *resp_p);
-void mlx4_free_context(struct ibv_context *ctx);
-
-int mlx4_alloc_buf(struct mlx4_buf *buf, int size, int page_size);
-void mlx4_free_buf(struct mlx4_buf *buf);
-
-uint32_t *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);
-void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, uint32_t *db);
-
-ib_api_status_t mlx4_poll_cq(const void * FUNC_PTR64 h_cq,
-                       ib_wc_t** const pp_free_wclist,
-                       ib_wc_t** const pp_done_wclist);
-ib_api_status_t mlx4_arm_cq(const      void * FUNC_PTR64 h_cq, const   boolean_t solicited);
-void mlx4_cq_event(struct ibv_cq *cq);
-void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn,
-                   struct mlx4_srq *srq);
-void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int new_cqe);
-
-int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
-                       struct mlx4_srq *srq);
-void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
-ib_api_status_t mlx4_post_srq_recv(const void* FUNC_PTR64 h_srq,
-                       ib_recv_wr_t* const wr,
-                       ib_recv_wr_t** bad_wr);
-
-#ifdef XRC_SUPPORT
-struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
-int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
-                      struct mlx4_srq *srq);
-void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
-#endif
-
-void mlx4_init_qp_indices(struct mlx4_qp *qp);
-void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp);
-ib_api_status_t mlx4_post_send(const void* FUNC_PTR64 h_qp,
-                       ib_send_wr_t* const wr,
-                       ib_send_wr_t** bad_wr);
-ib_api_status_t mlx4_post_recv(const void* FUNC_PTR64 h_qp,
-                       ib_recv_wr_t* const wr,
-                       ib_recv_wr_t** bad_wr);
-
-void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
-                          struct mlx4_qp *qp);
-int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
-                      enum ibv_qp_type type, struct mlx4_qp *qp);
-void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
-                      enum ibv_qp_type type);
-struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn);
-int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp);
-void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn);
-
-#endif /* MLX4_H */
+/*\r
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.\r
+ * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#ifndef MLX4_H\r
+#define MLX4_H\r
+\r
+\r
+#include <iba\ib_types.h>\r
+#include <iba\ib_uvp.h>\r
+\r
+#include "verbs.h"\r
+#include "mx_abi.h"\r
+\r
+#include "l2w.h"\r
+\r
+#define PFX            "mlx4: "\r
+\r
+#ifndef max\r
+#define max(a,b) \\r
+       ({ typeof (a) _a = (a); \\r
+          typeof (b) _b = (b); \\r
+          _a > _b ? _a : _b; })\r
+#endif\r
+\r
+#ifndef min\r
+#define min(a,b) \\r
+       ({ typeof (a) _a = (a); \\r
+          typeof (b) _b = (b); \\r
+          _a < _b ? _a : _b; })\r
+#endif\r
+\r
+enum {\r
+       MLX4_CQ_ENTRY_SIZE              = 0x20\r
+};\r
+\r
+enum {\r
+       MLX4_STAT_RATE_OFFSET           = 5\r
+};\r
+\r
+enum {\r
+       MLX4_QP_TABLE_BITS              = 8,\r
+       MLX4_QP_TABLE_SIZE              = 1 << MLX4_QP_TABLE_BITS,\r
+       MLX4_QP_TABLE_MASK              = MLX4_QP_TABLE_SIZE - 1\r
+};\r
+\r
+#ifdef XRC_SUPPORT\r
+enum {\r
+       MLX4_XRC_SRQ_TABLE_BITS         = 8,\r
+       MLX4_XRC_SRQ_TABLE_SIZE         = 1 << MLX4_XRC_SRQ_TABLE_BITS,\r
+       MLX4_XRC_SRQ_TABLE_MASK         = MLX4_XRC_SRQ_TABLE_SIZE - 1\r
+};\r
+\r
+enum {\r
+       MLX4_XRC_QPN_BIT                = (1 << 23)\r
+};\r
+#endif\r
+\r
+enum mlx4_db_type {\r
+       MLX4_DB_TYPE_CQ,\r
+       MLX4_DB_TYPE_RQ,\r
+       MLX4_NUM_DB_TYPE\r
+};\r
+\r
+enum mlx4_opcode_type {\r
+       MLX4_OPCODE_NOP                 = 0x00,\r
+       MLX4_OPCODE_SEND_INVAL          = 0x01,\r
+       MLX4_OPCODE_RDMA_WRITE          = 0x08,\r
+       MLX4_OPCODE_RDMA_WRITE_IMM      = 0x09,\r
+       MLX4_OPCODE_SEND                = 0x0a,\r
+       MLX4_OPCODE_SEND_IMM            = 0x0b,\r
+       MLX4_OPCODE_LSO                 = 0x0e,\r
+       MLX4_OPCODE_RDMA_READ           = 0x10,\r
+       MLX4_OPCODE_ATOMIC_CS           = 0x11,\r
+       MLX4_OPCODE_ATOMIC_FA           = 0x12,\r
+       MLX4_OPCODE_ATOMIC_MASK_CS      = 0x14,\r
+       MLX4_OPCODE_ATOMIC_MASK_FA      = 0x15,\r
+       MLX4_OPCODE_BIND_MW             = 0x18,\r
+       MLX4_OPCODE_FMR                 = 0x19,\r
+       MLX4_OPCODE_LOCAL_INVAL         = 0x1b,\r
+       MLX4_OPCODE_CONFIG_CMD          = 0x1f,\r
+\r
+       MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00,\r
+       MLX4_RECV_OPCODE_SEND           = 0x01,\r
+       MLX4_RECV_OPCODE_SEND_IMM       = 0x02,\r
+       MLX4_RECV_OPCODE_SEND_INVAL     = 0x03,\r
+\r
+       MLX4_CQE_OPCODE_ERROR           = 0x1e,\r
+       MLX4_CQE_OPCODE_RESIZE          = 0x16,\r
+\r
+       MLX4_OPCODE_INVALID                     = 0xff\r
+};\r
+\r
+struct mlx4_db_page;\r
+\r
+struct mlx4_context {\r
+       struct ibv_context              ibv_ctx;\r
+\r
+       uint8_t                        *uar;\r
+       pthread_spinlock_t              uar_lock;\r
+\r
+       uint8_t                        *bf_page;\r
+       int                             bf_buf_size;\r
+       int                             bf_offset;\r
+       pthread_spinlock_t              bf_lock;\r
+\r
+       struct {\r
+               struct mlx4_qp        **table;\r
+               int                     refcnt;\r
+       }                               qp_table[MLX4_QP_TABLE_SIZE];\r
+       pthread_mutex_t                 qp_table_mutex;\r
+       int                             num_qps;\r
+       int                             qp_table_shift;\r
+       int                             qp_table_mask;\r
+       int                             max_qp_wr;\r
+       int                             max_sge;\r
+       int                             max_cqe;\r
+\r
+#ifdef XRC_SUPPORT\r
+       struct {\r
+               struct mlx4_srq       **table;\r
+               int                     refcnt;\r
+       }                               xrc_srq_table[MLX4_XRC_SRQ_TABLE_SIZE];\r
+       pthread_mutex_t                 xrc_srq_table_mutex;\r
+       int                             num_xrc_srqs;\r
+       int                             xrc_srq_table_shift;\r
+       int                             xrc_srq_table_mask;\r
+#endif\r
+\r
+       struct mlx4_db_page            *db_list[MLX4_NUM_DB_TYPE];\r
+       pthread_mutex_t                 db_list_mutex;\r
+};\r
+\r
+struct mlx4_buf {\r
+       uint8_t                        *buf;\r
+       int                                     length;\r
+};\r
+\r
+struct mlx4_pd {\r
+       struct ibv_pd                   ibv_pd;\r
+       uint32_t                        pdn;\r
+};\r
+\r
+struct mlx4_cq {\r
+       struct ibv_cq                   ibv_cq;\r
+       struct mlx4_buf                 buf;\r
+       pthread_spinlock_t              lock;\r
+       uint32_t                        cqn;\r
+       uint32_t                        cons_index;\r
+       uint32_t                       *set_ci_db;\r
+       uint32_t                       *arm_db;\r
+       int                             arm_sn;\r
+};\r
+\r
+struct mlx4_srq {\r
+       struct ibv_srq                  ibv_srq;\r
+       struct mlx4_buf                 buf;\r
+       pthread_spinlock_t              lock;\r
+       uint64_t                       *wrid;\r
+       uint32_t                        srqn;\r
+       int                             max;\r
+       int                             max_gs;\r
+       int                             wqe_shift;\r
+       int                             head;\r
+       int                             tail;\r
+       uint32_t                       *db;\r
+       uint16_t                        counter;\r
+};\r
+\r
+struct mlx4_wq {\r
+       uint64_t                       *wrid;\r
+       pthread_spinlock_t              lock;\r
+       int                             wqe_cnt;\r
+       int                             max_post;\r
+       unsigned                        head;\r
+       unsigned                        tail;\r
+       int                             max_gs;\r
+       int                             wqe_shift;\r
+       int                             offset;\r
+};\r
+\r
+struct mlx4_qp {\r
+       struct ibv_qp                   ibv_qp;\r
+       struct mlx4_buf                 buf;\r
+       int                             max_inline_data;\r
+       int                             buf_size;\r
+\r
+       uint32_t                        doorbell_qpn;\r
+       uint32_t                        sq_signal_bits;\r
+       int                             sq_spare_wqes;\r
+       struct mlx4_wq                  sq;\r
+\r
+       uint32_t                       *db;\r
+       struct mlx4_wq                  rq;\r
+};\r
+\r
+struct mlx4_av {\r
+       uint32_t                        port_pd;\r
+       uint8_t                         reserved1;\r
+       uint8_t                         g_slid;\r
+       uint16_t                        dlid;\r
+       uint8_t                         reserved2;\r
+       uint8_t                         gid_index;\r
+       uint8_t                         stat_rate;\r
+       uint8_t                         hop_limit;\r
+       uint32_t                        sl_tclass_flowlabel;\r
+       uint8_t                         dgid[16];\r
+};\r
+\r
+struct mlx4_ah {\r
+       struct ibv_ah                   ibv_ah;\r
+       struct mlx4_av                  av;\r
+};\r
+\r
+#ifdef XRC_SUPPORT\r
+struct mlx4_xrc_domain {\r
+       struct ibv_xrc_domain           ibv_xrcd;\r
+       uint32_t                        xrcdn;\r
+};\r
+#endif\r
+\r
+static inline unsigned long align(unsigned long val, unsigned long align)\r
+{\r
+       return (val + align - 1) & ~(align - 1);\r
+}\r
+\r
+#define to_mxxx(xxx, type)                                             \\r
+       ((struct mlx4_##type *)                                 \\r
+        ((uint8_t *) ib##xxx - offsetof(struct mlx4_##type, ibv_##xxx)))\r
+\r
+static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)\r
+{\r
+       return to_mxxx(ctx, context);\r
+}\r
+\r
+static inline struct mlx4_pd *to_mpd(struct ibv_pd *ibpd)\r
+{\r
+       return to_mxxx(pd, pd);\r
+}\r
+\r
+static inline struct mlx4_cq *to_mcq(struct ibv_cq *ibcq)\r
+{\r
+       return to_mxxx(cq, cq);\r
+}\r
+\r
+static inline struct mlx4_srq *to_msrq(struct ibv_srq *ibsrq)\r
+{\r
+       return to_mxxx(srq, srq);\r
+}\r
+\r
+static inline struct mlx4_qp *to_mqp(struct ibv_qp *ibqp)\r
+{\r
+       return to_mxxx(qp, qp);\r
+}\r
+\r
+static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)\r
+{\r
+       return to_mxxx(ah, ah);\r
+}\r
+\r
+#ifdef XRC_SUPPORT\r
+static inline struct mlx4_xrc_domain *to_mxrcd(struct ibv_xrc_domain *ibxrcd)\r
+{\r
+       return to_mxxx(xrcd, xrc_domain);\r
+}\r
+#endif\r
+\r
+struct ibv_context * mlx4_alloc_context();\r
+struct ibv_context * mlx4_fill_context(struct ibv_context *ctx,\r
+                                                                               struct ibv_get_context_resp *resp_p);\r
+void mlx4_free_context(struct ibv_context *ctx);\r
+\r
+int mlx4_alloc_buf(struct mlx4_buf *buf, int size, int page_size);\r
+void mlx4_free_buf(struct mlx4_buf *buf);\r
+\r
+uint32_t *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);\r
+void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, uint32_t *db);\r
+\r
+ib_api_status_t mlx4_poll_cq(const void* h_cq,\r
+                       ib_wc_t** const pp_free_wclist,\r
+                       ib_wc_t** const pp_done_wclist);\r
+ib_api_status_t mlx4_arm_cq(const      void* h_cq, const       boolean_t solicited);\r
+void mlx4_cq_event(struct ibv_cq *cq);\r
+void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn,\r
+                   struct mlx4_srq *srq);\r
+void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int new_cqe);\r
+\r
+int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,\r
+                       struct mlx4_srq *srq);\r
+void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);\r
+ib_api_status_t mlx4_post_srq_recv(const void* h_srq,\r
+                       ib_recv_wr_t* const wr,\r
+                       ib_recv_wr_t** bad_wr);\r
+\r
+#ifdef XRC_SUPPORT\r
+struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);\r
+int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,\r
+                      struct mlx4_srq *srq);\r
+void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);\r
+#endif\r
+\r
+void mlx4_init_qp_indices(struct mlx4_qp *qp);\r
+void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp);\r
+ib_api_status_t mlx4_post_send(const void* h_qp,\r
+                       ib_send_wr_t* const wr,\r
+                       ib_send_wr_t** bad_wr);\r
+ib_api_status_t mlx4_post_recv(const void* h_qp,\r
+                       ib_recv_wr_t* const wr,\r
+                       ib_recv_wr_t** bad_wr);\r
+\r
+void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,\r
+                          struct mlx4_qp *qp);\r
+int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,\r
+                      enum ibv_qp_type type, struct mlx4_qp *qp);\r
+void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,\r
+                      enum ibv_qp_type type);\r
+struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn);\r
+int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp);\r
+void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn);\r
+\r
+#endif /* MLX4_H */\r
index 1e2a027..12c1ff9 100644 (file)
-/*
- * Copyright (c) 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
- * Copyright (c) 2007 Cisco, Inc.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "mlx4.h"
-#include "doorbell.h"
-#include "wqe.h"
-#include "mlx4_debug.h"
-
-#if defined(EVENT_TRACING)
-#include "qp.tmh"
-#endif
-
-static enum mlx4_opcode_type __to_opcode(ib_send_wr_t *wr)
-{
-
-       enum mlx4_opcode_type opcode = MLX4_OPCODE_INVALID;
-
-       switch (wr->wr_type) {
-       case WR_SEND: 
-               opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? 
-                                       MLX4_OPCODE_SEND_IMM : MLX4_OPCODE_SEND;
-               break;
-       case WR_RDMA_WRITE:     
-               opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ?
-                                       MLX4_OPCODE_RDMA_WRITE_IMM : MLX4_OPCODE_RDMA_WRITE;
-               break;
-       case WR_RDMA_READ:
-               opcode = MLX4_OPCODE_RDMA_READ;
-               break;
-       case WR_COMPARE_SWAP:
-               opcode = MLX4_OPCODE_ATOMIC_CS;
-               break;
-       case WR_FETCH_ADD:
-               opcode = MLX4_OPCODE_ATOMIC_FA;
-               break;
-       default:
-               opcode = MLX4_OPCODE_INVALID;
-               break;
-       }
-
-       return opcode;
-}
-
-static void *get_recv_wqe(struct mlx4_qp *qp, int n)
-{
-       return qp->buf.buf + qp->rq.offset + (n << qp->rq.wqe_shift);
-}
-
-static void *get_send_wqe(struct mlx4_qp *qp, int n)
-{
-       return qp->buf.buf + qp->sq.offset + (n << qp->sq.wqe_shift);
-}
-
-/*
- * Stamp a SQ WQE so that it is invalid if prefetched by marking the
- * first four bytes of every 64 byte chunk with 0xffffffff, except for
- * the very first chunk of the WQE.
- */
-static void stamp_send_wqe(struct mlx4_qp *qp, int n)
-{
-       uint32_t *wqe = get_send_wqe(qp, n);
-       int i;
-
-       for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16)
-               wqe[i] = 0xffffffff;
-}
-
-void mlx4_init_qp_indices(struct mlx4_qp *qp)
-{
-       qp->sq.head      = 0;
-       qp->sq.tail      = 0;
-       qp->rq.head      = 0;
-       qp->rq.tail      = 0;
-}
-
-void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp)
-{
-       struct mlx4_wqe_ctrl_seg *ctrl;
-       int i;
-
-       for (i = 0; i < qp->sq.wqe_cnt; ++i) {
-               ctrl = get_send_wqe(qp, i);
-               ctrl->owner_opcode = htonl((uint32_t)1 << 31);
-
-               stamp_send_wqe(qp, i);
-       }
-}
-
-static int wq_overflow(struct mlx4_wq *wq, int nreq, struct mlx4_cq *cq)
-{
-       int cur;
-
-       cur = wq->head - wq->tail;
-       if (cur + nreq < wq->max_post)
-               return 0;
-
-       pthread_spin_lock(&cq->lock);
-       cur = wq->head - wq->tail;
-       pthread_spin_unlock(&cq->lock);
-
-       return cur + nreq >= wq->max_post;
-}
-
-static inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
-                                uint64_t remote_addr, uint32_t rkey)
-{
-       rseg->raddr    = cl_hton64(remote_addr);
-       rseg->rkey     = rkey;
-       rseg->reserved = 0;
-}
-
-static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr)
-{
-       if (wr->wr_type == WR_COMPARE_SWAP) {
-               aseg->swap_add = wr->remote_ops.atomic2;
-               aseg->compare  = wr->remote_ops.atomic1;
-       } else {
-               aseg->swap_add = wr->remote_ops.atomic1;
-               aseg->compare  = 0;
-       }
-}
-
-static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, ib_send_wr_t *wr)
-{
-       memcpy(dseg->av, &to_mah((struct ibv_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av));
-       dseg->dqpn = wr->dgrm.ud.remote_qp;
-       dseg->qkey = wr->dgrm.ud.remote_qkey;
-}
-
-static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *ds)
-{
-       dseg->byte_count = cl_hton32(ds->length);
-       dseg->lkey       = cl_hton32(ds->lkey);
-       dseg->addr       = cl_hton64(ds->vaddr);
-}
-
-static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *ds)
-{
-       dseg->lkey       = cl_hton32(ds->lkey);
-       dseg->addr       = cl_hton64(ds->vaddr);
-
-       /*
-        * Need a barrier here before writing the byte_count field to
-        * make sure that all the data is visible before the
-        * byte_count field is set.  Otherwise, if the segment begins
-        * a new cacheline, the HCA prefetcher could grab the 64-byte
-        * chunk and get a valid (!= * 0xffffffff) byte count but
-        * stale data, and end up sending the wrong data.
-        */
-       wmb();
-
-       dseg->byte_count = cl_hton32(ds->length);
-}
-
-/*
- * Avoid using memcpy() to copy to BlueFlame page, since memcpy()
- * implementations may use move-string-buffer assembler instructions,
- * which do not guarantee order of copying.
- */
-static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, unsigned bytecnt)
-{
-#ifdef _WIN64
-       uint64_t *d = (uint64_t *)dst;
-       uint64_t *s = (uint64_t *)src;
-
-       while (bytecnt > 0) {
-               *d++ = *s++;
-               *d++ = *s++;
-               bytecnt -= 2 * sizeof (uint64_t);
-       }
-#else
-       while (bytecnt > 0) {
-               *dst++ = *src++;
-               *dst++ = *src++;
-               bytecnt -= 2 * sizeof (unsigned long);
-       }
-#endif 
-}
-
-ib_api_status_t
-mlx4_post_send(
-       IN              const   void* FUNC_PTR64                                h_qp,
-       IN                              ib_send_wr_t*   const           p_wr,
-               OUT                     ib_send_wr_t**                          bad_wr)
-{
-       struct ibv_qp *ibqp = (struct ibv_qp *)/*Ptr64ToPtr(*/h_qp/*)*/;
-       struct mlx4_qp *qp = to_mqp(ibqp);
-       struct mlx4_context *ctx;
-       uint8_t *wqe;
-       struct mlx4_wqe_ctrl_seg *ctrl = NULL;
-       enum mlx4_opcode_type opcode;
-       int ind;
-       int nreq;
-       int inl = 0;
-       ib_api_status_t status = IB_SUCCESS;
-       ib_send_wr_t *wr = p_wr;
-       int size = 0;
-       uint32_t i;
-
-       pthread_spin_lock(&qp->sq.lock);
-
-       /* XXX check that state is OK to post send */
-       if(ibqp->state == IBV_QPS_RESET) {
-               status = IB_INVALID_QP_STATE;
-               if (bad_wr)
-                       *bad_wr = wr;
-               goto err_state;
-       }
-
-       ind = qp->sq.head;
-
-       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
-               if (wq_overflow(&qp->sq, nreq, to_mcq(qp->ibv_qp.send_cq))) {
-                       status = IB_INSUFFICIENT_RESOURCES;
-                       if (bad_wr)
-                               *bad_wr = wr;
-                       goto out;
-               }
-
-               if (wr->num_ds > (uint32_t)qp->sq.max_gs) {
-                       status = IB_INVALID_MAX_SGE;
-                       if (bad_wr)
-                               *bad_wr = wr;
-                       goto out;
-               }
-
-               opcode = __to_opcode(wr);
-               if (opcode == MLX4_OPCODE_INVALID) {
-                       status = IB_INVALID_WR_TYPE;
-                       if (bad_wr)
-                               *bad_wr = wr;
-                       goto out;
-               }
-
-               wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
-               ctrl = (struct mlx4_wqe_ctrl_seg *)wqe;
-               qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
-
-               ctrl->xrcrb_flags =
-                       (wr->send_opt & IB_SEND_OPT_SIGNALED ?
-                        cl_hton32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
-                       (wr->send_opt & IB_SEND_OPT_SOLICITED ?
-                        cl_hton32(MLX4_WQE_CTRL_SOLICIT) : 0)   |
-                       qp->sq_signal_bits;
-
-               if (opcode == MLX4_OPCODE_SEND_IMM ||
-                   opcode == MLX4_OPCODE_RDMA_WRITE_IMM)
-                       ctrl->imm = wr->immediate_data;
-               else
-                       ctrl->imm = 0;
-
-               wqe += sizeof *ctrl;
-               size = sizeof *ctrl / 16;
-
-               switch (ibqp->qp_type) {
-#ifdef XRC_SUPPORT
-               case IBV_QPT_XRC:
-                       // TODO: why is the following line outcommented ?
-                       //ctrl->xrcrb_flags |= cl_hton32(wr->xrc_remote_srq_num << 8);
-                       /* fall thru */
-#endif                 
-               case IBV_QPT_RC:
-               case IBV_QPT_UC:
-                       switch (opcode) {
-                       case MLX4_OPCODE_ATOMIC_CS:
-                       case MLX4_OPCODE_ATOMIC_FA:
-                               set_raddr_seg((struct mlx4_wqe_raddr_seg *)wqe, wr->remote_ops.vaddr,
-                                                               wr->remote_ops.rkey);
-                               wqe  += sizeof (struct mlx4_wqe_raddr_seg);
-
-                               set_atomic_seg((struct mlx4_wqe_atomic_seg *)wqe, wr);
-                               wqe  += sizeof (struct mlx4_wqe_atomic_seg);
-                               size += (sizeof (struct mlx4_wqe_raddr_seg) +
-                                        sizeof (struct mlx4_wqe_atomic_seg)) / 16;
-
-                               break;
-
-                       case MLX4_OPCODE_RDMA_READ:
-                               inl = 1;
-                               /* fall through */
-                       case MLX4_OPCODE_RDMA_WRITE:
-                       case MLX4_OPCODE_RDMA_WRITE_IMM:
-                               set_raddr_seg((struct mlx4_wqe_raddr_seg *)wqe, wr->remote_ops.vaddr,
-                                                               wr->remote_ops.rkey);
-                               wqe  += sizeof (struct mlx4_wqe_raddr_seg);
-                               size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
-
-                               break;
-
-                       default:
-                               /* No extra segments required for sends */
-                               break;
-                       }
-                       break;
-
-               case IBV_QPT_UD:
-                       set_datagram_seg((struct mlx4_wqe_datagram_seg *)wqe, wr);
-                       wqe  += sizeof (struct mlx4_wqe_datagram_seg);
-                       size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
-                       break;
-
-               default:
-                       break;
-               }
-
-               if (wr->send_opt & IB_SEND_OPT_INLINE && wr->num_ds) {
-                       struct mlx4_wqe_inline_seg *seg;
-                       uint8_t *addr;
-                       int len, seg_len;
-                       int num_seg;
-                       int off, to_copy;
-
-                       inl = 0;
-
-                       seg = (struct mlx4_wqe_inline_seg *)wqe;
-                       wqe += sizeof *seg;
-                       off = (int)(((uintptr_t) wqe) & (MLX4_INLINE_ALIGN - 1));
-                       num_seg = 0;
-                       seg_len = 0;
-
-                       for (i = 0; i < wr->num_ds; ++i) {
-                               addr = (uint8_t *)(uintptr_t)wr->ds_array[i].vaddr;
-                               len  = wr->ds_array[i].length;
-                               inl += len;
-
-                               if ((uint32_t)inl > (uint32_t)qp->max_inline_data) {
-                                       inl = 0;
-                                       status = IB_INVALID_PARAMETER;
-                                       *bad_wr = wr;
-                                       goto out;
-                               }
-
-                               while (len >= MLX4_INLINE_ALIGN - off) {
-                                       to_copy = MLX4_INLINE_ALIGN - off;
-                                       memcpy(wqe, addr, to_copy);
-                                       len -= to_copy;
-                                       wqe += to_copy;
-                                       addr += to_copy;
-                                       seg_len += to_copy;
-                                       wmb(); /* see comment below */
-                                       seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);
-                                       seg_len = 0;
-                                       seg = (struct mlx4_wqe_inline_seg *)wqe;
-                                       wqe += sizeof *seg;
-                                       off = sizeof *seg;
-                                       ++num_seg;
-                               }
-
-                               memcpy(wqe, addr, len);
-                               wqe += len;
-                               seg_len += len;
-                               off += len;
-                       }
-
-                       if (seg_len) {
-                               ++num_seg;
-                               /*
-                                * Need a barrier here to make sure
-                                * all the data is visible before the
-                                * byte_count field is set.  Otherwise
-                                * the HCA prefetcher could grab the
-                                * 64-byte chunk with this inline
-                                * segment and get a valid (!=
-                                * 0xffffffff) byte count but stale
-                                * data, and end up sending the wrong
-                                * data.
-                                */
-                               wmb();
-                               seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);
-                       }
-
-                       size += (inl + num_seg * sizeof * seg + 15) / 16;
-               } else {
-                       struct mlx4_wqe_data_seg *seg = (struct mlx4_wqe_data_seg *)wqe;
-
-                       for (i = wr->num_ds; i > 0; --i)
-                               set_data_seg(seg + i - 1, wr->ds_array + i - 1);
-
-                       size += wr->num_ds * (sizeof *seg / 16);
-               }
-
-               ctrl->fence_size = (uint8_t)((wr->send_opt & IB_SEND_OPT_FENCE ?
-                                                                       MLX4_WQE_CTRL_FENCE : 0) | size);
-
-               /*
-                * Make sure descriptor is fully written before
-                * setting ownership bit (because HW can start
-                * executing as soon as we do).
-                */
-               wmb();
-
-               ctrl->owner_opcode = htonl(opcode) |
-                       (ind & qp->sq.wqe_cnt ? htonl((uint32_t)1 << 31) : 0);
-
-               /*
-                * We can improve latency by not stamping the last
-                * send queue WQE until after ringing the doorbell, so
-                * only stamp here if there are still more WQEs to post.
-                */
-               if (wr->p_next)
-                       stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &
-                                      (qp->sq.wqe_cnt - 1));
-
-               ++ind;
-
-               MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_QP, ("qpn %#x, wr_id %#I64x, ix %d, solicited %d\n", 
-                       qp->ibv_qp.qp_num, wr->wr_id, ind - 1, wr->send_opt & IB_SEND_OPT_SOLICITED)); 
-       }
-
-out:
-       ctx = to_mctx(ibqp->context);
-
-       if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) {
-               ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8);
-               *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn;
-               /*
-                * Make sure that descriptor is written to memory
-                * before writing to BlueFlame page.
-                */
-               wmb();
-
-               ++qp->sq.head;
-
-               pthread_spin_lock(&ctx->bf_lock);
-
-               mlx4_bf_copy((unsigned long *) (ctx->bf_page + ctx->bf_offset),
-                                               (unsigned long *) ctrl, align(size * 16, 64));
-
-               wc_wmb();
-
-               ctx->bf_offset ^= ctx->bf_buf_size;
-
-               pthread_spin_unlock(&ctx->bf_lock);
-       }else if (nreq) {
-               qp->sq.head += nreq;
-
-               /*
-                * Make sure that descriptors are written before
-                * doorbell record.
-                */
-               wmb();
-
-               *(uint32_t *) (ctx->uar + MLX4_SEND_DOORBELL) = qp->doorbell_qpn;
-       }
-
-       if (nreq)
-               stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &
-                              (qp->sq.wqe_cnt - 1));
-
-err_state:
-       pthread_spin_unlock(&qp->sq.lock);
-
-       return status;
-}
-
-
-ib_api_status_t
-mlx4_post_recv(
-       IN              const   void* FUNC_PTR64                                h_qp,
-       IN                              ib_recv_wr_t*   const           p_wr,
-               OUT                     ib_recv_wr_t**                          bad_wr)
-{
-       struct mlx4_qp *qp = to_mqp((struct ibv_qp *)/*Ptr64ToPtr(*/h_qp/*)*/);
-       struct mlx4_wqe_data_seg *scat;
-       ib_api_status_t status = IB_SUCCESS;
-       ib_recv_wr_t *wr = p_wr;
-       int nreq;
-       int ind;
-       uint32_t i;
-
-       pthread_spin_lock(&qp->rq.lock);
-
-       /* XXX check that state is OK to post receive */
-       if(qp->ibv_qp.state == IBV_QPS_RESET) {
-               status = IB_INVALID_QP_STATE;
-               if (bad_wr)
-                       *bad_wr = wr;
-               goto err_state;
-       }
-
-       ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
-
-       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
-               if (wq_overflow(&qp->rq, nreq, to_mcq(qp->ibv_qp.recv_cq))) {
-                       status = IB_INSUFFICIENT_RESOURCES;
-                       if (bad_wr)
-                               *bad_wr = wr;
-                       goto out;
-               }
-
-               if (wr->num_ds > (uint32_t)qp->rq.max_gs) {
-                       status = IB_INVALID_MAX_SGE;
-                       if (bad_wr)
-                               *bad_wr = wr;
-                       goto out;
-               }
-
-               scat = get_recv_wqe(qp, ind);
-
-               for (i = 0; i < wr->num_ds; ++i)
-                       __set_data_seg(scat + i, wr->ds_array + i);
-
-               if (i < (uint32_t)qp->rq.max_gs) {
-                       scat[i].byte_count = 0;
-                       scat[i].lkey       = htonl(MLX4_INVALID_LKEY);
-                       scat[i].addr       = 0;
-               }
-
-               qp->rq.wrid[ind] = wr->wr_id;
-
-               ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
-
-               MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_QP, ("qpn %#x, wr_id %#I64x, ix %d, \n", 
-                       qp->ibv_qp.qp_num, wr->wr_id, ind - 1)); 
-       }
-
-out:
-       if (nreq) {
-               qp->rq.head += nreq;
-
-               /*
-                * Make sure that descriptors are written before
-                * doorbell record.
-                */
-               wmb();
-
-               *qp->db = htonl(qp->rq.head & 0xffff);
-       }
-
-err_state:
-       pthread_spin_unlock(&qp->rq.lock);
-
-       return status;
-}
-
-static int num_inline_segs(int data, enum ibv_qp_type type)
-{
-       /*
-        * Inline data segments are not allowed to cross 64 byte
-        * boundaries.  For UD QPs, the data segments always start
-        * aligned to 64 bytes (16 byte control segment + 48 byte
-        * datagram segment); for other QPs, there will be a 16 byte
-        * control segment and possibly a 16 byte remote address
-        * segment, so in the worst case there will be only 32 bytes
-        * available for the first data segment.
-        */
-       if (type == IBV_QPT_UD)
-               data += (sizeof (struct mlx4_wqe_ctrl_seg) +
-                        sizeof (struct mlx4_wqe_datagram_seg)) %
-                       MLX4_INLINE_ALIGN;
-       else
-               data += (sizeof (struct mlx4_wqe_ctrl_seg) +
-                        sizeof (struct mlx4_wqe_raddr_seg)) %
-                       MLX4_INLINE_ALIGN;
-
-       return (int)(data + MLX4_INLINE_ALIGN - sizeof (struct mlx4_wqe_inline_seg) - 1) /
-               (MLX4_INLINE_ALIGN - sizeof (struct mlx4_wqe_inline_seg));
-}
-
-void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
-                          struct mlx4_qp *qp)
-{
-       int size;
-       unsigned max_sq_sge;
-
-       max_sq_sge       = align(cap->max_inline_data +
-                                num_inline_segs(cap->max_inline_data, type) *
-                                sizeof (struct mlx4_wqe_inline_seg),
-                                sizeof (struct mlx4_wqe_data_seg)) /
-               sizeof (struct mlx4_wqe_data_seg);
-       if (max_sq_sge < cap->max_send_sge)
-               max_sq_sge = cap->max_send_sge;
-
-       size = max_sq_sge * sizeof (struct mlx4_wqe_data_seg);
-       switch (type) {
-       case IBV_QPT_UD:
-               size += sizeof (struct mlx4_wqe_datagram_seg);
-               break;
-
-       case IBV_QPT_UC:
-               size += sizeof (struct mlx4_wqe_raddr_seg);
-               break;
-
-#ifdef XRC_SUPPORT
-       case IBV_QPT_XRC:
-#endif         
-       case IBV_QPT_RC:
-               size += sizeof (struct mlx4_wqe_raddr_seg);
-               /*
-                * An atomic op will require an atomic segment, a
-                * remote address segment and one scatter entry.
-                */
-               if (size < (sizeof (struct mlx4_wqe_atomic_seg) +
-                           sizeof (struct mlx4_wqe_raddr_seg) +
-                           sizeof (struct mlx4_wqe_data_seg)))
-                       size = (sizeof (struct mlx4_wqe_atomic_seg) +
-                               sizeof (struct mlx4_wqe_raddr_seg) +
-                               sizeof (struct mlx4_wqe_data_seg));
-               break;
-
-       default:
-               break;
-       }
-
-       /* Make sure that we have enough space for a bind request */
-       if (size < sizeof (struct mlx4_wqe_bind_seg))
-               size = sizeof (struct mlx4_wqe_bind_seg);
-
-       size += sizeof (struct mlx4_wqe_ctrl_seg);
-
-       for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;
-            qp->sq.wqe_shift++)
-               ; /* nothing */
-}
-
-int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
-                      enum ibv_qp_type type, struct mlx4_qp *qp)
-{
-       UNREFERENCED_PARAMETER(type);
-       
-       qp->rq.max_gs    = cap->max_recv_sge;
-
-       qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof (uint64_t));
-       if (!qp->sq.wrid)
-               return -1;
-
-       if (qp->rq.wqe_cnt) {
-               qp->rq.wrid = malloc(qp->rq.wqe_cnt * sizeof (uint64_t));
-               if (!qp->rq.wrid) {
-                       free(qp->sq.wrid);
-                       return -1;
-               }
-       }
-
-       for (qp->rq.wqe_shift = 4;
-               (1 << qp->rq.wqe_shift) < qp->rq.max_gs * (int) sizeof (struct mlx4_wqe_data_seg);
-               qp->rq.wqe_shift++)
-               ; /* nothing */
-
-       qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
-               (qp->sq.wqe_cnt << qp->sq.wqe_shift);
-       if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
-               qp->rq.offset = 0;
-               qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
-       } else {
-               qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
-               qp->sq.offset = 0;
-       }
-
-       if (mlx4_alloc_buf(&qp->buf, qp->buf_size, pd->context->page_size)) {
-               free(qp->sq.wrid);
-               if (qp->rq.wqe_cnt)
-                       free(qp->rq.wrid);
-               return -1;
-       }
-
-       mlx4_qp_init_sq_ownership(qp);
-
-       return 0;
-}
-
-void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
-                      enum ibv_qp_type type)
-{
-       int wqe_size;
-       struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);
-
-       wqe_size = (1 << qp->sq.wqe_shift) - (int) sizeof (struct mlx4_wqe_ctrl_seg);
-       
-       switch (type) {
-       case IBV_QPT_UD:
-               wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);
-               break;
-
-       case IBV_QPT_UC:
-       case IBV_QPT_RC:
-#ifdef XRC_SUPPORT
-       case IBV_QPT_XRC:
-#endif         
-               wqe_size -= sizeof (struct mlx4_wqe_raddr_seg);
-               break;
-
-       default:
-               break;
-       }
-
-       qp->sq.max_gs        = wqe_size / sizeof (struct mlx4_wqe_data_seg);
-       cap->max_send_sge    = min(ctx->max_sge, qp->sq.max_gs);
-       qp->sq.max_post      = min(ctx->max_qp_wr,
-                                  qp->sq.wqe_cnt - qp->sq_spare_wqes);
-       cap->max_send_wr     = qp->sq.max_post;
-
-       /*
-        * Inline data segments can't cross a 64 byte boundary.  So
-        * subtract off one segment header for each 64-byte chunk,
-        * taking into account the fact that wqe_size will be 32 mod
-        * 64 for non-UD QPs.
-        */
-       qp->max_inline_data  = wqe_size -
-               (int) sizeof (struct mlx4_wqe_inline_seg) *
-               (align(wqe_size, MLX4_INLINE_ALIGN) / MLX4_INLINE_ALIGN);
-       cap->max_inline_data = qp->max_inline_data;
-}
-
-struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn)
-{
-       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
-
-       if (ctx->qp_table[tind].refcnt)
-               return ctx->qp_table[tind].table[qpn & ctx->qp_table_mask];
-       else
-               return NULL;
-}
-
-int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp)
-{
-       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
-       int ret = 0;
-
-       pthread_mutex_lock(&ctx->qp_table_mutex);
-
-       if (!ctx->qp_table[tind].refcnt) {
-               ctx->qp_table[tind].table = calloc(ctx->qp_table_mask + 1,
-                                                  sizeof (struct mlx4_qp *));
-               if (!ctx->qp_table[tind].table) {
-                       ret = -1;
-                       goto out;
-               }
-       }
-
-       ++ctx->qp_table[tind].refcnt;
-       ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = qp;
-
-out:
-       pthread_mutex_unlock(&ctx->qp_table_mutex);
-       return ret;
-}
-
-void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn)
-{
-       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
-
-       pthread_mutex_lock(&ctx->qp_table_mutex);
-
-       if (!--ctx->qp_table[tind].refcnt)
-               free(ctx->qp_table[tind].table);
-       else
-               ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = NULL;
-
-       pthread_mutex_unlock(&ctx->qp_table_mutex);
-}
+/*\r
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.\r
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.\r
+ * Copyright (c) 2007 Cisco, Inc.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include "mlx4.h"\r
+#include "doorbell.h"\r
+#include "wqe.h"\r
+#include "mlx4_debug.h"\r
+\r
+#if defined(EVENT_TRACING)\r
+#include "qp.tmh"\r
+#endif\r
+\r
+static enum mlx4_opcode_type __to_opcode(ib_send_wr_t *wr)\r
+{\r
+\r
+       enum mlx4_opcode_type opcode = MLX4_OPCODE_INVALID;\r
+\r
+       switch (wr->wr_type) {\r
+       case WR_SEND: \r
+               opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? \r
+                                       MLX4_OPCODE_SEND_IMM : MLX4_OPCODE_SEND;\r
+               break;\r
+       case WR_RDMA_WRITE:     \r
+               opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ?\r
+                                       MLX4_OPCODE_RDMA_WRITE_IMM : MLX4_OPCODE_RDMA_WRITE;\r
+               break;\r
+       case WR_RDMA_READ:\r
+               opcode = MLX4_OPCODE_RDMA_READ;\r
+               break;\r
+       case WR_COMPARE_SWAP:\r
+               opcode = MLX4_OPCODE_ATOMIC_CS;\r
+               break;\r
+       case WR_FETCH_ADD:\r
+               opcode = MLX4_OPCODE_ATOMIC_FA;\r
+               break;\r
+       default:\r
+               opcode = MLX4_OPCODE_INVALID;\r
+               break;\r
+       }\r
+\r
+       return opcode;\r
+}\r
+\r
+static void *get_recv_wqe(struct mlx4_qp *qp, int n)\r
+{\r
+       return qp->buf.buf + qp->rq.offset + (n << qp->rq.wqe_shift);\r
+}\r
+\r
+static void *get_send_wqe(struct mlx4_qp *qp, int n)\r
+{\r
+       return qp->buf.buf + qp->sq.offset + (n << qp->sq.wqe_shift);\r
+}\r
+\r
+/*\r
+ * Stamp a SQ WQE so that it is invalid if prefetched by marking the\r
+ * first four bytes of every 64 byte chunk with 0xffffffff, except for\r
+ * the very first chunk of the WQE.\r
+ */\r
+static void stamp_send_wqe(struct mlx4_qp *qp, int n)\r
+{\r
+       uint32_t *wqe = get_send_wqe(qp, n);\r
+       int i;\r
+\r
+       for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16)\r
+               wqe[i] = 0xffffffff;\r
+}\r
+\r
+void mlx4_init_qp_indices(struct mlx4_qp *qp)\r
+{\r
+       qp->sq.head      = 0;\r
+       qp->sq.tail      = 0;\r
+       qp->rq.head      = 0;\r
+       qp->rq.tail      = 0;\r
+}\r
+\r
+void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp)\r
+{\r
+       struct mlx4_wqe_ctrl_seg *ctrl;\r
+       int i;\r
+\r
+       for (i = 0; i < qp->sq.wqe_cnt; ++i) {\r
+               ctrl = get_send_wqe(qp, i);\r
+               ctrl->owner_opcode = htonl((uint32_t)1 << 31);\r
+\r
+               stamp_send_wqe(qp, i);\r
+       }\r
+}\r
+\r
+static int wq_overflow(struct mlx4_wq *wq, int nreq, struct mlx4_cq *cq)\r
+{\r
+       int cur;\r
+\r
+       cur = wq->head - wq->tail;\r
+       if (cur + nreq < wq->max_post)\r
+               return 0;\r
+\r
+       pthread_spin_lock(&cq->lock);\r
+       cur = wq->head - wq->tail;\r
+       pthread_spin_unlock(&cq->lock);\r
+\r
+       return cur + nreq >= wq->max_post;\r
+}\r
+\r
+static inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,\r
+                                uint64_t remote_addr, uint32_t rkey)\r
+{\r
+       rseg->raddr    = cl_hton64(remote_addr);\r
+       rseg->rkey     = rkey;\r
+       rseg->reserved = 0;\r
+}\r
+\r
+static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr)\r
+{\r
+       if (wr->wr_type == WR_COMPARE_SWAP) {\r
+               aseg->swap_add = wr->remote_ops.atomic2;\r
+               aseg->compare  = wr->remote_ops.atomic1;\r
+       } else {\r
+               aseg->swap_add = wr->remote_ops.atomic1;\r
+               aseg->compare  = 0;\r
+       }\r
+}\r
+\r
+static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, ib_send_wr_t *wr)\r
+{\r
+       memcpy(dseg->av, &to_mah((struct ibv_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av));\r
+       dseg->dqpn = wr->dgrm.ud.remote_qp;\r
+       dseg->qkey = wr->dgrm.ud.remote_qkey;\r
+}\r
+\r
+static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *ds)\r
+{\r
+       dseg->byte_count = cl_hton32(ds->length);\r
+       dseg->lkey       = cl_hton32(ds->lkey);\r
+       dseg->addr       = cl_hton64(ds->vaddr);\r
+}\r
+\r
+static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *ds)\r
+{\r
+       dseg->lkey       = cl_hton32(ds->lkey);\r
+       dseg->addr       = cl_hton64(ds->vaddr);\r
+\r
+       /*\r
+        * Need a barrier here before writing the byte_count field to\r
+        * make sure that all the data is visible before the\r
+        * byte_count field is set.  Otherwise, if the segment begins\r
+        * a new cacheline, the HCA prefetcher could grab the 64-byte\r
+        * chunk and get a valid (!= * 0xffffffff) byte count but\r
+        * stale data, and end up sending the wrong data.\r
+        */\r
+       wmb();\r
+\r
+       dseg->byte_count = cl_hton32(ds->length);\r
+}\r
+\r
+/*\r
+ * Avoid using memcpy() to copy to BlueFlame page, since memcpy()\r
+ * implementations may use move-string-buffer assembler instructions,\r
+ * which do not guarantee order of copying.\r
+ */\r
+static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, unsigned bytecnt)\r
+{\r
+#ifdef _WIN64\r
+       uint64_t *d = (uint64_t *)dst;\r
+       uint64_t *s = (uint64_t *)src;\r
+\r
+       while (bytecnt > 0) {\r
+               *d++ = *s++;\r
+               *d++ = *s++;\r
+               bytecnt -= 2 * sizeof (uint64_t);\r
+       }\r
+#else\r
+       while (bytecnt > 0) {\r
+               *dst++ = *src++;\r
+               *dst++ = *src++;\r
+               bytecnt -= 2 * sizeof (unsigned long);\r
+       }\r
+#endif \r
+}\r
+\r
+ib_api_status_t\r
+mlx4_post_send(\r
+       IN              const   void*                                           h_qp,\r
+       IN                              ib_send_wr_t*   const           p_wr,\r
+               OUT                     ib_send_wr_t**                          bad_wr)\r
+{\r
+       struct ibv_qp *ibqp = (struct ibv_qp *)/*Ptr64ToPtr(*/h_qp/*)*/;\r
+       struct mlx4_qp *qp = to_mqp(ibqp);\r
+       struct mlx4_context *ctx;\r
+       uint8_t *wqe;\r
+       struct mlx4_wqe_ctrl_seg *ctrl = NULL;\r
+       enum mlx4_opcode_type opcode;\r
+       int ind;\r
+       int nreq;\r
+       int inl = 0;\r
+       ib_api_status_t status = IB_SUCCESS;\r
+       ib_send_wr_t *wr = p_wr;\r
+       int size = 0;\r
+       uint32_t i;\r
+\r
+       pthread_spin_lock(&qp->sq.lock);\r
+\r
+       /* XXX check that state is OK to post send */\r
+       if(ibqp->state == IBV_QPS_RESET) {\r
+               status = IB_INVALID_QP_STATE;\r
+               if (bad_wr)\r
+                       *bad_wr = wr;\r
+               goto err_state;\r
+       }\r
+\r
+       ind = qp->sq.head;\r
+\r
+       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+               if (wq_overflow(&qp->sq, nreq, to_mcq(qp->ibv_qp.send_cq))) {\r
+                       status = IB_INSUFFICIENT_RESOURCES;\r
+                       if (bad_wr)\r
+                               *bad_wr = wr;\r
+                       goto out;\r
+               }\r
+\r
+               if (wr->num_ds > (uint32_t)qp->sq.max_gs) {\r
+                       status = IB_INVALID_MAX_SGE;\r
+                       if (bad_wr)\r
+                               *bad_wr = wr;\r
+                       goto out;\r
+               }\r
+\r
+               opcode = __to_opcode(wr);\r
+               if (opcode == MLX4_OPCODE_INVALID) {\r
+                       status = IB_INVALID_WR_TYPE;\r
+                       if (bad_wr)\r
+                               *bad_wr = wr;\r
+                       goto out;\r
+               }\r
+\r
+               wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));\r
+               ctrl = (struct mlx4_wqe_ctrl_seg *)wqe;\r
+               qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;\r
+\r
+               ctrl->xrcrb_flags =\r
+                       (wr->send_opt & IB_SEND_OPT_SIGNALED ?\r
+                        cl_hton32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |\r
+                       (wr->send_opt & IB_SEND_OPT_SOLICITED ?\r
+                        cl_hton32(MLX4_WQE_CTRL_SOLICIT) : 0)   |\r
+                       qp->sq_signal_bits;\r
+\r
+               if (opcode == MLX4_OPCODE_SEND_IMM ||\r
+                   opcode == MLX4_OPCODE_RDMA_WRITE_IMM)\r
+                       ctrl->imm = wr->immediate_data;\r
+               else\r
+                       ctrl->imm = 0;\r
+\r
+               wqe += sizeof *ctrl;\r
+               size = sizeof *ctrl / 16;\r
+\r
+               switch (ibqp->qp_type) {\r
+#ifdef XRC_SUPPORT\r
+               case IBV_QPT_XRC:\r
+                       // TODO: why is the following line outcommented ?\r
+                       //ctrl->xrcrb_flags |= cl_hton32(wr->xrc_remote_srq_num << 8);\r
+                       /* fall thru */\r
+#endif                 \r
+               case IBV_QPT_RC:\r
+               case IBV_QPT_UC:\r
+                       switch (opcode) {\r
+                       case MLX4_OPCODE_ATOMIC_CS:\r
+                       case MLX4_OPCODE_ATOMIC_FA:\r
+                               set_raddr_seg((struct mlx4_wqe_raddr_seg *)wqe, wr->remote_ops.vaddr,\r
+                                                               wr->remote_ops.rkey);\r
+                               wqe  += sizeof (struct mlx4_wqe_raddr_seg);\r
+\r
+                               set_atomic_seg((struct mlx4_wqe_atomic_seg *)wqe, wr);\r
+                               wqe  += sizeof (struct mlx4_wqe_atomic_seg);\r
+                               size += (sizeof (struct mlx4_wqe_raddr_seg) +\r
+                                        sizeof (struct mlx4_wqe_atomic_seg)) / 16;\r
+\r
+                               break;\r
+\r
+                       case MLX4_OPCODE_RDMA_READ:\r
+                               inl = 1;\r
+                               /* fall through */\r
+                       case MLX4_OPCODE_RDMA_WRITE:\r
+                       case MLX4_OPCODE_RDMA_WRITE_IMM:\r
+                               set_raddr_seg((struct mlx4_wqe_raddr_seg *)wqe, wr->remote_ops.vaddr,\r
+                                                               wr->remote_ops.rkey);\r
+                               wqe  += sizeof (struct mlx4_wqe_raddr_seg);\r
+                               size += sizeof (struct mlx4_wqe_raddr_seg) / 16;\r
+\r
+                               break;\r
+\r
+                       default:\r
+                               /* No extra segments required for sends */\r
+                               break;\r
+                       }\r
+                       break;\r
+\r
+               case IBV_QPT_UD:\r
+                       set_datagram_seg((struct mlx4_wqe_datagram_seg *)wqe, wr);\r
+                       wqe  += sizeof (struct mlx4_wqe_datagram_seg);\r
+                       size += sizeof (struct mlx4_wqe_datagram_seg) / 16;\r
+                       break;\r
+\r
+               default:\r
+                       break;\r
+               }\r
+\r
+               if (wr->send_opt & IB_SEND_OPT_INLINE && wr->num_ds) {\r
+                       struct mlx4_wqe_inline_seg *seg;\r
+                       uint8_t *addr;\r
+                       int len, seg_len;\r
+                       int num_seg;\r
+                       int off, to_copy;\r
+\r
+                       inl = 0;\r
+\r
+                       seg = (struct mlx4_wqe_inline_seg *)wqe;\r
+                       wqe += sizeof *seg;\r
+                       off = (int)(((uintptr_t) wqe) & (MLX4_INLINE_ALIGN - 1));\r
+                       num_seg = 0;\r
+                       seg_len = 0;\r
+\r
+                       for (i = 0; i < wr->num_ds; ++i) {\r
+                               addr = (uint8_t *)(uintptr_t)wr->ds_array[i].vaddr;\r
+                               len  = wr->ds_array[i].length;\r
+                               inl += len;\r
+\r
+                               if ((uint32_t)inl > (uint32_t)qp->max_inline_data) {\r
+                                       inl = 0;\r
+                                       status = IB_INVALID_PARAMETER;\r
+                                       *bad_wr = wr;\r
+                                       goto out;\r
+                               }\r
+\r
+                               while (len >= MLX4_INLINE_ALIGN - off) {\r
+                                       to_copy = MLX4_INLINE_ALIGN - off;\r
+                                       memcpy(wqe, addr, to_copy);\r
+                                       len -= to_copy;\r
+                                       wqe += to_copy;\r
+                                       addr += to_copy;\r
+                                       seg_len += to_copy;\r
+                                       wmb(); /* see comment below */\r
+                                       seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);\r
+                                       seg_len = 0;\r
+                                       seg = (struct mlx4_wqe_inline_seg *)wqe;\r
+                                       wqe += sizeof *seg;\r
+                                       off = sizeof *seg;\r
+                                       ++num_seg;\r
+                               }\r
+\r
+                               memcpy(wqe, addr, len);\r
+                               wqe += len;\r
+                               seg_len += len;\r
+                               off += len;\r
+                       }\r
+\r
+                       if (seg_len) {\r
+                               ++num_seg;\r
+                               /*\r
+                                * Need a barrier here to make sure\r
+                                * all the data is visible before the\r
+                                * byte_count field is set.  Otherwise\r
+                                * the HCA prefetcher could grab the\r
+                                * 64-byte chunk with this inline\r
+                                * segment and get a valid (!=\r
+                                * 0xffffffff) byte count but stale\r
+                                * data, and end up sending the wrong\r
+                                * data.\r
+                                */\r
+                               wmb();\r
+                               seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);\r
+                       }\r
+\r
+                       size += (inl + num_seg * sizeof * seg + 15) / 16;\r
+               } else {\r
+                       struct mlx4_wqe_data_seg *seg = (struct mlx4_wqe_data_seg *)wqe;\r
+\r
+                       for (i = wr->num_ds; i > 0; --i)\r
+                               set_data_seg(seg + i - 1, wr->ds_array + i - 1);\r
+\r
+                       size += wr->num_ds * (sizeof *seg / 16);\r
+               }\r
+\r
+               ctrl->fence_size = (uint8_t)((wr->send_opt & IB_SEND_OPT_FENCE ?\r
+                                                                       MLX4_WQE_CTRL_FENCE : 0) | size);\r
+\r
+               /*\r
+                * Make sure descriptor is fully written before\r
+                * setting ownership bit (because HW can start\r
+                * executing as soon as we do).\r
+                */\r
+               wmb();\r
+\r
+               ctrl->owner_opcode = htonl(opcode) |\r
+                       (ind & qp->sq.wqe_cnt ? htonl((uint32_t)1 << 31) : 0);\r
+\r
+               /*\r
+                * We can improve latency by not stamping the last\r
+                * send queue WQE until after ringing the doorbell, so\r
+                * only stamp here if there are still more WQEs to post.\r
+                */\r
+               if (wr->p_next)\r
+                       stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &\r
+                                      (qp->sq.wqe_cnt - 1));\r
+\r
+               ++ind;\r
+\r
+               MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_QP, ("qpn %#x, wr_id %#I64x, ix %d, solicited %d\n", \r
+                       qp->ibv_qp.qp_num, wr->wr_id, ind - 1, wr->send_opt & IB_SEND_OPT_SOLICITED)); \r
+       }\r
+\r
+out:\r
+       ctx = to_mctx(ibqp->context);\r
+\r
+       if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) {\r
+               ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8);\r
+               *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn;\r
+               /*\r
+                * Make sure that descriptor is written to memory\r
+                * before writing to BlueFlame page.\r
+                */\r
+               wmb();\r
+\r
+               ++qp->sq.head;\r
+\r
+               pthread_spin_lock(&ctx->bf_lock);\r
+\r
+               mlx4_bf_copy((unsigned long *) (ctx->bf_page + ctx->bf_offset),\r
+                                               (unsigned long *) ctrl, align(size * 16, 64));\r
+\r
+               wc_wmb();\r
+\r
+               ctx->bf_offset ^= ctx->bf_buf_size;\r
+\r
+               pthread_spin_unlock(&ctx->bf_lock);\r
+       }else if (nreq) {\r
+               qp->sq.head += nreq;\r
+\r
+               /*\r
+                * Make sure that descriptors are written before\r
+                * doorbell record.\r
+                */\r
+               wmb();\r
+\r
+               *(uint32_t *) (ctx->uar + MLX4_SEND_DOORBELL) = qp->doorbell_qpn;\r
+       }\r
+\r
+       if (nreq)\r
+               stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &\r
+                              (qp->sq.wqe_cnt - 1));\r
+\r
+err_state:\r
+       pthread_spin_unlock(&qp->sq.lock);\r
+\r
+       return status;\r
+}\r
+\r
+\r
+ib_api_status_t\r
+mlx4_post_recv(\r
+       IN              const   void*                                           h_qp,\r
+       IN                              ib_recv_wr_t*   const           p_wr,\r
+               OUT                     ib_recv_wr_t**                          bad_wr)\r
+{\r
+       struct mlx4_qp *qp = to_mqp((struct ibv_qp *)/*Ptr64ToPtr(*/h_qp/*)*/);\r
+       struct mlx4_wqe_data_seg *scat;\r
+       ib_api_status_t status = IB_SUCCESS;\r
+       ib_recv_wr_t *wr = p_wr;\r
+       int nreq;\r
+       int ind;\r
+       uint32_t i;\r
+\r
+       pthread_spin_lock(&qp->rq.lock);\r
+\r
+       /* XXX check that state is OK to post receive */\r
+       if(qp->ibv_qp.state == IBV_QPS_RESET) {\r
+               status = IB_INVALID_QP_STATE;\r
+               if (bad_wr)\r
+                       *bad_wr = wr;\r
+               goto err_state;\r
+       }\r
+\r
+       ind = qp->rq.head & (qp->rq.wqe_cnt - 1);\r
+\r
+       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+               if (wq_overflow(&qp->rq, nreq, to_mcq(qp->ibv_qp.recv_cq))) {\r
+                       status = IB_INSUFFICIENT_RESOURCES;\r
+                       if (bad_wr)\r
+                               *bad_wr = wr;\r
+                       goto out;\r
+               }\r
+\r
+               if (wr->num_ds > (uint32_t)qp->rq.max_gs) {\r
+                       status = IB_INVALID_MAX_SGE;\r
+                       if (bad_wr)\r
+                               *bad_wr = wr;\r
+                       goto out;\r
+               }\r
+\r
+               scat = get_recv_wqe(qp, ind);\r
+\r
+               for (i = 0; i < wr->num_ds; ++i)\r
+                       __set_data_seg(scat + i, wr->ds_array + i);\r
+\r
+               if (i < (uint32_t)qp->rq.max_gs) {\r
+                       scat[i].byte_count = 0;\r
+                       scat[i].lkey       = htonl(MLX4_INVALID_LKEY);\r
+                       scat[i].addr       = 0;\r
+               }\r
+\r
+               qp->rq.wrid[ind] = wr->wr_id;\r
+\r
+               ind = (ind + 1) & (qp->rq.wqe_cnt - 1);\r
+\r
+               MLX4_PRINT( TRACE_LEVEL_INFORMATION, MLX4_DBG_QP, ("qpn %#x, wr_id %#I64x, ix %d, \n", \r
+                       qp->ibv_qp.qp_num, wr->wr_id, ind - 1)); \r
+       }\r
+\r
+out:\r
+       if (nreq) {\r
+               qp->rq.head += nreq;\r
+\r
+               /*\r
+                * Make sure that descriptors are written before\r
+                * doorbell record.\r
+                */\r
+               wmb();\r
+\r
+               *qp->db = htonl(qp->rq.head & 0xffff);\r
+       }\r
+\r
+err_state:\r
+       pthread_spin_unlock(&qp->rq.lock);\r
+\r
+       return status;\r
+}\r
+\r
+static int num_inline_segs(int data, enum ibv_qp_type type)\r
+{\r
+       /*\r
+        * Inline data segments are not allowed to cross 64 byte\r
+        * boundaries.  For UD QPs, the data segments always start\r
+        * aligned to 64 bytes (16 byte control segment + 48 byte\r
+        * datagram segment); for other QPs, there will be a 16 byte\r
+        * control segment and possibly a 16 byte remote address\r
+        * segment, so in the worst case there will be only 32 bytes\r
+        * available for the first data segment.\r
+        */\r
+       if (type == IBV_QPT_UD)\r
+               data += (sizeof (struct mlx4_wqe_ctrl_seg) +\r
+                        sizeof (struct mlx4_wqe_datagram_seg)) %\r
+                       MLX4_INLINE_ALIGN;\r
+       else\r
+               data += (sizeof (struct mlx4_wqe_ctrl_seg) +\r
+                        sizeof (struct mlx4_wqe_raddr_seg)) %\r
+                       MLX4_INLINE_ALIGN;\r
+\r
+       return (int)(data + MLX4_INLINE_ALIGN - sizeof (struct mlx4_wqe_inline_seg) - 1) /\r
+               (MLX4_INLINE_ALIGN - sizeof (struct mlx4_wqe_inline_seg));\r
+}\r
+\r
+void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,\r
+                          struct mlx4_qp *qp)\r
+{\r
+       int size;\r
+       unsigned max_sq_sge;\r
+\r
+       max_sq_sge       = align(cap->max_inline_data +\r
+                                num_inline_segs(cap->max_inline_data, type) *\r
+                                sizeof (struct mlx4_wqe_inline_seg),\r
+                                sizeof (struct mlx4_wqe_data_seg)) /\r
+               sizeof (struct mlx4_wqe_data_seg);\r
+       if (max_sq_sge < cap->max_send_sge)\r
+               max_sq_sge = cap->max_send_sge;\r
+\r
+       size = max_sq_sge * sizeof (struct mlx4_wqe_data_seg);\r
+       switch (type) {\r
+       case IBV_QPT_UD:\r
+               size += sizeof (struct mlx4_wqe_datagram_seg);\r
+               break;\r
+\r
+       case IBV_QPT_UC:\r
+               size += sizeof (struct mlx4_wqe_raddr_seg);\r
+               break;\r
+\r
+#ifdef XRC_SUPPORT\r
+       case IBV_QPT_XRC:\r
+#endif         \r
+       case IBV_QPT_RC:\r
+               size += sizeof (struct mlx4_wqe_raddr_seg);\r
+               /*\r
+                * An atomic op will require an atomic segment, a\r
+                * remote address segment and one scatter entry.\r
+                */\r
+               if (size < (sizeof (struct mlx4_wqe_atomic_seg) +\r
+                           sizeof (struct mlx4_wqe_raddr_seg) +\r
+                           sizeof (struct mlx4_wqe_data_seg)))\r
+                       size = (sizeof (struct mlx4_wqe_atomic_seg) +\r
+                               sizeof (struct mlx4_wqe_raddr_seg) +\r
+                               sizeof (struct mlx4_wqe_data_seg));\r
+               break;\r
+\r
+       default:\r
+               break;\r
+       }\r
+\r
+       /* Make sure that we have enough space for a bind request */\r
+       if (size < sizeof (struct mlx4_wqe_bind_seg))\r
+               size = sizeof (struct mlx4_wqe_bind_seg);\r
+\r
+       size += sizeof (struct mlx4_wqe_ctrl_seg);\r
+\r
+       for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;\r
+            qp->sq.wqe_shift++)\r
+               ; /* nothing */\r
+}\r
+\r
+int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,\r
+                      enum ibv_qp_type type, struct mlx4_qp *qp)\r
+{\r
+       UNREFERENCED_PARAMETER(type);\r
+       \r
+       qp->rq.max_gs    = cap->max_recv_sge;\r
+\r
+       qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof (uint64_t));\r
+       if (!qp->sq.wrid)\r
+               return -1;\r
+\r
+       if (qp->rq.wqe_cnt) {\r
+               qp->rq.wrid = malloc(qp->rq.wqe_cnt * sizeof (uint64_t));\r
+               if (!qp->rq.wrid) {\r
+                       free(qp->sq.wrid);\r
+                       return -1;\r
+               }\r
+       }\r
+\r
+       for (qp->rq.wqe_shift = 4;\r
+               (1 << qp->rq.wqe_shift) < qp->rq.max_gs * (int) sizeof (struct mlx4_wqe_data_seg);\r
+               qp->rq.wqe_shift++)\r
+               ; /* nothing */\r
+\r
+       qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +\r
+               (qp->sq.wqe_cnt << qp->sq.wqe_shift);\r
+       if (qp->rq.wqe_shift > qp->sq.wqe_shift) {\r
+               qp->rq.offset = 0;\r
+               qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;\r
+       } else {\r
+               qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;\r
+               qp->sq.offset = 0;\r
+       }\r
+\r
+       if (mlx4_alloc_buf(&qp->buf, qp->buf_size, pd->context->page_size)) {\r
+               free(qp->sq.wrid);\r
+               if (qp->rq.wqe_cnt)\r
+                       free(qp->rq.wrid);\r
+               return -1;\r
+       }\r
+\r
+       mlx4_qp_init_sq_ownership(qp);\r
+\r
+       return 0;\r
+}\r
+\r
+void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,\r
+                      enum ibv_qp_type type)\r
+{\r
+       int wqe_size;\r
+       struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);\r
+\r
+       wqe_size = (1 << qp->sq.wqe_shift) - (int) sizeof (struct mlx4_wqe_ctrl_seg);\r
+       \r
+       switch (type) {\r
+       case IBV_QPT_UD:\r
+               wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);\r
+               break;\r
+\r
+       case IBV_QPT_UC:\r
+       case IBV_QPT_RC:\r
+#ifdef XRC_SUPPORT\r
+       case IBV_QPT_XRC:\r
+#endif         \r
+               wqe_size -= sizeof (struct mlx4_wqe_raddr_seg);\r
+               break;\r
+\r
+       default:\r
+               break;\r
+       }\r
+\r
+       qp->sq.max_gs        = wqe_size / sizeof (struct mlx4_wqe_data_seg);\r
+       cap->max_send_sge    = min(ctx->max_sge, qp->sq.max_gs);\r
+       qp->sq.max_post      = min(ctx->max_qp_wr,\r
+                                  qp->sq.wqe_cnt - qp->sq_spare_wqes);\r
+       cap->max_send_wr     = qp->sq.max_post;\r
+\r
+       /*\r
+        * Inline data segments can't cross a 64 byte boundary.  So\r
+        * subtract off one segment header for each 64-byte chunk,\r
+        * taking into account the fact that wqe_size will be 32 mod\r
+        * 64 for non-UD QPs.\r
+        */\r
+       qp->max_inline_data  = wqe_size -\r
+               (int) sizeof (struct mlx4_wqe_inline_seg) *\r
+               (align(wqe_size, MLX4_INLINE_ALIGN) / MLX4_INLINE_ALIGN);\r
+       cap->max_inline_data = qp->max_inline_data;\r
+}\r
+\r
+struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn)\r
+{\r
+       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;\r
+\r
+       if (ctx->qp_table[tind].refcnt)\r
+               return ctx->qp_table[tind].table[qpn & ctx->qp_table_mask];\r
+       else\r
+               return NULL;\r
+}\r
+\r
+int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp)\r
+{\r
+       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;\r
+       int ret = 0;\r
+\r
+       pthread_mutex_lock(&ctx->qp_table_mutex);\r
+\r
+       if (!ctx->qp_table[tind].refcnt) {\r
+               ctx->qp_table[tind].table = calloc(ctx->qp_table_mask + 1,\r
+                                                  sizeof (struct mlx4_qp *));\r
+               if (!ctx->qp_table[tind].table) {\r
+                       ret = -1;\r
+                       goto out;\r
+               }\r
+       }\r
+\r
+       ++ctx->qp_table[tind].refcnt;\r
+       ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = qp;\r
+\r
+out:\r
+       pthread_mutex_unlock(&ctx->qp_table_mutex);\r
+       return ret;\r
+}\r
+\r
+void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn)\r
+{\r
+       int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;\r
+\r
+       pthread_mutex_lock(&ctx->qp_table_mutex);\r
+\r
+       if (!--ctx->qp_table[tind].refcnt)\r
+               free(ctx->qp_table[tind].table);\r
+       else\r
+               ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = NULL;\r
+\r
+       pthread_mutex_unlock(&ctx->qp_table_mutex);\r
+}\r
index e0c2f4e..c5e9ac8 100644 (file)
-/*
- * Copyright (c) 2007 Cisco, Inc.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "mlx4.h"
-#include "doorbell.h"
-#include "wqe.h"
-
-static void *get_wqe(struct mlx4_srq *srq, int n)
-{
-       return srq->buf.buf + (n << srq->wqe_shift);
-}
-
-void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind)
-{
-       struct mlx4_wqe_srq_next_seg *next;
-
-       pthread_spin_lock(&srq->lock);
-
-       next = get_wqe(srq, srq->tail);
-       next->next_wqe_index = htons((uint16_t)ind);
-       srq->tail = ind;
-
-       pthread_spin_unlock(&srq->lock);
-}
-
-ib_api_status_t
-mlx4_post_srq_recv(
-       IN              const   void* FUNC_PTR64                                        h_srq,
-       IN                              ib_recv_wr_t*           const           p_wr,
-               OUT                     ib_recv_wr_t**                          bad_wr)
-{
-       struct mlx4_srq *srq = to_msrq((struct ibv_srq *)/*Ptr64ToPtr(*/h_srq/*)*/);
-       struct mlx4_wqe_srq_next_seg *next;
-       struct mlx4_wqe_data_seg *scat;
-       ib_api_status_t status = IB_SUCCESS;
-       ib_recv_wr_t *wr = p_wr;
-       uint16_t nreq;
-       uint32_t i;
-
-       pthread_spin_lock(&srq->lock);
-
-       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
-               if (wr->num_ds > (uint32_t)srq->max_gs) {
-                       status = IB_INVALID_MAX_SGE;
-                       *bad_wr = wr;
-                       break;
-               }
-
-               if (srq->head == srq->tail) {
-                       /* SRQ is full*/
-                       status = IB_INSUFFICIENT_RESOURCES;
-                       *bad_wr = wr;
-                       break;
-               }
-
-               srq->wrid[srq->head] = wr->wr_id;
-
-               next      = get_wqe(srq, srq->head);
-               srq->head = ntohs(next->next_wqe_index);
-               scat      = (struct mlx4_wqe_data_seg *) (next + 1);
-
-               for (i = 0; i < wr->num_ds; ++i) {
-                       scat[i].byte_count = htonl(wr->ds_array[i].length);
-                       scat[i].lkey       = htonl(wr->ds_array[i].lkey);
-                       scat[i].addr       = htonll(wr->ds_array[i].vaddr);
-               }
-
-               if (i < (uint32_t)srq->max_gs) {
-                       scat[i].byte_count = 0;
-                       scat[i].lkey       = htonl(MLX4_INVALID_LKEY);
-                       scat[i].addr       = 0;
-               }
-       }
-
-       if (nreq) {
-               srq->counter = srq->counter + nreq;
-
-               /*
-                * Make sure that descriptors are written before
-                * we write doorbell record.
-                */
-               wmb();
-
-               *srq->db = htonl(srq->counter);
-       }
-
-       pthread_spin_unlock(&srq->lock);
-
-       return status;
-}
-
-int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
-                      struct mlx4_srq *srq)
-{
-       struct mlx4_wqe_srq_next_seg *next;
-       int size;
-       int buf_size;
-       int i;
-
-       UNREFERENCED_PARAMETER(attr);
-       
-       srq->wrid = malloc(srq->max * sizeof (uint64_t));
-       if (!srq->wrid)
-               return -1;
-
-       size = sizeof (struct mlx4_wqe_srq_next_seg) +
-               srq->max_gs * sizeof (struct mlx4_wqe_data_seg);
-
-       for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
-               ; /* nothing */
-
-       buf_size = srq->max << srq->wqe_shift;
-
-       if (mlx4_alloc_buf(&srq->buf, buf_size,
-                          pd->context->page_size)) {
-               free(srq->wrid);
-               return -1;
-       }
-
-       /*
-        * Now initialize the SRQ buffer so that all of the WQEs are
-        * linked into the list of free WQEs.
-        */
-
-       for (i = 0; i < srq->max; ++i) {
-               next = get_wqe(srq, i);
-               next->next_wqe_index = htons((uint16_t)((i + 1) & (srq->max - 1)));
-       }
-
-       srq->head = 0;
-       srq->tail = srq->max - 1;
-
-       return 0;
-}
-
-#ifdef XRC_SUPPORT
-struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)
-{
-       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
-
-       if (ctx->xrc_srq_table[tind].refcnt)
-               return ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask];
-       else
-               return NULL;
-}
-
-int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
-                      struct mlx4_srq *srq)
-{
-       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
-       int ret = 0;
-
-       pthread_mutex_lock(&ctx->xrc_srq_table_mutex);
-
-       if (!ctx->xrc_srq_table[tind].refcnt) {
-               ctx->xrc_srq_table[tind].table = calloc(ctx->xrc_srq_table_mask + 1,
-                                                  sizeof (struct mlx4_srq *));
-               if (!ctx->xrc_srq_table[tind].table) {
-                       ret = -1;
-                       goto out;
-               }
-       }
-
-       ++ctx->xrc_srq_table[tind].refcnt;
-       ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = srq;
-
-out:
-       pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);
-       return ret;
-}
-
-void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)
-{
-       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
-
-       pthread_mutex_lock(&ctx->xrc_srq_table_mutex);
-
-       if (!--ctx->xrc_srq_table[tind].refcnt)
-               free(ctx->xrc_srq_table[tind].table);
-       else
-               ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = NULL;
-
-       pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);
-}
-#endif
+/*\r
+ * Copyright (c) 2007 Cisco, Inc.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include "mlx4.h"\r
+#include "doorbell.h"\r
+#include "wqe.h"\r
+\r
+static void *get_wqe(struct mlx4_srq *srq, int n)\r
+{\r
+       return srq->buf.buf + (n << srq->wqe_shift);\r
+}\r
+\r
+void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind)\r
+{\r
+       struct mlx4_wqe_srq_next_seg *next;\r
+\r
+       pthread_spin_lock(&srq->lock);\r
+\r
+       next = get_wqe(srq, srq->tail);\r
+       next->next_wqe_index = htons((uint16_t)ind);\r
+       srq->tail = ind;\r
+\r
+       pthread_spin_unlock(&srq->lock);\r
+}\r
+\r
+ib_api_status_t\r
+mlx4_post_srq_recv(\r
+       IN              const   void*                                                   h_srq,\r
+       IN                              ib_recv_wr_t*           const           p_wr,\r
+               OUT                     ib_recv_wr_t**                          bad_wr)\r
+{\r
+       struct mlx4_srq *srq = to_msrq((struct ibv_srq *)/*Ptr64ToPtr(*/h_srq/*)*/);\r
+       struct mlx4_wqe_srq_next_seg *next;\r
+       struct mlx4_wqe_data_seg *scat;\r
+       ib_api_status_t status = IB_SUCCESS;\r
+       ib_recv_wr_t *wr = p_wr;\r
+       uint16_t nreq;\r
+       uint32_t i;\r
+\r
+       pthread_spin_lock(&srq->lock);\r
+\r
+       for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+               if (wr->num_ds > (uint32_t)srq->max_gs) {\r
+                       status = IB_INVALID_MAX_SGE;\r
+                       *bad_wr = wr;\r
+                       break;\r
+               }\r
+\r
+               if (srq->head == srq->tail) {\r
+                       /* SRQ is full*/\r
+                       status = IB_INSUFFICIENT_RESOURCES;\r
+                       *bad_wr = wr;\r
+                       break;\r
+               }\r
+\r
+               srq->wrid[srq->head] = wr->wr_id;\r
+\r
+               next      = get_wqe(srq, srq->head);\r
+               srq->head = ntohs(next->next_wqe_index);\r
+               scat      = (struct mlx4_wqe_data_seg *) (next + 1);\r
+\r
+               for (i = 0; i < wr->num_ds; ++i) {\r
+                       scat[i].byte_count = htonl(wr->ds_array[i].length);\r
+                       scat[i].lkey       = htonl(wr->ds_array[i].lkey);\r
+                       scat[i].addr       = htonll(wr->ds_array[i].vaddr);\r
+               }\r
+\r
+               if (i < (uint32_t)srq->max_gs) {\r
+                       scat[i].byte_count = 0;\r
+                       scat[i].lkey       = htonl(MLX4_INVALID_LKEY);\r
+                       scat[i].addr       = 0;\r
+               }\r
+       }\r
+\r
+       if (nreq) {\r
+               srq->counter = srq->counter + nreq;\r
+\r
+               /*\r
+                * Make sure that descriptors are written before\r
+                * we write doorbell record.\r
+                */\r
+               wmb();\r
+\r
+               *srq->db = htonl(srq->counter);\r
+       }\r
+\r
+       pthread_spin_unlock(&srq->lock);\r
+\r
+       return status;\r
+}\r
+\r
+int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,\r
+                      struct mlx4_srq *srq)\r
+{\r
+       struct mlx4_wqe_srq_next_seg *next;\r
+       int size;\r
+       int buf_size;\r
+       int i;\r
+\r
+       UNREFERENCED_PARAMETER(attr);\r
+       \r
+       srq->wrid = malloc(srq->max * sizeof (uint64_t));\r
+       if (!srq->wrid)\r
+               return -1;\r
+\r
+       size = sizeof (struct mlx4_wqe_srq_next_seg) +\r
+               srq->max_gs * sizeof (struct mlx4_wqe_data_seg);\r
+\r
+       for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift)\r
+               ; /* nothing */\r
+\r
+       buf_size = srq->max << srq->wqe_shift;\r
+\r
+       if (mlx4_alloc_buf(&srq->buf, buf_size,\r
+                          pd->context->page_size)) {\r
+               free(srq->wrid);\r
+               return -1;\r
+       }\r
+\r
+       /*\r
+        * Now initialize the SRQ buffer so that all of the WQEs are\r
+        * linked into the list of free WQEs.\r
+        */\r
+\r
+       for (i = 0; i < srq->max; ++i) {\r
+               next = get_wqe(srq, i);\r
+               next->next_wqe_index = htons((uint16_t)((i + 1) & (srq->max - 1)));\r
+       }\r
+\r
+       srq->head = 0;\r
+       srq->tail = srq->max - 1;\r
+\r
+       return 0;\r
+}\r
+\r
+#ifdef XRC_SUPPORT\r
+struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)\r
+{\r
+       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;\r
+\r
+       if (ctx->xrc_srq_table[tind].refcnt)\r
+               return ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask];\r
+       else\r
+               return NULL;\r
+}\r
+\r
+int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,\r
+                      struct mlx4_srq *srq)\r
+{\r
+       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;\r
+       int ret = 0;\r
+\r
+       pthread_mutex_lock(&ctx->xrc_srq_table_mutex);\r
+\r
+       if (!ctx->xrc_srq_table[tind].refcnt) {\r
+               ctx->xrc_srq_table[tind].table = calloc(ctx->xrc_srq_table_mask + 1,\r
+                                                  sizeof (struct mlx4_srq *));\r
+               if (!ctx->xrc_srq_table[tind].table) {\r
+                       ret = -1;\r
+                       goto out;\r
+               }\r
+       }\r
+\r
+       ++ctx->xrc_srq_table[tind].refcnt;\r
+       ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = srq;\r
+\r
+out:\r
+       pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);\r
+       return ret;\r
+}\r
+\r
+void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)\r
+{\r
+       int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;\r
+\r
+       pthread_mutex_lock(&ctx->xrc_srq_table_mutex);\r
+\r
+       if (!--ctx->xrc_srq_table[tind].refcnt)\r
+               free(ctx->xrc_srq_table[tind].table);\r
+       else\r
+               ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = NULL;\r
+\r
+       pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);\r
+}\r
+#endif\r
index e3070ce..8ebcda9 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -315,7 +316,7 @@ mlnx_conv_hca_cap(
 \r
 ib_api_status_t\r
 mlnx_local_mad (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN              const   uint8_t                                         port_num,\r
        IN              const   ib_av_attr_t                            *p_src_av_attr,\r
        IN              const   ib_mad_t                                        *p_mad_in,\r
@@ -343,8 +344,8 @@ mlnx_mcast_if(
 \r
 ib_api_status_t\r
 fw_access_ctrl(\r
-       IN              const   void* FUNC_PTR64                                context,\r
-       IN              const   void* FUNC_PTR64* const         handle_array    OPTIONAL,\r
+       IN              const   void*                                           context,\r
+       IN              const   void** const                            handle_array    OPTIONAL,\r
        IN                              uint32_t                                        num_handles,\r
        IN                              ib_ci_op_t* const                       p_ci_op,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf              OPTIONAL);\r
index d4f6e5d..f937200 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -56,7 +57,7 @@
 \r
 ib_api_status_t\r
 mlnx_post_send (\r
-       IN      const   ib_qp_handle_t FUNC_PTR64                                       h_qp,\r
+       IN      const   ib_qp_handle_t                                  h_qp,\r
        IN                      ib_send_wr_t                                    *p_send_wr,\r
                OUT             ib_send_wr_t                                    **pp_failed )\r
 {\r
@@ -94,7 +95,7 @@ err_post_send:
 \r
 ib_api_status_t \r
 mlnx_post_recv (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN                              ib_recv_wr_t                            *p_recv_wr,\r
                OUT                     ib_recv_wr_t                            **pp_failed OPTIONAL )\r
 {\r
@@ -131,7 +132,7 @@ err_post_recv:
 \r
 ib_api_status_t \r
 mlnx_post_srq_recv (\r
-       IN              const   ib_srq_handle_t FUNC_PTR64                              h_srq,\r
+       IN              const   ib_srq_handle_t                         h_srq,\r
        IN                              ib_recv_wr_t                            *p_recv_wr,\r
                OUT                     ib_recv_wr_t                            **pp_failed OPTIONAL )\r
 {\r
@@ -172,7 +173,7 @@ err_post_recv:
 \r
 ib_api_status_t\r
 mlnx_peek_cq(\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
        OUT                             uint32_t* const                         p_n_cqes )\r
 {\r
        UNREFERENCED_PARAMETER(h_cq);\r
@@ -183,7 +184,7 @@ mlnx_peek_cq(
 \r
 ib_api_status_t\r
 mlnx_poll_cq (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
        IN      OUT                     ib_wc_t** const                         pp_free_wclist,\r
                OUT                     ib_wc_t** const                         pp_done_wclist )\r
 {\r
@@ -222,7 +223,7 @@ err_invalid_params:
 \r
 ib_api_status_t\r
 mlnx_enable_cq_notify (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
        IN              const   boolean_t                                       solicited )\r
 {\r
        int err;\r
@@ -251,7 +252,7 @@ mlnx_enable_cq_notify (
 \r
 ib_api_status_t\r
 mlnx_enable_ncomp_cq_notify (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
        IN              const   uint32_t                                        n_cqes )\r
 {\r
        int err;\r
@@ -279,8 +280,8 @@ mlnx_enable_ncomp_cq_notify (
 \r
 ib_api_status_t\r
 mlnx_bind_mw (\r
-       IN              const   ib_mw_handle_t FUNC_PTR64                               h_mw,\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_mw_handle_t                          h_mw,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN                              ib_bind_wr_t* const                     p_mw_bind,\r
                OUT                     net32_t* const                          p_rkey )\r
 {\r
index 754c19b..d41a325 100644 (file)
@@ -1,5 +1,6 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -650,8 +651,8 @@ unmap_crspace_for_all( struct ib_ucontext *p_context )
 \r
 ib_api_status_t\r
 fw_access_ctrl(\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
-       IN              const   void* FUNC_PTR64* const         handle_array    OPTIONAL,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
+       IN              const   void** const                            handle_array    OPTIONAL,\r
        IN                              uint32_t                                        num_handles,\r
        IN                              ib_ci_op_t* const                       p_ci_op,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
index 35a9658..f7c744c 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
 */\r
 ib_api_status_t\r
 mlnx_attach_mcast (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN              const   ib_gid_t                                        *p_mcast_gid,\r
        IN              const   uint16_t                                        mcast_lid,\r
-               OUT                     ib_mcast_handle_t FUNC_PTR64                    *ph_mcast,\r
+               OUT                     ib_mcast_handle_t                       *ph_mcast,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err;\r
@@ -136,7 +137,7 @@ end:
 \r
 ib_api_status_t\r
 mlnx_detach_mcast (\r
-       IN              const   ib_mcast_handle_t FUNC_PTR64                    h_mcast)\r
+       IN              const   ib_mcast_handle_t                       h_mcast)\r
 {\r
        ib_api_status_t         status = IB_INVALID_PARAMETER;\r
        int err;\r
index f73aee5..434e5fb 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
 \r
 ib_api_status_t\r
 mlnx_register_mr (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   ib_mr_create_t                          *p_mr_create,\r
        OUT                     net32_t* const                          p_lkey,\r
        OUT                     net32_t* const                          p_rkey,\r
-       OUT                     ib_mr_handle_t FUNC_PTR64                               *ph_mr,\r
+       OUT                     ib_mr_handle_t                          *ph_mr,\r
        IN                              boolean_t                                       um_call )\r
 {\r
        ib_api_status_t         status;\r
@@ -118,12 +119,12 @@ err_unsupported:
 \r
 ib_api_status_t\r
 mlnx_register_pmr (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   ib_phys_create_t* const         p_pmr_create,\r
        IN      OUT                     uint64_t* const                         p_vaddr,\r
                OUT                     net32_t* const                          p_lkey,\r
                OUT                     net32_t* const                          p_rkey,\r
-               OUT                     ib_mr_handle_t FUNC_PTR64* const                ph_mr,\r
+               OUT                     ib_mr_handle_t* const           ph_mr,\r
        IN                              boolean_t                                       um_call )\r
 {\r
        ib_api_status_t         status;\r
@@ -205,7 +206,7 @@ err_mem:
 \r
 ib_api_status_t\r
 mlnx_query_mr (\r
-       IN              const   ib_mr_handle_t FUNC_PTR64                               h_mr,\r
+       IN              const   ib_mr_handle_t                          h_mr,\r
                OUT                     ib_mr_attr_t                            *p_mr_query )\r
 {\r
        UNREFERENCED_PARAMETER(h_mr);\r
@@ -217,12 +218,12 @@ mlnx_query_mr (
 \r
 ib_api_status_t\r
 mlnx_modify_mr (\r
-       IN              const   ib_mr_handle_t FUNC_PTR64                               h_mr,\r
+       IN              const   ib_mr_handle_t                          h_mr,\r
        IN              const   ib_mr_mod_t                                     mem_modify_req,\r
        IN              const   ib_mr_create_t                          *p_mr_create,\r
                OUT                     uint32_t                                        *p_lkey,\r
                OUT                     uint32_t                                        *p_rkey,\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd OPTIONAL,\r
+       IN              const   ib_pd_handle_t                          h_pd OPTIONAL,\r
        IN                              boolean_t                                       um_call )\r
 {\r
        UNREFERENCED_PARAMETER(h_mr);\r
@@ -239,13 +240,13 @@ mlnx_modify_mr (
 \r
 ib_api_status_t\r
 mlnx_modify_pmr (\r
-       IN              const   ib_mr_handle_t FUNC_PTR64                               h_mr,\r
+       IN              const   ib_mr_handle_t                          h_mr,\r
        IN              const   ib_mr_mod_t                                     mem_modify_req,\r
        IN              const   ib_phys_create_t* const         p_pmr_create,\r
        IN      OUT                     uint64_t* const                         p_vaddr,\r
                OUT                     uint32_t* const                         p_lkey,\r
                OUT                     uint32_t* const                         p_rkey,\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd OPTIONAL,\r
+       IN              const   ib_pd_handle_t                          h_pd OPTIONAL,\r
        IN                              boolean_t                                       um_call )\r
 {\r
        UNREFERENCED_PARAMETER(h_mr);\r
@@ -262,13 +263,13 @@ mlnx_modify_pmr (
 \r
 ib_api_status_t\r
 mlnx_register_smr (\r
-       IN              const   ib_mr_handle_t FUNC_PTR64                               h_mr,\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_mr_handle_t                          h_mr,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   ib_access_t                                     access_ctrl,\r
        IN      OUT                     uint64_t* const                         p_vaddr,\r
                OUT                     net32_t* const                          p_lkey,\r
                OUT                     net32_t* const                          p_rkey,\r
-               OUT                     ib_mr_handle_t FUNC_PTR64* const                ph_mr,\r
+               OUT                     ib_mr_handle_t* const           ph_mr,\r
        IN                              boolean_t                                       um_call )\r
 {\r
        UNREFERENCED_PARAMETER(h_mr);\r
@@ -285,7 +286,7 @@ mlnx_register_smr (
 \r
 ib_api_status_t\r
 mlnx_deregister_mr (\r
-       IN              const   ib_mr_handle_t FUNC_PTR64                               h_mr)\r
+       IN              const   ib_mr_handle_t                          h_mr)\r
 {\r
        ib_api_status_t         status;\r
        int err;\r
@@ -331,9 +332,9 @@ err_unsupported:
 \r
 ib_api_status_t\r
 mlnx_alloc_fmr(\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   mlnx_fmr_create_t* const        p_fmr_create,\r
-               OUT                     mlnx_fmr_handle_t FUNC_PTR64*   const   ph_fmr\r
+               OUT                     mlnx_fmr_handle_t*      const   ph_fmr\r
        )\r
 {\r
        ib_api_status_t         status;\r
@@ -392,7 +393,7 @@ err_unsupported:
 \r
 ib_api_status_t\r
 mlnx_map_phys_fmr (\r
-       IN              const   mlnx_fmr_handle_t FUNC_PTR64                    h_fmr,\r
+       IN              const   mlnx_fmr_handle_t                       h_fmr,\r
        IN              const   uint64_t* const                         page_list,\r
        IN              const   int                                                     list_len,\r
        IN      OUT                     uint64_t* const                         p_vaddr,\r
@@ -438,7 +439,7 @@ err_dealloc_fmr:
 \r
 ib_api_status_t\r
 mlnx_unmap_fmr (\r
-       IN              const   mlnx_fmr_handle_t FUNC_PTR64                    *ph_fmr)\r
+       IN              const   mlnx_fmr_handle_t                       *ph_fmr)\r
 {\r
        ib_api_status_t         status;\r
        int err;\r
@@ -489,7 +490,7 @@ err_unsupported:
        \r
 ib_api_status_t\r
 mlnx_dealloc_fmr (\r
-       IN              const   mlnx_fmr_handle_t FUNC_PTR64                    h_fmr\r
+       IN              const   mlnx_fmr_handle_t                       h_fmr\r
        )\r
 {\r
        ib_api_status_t         status;\r
@@ -538,9 +539,9 @@ err_unsupported:
 \r
 ib_api_status_t\r
 mlnx_create_mw (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
                OUT                     net32_t* const                          p_rkey,\r
-               OUT                     ib_mw_handle_t FUNC_PTR64                               *ph_mw,\r
+               OUT                     ib_mw_handle_t                          *ph_mw,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        UNREFERENCED_PARAMETER(h_pd);\r
@@ -553,8 +554,8 @@ mlnx_create_mw (
 \r
 ib_api_status_t\r
 mlnx_query_mw (\r
-       IN              const   ib_mw_handle_t FUNC_PTR64                               h_mw,\r
-               OUT                     ib_pd_handle_t FUNC_PTR64                               *ph_pd,\r
+       IN              const   ib_mw_handle_t                          h_mw,\r
+               OUT                     ib_pd_handle_t                          *ph_pd,\r
                OUT                     net32_t* const                          p_rkey,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
@@ -568,7 +569,7 @@ mlnx_query_mw (
 \r
 ib_api_status_t\r
 mlnx_destroy_mw (\r
-       IN              const   ib_mw_handle_t FUNC_PTR64                               h_mw)\r
+       IN              const   ib_mw_handle_t                          h_mw)\r
 {\r
        UNREFERENCED_PARAMETER(h_mw);\r
        HCA_PRINT(TRACE_LEVEL_ERROR  , HCA_DBG_MEMORY  ,("mlnx_destroy_mw not implemented\n"));\r
index 9b9e057..0ca785d 100644 (file)
@@ -1,6 +1,7 @@
 /*\r
  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
  * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. \r
+ * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
  *\r
  * This software is available to you under the OpenIB.org BSD license\r
  * below:\r
@@ -48,7 +49,7 @@
 // Local declarations\r
 ib_api_status_t\r
 mlnx_query_qp (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf );\r
 \r
@@ -61,7 +62,7 @@ mlnx_open_ca (
        IN              const   ci_completion_cb_t                      pfn_completion_cb,\r
        IN              const   ci_async_event_cb_t                     pfn_async_event_cb,\r
        IN              const   void*const                                      ca_context,\r
-               OUT                     ib_ca_handle_t FUNC_PTR64                               *ph_ca)\r
+               OUT                     ib_ca_handle_t                          *ph_ca)\r
 {\r
        mlnx_hca_t                              *p_hca;\r
        ib_api_status_t status = IB_NOT_FOUND;\r
@@ -119,7 +120,7 @@ err_set_cb:
 \r
 ib_api_status_t\r
 mlnx_query_ca (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
                OUT                     ib_ca_attr_t                            *p_ca_attr,\r
        IN      OUT                     uint32_t                                        *p_byte_count,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
@@ -322,7 +323,7 @@ err_user_unsupported:
 \r
 ib_api_status_t\r
 mlnx_modify_ca (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN              const   uint8_t                                         port_num,\r
        IN              const   ib_ca_mod_t                                     modca_cmd,\r
        IN              const   ib_port_attr_mod_t                      *p_port_attr)\r
@@ -388,7 +389,7 @@ err_unsupported:
 \r
 ib_api_status_t\r
 mlnx_close_ca (\r
-       IN                              ib_ca_handle_t FUNC_PTR64                               h_ca)\r
+       IN                              ib_ca_handle_t                          h_ca)\r
 {\r
        mlnx_hob_t                      *hob_p = (mlnx_hob_t *)h_ca;\r
        HCA_ENTER(HCA_DBG_SHIM);\r
@@ -407,9 +408,9 @@ done:
 \r
 static ib_api_status_t\r
 mlnx_um_open(\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN      OUT                     ci_umv_buf_t* const                     p_umv_buf,\r
-               OUT                     ib_ca_handle_t FUNC_PTR64* const                ph_um_ca )\r
+               OUT                     ib_ca_handle_t* const           ph_um_ca )\r
 {\r
        int err;\r
        ib_api_status_t         status;\r
@@ -507,8 +508,8 @@ end:
 \r
 static void\r
 mlnx_um_close(\r
-       IN                              ib_ca_handle_t FUNC_PTR64                               h_ca,\r
-       IN                              ib_ca_handle_t FUNC_PTR64                               h_um_ca )\r
+       IN                              ib_ca_handle_t                          h_ca,\r
+       IN                              ib_ca_handle_t                          h_um_ca )\r
 {\r
        struct ib_ucontext *p_ucontext = (struct ib_ucontext *)h_um_ca;\r
        mlnx_hob_t                      *hob_p = (mlnx_hob_t *)h_ca;\r
@@ -537,9 +538,9 @@ done:
 \r
 ib_api_status_t\r
 mlnx_allocate_pd (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN              const   ib_pd_type_t                            type,\r
-               OUT                     ib_pd_handle_t FUNC_PTR64                               *ph_pd,\r
+               OUT                     ib_pd_handle_t                          *ph_pd,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        ib_api_status_t         status;\r
@@ -592,7 +593,7 @@ err_alloc_pd:
 \r
 ib_api_status_t\r
 mlnx_deallocate_pd (\r
-       IN                              ib_pd_handle_t FUNC_PTR64                               h_pd)\r
+       IN                              ib_pd_handle_t                          h_pd)\r
 {\r
        ib_api_status_t         status;\r
        int err;\r
@@ -629,9 +630,9 @@ err_dealloc_pd:
 */\r
 ib_api_status_t\r
 mlnx_create_av (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   ib_av_attr_t                            *p_addr_vector,\r
-               OUT                     ib_av_handle_t FUNC_PTR64                               *ph_av,\r
+               OUT                     ib_av_handle_t                          *ph_av,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err = 0;\r
@@ -690,9 +691,9 @@ err_inval_params:
 \r
 ib_api_status_t\r
 mlnx_query_av (\r
-       IN              const   ib_av_handle_t FUNC_PTR64                               h_av,\r
+       IN              const   ib_av_handle_t                          h_av,\r
                OUT                     ib_av_attr_t                            *p_addr_vector,\r
-               OUT                     ib_pd_handle_t FUNC_PTR64                               *ph_pd,\r
+               OUT                     ib_pd_handle_t                          *ph_pd,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err;\r
@@ -749,7 +750,7 @@ err_user_unsupported:
 \r
 ib_api_status_t\r
 mlnx_modify_av (\r
-       IN              const   ib_av_handle_t FUNC_PTR64                               h_av,\r
+       IN              const   ib_av_handle_t                          h_av,\r
        IN              const   ib_av_attr_t                            *p_addr_vector,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
@@ -799,7 +800,7 @@ err_user_unsupported:
 \r
 ib_api_status_t\r
 mlnx_destroy_av (\r
-       IN              const   ib_av_handle_t FUNC_PTR64                               h_av)\r
+       IN              const   ib_av_handle_t                          h_av)\r
 {\r
        int err;\r
        ib_api_status_t         status = IB_SUCCESS;\r
@@ -834,10 +835,10 @@ err_destroy_ah:
 \r
 ib_api_status_t\r
 mlnx_create_srq (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                       h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   void                                            *srq_context,\r
        IN              const   ib_srq_attr_t * const           p_srq_attr,\r
-               OUT                     ib_srq_handle_t FUNC_PTR64                      *ph_srq,\r
+               OUT                     ib_srq_handle_t                         *ph_srq,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err;\r
@@ -904,7 +905,7 @@ err_inval_params:
 \r
 ib_api_status_t\r
 mlnx_modify_srq (\r
-               IN              const   ib_srq_handle_t FUNC_PTR64                      h_srq,\r
+               IN              const   ib_srq_handle_t                         h_srq,\r
                IN              const   ib_srq_attr_t* const                    p_srq_attr,\r
                IN              const   ib_srq_attr_mask_t                      srq_attr_mask,\r
                IN      OUT             ci_umv_buf_t                            *p_umv_buf OPTIONAL )\r
@@ -936,7 +937,7 @@ mlnx_modify_srq (
 \r
 ib_api_status_t\r
 mlnx_query_srq (\r
-       IN              const   ib_srq_handle_t FUNC_PTR64                              h_srq,\r
+       IN              const   ib_srq_handle_t                         h_srq,\r
                OUT                     ib_srq_attr_t* const                    p_srq_attr,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf OPTIONAL )\r
 {\r
@@ -967,7 +968,7 @@ mlnx_query_srq (
 \r
 ib_api_status_t\r
 mlnx_destroy_srq (\r
-       IN      const   ib_srq_handle_t FUNC_PTR64              h_srq )\r
+       IN      const   ib_srq_handle_t         h_srq )\r
 {\r
        int err;\r
        ib_api_status_t         status = IB_SUCCESS;\r
@@ -1000,12 +1001,12 @@ mlnx_destroy_srq (
 \r
 static ib_api_status_t\r
 _create_qp (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   uint8_t                                         port_num,\r
        IN              const   void                                            *qp_context,\r
        IN              const   ib_qp_create_t                          *p_create_attr,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr,\r
-               OUT                     ib_qp_handle_t FUNC_PTR64                               *ph_qp,\r
+               OUT                     ib_qp_handle_t                          *ph_qp,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err;\r
@@ -1093,12 +1094,12 @@ end:
 \r
 ib_api_status_t\r
 mlnx_create_spl_qp (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   uint8_t                                         port_num,\r
        IN              const   void                                            *qp_context,\r
        IN              const   ib_qp_create_t                          *p_create_attr,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr,\r
-               OUT                     ib_qp_handle_t FUNC_PTR64                               *ph_qp )\r
+               OUT                     ib_qp_handle_t                          *ph_qp )\r
 {\r
        ib_api_status_t         status;\r
        PREP_IBDEV_FOR_PRINT(((struct ib_pd*)h_pd)->device);\r
@@ -1119,11 +1120,11 @@ mlnx_create_spl_qp (
 \r
 ib_api_status_t\r
 mlnx_create_qp (\r
-       IN              const   ib_pd_handle_t FUNC_PTR64                               h_pd,\r
+       IN              const   ib_pd_handle_t                          h_pd,\r
        IN              const   void                                            *qp_context,\r
        IN              const   ib_qp_create_t                          *p_create_attr,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr,\r
-               OUT                     ib_qp_handle_t FUNC_PTR64                               *ph_qp,\r
+               OUT                     ib_qp_handle_t                          *ph_qp,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        ib_api_status_t         status;\r
@@ -1150,7 +1151,7 @@ mlnx_create_qp (
 \r
 ib_api_status_t\r
 mlnx_modify_qp (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN              const   ib_qp_mod_t                                     *p_modify_attr,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr OPTIONAL,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf OPTIONAL )\r
@@ -1231,7 +1232,7 @@ err_inval_params:
 \r
 ib_api_status_t\r
 mlnx_ndi_modify_qp (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN              const   ib_qp_mod_t                                     *p_modify_attr,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr OPTIONAL,\r
        IN              const   uint32_t                                        buf_size,\r
@@ -1263,7 +1264,7 @@ mlnx_ndi_modify_qp (
 \r
 ib_api_status_t\r
 mlnx_query_qp (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
                OUT                     ib_qp_attr_t                            *p_qp_attr,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
@@ -1319,7 +1320,7 @@ mlnx_query_qp (
 \r
 ib_api_status_t\r
 mlnx_destroy_qp (\r
-       IN              const   ib_qp_handle_t FUNC_PTR64                               h_qp,\r
+       IN              const   ib_qp_handle_t                          h_qp,\r
        IN              const   uint64_t                                        timewait )\r
 {\r
        ib_api_status_t         status;\r
@@ -1360,10 +1361,10 @@ err_destroy_qp:
 \r
 ib_api_status_t\r
 mlnx_create_cq (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN              const   void                                            *cq_context,\r
        IN      OUT                     uint32_t                                        *p_size,\r
-               OUT                     ib_cq_handle_t FUNC_PTR64                               *ph_cq,\r
+               OUT                     ib_cq_handle_t                          *ph_cq,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
        int err;\r
@@ -1441,7 +1442,7 @@ err_cqe:
 \r
 ib_api_status_t\r
 mlnx_resize_cq (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
        IN      OUT                     uint32_t                                        *p_size,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
@@ -1456,7 +1457,7 @@ mlnx_resize_cq (
 \r
 ib_api_status_t\r
 mlnx_query_cq (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
+       IN              const   ib_cq_handle_t                          h_cq,\r
                OUT                     uint32_t                                        *p_size,\r
        IN      OUT                     ci_umv_buf_t                            *p_umv_buf )\r
 {\r
@@ -1471,7 +1472,7 @@ mlnx_query_cq (
 \r
 ib_api_status_t\r
 mlnx_destroy_cq (\r
-       IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq)\r
+       IN              const   ib_cq_handle_t                          h_cq)\r
 {\r
                                                                                                                                                                \r
        ib_api_status_t         status;\r
@@ -1508,7 +1509,7 @@ err_destroy_cq:
 \r
 ib_api_status_t\r
 mlnx_local_mad (\r
-       IN              const   ib_ca_handle_t FUNC_PTR64                               h_ca,\r
+       IN              const   ib_ca_handle_t                          h_ca,\r
        IN              const   uint8_t                                         port_num,\r
        IN              const   ib_av_attr_t*                                   p_av_attr,\r
        IN              const   ib_mad_t                                        *p_mad_in,\r
index 20d5d70..ff02ded 100644 (file)
-/*
- * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
- * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
- * Copyright (c) 2004 Intel Corporation.  All rights reserved.
- * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
- * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
- * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2005 Cisco Systems.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * $Id$
- */
-
-#if !defined(IB_VERBS_H)
-#define IB_VERBS_H
-
-#include <iba/ib_types.h>
-#include <iba/ib_ci.h>
-#include <mt_l2w.h>
-
-union ib_gid {
-       u8      raw[16];
-       struct {
-               __be64  subnet_prefix;
-               __be64  interface_id;
-       } global;
-};
-
-enum ib_node_type {
-       IB_NODE_CA      = 1,
-       IB_NODE_SWITCH,
-       IB_NODE_ROUTER
-};
-
-enum ib_device_cap_flags {
-       IB_DEVICE_RESIZE_MAX_WR         = 1,
-       IB_DEVICE_BAD_PKEY_CNTR         = (1<<1),
-       IB_DEVICE_BAD_QKEY_CNTR         = (1<<2),
-       IB_DEVICE_RAW_MULTI             = (1<<3),
-       IB_DEVICE_AUTO_PATH_MIG         = (1<<4),
-       IB_DEVICE_CHANGE_PHY_PORT       = (1<<5),
-       IB_DEVICE_UD_AV_PORT_ENFORCE    = (1<<6),
-       IB_DEVICE_CURR_QP_STATE_MOD     = (1<<7),
-       IB_DEVICE_SHUTDOWN_PORT         = (1<<8),
-       IB_DEVICE_INIT_TYPE             = (1<<9),
-       IB_DEVICE_PORT_ACTIVE_EVENT     = (1<<10),
-       IB_DEVICE_SYS_IMAGE_GUID        = (1<<11),
-       IB_DEVICE_RC_RNR_NAK_GEN        = (1<<12),
-       IB_DEVICE_SRQ_RESIZE            = (1<<13),
-       IB_DEVICE_N_NOTIFY_CQ           = (1<<14),
-};
-
-struct ib_device_attr {
-       u64                     fw_ver;
-       __be64                  sys_image_guid;
-       u64                     max_mr_size;
-       u64                     page_size_cap;
-       u32                     vendor_id;
-       u32                     vendor_part_id;
-       u32                     hw_ver;
-       int                     max_qp;
-       int                     max_qp_wr;
-       int                     device_cap_flags;
-       int                     max_sge;
-       int                     max_sge_rd;
-       int                     max_cq;
-       int                     max_cqe;
-       int                     max_mr;
-       int                     max_pd;
-       int                     max_qp_rd_atom;
-       int                     max_ee_rd_atom;
-       int                     max_res_rd_atom;
-       int                     max_qp_init_rd_atom;
-       int                     max_ee_init_rd_atom;
-       enum ib_atomic_cap      atomic_cap;
-       int                     max_ee;
-       int                     max_rdd;
-       int                     max_mw;
-       int                     max_raw_ipv6_qp;
-       int                     max_raw_ethy_qp;
-       int                     max_mcast_grp;
-       int                     max_mcast_qp_attach;
-       int                     max_total_mcast_qp_attach;
-       int                     max_ah;
-       int                     max_fmr;
-       int                     max_map_per_fmr;
-       int                     max_srq;
-       int                     max_srq_wr;
-       int                     max_srq_sge;
-       u16                     max_pkeys;
-       u8                      local_ca_ack_delay;
-};
-
-static inline int ib_mtu_enum_to_int(int mtu)
-{
-       switch (mtu) {
-       case IB_MTU_LEN_256:  return  256;
-       case IB_MTU_LEN_512:  return  512;
-       case IB_MTU_LEN_1024: return 1024;
-       case IB_MTU_LEN_2048: return 2048;
-       case IB_MTU_LEN_4096: return 4096;
-       default:          return -1;
-       }
-}
-
-enum ib_port_state {
-       IB_PORT_NOP             = 0,
-       IB_PORT_DOWN            = 1,
-       IB_PORT_INIT            = 2,
-       IB_PORT_ARMED           = 3,
-       IB_PORT_ACTIVE          = 4,
-       IB_PORT_ACTIVE_DEFER    = 5
-};
-
-enum ib_port_cap_flags {
-       IB_PORT_SM                              = 1 <<  1,
-       IB_PORT_NOTICE_SUP                      = 1 <<  2,
-       IB_PORT_TRAP_SUP                        = 1 <<  3,
-       IB_PORT_OPT_IPD_SUP                     = 1 <<  4,
-       IB_PORT_AUTO_MIGR_SUP                   = 1 <<  5,
-       IB_PORT_SL_MAP_SUP                      = 1 <<  6,
-       IB_PORT_MKEY_NVRAM                      = 1 <<  7,
-       IB_PORT_PKEY_NVRAM                      = 1 <<  8,
-       IB_PORT_LED_INFO_SUP                    = 1 <<  9,
-       IB_PORT_SM_DISABLED                     = 1 << 10,
-       IB_PORT_SYS_IMAGE_GUID_SUP              = 1 << 11,
-       IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP       = 1 << 12,
-       IB_PORT_CM_SUP                          = 1 << 16,
-       IB_PORT_SNMP_TUNNEL_SUP                 = 1 << 17,
-       IB_PORT_REINIT_SUP                      = 1 << 18,
-       IB_PORT_DEVICE_MGMT_SUP                 = 1 << 19,
-       IB_PORT_VENDOR_CLASS_SUP                = 1 << 20,
-       IB_PORT_DR_NOTICE_SUP                   = 1 << 21,
-       IB_PORT_CAP_MASK_NOTICE_SUP             = 1 << 22,
-       IB_PORT_BOOT_MGMT_SUP                   = 1 << 23,
-       IB_PORT_LINK_LATENCY_SUP                = 1 << 24,
-       IB_PORT_CLIENT_REG_SUP                  = 1 << 25
-};
-
-enum ib_port_width {
-       IB_WIDTH_1X     = 1,
-       IB_WIDTH_4X     = 2,
-       IB_WIDTH_8X     = 4,
-       IB_WIDTH_12X    = 8
-};
-
-static inline int ib_width_enum_to_int(enum ib_port_width width)
-{
-       switch (width) {
-       case IB_WIDTH_1X:  return  1;
-       case IB_WIDTH_4X:  return  4;
-       case IB_WIDTH_8X:  return  8;
-       case IB_WIDTH_12X: return 12;
-       default:          return -1;
-       }
-}
-
-struct ib_port_attr {
-       enum ib_port_state      state;
-       enum ib_mtu             max_mtu;
-       enum ib_mtu             active_mtu;
-       int                     gid_tbl_len;
-       u32                     port_cap_flags;
-       u32                     max_msg_sz;
-       u32                     bad_pkey_cntr;
-       u32                     qkey_viol_cntr;
-       u16                     pkey_tbl_len;
-       u16                     lid;
-       u16                     sm_lid;
-       u8                      lmc;
-       u8                      max_vl_num;
-       u8                      sm_sl;
-       u8                      subnet_timeout;
-       u8                      init_type_reply;
-       u8                      active_width;
-       u8                      active_speed;
-       u8                      phys_state;
-};
-
-enum ib_device_modify_flags {
-       IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1
-};
-
-struct ib_device_modify {
-       u64     sys_image_guid;
-};
-
-enum ib_port_modify_flags {
-       IB_PORT_SHUTDOWN                = 1,
-       IB_PORT_INIT_TYPE               = (1<<2),
-       IB_PORT_RESET_QKEY_CNTR         = (1<<3)
-};
-
-struct ib_port_modify {
-       u32     set_port_cap_mask;
-       u32     clr_port_cap_mask;
-       u8      init_type;
-};
-
-enum ib_event_type {
-       IB_EVENT_CQ_ERR                                                                 = IB_AE_CQ_ERROR,
-       IB_EVENT_QP_FATAL                                                               = IB_AE_QP_FATAL,
-       IB_EVENT_QP_REQ_ERR                                                     = IB_AE_WQ_REQ_ERROR,
-       IB_EVENT_QP_ACCESS_ERR                                  = IB_AE_WQ_ACCESS_ERROR,
-       IB_EVENT_COMM_EST                                                       = IB_AE_QP_COMM,
-       IB_EVENT_SQ_DRAINED                                             = IB_AE_SQ_DRAINED,
-       IB_EVENT_PATH_MIG                                                               = IB_AE_QP_APM,
-       IB_EVENT_PATH_MIG_ERR                                   = IB_AE_QP_APM_ERROR,
-       IB_EVENT_DEVICE_FATAL                                           = IB_AE_LOCAL_FATAL,
-       IB_EVENT_PORT_ACTIVE                                            = IB_AE_PORT_ACTIVE,
-       IB_EVENT_PORT_ERR                                                               = IB_AE_PORT_DOWN,
-       IB_EVENT_SRQ_LIMIT_REACHED                              = IB_AE_SRQ_LIMIT_REACHED,
-       IB_EVENT_SRQ_CATAS_ERROR                                        = IB_AE_SRQ_CATAS_ERROR,
-       IB_EVENT_SRQ_QP_LAST_WQE_REACHED                = IB_AE_SRQ_QP_LAST_WQE_REACHED,
-       IB_EVENT_LID_CHANGE                                                     = IB_AE_UNKNOWN + 1,
-       IB_EVENT_PKEY_CHANGE,
-       IB_EVENT_SM_CHANGE
-};
-
-struct ib_event {
-       struct ib_device        *device;
-       union {
-               struct ib_cq    *cq;
-               struct ib_qp    *qp;
-               struct ib_srq   *srq;
-               u8              port_num;
-       } element;
-       enum ib_event_type      event;
-       uint64_t                vendor_specific;
-};
-
-struct ib_event_handler {
-       struct ib_device *device;
-       void            (*handler)(struct ib_event_handler *, struct ib_event *);
-       struct list_head  list;
-};
-
-#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler)         \
-       (_ptr)->device  = _device;                      \
-       (_ptr)->handler = _handler;                     \
-       INIT_LIST_HEAD(&(_ptr)->list)                   
-
-struct ib_global_route {
-       union ib_gid    dgid;
-       u32             flow_label;
-       u8              sgid_index;
-       u8              hop_limit;
-       u8              traffic_class;
-};
-
-struct ib_grh {
-       __be32          version_tclass_flow;
-       __be16          paylen;
-       u8              next_hdr;
-       u8              hop_limit;
-       union ib_gid    sgid;
-       union ib_gid    dgid;
-};
-
-enum {
-       IB_MULTICAST_QPN = 0xffffff
-};
-
-enum ib_ah_flags {
-       IB_AH_GRH       = 1
-};
-
-struct ib_ah_attr {
-       struct ib_global_route  grh;
-       u16                     dlid;
-       u8                      sl;
-       u8                      src_path_bits;
-       u8                      static_rate;
-       u8                      ah_flags;
-       u8                      port_num;
-};
-
-enum ib_cq_notify {
-       IB_CQ_SOLICITED,
-       IB_CQ_NEXT_COMP
-};
-
-struct ib_srq_init_attr {
-       void                                    (*event_handler)(struct ib_event *, void *);
-       void                                    *srq_context;
-       ib_srq_attr_t                   attr;
-};
-
-struct ib_qp_cap {
-       u32     max_send_wr;
-       u32     max_recv_wr;
-       u32     max_send_sge;
-       u32     max_recv_sge;
-       u32     max_inline_data;
-};
-
-enum ib_sig_type {
-       IB_SIGNAL_ALL_WR,
-       IB_SIGNAL_REQ_WR
-};
-
-struct ib_qp_init_attr {
-       void                  (*event_handler)(struct ib_event *, void *);
-       void                   *qp_context;
-       struct ib_cq           *send_cq;
-       struct ib_cq           *recv_cq;
-       struct ib_srq          *srq;
-       struct ib_qp_cap        cap;
-       enum ib_sig_type        sq_sig_type;
-       enum ib_qp_type_t               qp_type;
-       u8                      port_num; /* special QP types only */
-};
-
-enum ib_rnr_timeout {
-       IB_RNR_TIMER_655_36 =  0,
-       IB_RNR_TIMER_000_01 =  1,
-       IB_RNR_TIMER_000_02 =  2,
-       IB_RNR_TIMER_000_03 =  3,
-       IB_RNR_TIMER_000_04 =  4,
-       IB_RNR_TIMER_000_06 =  5,
-       IB_RNR_TIMER_000_08 =  6,
-       IB_RNR_TIMER_000_12 =  7,
-       IB_RNR_TIMER_000_16 =  8,
-       IB_RNR_TIMER_000_24 =  9,
-       IB_RNR_TIMER_000_32 = 10,
-       IB_RNR_TIMER_000_48 = 11,
-       IB_RNR_TIMER_000_64 = 12,
-       IB_RNR_TIMER_000_96 = 13,
-       IB_RNR_TIMER_001_28 = 14,
-       IB_RNR_TIMER_001_92 = 15,
-       IB_RNR_TIMER_002_56 = 16,
-       IB_RNR_TIMER_003_84 = 17,
-       IB_RNR_TIMER_005_12 = 18,
-       IB_RNR_TIMER_007_68 = 19,
-       IB_RNR_TIMER_010_24 = 20,
-       IB_RNR_TIMER_015_36 = 21,
-       IB_RNR_TIMER_020_48 = 22,
-       IB_RNR_TIMER_030_72 = 23,
-       IB_RNR_TIMER_040_96 = 24,
-       IB_RNR_TIMER_061_44 = 25,
-       IB_RNR_TIMER_081_92 = 26,
-       IB_RNR_TIMER_122_88 = 27,
-       IB_RNR_TIMER_163_84 = 28,
-       IB_RNR_TIMER_245_76 = 29,
-       IB_RNR_TIMER_327_68 = 30,
-       IB_RNR_TIMER_491_52 = 31
-};
-
-enum ib_qp_attr_mask {
-       IB_QP_STATE                     = 1,
-       IB_QP_CUR_STATE                 = (1<<1),
-       IB_QP_EN_SQD_ASYNC_NOTIFY       = (1<<2),
-       IB_QP_ACCESS_FLAGS              = (1<<3),
-       IB_QP_PKEY_INDEX                = (1<<4),
-       IB_QP_PORT                      = (1<<5),
-       IB_QP_QKEY                      = (1<<6),
-       IB_QP_AV                        = (1<<7),
-       IB_QP_PATH_MTU                  = (1<<8),
-       IB_QP_TIMEOUT                   = (1<<9),
-       IB_QP_RETRY_CNT                 = (1<<10),
-       IB_QP_RNR_RETRY                 = (1<<11),
-       IB_QP_RQ_PSN                    = (1<<12),
-       IB_QP_MAX_QP_RD_ATOMIC          = (1<<13),
-       IB_QP_ALT_PATH                  = (1<<14),
-       IB_QP_MIN_RNR_TIMER             = (1<<15),
-       IB_QP_SQ_PSN                    = (1<<16),
-       IB_QP_MAX_DEST_RD_ATOMIC        = (1<<17),
-       IB_QP_PATH_MIG_STATE            = (1<<18),
-       IB_QP_CAP                       = (1<<19),
-       IB_QP_DEST_QPN                  = (1<<20)
-};
-
-//TODO: these literals are also defined in ib_types.h and have there ANOTHER VALUES !!! 
-enum ib_qp_state {
-       IBQPS_RESET,
-       IBQPS_INIT,
-       IBQPS_RTR,
-       IBQPS_RTS,
-       IBQPS_SQD,
-       IBQPS_SQE,
-       IBQPS_ERR
-};
-
-
-struct ib_qp_attr {
-       enum ib_qp_state        qp_state;
-       enum ib_qp_state        cur_qp_state;
-       enum ib_mtu             path_mtu;
-       ib_apm_state_t  path_mig_state;
-       u32                     qkey;
-       u32                     rq_psn;
-       u32                     sq_psn;
-       u32                     dest_qp_num;
-       int                     qp_access_flags;
-       struct ib_qp_cap        cap;
-       struct ib_ah_attr       ah_attr;
-       struct ib_ah_attr       alt_ah_attr;
-       u16                     pkey_index;
-       u16                     alt_pkey_index;
-       u8                      en_sqd_async_notify;
-       u8                      sq_draining;
-       u8                      max_rd_atomic;
-       u8                      max_dest_rd_atomic;
-       u8                      min_rnr_timer;
-       u8                      port_num;
-       u8                      timeout;
-       u8                      retry_cnt;
-       u8                      rnr_retry;
-       u8                      alt_port_num;
-       u8                      alt_timeout;
-};
-
-struct ib_sge {
-       u64     addr;
-       u32     length;
-       u32     lkey;
-};
-
-
-typedef enum MTHCA_QP_ACCESS_FLAGS {
-       MTHCA_ACCESS_LOCAL_WRITE        = 1,
-       MTHCA_ACCESS_REMOTE_WRITE       = (1<<1),
-       MTHCA_ACCESS_REMOTE_READ        = (1<<2),
-       MTHCA_ACCESS_REMOTE_ATOMIC      = (1<<3),
-       MTHCA_ACCESS_MW_BIND    = (1<<4)
-} mthca_qp_access_t;
-
-struct ib_phys_buf {
-       u64      addr;
-       u64      size;
-};
-
-struct ib_mr_attr {
-       struct ib_pd    *pd;
-       u64             device_virt_addr;
-       u64             size;
-       mthca_qp_access_t               mr_access_flags;
-       u32             lkey;
-       u32             rkey;
-};
-
-enum ib_mr_rereg_flags {
-       IB_MR_REREG_TRANS       = 1,
-       IB_MR_REREG_PD          = (1<<1),
-       IB_MR_REREG_ACCESS      = (1<<2)
-};
-
-struct ib_mw_bind {
-       struct ib_mr   *mr;
-       u64             wr_id;
-       u64             addr;
-       u32             length;
-       int             send_flags;
-       int             mw_access_flags;
-};
-
-struct ib_fmr_attr {
-       int     max_pages;
-       int     max_maps;
-       u8      page_shift;
-};
-
-struct ib_ucontext {
-       struct ib_device  *device;
-       PVOID   user_uar;
-       struct ib_pd *pd;
-       atomic_t                usecnt; /* count all resources */
-       ULONG           is_removing;
-       cl_list_item_t list_item;                       // chain of user contexts
-       // for tools support
-       KMUTEX  mutex;
-       PMDL    p_mdl;
-       PVOID   va;
-       int     fw_if_open;
-};
-
-struct ib_uobject {
-       u64                     user_handle;    /* handle given to us by userspace */
-       struct ib_ucontext     *context;        /* associated user context */
-       struct list_head        list;           /* link to context's list */
-       u32                     id;             /* index into kernel idr */
-};
-
-struct ib_umem {
-       u64             user_base;
-       u64             virt_base;
-       u64                     length;
-       int                     offset;
-       int                     page_size;
-       int                     writable;
-       struct list_head        chunk_list;
-};
-
-#pragma warning( disable : 4200 )
-struct ib_umem_chunk {
-       struct list_head        list;
-       int                     nents;
-       int                     nmap;
-       struct scatterlist      page_list[0];
-};
-#pragma warning( default : 4200 )
-
-#define IB_UMEM_MAX_PAGE_CHUNK                                         \
-       ((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) /      \
-        ((char *) &((struct ib_umem_chunk *) 0)->page_list[1] -        \
-         (char *) &((struct ib_umem_chunk *) 0)->page_list[0]))
-
-struct ib_pd {
-       struct list_head        list;           /* for chaining AV MRs (for user mode only) */
-       struct ib_device       *device;
-       struct ib_ucontext      *ucontext;
-       atomic_t                usecnt; /* count all resources */
-       KMUTEX mutex;   /* for chaining AV MRs (for user mode only) */
-};
-
-struct ib_ah {
-       struct ib_device        *device;
-       struct ib_pd            *pd;
-       struct ib_ucontext      *ucontext;
-};
-
-typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);
-
-struct ib_cq {
-       struct ib_device       *device;
-       struct ib_ucontext      *ucontext;
-       struct ib_mr *ib_mr;
-       ib_comp_handler         comp_handler;
-       void                  (*event_handler)(struct ib_event *, void *);
-       void *                  cq_context;
-       int                     cqe;
-       atomic_t                usecnt; /* count number of work queues */
-};
-
-struct ib_srq {
-       struct ib_device       *device;
-       struct ib_pd           *pd;
-       struct ib_ucontext      *ucontext;
-       struct ib_mr *ib_mr;
-       void                  (*event_handler)(struct ib_event *, void *);
-       void                   *srq_context;
-       atomic_t                usecnt; /* count number of work queues */
-};
-
-struct ib_qp {
-       struct ib_device       *device;
-       struct ib_pd           *pd;
-       struct ib_cq           *send_cq;
-       struct ib_cq           *recv_cq;
-       struct ib_srq          *srq;
-       struct ib_ucontext      *ucontext;
-       struct ib_mr *ib_mr;
-       void                  (*event_handler)(struct ib_event *, void *);
-       void                   *qp_context;
-       u32                     qp_num;
-       enum ib_qp_type_t               qp_type;
-};
-
-struct ib_mr {
-       struct list_head        list;           /* for chaining AV MRs (for user mode only) */
-       struct ib_device  *device;
-       struct ib_pd      *pd;
-       u32                lkey;
-       u32                rkey;
-       atomic_t           usecnt; /* count number of MWs */
-};
-
-struct ib_mw {
-       struct ib_device        *device;
-       struct ib_pd            *pd;
-       u32                     rkey;
-};
-
-struct ib_fmr {
-       struct ib_device        *device;
-       struct ib_pd            *pd;
-       struct list_head        list;
-       u32                     lkey;
-       u32                     rkey;
-};
-
-struct ib_mad;
-struct ib_grh;
-
-enum ib_process_mad_flags {
-       IB_MAD_IGNORE_MKEY      = 1,
-       IB_MAD_IGNORE_BKEY      = 2,
-       IB_MAD_IGNORE_ALL       = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
-};
-
-enum ib_mad_result {
-       IB_MAD_RESULT_FAILURE  = 0,      /* (!SUCCESS is the important flag) */
-       IB_MAD_RESULT_SUCCESS  = 1 << 0, /* MAD was successfully processed   */
-       IB_MAD_RESULT_REPLY    = 1 << 1, /* Reply packet needs to be sent    */
-       IB_MAD_RESULT_CONSUMED = 1 << 2  /* Packet consumed: stop processing */
-};
-
-#define IB_DEVICE_NAME_MAX 64
-
-struct ib_cache {
-       rwlock_t                lock;
-       struct ib_event_handler event_handler;
-       struct ib_pkey_cache  **pkey_cache;
-       struct ib_gid_cache   **gid_cache;
-};
-
-struct mthca_dev;
-
-struct ib_device {
-       struct mthca_dev                *mdev;
-
-       char                          name[IB_DEVICE_NAME_MAX];
-
-       struct list_head              event_handler_list;
-       spinlock_t                    event_handler_lock;
-
-       struct list_head              core_list;
-       struct list_head              client_data_list;
-       spinlock_t                    client_data_lock;
-
-       struct ib_cache               cache;
-
-       u32                           flags;
-
-       int                        (*query_device)(struct ib_device *device,
-                                                  struct ib_device_attr *device_attr);
-       int                        (*query_port)(struct ib_device *device,
-                                                u8 port_num,
-                                                struct ib_port_attr *port_attr);
-       int                        (*query_gid_chunk)(struct ib_device *device,
-                                               u8 port_num, int index,
-                                               union ib_gid gid[8]);
-       int                        (*query_pkey_chunk)(struct ib_device *device,
-                                                u8 port_num, u16 index, __be16 pkey[32]);
-       int                        (*modify_device)(struct ib_device *device,
-                                                   int device_modify_mask,
-                                                   struct ib_device_modify *device_modify);
-       int                        (*modify_port)(struct ib_device *device,
-                                                 u8 port_num, int port_modify_mask,
-                                                 struct ib_port_modify *port_modify);
-       struct ib_ucontext *       (*alloc_ucontext)(struct ib_device *device,
-                                                    ci_umv_buf_t* const        p_umv_buf);
-       int                        (*dealloc_ucontext)(struct ib_ucontext *context);
-       struct ib_pd *             (*alloc_pd)(struct ib_device *device,
-                                              struct ib_ucontext *context,
-                                              ci_umv_buf_t* const      p_umv_buf);
-       int                        (*dealloc_pd)(struct ib_pd *pd);
-       struct ib_ah *             (*create_ah)(struct ib_pd *pd,
-                                               struct ib_ah_attr *ah_attr);
-       int                        (*modify_ah)(struct ib_ah *ah,
-                                               struct ib_ah_attr *ah_attr);
-       int                        (*query_ah)(struct ib_ah *ah,
-                                              struct ib_ah_attr *ah_attr);
-       int                        (*destroy_ah)(struct ib_ah *ah);
-       struct ib_srq *            (*create_srq)(struct ib_pd *pd,
-                                                struct ib_srq_init_attr *srq_init_attr,
-                                                ci_umv_buf_t* const    p_umv_buf);
-       int                        (*modify_srq)(struct ib_srq *srq,
-                                                ib_srq_attr_t *srq_attr,
-                                                ib_srq_attr_mask_t srq_attr_mask);
-       int                        (*query_srq)(struct ib_srq *srq,
-                                               ib_srq_attr_t *srq_attr);
-       int                        (*destroy_srq)(struct ib_srq *srq);
-       int                        (*post_srq_recv)(struct ib_srq *srq,
-                                                   struct _ib_recv_wr *recv_wr,
-                                                   struct _ib_recv_wr **bad_recv_wr);
-       struct ib_qp *             (*create_qp)(struct ib_pd *pd,
-                                               struct ib_qp_init_attr *qp_init_attr,
-                                               ci_umv_buf_t* const     p_umv_buf);
-       int                        (*modify_qp)(struct ib_qp *qp,
-                                               struct ib_qp_attr *qp_attr,
-                                               int qp_attr_mask);
-       int                        (*query_qp)(struct ib_qp *qp,
-                                              struct ib_qp_attr *qp_attr,
-                                              int qp_attr_mask,
-                                              struct ib_qp_init_attr *qp_init_attr);
-       int                        (*destroy_qp)(struct ib_qp *qp);
-       int                        (*post_send)(struct ib_qp *qp,
-                                               struct _ib_send_wr *send_wr,
-                                               struct _ib_send_wr **bad_send_wr);
-       int                        (*post_recv)(struct ib_qp *qp,
-                                               struct _ib_recv_wr *recv_wr,
-                                               struct _ib_recv_wr **bad_recv_wr);
-       struct ib_cq *             (*create_cq)(struct ib_device *device, int cqe,
-                                               struct ib_ucontext *context,
-                                               ci_umv_buf_t* const     p_umv_buf);
-       int                        (*destroy_cq)(struct ib_cq *cq);
-       int                        (*resize_cq)(struct ib_cq *cq, int *cqe);
-       int                        (*poll_cq)(struct ib_cq *cq, int num_entries,
-                                             struct _ib_wc *wc);
-       int                        (*peek_cq)(struct ib_cq *cq, int wc_cnt);
-       int                        (*req_notify_cq)(struct ib_cq *cq,
-                                                   enum ib_cq_notify cq_notify);
-       int                        (*req_ncomp_notif)(struct ib_cq *cq,
-                                                     int wc_cnt);
-       struct ib_mr *             (*get_dma_mr)(struct ib_pd *pd,
-                                                mthca_qp_access_t mr_access_flags);
-       struct ib_mr *             (*reg_phys_mr)(struct ib_pd *pd,
-                                                 struct ib_phys_buf *phys_buf_array,
-                                                 int num_phys_buf,
-                                                 mthca_qp_access_t mr_access_flags,
-                                                 u64 *iova_start);
-       struct ib_mr *                     (*reg_virt_mr)(struct ib_pd *pd, 
-                                               void* FUNC_PTR64        vaddr, uint64_t length, uint64_t hca_va,
-                                               mthca_qp_access_t acc, boolean_t um_call, boolean_t secure);
-       int                        (*query_mr)(struct ib_mr *mr,
-                                              struct ib_mr_attr *mr_attr);
-       int                        (*dereg_mr)(struct ib_mr *mr);
-       int                        (*rereg_phys_mr)(struct ib_mr *mr,
-                                                   int mr_rereg_mask,
-                                                   struct ib_pd *pd,
-                                                   struct ib_phys_buf *phys_buf_array,
-                                                   int num_phys_buf,
-                                                   mthca_qp_access_t mr_access_flags,
-                                                   u64 *iova_start);
-       struct ib_mw *             (*alloc_mw)(struct ib_pd *pd);
-       int                        (*bind_mw)(struct ib_qp *qp,
-                                             struct ib_mw *mw,
-                                             struct ib_mw_bind *mw_bind);
-       int                        (*dealloc_mw)(struct ib_mw *mw);
-       struct ib_fmr *            (*alloc_fmr)(struct ib_pd *pd,
-                                               mthca_qp_access_t mr_access_flags,
-                                               struct ib_fmr_attr *fmr_attr);
-       int                        (*map_phys_fmr)(struct ib_fmr *fmr,
-                                                  u64 *page_list, int list_len,
-                                                  u64 iova);
-       int                        (*unmap_fmr)(struct list_head *fmr_list);
-       int                        (*dealloc_fmr)(struct ib_fmr *fmr);
-       int                        (*attach_mcast)(struct ib_qp *qp,
-                                                  union ib_gid *gid,
-                                                  u16 lid);
-       int                        (*detach_mcast)(struct ib_qp *qp,
-                                                  union ib_gid *gid,
-                                                  u16 lid);
-       int                        (*process_mad)(struct ib_device *device,
-                                                 int process_mad_flags,
-                                                 u8 port_num,
-                                                 struct _ib_wc *in_wc,
-                                                 struct _ib_grh *in_grh,
-                                                 struct ib_mad *in_mad,
-                                                 struct ib_mad *out_mad);
-
-       struct list_head             port_list;
-
-       u64                                                             uverbs_cmd_mask;
-       __be64                                  node_guid;
-       u8                           node_type;
-       u8                           phys_port_cnt;
-};
-
-struct ib_client {
-       char  *name;
-       void (*add)   (struct ib_device *);
-       void (*remove)(struct ib_device *);
-
-       struct list_head list;
-};
-
-struct ib_device *ib_alloc_device(size_t size);
-void ib_dealloc_device(struct ib_device *device);
-
-int ib_register_device   (struct ib_device *device);
-void ib_unregister_device(struct ib_device *device);
-
-int ib_register_client   (struct ib_client *client);
-void ib_unregister_client(struct ib_client *client);
-
-void *ib_get_client_data(struct ib_device *device, struct ib_client *client);
-void  ib_set_client_data(struct ib_device *device, struct ib_client *client,
-                        void *data);
-
-int ib_core_init(void);
-
-void ib_core_cleanup(void);
-
-int ib_register_event_handler  (struct ib_event_handler *event_handler);
-int ib_unregister_event_handler(struct ib_event_handler *event_handler);
-void ib_dispatch_event(struct ib_event *event);
-
-int ib_query_device(struct ib_device *device,
-                   struct ib_device_attr *device_attr);
-
-int ib_query_port(struct ib_device *device,
-                 u8 port_num, struct ib_port_attr *port_attr);
-
-int ib_query_gid_chunk(struct ib_device *device,
-                u8 port_num, int index, union ib_gid gid[8]);
-
-int ib_query_pkey_chunk(struct ib_device *device,
-                 u8 port_num, u16 index, __be16 pkey[32]);
-
-int ib_modify_device(struct ib_device *device,
-                    int device_modify_mask,
-                    struct ib_device_modify *device_modify);
-
-int ib_modify_port(struct ib_device *device,
-                  u8 port_num, int port_modify_mask,
-                  struct ib_port_modify *port_modify);
-
-/**
- * ibv_alloc_pd - Allocates an unused protection domain.
- * @device: The device on which to allocate the protection domain.
- * @context: user process context (for application calls only)
- * @p_umv_buf: parameters structure (for application calls only)
- *
- * A protection domain object provides an association between QPs, shared
- * receive queues, address handles, memory regions, and memory windows.
- */
-struct ib_pd *ibv_alloc_pd(struct ib_device *device,
-       struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf);
-
-/**
- * ibv_dealloc_pd - Deallocates a protection domain.
- * @pd: The protection domain to deallocate.
- */
-int ibv_dealloc_pd(struct ib_pd *pd);
-
-/**
- * ibv_create_ah - Creates an address handle for the given address vector.
- * @pd: The protection domain associated with the address handle.
- * @ah_attr: The attributes of the address vector.
- * @context: user process context (for application calls only)
- * @p_umv_buf: parameters structure (for application calls only)
- *
- * The address handle is used to reference a local or global destination
- * in all UD QP post sends.
- */
-struct ib_ah *ibv_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
-       struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf);
-
-/**
- * ibv_create_ah_from_wc - Creates an address handle associated with the
- *   sender of the specified work completion.
- * @pd: The protection domain associated with the address handle.
- * @wc: Work completion information associated with a received message.
- * @grh: References the received global route header.  This parameter is
- *   ignored unless the work completion indicates that the GRH is valid.
- * @port_num: The outbound port number to associate with the address.
- *
- * The address handle is used to reference a local or global destination
- * in all UD QP post sends.
- */
-struct ib_ah *ibv_create_ah_from_wc(struct ib_pd *pd, struct _ib_wc *wc,
-                                  struct ib_grh *grh, u8 port_num);
-
-/**
- * ibv_modify_ah - Modifies the address vector associated with an address
- *   handle.
- * @ah: The address handle to modify.
- * @ah_attr: The new address vector attributes to associate with the
- *   address handle.
- */
-int ibv_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-/**
- * ibv_query_ah - Queries the address vector associated with an address
- *   handle.
- * @ah: The address handle to query.
- * @ah_attr: The address vector attributes associated with the address
- *   handle.
- */
-int ibv_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-/**
- * ibv_destroy_ah - Destroys an address handle.
- * @ah: The address handle to destroy.
- */
-int ibv_destroy_ah(struct ib_ah *ah);
-
-/**
- * ibv_create_srq - Creates a SRQ associated with the specified protection
- *   domain.
- * @pd: The protection domain associated with the SRQ.
- * @srq_init_attr: A list of initial attributes required to create the
- *   SRQ.  If SRQ creation succeeds, then the attributes are updated to 
- *   the actual capabilities of the created SRQ.
- * @context: user process context (for application calls only)
- * @p_umv_buf: parameters structure (for application calls only)
- *
- * srq_attr->max_wr and srq_attr->max_sge are read the determine the
- * requested size of the SRQ, and set to the actual values allocated
- * on return.  If ibv_create_srq() succeeds, then max_wr and max_sge
- * will always be at least as large as the requested values.
- */
-struct ib_srq *ibv_create_srq(struct ib_pd *pd,
-       struct ib_srq_init_attr *srq_init_attr,
-       struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf);
-
-
-/**
- * ibv_modify_srq - Modifies the attributes for the specified SRQ.
- * @srq: The SRQ to modify.
- * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,
- *   the current values of selected SRQ attributes are returned.
- * @srq_attr_mask: A bit-mask used to specify which attributes of