[hermon] Add support for multiple ports and detecting non-IB ports

Originally-fixed-by: Itay Gazit <itaygazit@gmail.com>

diff --git a/src/drivers/infiniband/MT25408_PRM.h b/src/drivers/infiniband/MT25408_PRM.h
index 39ecd17..419e25a 100644 (file)
--- a/src/drivers/infiniband/MT25408_PRM.h
+++ b/src/drivers/infiniband/MT25408_PRM.h
@@ -2071,7 +2071,11 @@ struct hermonprm_query_dev_cap_st {      /* Little Endian */
     pseudo_bit_t       pkv[0x00001];          /* PKey Violation Counter Supported */
     pseudo_bit_t       qkv[0x00001];          /* QKey Violation Coutner Supported */
     pseudo_bit_t       vmm[0x00001];          /* Hermon New */
-    pseudo_bit_t       reserved27[0x00005];
+    pseudo_bit_t       fcoe[0x00001];
+    pseudo_bit_t       dpdp[0x00001];         /* Dual Port Different Protocols */
+    pseudo_bit_t       raw_ethertype[0x00001];
+    pseudo_bit_t       raw_ipv6[0x00001];
+    pseudo_bit_t       blh[0x00001];
     pseudo_bit_t       mw[0x00001];           /* Memory windows supported */
     pseudo_bit_t       apm[0x00001];          /* Automatic Path Migration Supported */
     pseudo_bit_t       atm[0x00001];          /* Atomic operations supported (atomicity is guaranteed between QPs on this HCA) */

diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index f4e228e..136b843 100644 (file)
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -541,6 +541,16 @@ hermon_cmd_map_fa ( struct hermon *hermon,
                            0, map, 1, NULL );
 }
 
+static inline int
+hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
+                       struct hermonprm_sense_port *port_type ) {
+       return hermon_cmd ( hermon,
+                            HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
+                                                 1, sizeof ( *port_type ) ),
+                            0, NULL, port, port_type );
+}
+
+
 /***************************************************************************
  *
  * Memory translation table operations
@@ -1664,7 +1674,7 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
               ( link_up ? "up" : "down" ) );
 
        /* Sanity check */
-       if ( port >= HERMON_NUM_PORTS ) {
+       if ( port >= hermon->cap.num_ports ) {
                DBGC ( hermon, "Hermon %p port %d does not exist!\n",
                       hermon, ( port + 1 ) );
                return;
@@ -1735,6 +1745,36 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
  ***************************************************************************
  */
 
+/**
+ * Sense port type
+ *
+ * @v ibdev            Infiniband device
+ * @ret port_type      Port type, or negative error
+ */
+static int hermon_sense_port_type ( struct ib_device *ibdev ) {
+       struct hermon *hermon = ib_get_drvdata ( ibdev );
+       struct hermonprm_sense_port sense_port;
+       int port_type;
+       int rc;
+
+       /* If DPDP is not supported, always assume Infiniband */
+       if ( ! hermon->cap.dpdp )
+               return HERMON_PORT_TYPE_IB;
+
+       /* Sense the port type */
+       if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
+                                           &sense_port ) ) != 0 ) {
+               DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
+                      hermon, ibdev->port, strerror ( rc ) );
+               return rc;
+       }
+       port_type = MLX_GET ( &sense_port, port_type );
+
+       DBGC ( hermon, "Hermon %p port %d type %d\n",
+              hermon, ibdev->port, port_type );
+       return port_type;
+}
+
 /**
  * Initialise Infiniband link
  *
@@ -1744,8 +1784,19 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
 static int hermon_open ( struct ib_device *ibdev ) {
        struct hermon *hermon = ib_get_drvdata ( ibdev );
        struct hermonprm_init_port init_port;
+       int port_type;
        int rc;
 
+       /* Check we are connected to an Infiniband network */
+       if ( ( rc = port_type = hermon_sense_port_type ( ibdev ) ) < 0 )
+               return rc;
+       if ( port_type != HERMON_PORT_TYPE_IB ) {
+               DBGC ( hermon, "Hermon %p port %d not connected to an "
+                      "Infiniband network", hermon, ibdev->port );
+               return -ENOTCONN;
+        }
+
+       /* Init Port */
        memset ( &init_port, 0, sizeof ( init_port ) );
        MLX_FILL_2 ( &init_port, 0,
                     port_width_cap, 3,
@@ -2099,6 +2150,15 @@ static int hermon_get_cap ( struct hermon *hermon ) {
                ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
        hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
        hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
+       hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
+       hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );
+
+       /* Sanity check */
+       if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
+               DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
+                      hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
+               hermon->cap.num_ports = HERMON_MAX_PORTS;
+       }
 
        return 0;
 }
@@ -2478,7 +2538,7 @@ static int hermon_probe ( struct pci_device *pci,
        struct hermon *hermon;
        struct ib_device *ibdev;
        struct hermonprm_init_hca init_hca;
-       int i;
+       unsigned int i;
        int rc;
 
        /* Allocate Hermon device */
@@ -2489,20 +2549,6 @@ static int hermon_probe ( struct pci_device *pci,
        }
        pci_set_drvdata ( pci, hermon );
 
-       /* Allocate Infiniband devices */
-       for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
-               ibdev = alloc_ibdev ( 0 );
-               if ( ! ibdev ) {
-                       rc = -ENOMEM;
-                       goto err_alloc_ibdev;
-               }
-               hermon->ibdev[i] = ibdev;
-               ibdev->op = &hermon_ib_operations;
-               ibdev->dev = &pci->dev;
-               ibdev->port = ( HERMON_PORT_BASE + i );
-               ib_set_drvdata ( ibdev, hermon );
-       }
-
        /* Fix up PCI device */
        adjust_pci_device ( pci );
 
@@ -2534,6 +2580,20 @@ static int hermon_probe ( struct pci_device *pci,
        if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
                goto err_get_cap;
 
+       /* Allocate Infiniband devices */
+       for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
+               ibdev = alloc_ibdev ( 0 );
+               if ( ! ibdev ) {
+                       rc = -ENOMEM;
+                       goto err_alloc_ibdev;
+               }
+               hermon->ibdev[i] = ibdev;
+               ibdev->op = &hermon_ib_operations;
+               ibdev->dev = &pci->dev;
+               ibdev->port = ( HERMON_PORT_BASE + i );
+               ib_set_drvdata ( ibdev, hermon );
+       }
+
        /* Allocate ICM */
        memset ( &init_hca, 0, sizeof ( init_hca ) );
        if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 )
@@ -2552,7 +2612,7 @@ static int hermon_probe ( struct pci_device *pci,
        /* Set up memory protection */
        if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
                goto err_setup_mpt;
-       for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
+       for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
                hermon->ibdev[i]->rdma_key = hermon->lkey;
 
        /* Set up event queue */
@@ -2563,12 +2623,13 @@ static int hermon_probe ( struct pci_device *pci,
        if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
                goto err_conf_special_qps;
 
-       /* Update MAD parameters */
-       for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
+       /* Update IPoIB MAC address */
+       for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
                ib_smc_update ( hermon->ibdev[i], hermon_mad );
+       }
 
        /* Register Infiniband devices */
-       for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
+       for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
                if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) {
                        DBGC ( hermon, "Hermon %p could not register IB "
                               "device: %s\n", hermon, strerror ( rc ) );
@@ -2578,9 +2639,9 @@ static int hermon_probe ( struct pci_device *pci,
 
        return 0;
 
-       i = HERMON_NUM_PORTS;
+       i = hermon->cap.num_ports;
  err_register_ibdev:
-       for ( i-- ; i >= 0 ; i-- )
+       for ( i-- ; ( signed int ) i >= 0 ; i-- )
                unregister_ibdev ( hermon->ibdev[i] );
  err_conf_special_qps:
        hermon_destroy_eq ( hermon );
@@ -2590,6 +2651,10 @@ static int hermon_probe ( struct pci_device *pci,
  err_init_hca:
        hermon_free_icm ( hermon );
  err_alloc_icm:
+       i = hermon->cap.num_ports;
+ err_alloc_ibdev:
+       for ( i-- ; ( signed int ) i >= 0 ; i-- )
+               ibdev_put ( hermon->ibdev[i] );
  err_get_cap:
        hermon_stop_firmware ( hermon );
  err_start_firmware:
@@ -2597,10 +2662,6 @@ static int hermon_probe ( struct pci_device *pci,
  err_mailbox_out:
        free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
  err_mailbox_in:
-       i = HERMON_NUM_PORTS;
- err_alloc_ibdev:
-       for ( i-- ; i >= 0 ; i-- )
-               ibdev_put ( hermon->ibdev[i] );
        free ( hermon );
  err_alloc_hermon:
        return rc;
@@ -2615,7 +2676,7 @@ static void hermon_remove ( struct pci_device *pci ) {
        struct hermon *hermon = pci_get_drvdata ( pci );
        int i;
 
-       for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
+       for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
                unregister_ibdev ( hermon->ibdev[i] );
        hermon_destroy_eq ( hermon );
        hermon_cmd_close_hca ( hermon );
@@ -2624,7 +2685,7 @@ static void hermon_remove ( struct pci_device *pci ) {
        hermon_stop_firmware ( hermon );
        free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
        free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
-       for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
+       for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
                ibdev_put ( hermon->ibdev[i] );
        free ( hermon );
 }

diff --git a/src/drivers/infiniband/hermon.h b/src/drivers/infiniband/hermon.h
index 8e5d35e..ea973c0 100644 (file)
--- a/src/drivers/infiniband/hermon.h
+++ b/src/drivers/infiniband/hermon.h
@@ -21,7 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /* Ports in existence */
-#define HERMON_NUM_PORTS               2
+#define HERMON_MAX_PORTS               2
 #define HERMON_PORT_BASE               1
 
 /* PCI BARs */
@@ -61,6 +61,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define HERMON_HCR_READ_MCG            0x0025
 #define HERMON_HCR_WRITE_MCG           0x0026
 #define HERMON_HCR_MGID_HASH           0x0027
+#define HERMON_HCR_SENSE_PORT          0x004d
 #define HERMON_HCR_RUN_FW              0x0ff6
 #define HERMON_HCR_DISABLE_LAM         0x0ff7
 #define HERMON_HCR_ENABLE_LAM          0x0ff8
@@ -164,6 +165,14 @@ struct hermonprm_port_state_change_event_st {
        struct hermonprm_port_state_change_st data;
 } __attribute__ (( packed ));
 
+/** Hermon sense port */
+struct hermonprm_sense_port_st {
+       pseudo_bit_t port_type[0x00020];
+/* -------------- */
+       pseudo_bit_t reserved[0x00020];
+};
+#define HERMON_PORT_TYPE_IB            1
+
 /*
  * Wrapper structures for hardware datatypes
  *
@@ -192,6 +201,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_query_dev_cap );
 struct MLX_DECLARE_STRUCT ( hermonprm_query_fw );
 struct MLX_DECLARE_STRUCT ( hermonprm_queue_pair_ee_context_entry );
 struct MLX_DECLARE_STRUCT ( hermonprm_scalar_parameter );
+struct MLX_DECLARE_STRUCT ( hermonprm_sense_port );
 struct MLX_DECLARE_STRUCT ( hermonprm_send_db_register );
 struct MLX_DECLARE_STRUCT ( hermonprm_ud_address_vector );
 struct MLX_DECLARE_STRUCT ( hermonprm_virtual_physical_mapping );
@@ -296,6 +306,10 @@ struct hermon_dev_cap {
        size_t dmpt_entry_size;
        /** Number of reserved UARs */
        unsigned int reserved_uars;
+       /** Number of ports */
+       unsigned int num_ports;
+       /** Dual-port different protocol */
+       int dpdp;
 };
 
 /** Number of cMPT entries of each type */
@@ -523,7 +537,7 @@ struct hermon {
        unsigned long qpn_base;
 
        /** Infiniband devices */
-       struct ib_device *ibdev[HERMON_NUM_PORTS];
+       struct ib_device *ibdev[HERMON_MAX_PORTS];
 };
 
 /** Global protection domain */