Starting the firmware directly now works.

diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h
index 407f29d..564b669 100644 (file)
--- a/src/drivers/net/mlx_ipoib/arbel.h
+++ b/src/drivers/net/mlx_ipoib/arbel.h
@@ -7,6 +7,9 @@
  *
  */
 
+#include <stdint.h>
+#include <gpxe/uaccess.h>
+
 /*
  * Hardware constants
  *
@@ -45,6 +48,9 @@
 #define ARBEL_HCR_READ_MGM             0x0025
 #define ARBEL_HCR_WRITE_MGM            0x0026
 #define ARBEL_HCR_MGID_HASH            0x0027
+#define ARBEL_HCR_RUN_FW               0x0ff6
+#define ARBEL_HCR_UNMAP_FA             0x0ffe
+#define ARBEL_HCR_MAP_FA               0x0fff
 
 /* Service types */
 #define ARBEL_ST_UD                    0x03
@@ -87,6 +93,7 @@ struct MLX_DECLARE_STRUCT ( arbelprm_queue_pair_ee_context_entry );
 struct MLX_DECLARE_STRUCT ( arbelprm_recv_wqe_segment_next );
 struct MLX_DECLARE_STRUCT ( arbelprm_send_doorbell );
 struct MLX_DECLARE_STRUCT ( arbelprm_ud_address_vector );
+struct MLX_DECLARE_STRUCT ( arbelprm_virtual_physical_mapping );
 struct MLX_DECLARE_STRUCT ( arbelprm_wqe_segment_ctrl_send );
 struct MLX_DECLARE_STRUCT ( arbelprm_wqe_segment_data_ptr );
 struct MLX_DECLARE_STRUCT ( arbelprm_wqe_segment_next );
@@ -240,15 +247,19 @@ typedef uint32_t arbel_bitmask_t;
 
 /** An Arbel device */
 struct arbel {
-       /** Configuration registers */
+       /** PCI configuration registers */
        void *config;
+       /** PCI user Access Region */
+       void *uar;
+
        /** Command input mailbox */
        void *mailbox_in;
        /** Command output mailbox */
        void *mailbox_out;
 
-       /** User Access Region */
-       void *uar;
+       /** Firmware area in external memory */
+       userptr_t firmware_area;
+
        /** Doorbell records */
        union arbelprm_doorbell_record *db_rec;
        /** Reserved LKey

diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c
index c7cc8c7..26677ab 100644 (file)
--- a/src/drivers/net/mlx_ipoib/mt25218.c
+++ b/src/drivers/net/mlx_ipoib/mt25218.c
@@ -315,6 +315,29 @@ arbel_cmd_mgid_hash ( struct arbel *arbel, const struct ib_gid *gid,
                           0, gid, 0, hash );
 }
 
+static inline int
+arbel_cmd_run_fw ( struct arbel *arbel ) {
+       return arbel_cmd ( arbel,
+                          ARBEL_HCR_VOID_CMD ( ARBEL_HCR_RUN_FW ),
+                          0, NULL, 0, NULL );
+}
+
+static inline int
+arbel_cmd_unmap_fa ( struct arbel *arbel ) {
+       return arbel_cmd ( arbel,
+                          ARBEL_HCR_VOID_CMD ( ARBEL_HCR_UNMAP_FA ),
+                          0, NULL, 0, NULL );
+}
+
+static inline int
+arbel_cmd_map_fa ( struct arbel *arbel,
+                  const struct arbelprm_virtual_physical_mapping *map_fa ) {
+       return arbel_cmd ( arbel,
+                          ARBEL_HCR_IN_CMD ( ARBEL_HCR_MAP_FA,
+                                             1, sizeof ( *map_fa ) ),
+                          0, map_fa, 1, NULL );
+}
+
 /***************************************************************************
  *
  * Completion queue operations
@@ -958,15 +981,15 @@ static int arbel_complete ( struct ib_device *ibdev,
        }
        qp = wq->qp;
        arbel_qp = qp->dev_priv;
+       arbel_send_wq = &arbel_qp->send;
+       arbel_recv_wq = &arbel_qp->recv;
 
        /* Identify work queue entry index */
        if ( is_send ) {
-               arbel_send_wq = &arbel_qp->send;
                wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_send_wq->wqe ) ) /
                            sizeof ( arbel_send_wq->wqe[0] ) );
                assert ( wqe_idx < qp->send.num_wqes );
        } else {
-               arbel_recv_wq = &arbel_qp->recv;
                wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_recv_wq->wqe ) ) /
                            sizeof ( arbel_recv_wq->wqe[0] ) );
                assert ( wqe_idx < qp->recv.num_wqes );
@@ -1177,6 +1200,12 @@ static struct ib_device_operations arbel_ib_operations = {
        .mcast_detach   = arbel_mcast_detach,
 };
 
+/***************************************************************************
+ *
+ * MAD IFC operations
+ *
+ ***************************************************************************
+ */
 
 static int arbel_mad_ifc ( struct arbel *arbel,
                           union arbelprm_mad *mad ) {
@@ -1301,6 +1330,102 @@ static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) {
        return 0;
 }
 
+/***************************************************************************
+ *
+ * Firmware control
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Start firmware running
+ *
+ * @v arbel            Arbel device
+ * @ret rc             Return status code
+ */
+static int arbel_start_firmware ( struct arbel *arbel ) {
+       struct arbelprm_query_fw fw;
+       struct arbelprm_virtual_physical_mapping map_fa;
+       unsigned int fw_pages;
+       unsigned int log2_fw_pages;
+       size_t fw_size;
+       physaddr_t fw_base;
+       int rc;
+
+       /* Get firmware parameters */
+       if ( ( rc = arbel_cmd_query_fw ( arbel, &fw ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p could not query firmware: %s\n",
+                      arbel, strerror ( rc ) );
+               goto err_query_fw;
+       }
+       DBGC ( arbel, "Arbel %p firmware version %ld.%ld.%ld\n", arbel,
+              MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
+              MLX_GET ( &fw, fw_rev_subminor ) );
+       fw_pages = MLX_GET ( &fw, fw_pages );
+       log2_fw_pages = fls ( fw_pages - 1 );
+       fw_pages = ( 1 << log2_fw_pages );
+       DBGC ( arbel, "Arbel %p requires %d kB for firmware\n",
+              arbel, ( fw_pages * 4 ) );
+
+       /* Allocate firmware pages and map firmware area */
+       fw_size = ( fw_pages * 4096 );
+       arbel->firmware_area = umalloc ( fw_size );
+       if ( ! arbel->firmware_area ) {
+               rc = -ENOMEM;
+               goto err_alloc_fa;
+       }
+       fw_base = ( user_to_phys ( arbel->firmware_area, fw_size ) &
+                   ~( fw_size - 1 ) );
+       DBGC ( arbel, "Arbel %p firmware area at physical [%lx,%lx)\n",
+              arbel, fw_base, ( fw_base + fw_size ) );
+       memset ( &map_fa, 0, sizeof ( map_fa ) );
+       MLX_FILL_2 ( &map_fa, 3,
+                    log2size, log2_fw_pages,
+                    pa_l, ( fw_base >> 12 ) );
+       if ( ( rc = arbel_cmd_map_fa ( arbel, &map_fa ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p could not map firmware: %s\n",
+                      arbel, strerror ( rc ) );
+               goto err_map_fa;
+       }
+
+       /* Start firmware */
+       if ( ( rc = arbel_cmd_run_fw ( arbel ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p could not run firmware: %s\n",
+                      arbel, strerror ( rc ) );
+               goto err_run_fw;
+       }
+
+       DBGC ( arbel, "Arbel %p firmware started\n", arbel );
+       return 0;
+
+ err_run_fw:
+       arbel_cmd_unmap_fa ( arbel );
+ err_map_fa:
+       ufree ( arbel->firmware_area );
+       arbel->firmware_area = UNULL;
+ err_alloc_fa:
+ err_query_fw:
+       return rc;
+}
+
+/**
+ * Stop firmware running
+ *
+ * @v arbel            Arbel device
+ */
+static void arbel_stop_firmware ( struct arbel *arbel ) {
+       int rc;
+
+       if ( ( rc = arbel_cmd_unmap_fa ( arbel ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p FATAL could not stop firmware: %s\n",
+                      arbel, strerror ( rc ) );
+               /* Leak memory and return; at least we avoid corruption */
+               return;
+       }
+       ufree ( arbel->firmware_area );
+       arbel->firmware_area = UNULL;
+}
+
 /**
  * Probe PCI device
  *
@@ -1311,7 +1436,6 @@ static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) {
 static int arbel_probe ( struct pci_device *pci,
                         const struct pci_device_id *id __unused ) {
        struct ib_device *ibdev;
-       struct arbelprm_query_fw fw;
        struct arbelprm_query_dev_lim dev_lim;
        struct arbel *arbel;
        udqp_t qph;
@@ -1329,6 +1453,16 @@ static int arbel_probe ( struct pci_device *pci,
        arbel = ibdev->dev_priv;
        memset ( arbel, 0, sizeof ( *arbel ) );
 
+       /* Fix up PCI device */
+       adjust_pci_device ( pci );
+
+       /* Get PCI BARs */
+       arbel->config = ioremap ( pci_bar_start ( pci, ARBEL_PCI_CONFIG_BAR ),
+                                 ARBEL_PCI_CONFIG_BAR_SIZE );
+       arbel->uar = ioremap ( ( pci_bar_start ( pci, ARBEL_PCI_UAR_BAR ) +
+                                ARBEL_PCI_UAR_IDX * ARBEL_PCI_UAR_SIZE ),
+                              ARBEL_PCI_UAR_SIZE );
+
        /* Allocate space for mailboxes */
        arbel->mailbox_in = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
        if ( ! arbel->mailbox_in ) {
@@ -1341,25 +1475,12 @@ static int arbel_probe ( struct pci_device *pci,
                goto err_mailbox_out;
        }
 
-       /* Fix up PCI device */
-       adjust_pci_device ( pci );
+       /* Start firmware */
+       if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
+               goto err_start_firmware;
 
-       /* Get PCI BARs */
-       arbel->config = ioremap ( pci_bar_start ( pci, ARBEL_PCI_CONFIG_BAR ),
-                                 ARBEL_PCI_CONFIG_BAR_SIZE );
-       arbel->uar = ioremap ( ( pci_bar_start ( pci, ARBEL_PCI_UAR_BAR ) +
-                                ARBEL_PCI_UAR_IDX * ARBEL_PCI_UAR_SIZE ),
-                              ARBEL_PCI_UAR_SIZE );
 
-       /* Initialise firmware */
-       if ( ( rc = arbel_cmd_query_fw ( arbel, &fw ) ) != 0 ) {
-               DBGC ( arbel, "Arbel %p could not query firmware: %s\n",
-                      arbel, strerror ( rc ) );
-               goto err_query_fw;
-       }
-       DBGC ( arbel, "Arbel %p firmware version %ld.%ld.%ld\n", arbel,
-              MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
-              MLX_GET ( &fw, fw_rev_subminor ) );
+       while ( 1 ) {}
 
        /* Initialise hardware */
        if ( ( rc = ib_driver_init ( pci, &qph ) ) != 0 )
@@ -1425,7 +1546,9 @@ static int arbel_probe ( struct pci_device *pci,
  err_query_dev_lim:
        ib_driver_close ( 0 );
  err_ib_driver_init:
- err_query_fw:
+
+       arbel_stop_firmware ( arbel );
+ err_start_firmware:
        free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
  err_mailbox_out:
        free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
@@ -1446,6 +1569,7 @@ static void arbel_remove ( struct pci_device *pci ) {
 
        ipoib_remove ( ibdev );
        ib_driver_close ( 0 );
+       arbel_stop_firmware ( arbel );
        free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
        free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
        free_ibdev ( ibdev );