Started IB driver rewrite
[people/sha0/gpxe.git] / src / drivers / net / mlx_ipoib / ib_mt25218.c
1 /*
2   This software is available to you under a choice of one of two
3   licenses.  You may choose to be licensed under the terms of the GNU
4   General Public License (GPL) Version 2, available at
5   <http://www.fsf.org/copyleft/gpl.html>, or the OpenIB.org BSD
6   license, available in the LICENSE.TXT file accompanying this
7   software.  These details are also available at
8   <http://openib.org/license.html>.
9
10   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
11   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
12   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
13   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
14   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
15   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
16   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17   SOFTWARE.
18
19   Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
20 */
21
22 #include "mt25218.h"
23 #include "ib_driver.h"
24 #include <gpxe/pci.h>
25
26 #define MOD_INC(counter, max_count) (counter) = ((counter)+1) & ((max_count) - 1)
27
28 #define breakpoint {volatile __u32 *p=(__u32 *)0x1234;printf("breakpoint\n");do {} while((*p) != 0x1234);}
29
30 #define WRITE_BYTE_VOL(addr, off, val) \
31     do { \
32         (*((volatile __u8 *)(((volatile __u8 *)(addr)) + off))) = (val); \
33     } while(0)
34
35 #define WRITE_WORD_VOL(addr, off, val) \
36     do { \
37         (*((volatile __u16 *)(((volatile __u8 *)(addr)) + off))) = (val); \
38     } while(0)
39
40 #define WRITE_DWORD_VOL(addr, off, val) \
41     do { \
42         (*((volatile __u32 *)(((volatile __u8 *)(addr)) + off))) = (val); \
43     } while(0)
44
45 struct device_buffers_st {
46         /* inprm and outprm do not have alignnemet constraint sice that
47            is acheived programatically */
48         u8 inprm_buf[INPRM_BUF_SZ];
49         u8 outprm_buf[OUTPRM_BUF_SZ];
50         union recv_wqe_u mads_qp_rcv_queue[NUM_MADS_RCV_WQES]
51             __attribute__ ((aligned(RECV_WQE_U_ALIGN)));
52         union recv_wqe_u ipoib_qp_rcv_queue[NUM_IPOIB_RCV_WQES]
53             __attribute__ ((aligned(RECV_WQE_U_ALIGN)));
54         union ud_send_wqe_u mads_qp_snd_queue[NUM_MADS_SND_WQES]
55             __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN)));
56         union ud_send_wqe_u ipoib_qp_snd_queue[NUM_IPOIB_SND_WQES]
57             __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN)));
58         struct eqe_t eq_buf[1 << LOG2_EQ_SZ]
59             __attribute__ ((aligned(sizeof(struct eqe_t))));
60         union cqe_st mads_snd_cq_buf[NUM_MADS_SND_CQES]
61             __attribute__ ((aligned(sizeof(union cqe_st))));
62         union cqe_st ipoib_snd_cq_buf[NUM_IPOIB_SND_CQES]
63             __attribute__ ((aligned(sizeof(union cqe_st))));
64         union cqe_st mads_rcv_cq_buf[NUM_MADS_RCV_CQES]
65             __attribute__ ((aligned(sizeof(union cqe_st))));
66         union cqe_st ipoib_rcv_cq_buf[NUM_IPOIB_RCV_CQES]
67             __attribute__ ((aligned(sizeof(union cqe_st))));
68         union ud_av_u av_array[NUM_AVS];
69 } __attribute__ ((packed));
70
71 #define STRUCT_ALIGN_SZ 4096
72 #define SRC_BUF_SZ (sizeof(struct device_buffers_st) + STRUCT_ALIGN_SZ - 1)
73
74 /* the following must be kept in this order
75    for the memory region to cover the buffers */
76 static u8 src_buf[SRC_BUF_SZ];
77 static struct ib_buffers_st ib_buffers;
78 static __u32 memreg_size;
79 /* end of order constraint */
80
81 struct phys_mem_desc_st {
82         unsigned long base;
83         unsigned long offset;
84 };
85
86 static struct phys_mem_desc_st phys_mem;
87
88 static struct dev_pci_struct memfree_pci_dev;
89 static struct device_buffers_st *dev_buffers_p;
90 static struct device_ib_data_st dev_ib_data;
91
92
93
94 struct map_icm_st icm_map_obj;
95
96 static int gw_write_cr(__u32 addr, __u32 data)
97 {
98         writel(htonl(data), memfree_pci_dev.cr_space + addr);
99         return 0;
100 }
101
102 static int gw_read_cr(__u32 addr, __u32 * result)
103 {
104         *result = ntohl(readl(memfree_pci_dev.cr_space + addr));
105         return 0;
106 }
107
108 static int reset_hca(void)
109 {
110         return gw_write_cr(MEMFREE_RESET_OFFSET, 1);
111 }
112
113 static int ib_device_init(struct pci_device *dev)
114 {
115         int i;
116         int rc;
117
118         tprintf("");
119
120         memset(&dev_ib_data, 0, sizeof dev_ib_data);
121
122         /* save bars */
123         tprintf("bus=%d devfn=0x%x", dev->bus, dev->devfn);
124         for (i = 0; i < 6; ++i) {
125                 memfree_pci_dev.dev.bar[i] =
126                     pci_bar_start(dev, PCI_BASE_ADDRESS_0 + (i << 2));
127                 tprintf("bar[%d]= 0x%08lx", i, memfree_pci_dev.dev.bar[i]);
128         }
129
130         tprintf("");
131         /* save config space */
132         for (i = 0; i < 64; ++i) {
133                 rc = pci_read_config_dword(dev, i << 2,
134                                            &memfree_pci_dev.dev.
135                                            dev_config_space[i]);
136                 if (rc) {
137                         eprintf("");
138                         return rc;
139                 }
140                 tprintf("config[%d]= 0x%08lx", i << 2,
141                         memfree_pci_dev.dev.dev_config_space[i]);
142         }
143
144         tprintf("");
145         memfree_pci_dev.dev.dev = dev;
146
147         /* map cr-space */
148         memfree_pci_dev.cr_space =
149             ioremap(memfree_pci_dev.dev.bar[0], 0x100000);
150         if (!memfree_pci_dev.cr_space) {
151                 eprintf("");
152                 return -1;
153         }
154
155         /* map uar */
156         memfree_pci_dev.uar =
157             ioremap(memfree_pci_dev.dev.bar[2] + UAR_IDX * 0x1000, 0x1000);
158         if (!memfree_pci_dev.uar) {
159                 eprintf("");
160                 return -1;
161         }
162         tprintf("uar_base (pa:va) = 0x%lx %p",
163                 memfree_pci_dev.dev.bar[2] + UAR_IDX * 0x1000,
164                 memfree_pci_dev.uar);
165
166         tprintf("");
167
168         return 0;
169 }
170
171 static inline unsigned long lalign(unsigned long buf, unsigned long align)
172 {
173         return (unsigned long)((buf + align - 1) &
174                                (~(((unsigned long)align) - 1)));
175 }
176
177 #include <gpxe/umalloc.h>
178
179 static int init_dev_data(void)
180 {
181         unsigned long tmp;
182         unsigned long reserve_size = 32 * 1024 * 1024;
183
184         tmp = lalign(virt_to_bus(src_buf), STRUCT_ALIGN_SZ);
185
186         dev_buffers_p = bus_to_virt(tmp);
187         memreg_size = (__u32) (&memreg_size) - (__u32) dev_buffers_p;
188         tprintf("src_buf=%p, dev_buffers_p=%p, memreg_size=0x%lx", src_buf,
189                 dev_buffers_p, memreg_size);
190
191         tprintf("inprm: va=%p, pa=0x%lx", dev_buffers_p->inprm_buf,
192                 virt_to_bus(dev_buffers_p->inprm_buf));
193         tprintf("outprm: va=%p, pa=0x%lx", dev_buffers_p->outprm_buf,
194                 virt_to_bus(dev_buffers_p->outprm_buf));
195
196         userptr_t lotsofmem = umalloc ( reserve_size * 2 );
197         if ( ! lotsofmem ) {
198                 printf ( "Could not allocate large memblock\n" );
199                 return -1;
200         }
201         phys_mem.base = ( ( user_to_phys ( lotsofmem, 0 ) + reserve_size ) &
202                           ~( reserve_size - 1 ) );
203         phys_mem.offset = 0;
204
205         return 0;
206 }
207
208 static int restore_config(void)
209 {
210         int i;
211         int rc;
212
213         for (i = 0; i < 64; ++i) {
214                 if (i != 22 && i != 23) {
215                         rc = pci_write_config_dword(memfree_pci_dev.dev.dev,
216                                                     i << 2,
217                                                     memfree_pci_dev.dev.
218                                                     dev_config_space[i]);
219                         if (rc) {
220                                 return rc;
221                         }
222                 }
223         }
224         return 0;
225 }
226
227 static void prep_init_hca_buf(struct init_hca_st *init_hca_p, void *buf)
228 {
229         unsigned long ptr;
230         __u8 shift;
231
232         memset(buf, 0, MT_STRUCT_SIZE(arbelprm_init_hca_st));
233
234         ptr = (unsigned long)buf +
235             MT_BYTE_OFFSET(arbelprm_init_hca_st,
236                            qpc_eec_cqc_eqc_rdb_parameters);
237
238         shift = 32 - MT_BIT_SIZE(arbelprm_qpcbaseaddr_st, qpc_base_addr_l);
239         INS_FLD(init_hca_p->qpc_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
240                 qpc_base_addr_h);
241         INS_FLD(init_hca_p->qpc_base_addr_l >> shift, ptr,
242                 arbelprm_qpcbaseaddr_st, qpc_base_addr_l);
243         INS_FLD(init_hca_p->log_num_of_qp, ptr, arbelprm_qpcbaseaddr_st,
244                 log_num_of_qp);
245
246         shift = 32 - MT_BIT_SIZE(arbelprm_qpcbaseaddr_st, eec_base_addr_l);
247         INS_FLD(init_hca_p->eec_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
248                 eec_base_addr_h);
249         INS_FLD(init_hca_p->eec_base_addr_l >> shift, ptr,
250                 arbelprm_qpcbaseaddr_st, eec_base_addr_l);
251         INS_FLD(init_hca_p->log_num_of_ee, ptr, arbelprm_qpcbaseaddr_st,
252                 log_num_of_ee);
253
254         shift = 32 - MT_BIT_SIZE(arbelprm_qpcbaseaddr_st, srqc_base_addr_l);
255         INS_FLD(init_hca_p->srqc_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
256                 srqc_base_addr_h);
257         INS_FLD(init_hca_p->srqc_base_addr_l >> shift, ptr,
258                 arbelprm_qpcbaseaddr_st, srqc_base_addr_l);
259         INS_FLD(init_hca_p->log_num_of_srq, ptr, arbelprm_qpcbaseaddr_st,
260                 log_num_of_srq);
261
262         shift = 32 - MT_BIT_SIZE(arbelprm_qpcbaseaddr_st, cqc_base_addr_l);
263         INS_FLD(init_hca_p->cqc_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
264                 cqc_base_addr_h);
265         INS_FLD(init_hca_p->cqc_base_addr_l >> shift, ptr,
266                 arbelprm_qpcbaseaddr_st, cqc_base_addr_l);
267         INS_FLD(init_hca_p->log_num_of_cq, ptr, arbelprm_qpcbaseaddr_st,
268                 log_num_of_cq);
269
270         INS_FLD(init_hca_p->eqpc_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
271                 eqpc_base_addr_h);
272         INS_FLD(init_hca_p->eqpc_base_addr_l, ptr, arbelprm_qpcbaseaddr_st,
273                 eqpc_base_addr_l);
274
275         INS_FLD(init_hca_p->eeec_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
276                 eeec_base_addr_h);
277         INS_FLD(init_hca_p->eeec_base_addr_l, ptr, arbelprm_qpcbaseaddr_st,
278                 eeec_base_addr_l);
279
280         shift = 32 - MT_BIT_SIZE(arbelprm_qpcbaseaddr_st, eqc_base_addr_l);
281         INS_FLD(init_hca_p->eqc_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
282                 eqc_base_addr_h);
283         INS_FLD(init_hca_p->eqc_base_addr_l >> shift, ptr,
284                 arbelprm_qpcbaseaddr_st, eqc_base_addr_l);
285         INS_FLD(init_hca_p->log_num_of_eq, ptr, arbelprm_qpcbaseaddr_st,
286                 log_num_eq);
287
288         INS_FLD(init_hca_p->rdb_base_addr_h, ptr, arbelprm_qpcbaseaddr_st,
289                 rdb_base_addr_h);
290         INS_FLD(init_hca_p->rdb_base_addr_l, ptr, arbelprm_qpcbaseaddr_st,
291                 rdb_base_addr_l);
292
293         ptr = (unsigned long)buf +
294             MT_BYTE_OFFSET(arbelprm_init_hca_st, multicast_parameters);
295
296         INS_FLD(init_hca_p->mc_base_addr_h, ptr, arbelprm_multicastparam_st,
297                 mc_base_addr_h);
298         INS_FLD(init_hca_p->mc_base_addr_l, ptr, arbelprm_multicastparam_st,
299                 mc_base_addr_l);
300         INS_FLD(init_hca_p->log_mc_table_entry_sz, ptr,
301                 arbelprm_multicastparam_st, log_mc_table_entry_sz);
302         INS_FLD(init_hca_p->mc_table_hash_sz, ptr, arbelprm_multicastparam_st,
303                 mc_table_hash_sz);
304         INS_FLD(init_hca_p->log_mc_table_sz, ptr, arbelprm_multicastparam_st,
305                 log_mc_table_sz);
306
307         ptr = (unsigned long)buf +
308             MT_BYTE_OFFSET(arbelprm_init_hca_st, tpt_parameters);
309
310         INS_FLD(init_hca_p->mpt_base_addr_h, ptr, arbelprm_tptparams_st,
311                 mpt_base_adr_h);
312         INS_FLD(init_hca_p->mpt_base_addr_l, ptr, arbelprm_tptparams_st,
313                 mpt_base_adr_l);
314         INS_FLD(init_hca_p->log_mpt_sz, ptr, arbelprm_tptparams_st, log_mpt_sz);
315         INS_FLD(init_hca_p->mtt_base_addr_h, ptr, arbelprm_tptparams_st,
316                 mtt_base_addr_h);
317         INS_FLD(init_hca_p->mtt_base_addr_l, ptr, arbelprm_tptparams_st,
318                 mtt_base_addr_l);
319
320         ptr = (unsigned long)buf +
321             MT_BYTE_OFFSET(arbelprm_init_hca_st, uar_parameters);
322
323         INS_FLD(init_hca_p->log_max_uars, ptr, arbelprm_uar_params_st,
324                 log_max_uars);
325
326 }
327
328 static void prep_sw2hw_mpt_buf(void *buf, __u32 mkey)
329 {
330         INS_FLD(1, buf, arbelprm_mpt_st, lw);
331         INS_FLD(1, buf, arbelprm_mpt_st, lr);
332         INS_FLD(1, buf, arbelprm_mpt_st, pa);
333         INS_FLD(1, buf, arbelprm_mpt_st, r_w);
334         INS_FLD(mkey, buf, arbelprm_mpt_st, mem_key);
335         INS_FLD(GLOBAL_PD, buf, arbelprm_mpt_st, pd);
336         INS_FLD(virt_to_bus(dev_buffers_p), buf, arbelprm_mpt_st,
337                 start_address_l);
338         INS_FLD(memreg_size, buf, arbelprm_mpt_st, reg_wnd_len_l);
339 }
340
341 static void prep_sw2hw_eq_buf(void *buf, struct eqe_t *eq_buf)
342 {
343         memset(buf, 0, MT_STRUCT_SIZE(arbelprm_eqc_st));
344
345         INS_FLD(0xa, buf, arbelprm_eqc_st, st); /* fired */
346         INS_FLD(virt_to_bus(eq_buf), buf, arbelprm_eqc_st, start_address_l);
347         INS_FLD(LOG2_EQ_SZ, buf, arbelprm_eqc_st, log_eq_size);
348         INS_FLD(GLOBAL_PD, buf, arbelprm_eqc_st, pd);
349         INS_FLD(dev_ib_data.mkey, buf, arbelprm_eqc_st, lkey);
350 }
351
352 static void init_eq_buf(void *eq_buf)
353 {
354         struct eqe_t *eq = eq_buf;
355         int i, num_eqes = 1 << LOG2_EQ_SZ;
356
357         memset(eq, 0, num_eqes * sizeof eq[0]);
358         for (i = 0; i < num_eqes; ++i)
359                 WRITE_BYTE_VOL(&eq[i], EQE_OWNER_OFFSET, EQE_OWNER_VAL_HW);
360 }
361
362 static void prep_init_ib_buf(void *buf)
363 {
364         memset(buf, 0, MT_STRUCT_SIZE(arbelprm_init_ib_st));
365
366         INS_FLD(MTU_2048, buf, arbelprm_init_ib_st, mtu_cap);
367         INS_FLD(3, buf, arbelprm_init_ib_st, port_width_cap);
368         INS_FLD(1, buf, arbelprm_init_ib_st, vl_cap);
369         INS_FLD(1, buf, arbelprm_init_ib_st, max_gid);
370         INS_FLD(64, buf, arbelprm_init_ib_st, max_pkey);
371 }
372
373 static void prep_sw2hw_cq_buf(void *buf, __u8 eqn,
374                               __u32 cqn,
375                               union cqe_st *cq_buf,
376                               __u32 cq_ci_db_record, __u32 cq_state_db_record)
377 {
378         memset(buf, 0, MT_STRUCT_SIZE(arbelprm_completion_queue_context_st));
379
380         INS_FLD(0xA, buf, arbelprm_completion_queue_context_st, st);
381         INS_FLD(virt_to_bus(cq_buf), buf, arbelprm_completion_queue_context_st,
382                 start_address_l);
383         INS_FLD(LOG2_CQ_SZ, buf, arbelprm_completion_queue_context_st,
384                 log_cq_size);
385         INS_FLD(dev_ib_data.uar_idx, buf, arbelprm_completion_queue_context_st,
386                 usr_page);
387         INS_FLD(eqn, buf, arbelprm_completion_queue_context_st, c_eqn);
388         INS_FLD(GLOBAL_PD, buf, arbelprm_completion_queue_context_st, pd);
389         INS_FLD(dev_ib_data.mkey, buf, arbelprm_completion_queue_context_st,
390                 l_key);
391         INS_FLD(cqn, buf, arbelprm_completion_queue_context_st, cqn);
392         INS_FLD(cq_ci_db_record, buf, arbelprm_completion_queue_context_st,
393                 cq_ci_db_record);
394         INS_FLD(cq_state_db_record, buf, arbelprm_completion_queue_context_st,
395                 cq_state_db_record);
396 }
397
398 static void prep_rst2init_qpee_buf(void *buf,
399                                    __u32 snd_cqn,
400                                    __u32 rcv_cqn,
401                                    __u32 qkey,
402                                    __u32 log_rq_size,
403                                    __u32 log_rq_stride,
404                                    __u32 log_sq_size,
405                                    __u32 log_sq_stride,
406                                    __u32 snd_wqe_base_adr_l,
407                                    __u32 snd_db_record_index,
408                                    __u32 rcv_wqe_base_adr_l,
409                                    __u32 rcv_db_record_index)
410 {
411         void *tmp;
412         int shift;
413         struct qp_ee_state_tarnisition_st *prm = buf;
414
415         memset(buf, 0, sizeof *prm);
416
417         tprintf("snd_cqn=0x%lx", snd_cqn);
418         tprintf("rcv_cqn=0x%lx", rcv_cqn);
419         tprintf("qkey=0x%lx", qkey);
420         tprintf("log_rq_size=0x%lx", log_rq_size);
421         tprintf("log_rq_stride=0x%lx", log_rq_stride);
422         tprintf("log_sq_size=0x%lx", log_sq_size);
423         tprintf("log_sq_stride=0x%lx", log_sq_stride);
424         tprintf("snd_wqe_base_adr_l=0x%lx", snd_wqe_base_adr_l);
425         tprintf("snd_db_record_index=0x%lx", snd_db_record_index);
426         tprintf("rcv_wqe_base_adr_l=0x%lx", rcv_wqe_base_adr_l);
427         tprintf("rcv_db_record_index=0x%lx", rcv_db_record_index);
428
429         tmp = &prm->ctx;
430         INS_FLD(TS_UD, tmp, arbelprm_queue_pair_ee_context_entry_st, st);
431         INS_FLD(PM_STATE_MIGRATED, tmp, arbelprm_queue_pair_ee_context_entry_st,
432                 pm_state);
433         INS_FLD(1, tmp, arbelprm_queue_pair_ee_context_entry_st, de);
434         INS_FLD(MTU_2048, tmp, arbelprm_queue_pair_ee_context_entry_st, mtu);
435         INS_FLD(11, tmp, arbelprm_queue_pair_ee_context_entry_st, msg_max);
436         INS_FLD(log_rq_size, tmp, arbelprm_queue_pair_ee_context_entry_st,
437                 log_rq_size);
438         INS_FLD(log_rq_stride, tmp, arbelprm_queue_pair_ee_context_entry_st,
439                 log_rq_stride);
440         INS_FLD(log_sq_size, tmp, arbelprm_queue_pair_ee_context_entry_st,
441                 log_sq_size);
442         INS_FLD(log_sq_stride, tmp, arbelprm_queue_pair_ee_context_entry_st,
443                 log_sq_stride);
444         INS_FLD(dev_ib_data.uar_idx, tmp,
445                 arbelprm_queue_pair_ee_context_entry_st, usr_page);
446         INS_FLD(GLOBAL_PD, tmp, arbelprm_queue_pair_ee_context_entry_st, pd);
447         INS_FLD(dev_ib_data.mkey, tmp, arbelprm_queue_pair_ee_context_entry_st,
448                 wqe_lkey);
449         INS_FLD(1, tmp, arbelprm_queue_pair_ee_context_entry_st, ssc);
450         INS_FLD(snd_cqn, tmp, arbelprm_queue_pair_ee_context_entry_st, cqn_snd);
451         shift =
452             32 - MT_BIT_SIZE(arbelprm_queue_pair_ee_context_entry_st,
453                              snd_wqe_base_adr_l);
454         INS_FLD(snd_wqe_base_adr_l >> shift, tmp,
455                 arbelprm_queue_pair_ee_context_entry_st, snd_wqe_base_adr_l);
456         INS_FLD(snd_db_record_index, tmp,
457                 arbelprm_queue_pair_ee_context_entry_st, snd_db_record_index);
458         INS_FLD(1, tmp, arbelprm_queue_pair_ee_context_entry_st, rsc);
459         INS_FLD(rcv_cqn, tmp, arbelprm_queue_pair_ee_context_entry_st, cqn_rcv);
460         shift =
461             32 - MT_BIT_SIZE(arbelprm_queue_pair_ee_context_entry_st,
462                              rcv_wqe_base_adr_l);
463         INS_FLD(rcv_wqe_base_adr_l >> shift, tmp,
464                 arbelprm_queue_pair_ee_context_entry_st, rcv_wqe_base_adr_l);
465         INS_FLD(rcv_db_record_index, tmp,
466                 arbelprm_queue_pair_ee_context_entry_st, rcv_db_record_index);
467         INS_FLD(qkey, tmp, arbelprm_queue_pair_ee_context_entry_st, q_key);
468
469         tmp =
470             (__u8 *) (&prm->ctx) +
471             MT_BYTE_OFFSET(arbelprm_queue_pair_ee_context_entry_st,
472                            primary_address_path);
473         INS_FLD(dev_ib_data.port, tmp, arbelprm_address_path_st, port_number);
474
475 }
476
477 static void prep_init2rtr_qpee_buf(void *buf)
478 {
479         struct qp_ee_state_tarnisition_st *prm;
480
481         prm = (struct qp_ee_state_tarnisition_st *)buf;
482
483         memset(prm, 0, sizeof *prm);
484
485         INS_FLD(MTU_2048, &prm->ctx, arbelprm_queue_pair_ee_context_entry_st,
486                 mtu);
487         INS_FLD(11, &prm->ctx, arbelprm_queue_pair_ee_context_entry_st,
488                 msg_max);
489 }
490
491 static void init_av_array(void)
492 {
493 }
494
495 /*
496  * my_log2()
497  */
498 static int my_log2(unsigned long arg)
499 {
500         int i;
501         __u32 tmp;
502
503         if (arg == 0) {
504                 return INT_MIN; /* log2(0) = -infinity */
505         }
506
507         tmp = 1;
508         i = 0;
509         while (tmp < arg) {
510                 tmp = tmp << 1;
511                 ++i;
512         }
513
514         return i;
515 }
516
517 /*
518  * get_req_icm_pages
519  */
520 static unsigned long get_req_icm_pages(unsigned long log2_reserved,
521                                        unsigned long app_rsrc,
522                                        unsigned long entry_size,
523                                        unsigned long *log2_entries_p)
524 {
525         unsigned long size;
526         unsigned long log2_entries;
527
528         log2_entries = my_log2((1 << log2_reserved) + app_rsrc);
529         *log2_entries_p = log2_entries;
530         size = (1 << log2_entries) * entry_size;
531
532         return (size + 4095) >> 12;
533 }
534
535 static void init_uar_context(void *uar_context_va)
536 {
537         void *ptr;
538         /* clear all uar context */
539         memset(uar_context_va, 0, 4096);
540
541         ptr = uar_context_va + MADS_RCV_CQ_ARM_DB_IDX * 8;
542         INS_FLD_TO_BE(UAR_RES_CQ_ARM, ptr, arbelprm_cq_arm_db_record_st, res);
543         INS_FLD_TO_BE(dev_ib_data.mads_qp.rcv_cq.cqn, ptr,
544                       arbelprm_cq_arm_db_record_st, cq_number);
545
546         ptr = uar_context_va + MADS_SND_CQ_ARM_DB_IDX * 8;
547         INS_FLD_TO_BE(UAR_RES_CQ_ARM, ptr, arbelprm_cq_arm_db_record_st, res);
548         INS_FLD_TO_BE(dev_ib_data.mads_qp.snd_cq.cqn, ptr,
549                       arbelprm_cq_arm_db_record_st, cq_number);
550
551         ptr = uar_context_va + IPOIB_RCV_CQ_ARM_DB_IDX * 8;
552         INS_FLD_TO_BE(UAR_RES_CQ_ARM, ptr, arbelprm_cq_arm_db_record_st, res);
553         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.rcv_cq.cqn, ptr,
554                       arbelprm_cq_arm_db_record_st, cq_number);
555
556         ptr = uar_context_va + IPOIB_SND_CQ_ARM_DB_IDX * 8;
557         INS_FLD_TO_BE(UAR_RES_CQ_ARM, ptr, arbelprm_cq_arm_db_record_st, res);
558         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.snd_cq.cqn, ptr,
559                       arbelprm_cq_arm_db_record_st, cq_number);
560
561         ptr = uar_context_va + MADS_SND_QP_DB_IDX * 8;
562         INS_FLD_TO_BE(UAR_RES_SQ_DBELL, ptr, arbelprm_qp_db_record_st, res);
563         INS_FLD_TO_BE(dev_ib_data.mads_qp.qpn, ptr, arbelprm_qp_db_record_st,
564                       qp_number);
565
566         ptr = uar_context_va + IPOIB_SND_QP_DB_IDX * 8;
567         INS_FLD_TO_BE(UAR_RES_SQ_DBELL, ptr, arbelprm_qp_db_record_st, res);
568         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.qpn, ptr, arbelprm_qp_db_record_st,
569                       qp_number);
570
571         ptr = uar_context_va + GROUP_SEP_IDX * 8;
572         INS_FLD_TO_BE(UAR_RES_GROUP_SEP, ptr, arbelprm_cq_arm_db_record_st,
573                       res);
574
575         ptr = uar_context_va + MADS_RCV_QP_DB_IDX * 8;
576         INS_FLD_TO_BE(UAR_RES_RQ_DBELL, ptr, arbelprm_qp_db_record_st, res);
577         INS_FLD_TO_BE(dev_ib_data.mads_qp.qpn, ptr, arbelprm_qp_db_record_st,
578                       qp_number);
579
580         ptr = uar_context_va + IPOIB_RCV_QP_DB_IDX * 8;
581         INS_FLD_TO_BE(UAR_RES_RQ_DBELL, ptr, arbelprm_qp_db_record_st, res);
582         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.qpn, ptr, arbelprm_qp_db_record_st,
583                       qp_number);
584
585         ptr = uar_context_va + MADS_RCV_CQ_CI_DB_IDX * 8;
586         INS_FLD_TO_BE(UAR_RES_CQ_SET_CI, ptr, arbelprm_cq_ci_db_record_st, res);
587         INS_FLD_TO_BE(dev_ib_data.mads_qp.rcv_cq.cqn, ptr,
588                       arbelprm_cq_ci_db_record_st, cq_number);
589
590         ptr = uar_context_va + MADS_SND_CQ_CI_DB_IDX * 8;
591         INS_FLD_TO_BE(UAR_RES_CQ_SET_CI, ptr, arbelprm_cq_ci_db_record_st, res);
592         INS_FLD_TO_BE(dev_ib_data.mads_qp.snd_cq.cqn, ptr,
593                       arbelprm_cq_ci_db_record_st, cq_number);
594
595         ptr = uar_context_va + IPOIB_RCV_CQ_CI_DB_IDX * 8;
596         INS_FLD_TO_BE(UAR_RES_CQ_SET_CI, ptr, arbelprm_cq_ci_db_record_st, res);
597         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.rcv_cq.cqn, ptr,
598                       arbelprm_cq_ci_db_record_st, cq_number);
599
600         ptr = uar_context_va + IPOIB_SND_CQ_CI_DB_IDX * 8;
601         INS_FLD_TO_BE(UAR_RES_CQ_SET_CI, ptr, arbelprm_cq_ci_db_record_st, res);
602         INS_FLD_TO_BE(dev_ib_data.ipoib_qp.snd_cq.cqn, ptr,
603                       arbelprm_cq_ci_db_record_st, cq_number);
604
605 }
606
607 static int setup_hca(__u8 port, void **eq_p)
608 {
609         int ret;
610         int rc;
611         struct query_fw_st qfw;
612         struct map_icm_st map_obj;
613         struct dev_lim_st dev_lim;
614         struct init_hca_st init_hca;
615         __u8 log2_pages;
616         unsigned long icm_start, icm_size, tmp;
617         unsigned long log2_entries;
618         __u32 aux_pages;
619         __u32 mem_key, key, tmp_key;
620         __u8 eqn;
621         __u32 event_mask;
622         struct eqe_t *eq_buf;
623         void *inprm;
624         unsigned long bus_addr;
625         struct query_adapter_st qa;
626         __u8 log_max_uars = 1;
627         void *uar_context_va;
628         __u32 uar_context_pa;
629
630         tprintf("called");
631         init_dev_data();
632         inprm = get_inprm_buf();
633
634         rc = reset_hca();
635         if (rc) {
636                 eprintf("");
637                 return rc;
638         } else {
639                 tprintf("reset_hca() success");
640         }
641
642         mdelay(1000);           /* wait for 1 sec */
643
644         rc = restore_config();
645         if (rc) {
646                 eprintf("");
647                 return rc;
648         } else {
649                 tprintf("restore_config() success");
650         }
651
652         dev_ib_data.pd = GLOBAL_PD;
653         dev_ib_data.port = port;
654         dev_ib_data.qkey = GLOBAL_QKEY;
655
656         rc = cmd_query_fw(&qfw);
657         if (rc) {
658                 eprintf("");
659                 return rc;
660         }
661         else {
662                 tprintf("cmd_query_fw() success");
663
664                 if (print_info) {
665                         printf("FW ver = %d.%d.%d\n",
666                         qfw.fw_rev_major,
667                         qfw.fw_rev_minor,
668                         qfw.fw_rev_subminor);
669                 }
670
671                 tprintf("fw_rev_major=%d", qfw.fw_rev_major);
672                 tprintf("fw_rev_minor=%d", qfw.fw_rev_minor);
673                 tprintf("fw_rev_subminor=%d", qfw.fw_rev_subminor);
674                 tprintf("error_buf_start_h=0x%lx", qfw.error_buf_start_h);
675                 tprintf("error_buf_start_l=0x%lx", qfw.error_buf_start_l);
676                 tprintf("error_buf_size=%ld", qfw.error_buf_size);
677         }
678
679
680
681         bus_addr =
682             ((unsigned long)((u64) qfw.error_buf_start_h << 32) | qfw.
683              error_buf_start_l);
684     dev_ib_data.error_buf_addr= ioremap(bus_addr,
685                                                                                 qfw.error_buf_size*4);
686         dev_ib_data.error_buf_size= qfw.error_buf_size;
687         if (!dev_ib_data.error_buf_addr) {
688                 eprintf("");
689                 return -1;
690         }
691
692
693         bus_addr =
694             ((unsigned long)((u64) qfw.clear_int_addr.addr_h << 32) | qfw.
695              clear_int_addr.addr_l);
696         dev_ib_data.clr_int_addr = bus_to_virt(bus_addr);
697
698         rc = cmd_enable_lam();
699         if (rc == 0x22 /* LAM_NOT_PRE -- need to put a name here */ ) {
700                 // ??????
701         } else if (rc == 0) {
702                 // ??????
703         } else {
704                 eprintf("");
705                 return rc;
706         }
707
708         log2_pages = my_log2(qfw.fw_pages);
709
710         memset(&map_obj, 0, sizeof map_obj);
711         map_obj.num_vpm = 1;
712         map_obj.vpm_arr[0].log2_size = log2_pages;
713         map_obj.vpm_arr[0].pa_l = phys_mem.base + phys_mem.offset;
714         rc = cmd_map_fa(&map_obj);
715         if (rc) {
716                 eprintf("");
717                 return rc;
718         }
719         phys_mem.offset += 1 << (log2_pages + 12);
720
721         rc = cmd_run_fw();
722         if (rc) {
723                 ret = -1;
724                 eprintf("");
725                 goto undo_map_fa;
726         }
727
728         rc = cmd_mod_stat_cfg();
729         if (rc) {
730                 ret = -1;
731                 eprintf("");
732                 goto undo_map_fa;
733         }
734
735         rc = cmd_query_dev_lim(&dev_lim);
736         if (rc) {
737                 ret = -1;
738                 eprintf("");
739                 goto undo_map_fa;
740         }
741
742         dev_ib_data.uar_idx = dev_lim.num_rsvd_uars;
743
744         tprintf("max_icm_size_h=0x%lx", dev_lim.max_icm_size_h);
745         tprintf("max_icm_size_l=0x%lx", dev_lim.max_icm_size_l);
746
747         memset(&init_hca, 0, sizeof init_hca);
748         icm_start = 0;
749         icm_size = 0;
750
751         icm_start += ((dev_lim.num_rsvd_uars + 1) << 12);
752         icm_size += ((dev_lim.num_rsvd_uars + 1) << 12);
753
754         tmp = get_req_icm_pages(dev_lim.log2_rsvd_qps,
755                                 MAX_APP_QPS,
756                                 dev_lim.qpc_entry_sz, &log2_entries);
757         init_hca.qpc_base_addr_l = icm_start;
758         init_hca.log_num_of_qp = log2_entries;
759         icm_start += (tmp << 12);
760         icm_size += (tmp << 12);
761
762         init_hca.eqpc_base_addr_l = icm_start;
763         icm_start += (tmp << 12);
764         icm_size += (tmp << 12);
765
766         tmp = get_req_icm_pages(dev_lim.log2_rsvd_srqs,
767                                 0, dev_lim.srq_entry_sz, &log2_entries);
768         init_hca.srqc_base_addr_l = icm_start;
769         init_hca.log_num_of_srq = log2_entries;
770         icm_start += (tmp << 12);
771         icm_size += (tmp << 12);
772
773         tmp = get_req_icm_pages(dev_lim.log2_rsvd_ees,
774                                 0, dev_lim.eec_entry_sz, &log2_entries);
775         init_hca.eec_base_addr_l = icm_start;
776         init_hca.log_num_of_ee = log2_entries;
777         icm_start += (tmp << 12);
778         icm_size += (tmp << 12);
779
780         init_hca.eeec_base_addr_l = icm_start;
781         icm_start += (tmp << 12);
782         icm_size += (tmp << 12);
783
784         tmp = get_req_icm_pages(dev_lim.log2_rsvd_cqs,
785                                 MAX_APP_CQS,
786                                 dev_lim.cqc_entry_sz, &log2_entries);
787         init_hca.cqc_base_addr_l = icm_start;
788         init_hca.log_num_of_cq = log2_entries;
789         icm_start += (tmp << 12);
790         icm_size += (tmp << 12);
791
792         tmp = get_req_icm_pages(dev_lim.log2_rsvd_mtts,
793                                 0, dev_lim.mtt_entry_sz, &log2_entries);
794         init_hca.mtt_base_addr_l = icm_start;
795         icm_start += (tmp << 12);
796         icm_size += (tmp << 12);
797
798         tmp = get_req_icm_pages(dev_lim.log2_rsvd_mrws,
799                                 1, dev_lim.mpt_entry_sz, &log2_entries);
800         init_hca.mpt_base_addr_l = icm_start;
801         init_hca.log_mpt_sz = log2_entries;
802         icm_start += (tmp << 12);
803         icm_size += (tmp << 12);
804
805         tmp = get_req_icm_pages(dev_lim.log2_rsvd_rdbs, 1, 32,  /* size of rdb entry */
806                                 &log2_entries);
807         init_hca.rdb_base_addr_l = icm_start;
808         icm_start += (tmp << 12);
809         icm_size += (tmp << 12);
810
811         init_hca.eqc_base_addr_l = icm_start;
812         init_hca.log_num_of_eq = LOG2_EQS;
813         tmp = dev_lim.eqc_entry_sz * (1 << LOG2_EQS);
814         icm_start += tmp;
815         icm_size += tmp;
816
817         init_hca.mc_base_addr_l = icm_start;
818         init_hca.log_mc_table_entry_sz =
819             my_log2(MT_STRUCT_SIZE(arbelprm_mgm_entry_st));
820         init_hca.mc_table_hash_sz = 8;
821         init_hca.log_mc_table_sz = 3;
822         icm_size +=
823             (MT_STRUCT_SIZE(arbelprm_mgm_entry_st) * init_hca.mc_table_hash_sz);
824         icm_start +=
825             (MT_STRUCT_SIZE(arbelprm_mgm_entry_st) * init_hca.mc_table_hash_sz);
826
827         rc = cmd_set_icm_size(icm_size, &aux_pages);
828         if (rc) {
829                 ret = -1;
830                 eprintf("");
831                 goto undo_map_fa;
832         }
833
834         memset(&map_obj, 0, sizeof map_obj);
835         map_obj.num_vpm = 1;
836         map_obj.vpm_arr[0].pa_l = phys_mem.base + phys_mem.offset;
837         map_obj.vpm_arr[0].log2_size = my_log2(aux_pages);
838         rc = cmd_map_icm_aux(&map_obj);
839         if (rc) {
840                 ret = -1;
841                 eprintf("");
842                 goto undo_map_fa;
843         }
844         phys_mem.offset += (1 << (map_obj.vpm_arr[0].log2_size + 12));
845
846         uar_context_pa = phys_mem.base + phys_mem.offset +
847             dev_ib_data.uar_idx * 4096;
848         uar_context_va = phys_to_virt(uar_context_pa);
849         tprintf("uar_context: va=%p, pa=0x%lx", uar_context_va,
850                 uar_context_pa);
851         dev_ib_data.uar_context_base = uar_context_va;
852
853         memset(&map_obj, 0, sizeof map_obj);
854         map_obj.num_vpm = 1;
855         map_obj.vpm_arr[0].pa_l = phys_mem.base + phys_mem.offset;
856         map_obj.vpm_arr[0].log2_size = my_log2((icm_size + 4095) >> 12);
857         rc = cmd_map_icm(&map_obj);
858         if (rc) {
859                 ret = -1;
860                 eprintf("");
861                 goto undo_map_fa;
862         }
863         icm_map_obj = map_obj;
864
865         phys_mem.offset += (1 << (map_obj.vpm_arr[0].log2_size + 12));
866
867         init_hca.log_max_uars = log_max_uars;
868         tprintf("inprm: va=%p, pa=0x%lx", inprm, virt_to_bus(inprm));
869         prep_init_hca_buf(&init_hca, inprm);
870         rc = cmd_init_hca(inprm, MT_STRUCT_SIZE(arbelprm_init_hca_st));
871         if (rc) {
872                 ret = -1;
873                 eprintf("");
874                 goto undo_map_fa;
875         }
876
877         rc = cmd_query_adapter(&qa);
878         if (rc) {
879                 eprintf("");
880                 return rc;
881         }
882         dev_ib_data.clr_int_data = 1 << qa.intapin;
883
884         tmp_key = 1 << dev_lim.log2_rsvd_mrws | MKEY_PREFIX;
885         mem_key = 1 << (dev_lim.log2_rsvd_mrws + 8) | (MKEY_PREFIX >> 24);
886         prep_sw2hw_mpt_buf(inprm, tmp_key);
887         rc = cmd_sw2hw_mpt(&key, 1 << dev_lim.log2_rsvd_mrws, inprm,
888                            SW2HW_MPT_IBUF_SZ);
889         if (rc) {
890                 ret = -1;
891                 eprintf("");
892                 goto undo_map_fa;
893         } else {
894                 tprintf("cmd_sw2hw_mpt() success, key=0x%lx", mem_key);
895         }
896         dev_ib_data.mkey = mem_key;
897
898         eqn = EQN;
899         /* allocate a single EQ which will receive 
900            all the events */
901         eq_buf = dev_buffers_p->eq_buf;
902         init_eq_buf(eq_buf);    /* put in HW ownership */
903         prep_sw2hw_eq_buf(inprm, eq_buf);
904         rc = cmd_sw2hw_eq(SW2HW_EQ_IBUF_SZ);
905         if (rc) {
906                 ret = -1;
907                 eprintf("");
908                 goto undo_sw2hw_mpt;
909         } else
910                 tprintf("cmd_sw2hw_eq() success");
911
912         event_mask = (1 << XDEV_EV_TYPE_CQ_COMP) |
913             (1 << XDEV_EV_TYPE_CQ_ERR) |
914             (1 << XDEV_EV_TYPE_LOCAL_WQ_CATAS_ERR) |
915             (1 << XDEV_EV_TYPE_PORT_ERR) |
916             (1 << XDEV_EV_TYPE_LOCAL_WQ_INVALID_REQ_ERR) |
917             (1 << XDEV_EV_TYPE_LOCAL_WQ_ACCESS_VIOL_ERR) |
918             (1 << TAVOR_IF_EV_TYPE_OVERRUN);
919         rc = cmd_map_eq(eqn, event_mask, 1);
920         if (rc) {
921                 ret = -1;
922                 eprintf("");
923                 goto undo_sw2hw_eq;
924         } else
925                 tprintf("cmd_map_eq() success");
926
927         dev_ib_data.eq.eqn = eqn;
928         dev_ib_data.eq.eq_buf = eq_buf;
929         dev_ib_data.eq.cons_counter = 0;
930         dev_ib_data.eq.eq_size = 1 << LOG2_EQ_SZ;
931         bus_addr =
932             ((unsigned long)((u64) qfw.eq_ci_table.addr_h << 32) | qfw.
933              eq_ci_table.addr_l)
934             + eqn * 8;
935         dev_ib_data.eq.ci_base_base_addr = bus_to_virt(bus_addr);
936         *eq_p = &dev_ib_data.eq;
937
938         prep_init_ib_buf(inprm);
939         rc = cmd_init_ib(port, inprm, INIT_IB_IBUF_SZ);
940         if (rc) {
941                 ret = -1;
942                 eprintf("");
943                 goto undo_sw2hw_eq;
944         } else
945                 tprintf("cmd_init_ib() success");
946
947         init_av_array();
948         tprintf("init_av_array() done");
949
950         /* set the qp and cq numbers according
951            to the results of query_dev_lim */
952         dev_ib_data.mads_qp.qpn = (1 << dev_lim.log2_rsvd_qps) +
953             +QPN_BASE + MADS_QPN_SN;
954         dev_ib_data.ipoib_qp.qpn = (1 << dev_lim.log2_rsvd_qps) +
955             +QPN_BASE + IPOIB_QPN_SN;
956
957         dev_ib_data.mads_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) +
958             MADS_SND_CQN_SN;
959         dev_ib_data.mads_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) +
960             MADS_RCV_CQN_SN;
961
962         dev_ib_data.ipoib_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) +
963             IPOIB_SND_CQN_SN;
964         dev_ib_data.ipoib_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) +
965             IPOIB_RCV_CQN_SN;
966
967         init_uar_context(uar_context_va);
968
969         ret = 0;
970         goto exit;
971
972       undo_sw2hw_eq:
973         rc = cmd_hw2sw_eq(eqn);
974         if (rc)
975                 eprintf("");
976         else
977                 tprintf("cmd_hw2sw_eq() success");
978
979       undo_sw2hw_mpt:
980         rc = cmd_hw2sw_mpt(tmp_key);
981         if (rc)
982                 eprintf("");
983
984       undo_map_fa:
985         rc = cmd_unmap_fa();
986         if (rc)
987                 eprintf("");
988
989       exit:
990         return ret;
991 }
992
993
994 static int unset_hca(void)
995 {
996         int rc, ret = 0;
997
998         rc = cmd_unmap_icm(&icm_map_obj);
999         if (rc)
1000                 eprintf("");
1001         ret |= rc;
1002
1003
1004         rc = cmd_unmap_icm_aux();
1005         if (rc)
1006                 eprintf("");
1007         ret |= rc;
1008
1009         rc = cmd_unmap_fa();
1010         if (rc)
1011                 eprintf("");
1012         ret |= rc;
1013
1014         return ret;
1015 }
1016
1017 static void *get_inprm_buf(void)
1018 {
1019         return dev_buffers_p->inprm_buf;
1020 }
1021
1022 static void *get_outprm_buf(void)
1023 {
1024         return dev_buffers_p->outprm_buf;
1025 }
1026
1027 static void *get_send_wqe_buf(void *wqe, __u8 index)
1028 {
1029         struct ud_send_wqe_st *snd_wqe = wqe;
1030
1031         return bus_to_virt(be32_to_cpu(snd_wqe->mpointer[index].local_addr_l));
1032 }
1033
1034 static void *get_rcv_wqe_buf(void *wqe, __u8 index)
1035 {
1036         struct recv_wqe_st *rcv_wqe = wqe;
1037
1038         return bus_to_virt(be32_to_cpu(rcv_wqe->mpointer[index].local_addr_l));
1039 }
1040
1041 static void modify_av_params(struct ud_av_st *av,
1042                              __u16 dlid,
1043                              __u8 g,
1044                              __u8 sl, __u8 rate, union ib_gid_u *gid, __u32 qpn)
1045 {
1046         memset(&av->av, 0, sizeof av->av);
1047
1048         INS_FLD_TO_BE(dev_ib_data.port, &av->av, arbelprm_ud_address_vector_st,
1049                       port_number);
1050         INS_FLD_TO_BE(dev_ib_data.pd, &av->av, arbelprm_ud_address_vector_st,
1051                       pd);
1052         INS_FLD_TO_BE(dlid, &av->av, arbelprm_ud_address_vector_st, rlid);
1053         INS_FLD_TO_BE(g, &av->av, arbelprm_ud_address_vector_st, g);
1054         INS_FLD_TO_BE(sl, &av->av, arbelprm_ud_address_vector_st, sl);
1055         INS_FLD_TO_BE(3, &av->av, arbelprm_ud_address_vector_st, msg);
1056
1057         if (rate >= 3)
1058                 INS_FLD_TO_BE(0, &av->av, arbelprm_ud_address_vector_st, max_stat_rate);        /* 4x */
1059         else
1060                 INS_FLD_TO_BE(1, &av->av, arbelprm_ud_address_vector_st, max_stat_rate);        /* 1x */
1061
1062         if (g) {
1063                 if (gid) {
1064                         INS_FLD(*((__u32 *) (&gid->raw[0])), &av->av,
1065                                 arbelprm_ud_address_vector_st, rgid_127_96);
1066                         INS_FLD(*((__u32 *) (&gid->raw[4])), &av->av,
1067                                 arbelprm_ud_address_vector_st, rgid_95_64);
1068                         INS_FLD(*((__u32 *) (&gid->raw[8])), &av->av,
1069                                 arbelprm_ud_address_vector_st, rgid_63_32);
1070                         INS_FLD(*((__u32 *) (&gid->raw[12])), &av->av,
1071                                 arbelprm_ud_address_vector_st, rgid_31_0);
1072                 } else {
1073                         INS_FLD(0, &av->av, arbelprm_ud_address_vector_st,
1074                                 rgid_127_96);
1075                         INS_FLD(0, &av->av, arbelprm_ud_address_vector_st,
1076                                 rgid_95_64);
1077                         INS_FLD(0, &av->av, arbelprm_ud_address_vector_st,
1078                                 rgid_63_32);
1079                         INS_FLD(0, &av->av, arbelprm_ud_address_vector_st,
1080                                 rgid_31_0);
1081                 }
1082         } else {
1083                 INS_FLD(0, &av->av, arbelprm_ud_address_vector_st, rgid_127_96);
1084                 INS_FLD(0, &av->av, arbelprm_ud_address_vector_st, rgid_95_64);
1085                 INS_FLD(0, &av->av, arbelprm_ud_address_vector_st, rgid_63_32);
1086                 INS_FLD(2, &av->av, arbelprm_ud_address_vector_st, rgid_31_0);
1087         }
1088         av->dest_qp = qpn;
1089         av->qkey = dev_ib_data.qkey;
1090 }
1091
1092 static void init_cq_buf(union cqe_st *cq_buf, __u8 num_cqes)
1093 {
1094         int i;
1095
1096         memset(cq_buf, 0, sizeof(union cqe_st) * num_cqes);
1097         for (i = 0; i < num_cqes; ++i) {
1098                 WRITE_BYTE_VOL(&cq_buf[i], CQE_OWNER_OFFSET, CQE_OWNER_VAL_HW);
1099         }
1100 }
1101
1102 static int post_rcv_buf(struct udqp_st *qp, struct recv_wqe_st *rcv_wqe)
1103 {
1104         int i;
1105
1106         /* put a valid lkey */
1107         for (i = 0; i < MAX_SCATTER; ++i) {
1108                 rcv_wqe->mpointer[i].lkey = cpu_to_be32(dev_ib_data.mkey);
1109         }
1110
1111         qp->post_rcv_counter++;
1112         WRITE_WORD_VOL(qp->rcv_uar_context, 2, htons(qp->post_rcv_counter));
1113
1114         return 0;
1115 }
1116
1117 static int post_send_req(void *qph, void *wqeh, __u8 num_gather)
1118 {
1119         int rc;
1120         struct udqp_st *qp = qph;
1121         struct ud_send_wqe_st *snd_wqe = wqeh;
1122         struct send_doorbell_st dbell;
1123         __u32 nds;
1124
1125         qp->post_send_counter++;
1126
1127         WRITE_WORD_VOL(qp->send_uar_context, 2, htons(qp->post_send_counter));
1128
1129         memset(&dbell, 0, sizeof dbell);
1130         INS_FLD(XDEV_NOPCODE_SEND, &dbell, arbelprm_send_doorbell_st, nopcode);
1131         INS_FLD(1, &dbell, arbelprm_send_doorbell_st, f);
1132         INS_FLD(qp->post_send_counter - 1, &dbell, arbelprm_send_doorbell_st,
1133                 wqe_counter);
1134         INS_FLD(1, &dbell, arbelprm_send_doorbell_st, wqe_cnt);
1135         nds = (sizeof(snd_wqe->next) +
1136                sizeof(snd_wqe->udseg) +
1137                sizeof(snd_wqe->mpointer[0]) * num_gather) >> 4;
1138         INS_FLD(nds, &dbell, arbelprm_send_doorbell_st, nds);
1139         INS_FLD(qp->qpn, &dbell, arbelprm_send_doorbell_st, qpn);
1140
1141         if (qp->last_posted_snd_wqe) {
1142                 INS_FLD_TO_BE(nds,
1143                               &qp->last_posted_snd_wqe->next.next,
1144                               arbelprm_wqe_segment_next_st, nds);
1145                 INS_FLD_TO_BE(1,
1146                               &qp->last_posted_snd_wqe->next.next,
1147                               arbelprm_wqe_segment_next_st, f);
1148                 INS_FLD_TO_BE(XDEV_NOPCODE_SEND,
1149                               &qp->last_posted_snd_wqe->next.next,
1150                               arbelprm_wqe_segment_next_st, nopcode);
1151         }
1152
1153         rc = cmd_post_doorbell(&dbell, POST_SND_OFFSET);
1154         if (!rc) {
1155                 qp->last_posted_snd_wqe = snd_wqe;
1156         }
1157
1158         return rc;
1159
1160 }
1161
1162 static int create_mads_qp(void **qp_pp, void **snd_cq_pp, void **rcv_cq_pp)
1163 {
1164         __u8 i, next_i, j, k;
1165         int rc;
1166         struct udqp_st *qp;
1167         __u32 bus_addr;
1168         __u8 nds;
1169         void *ptr;
1170
1171         qp = &dev_ib_data.mads_qp;
1172
1173         /* set the pointer to the receive WQEs buffer */
1174         qp->rcv_wq = dev_buffers_p->mads_qp_rcv_queue;
1175
1176         qp->send_buf_sz = MAD_BUF_SZ;
1177         qp->rcv_buf_sz = MAD_BUF_SZ;
1178
1179         qp->max_recv_wqes = NUM_MADS_RCV_WQES;  /* max wqes in this work queue */
1180         qp->recv_wqe_cur_free = NUM_MADS_RCV_WQES;      /* current free wqes */
1181         qp->recv_wqe_alloc_idx = 0;     /* index from wqes can be allocated if there are free wqes */
1182
1183         qp->rcv_uar_context =
1184             dev_ib_data.uar_context_base + 8 * MADS_RCV_QP_DB_IDX;
1185         qp->send_uar_context =
1186             dev_ib_data.uar_context_base + 8 * MADS_SND_QP_DB_IDX;
1187
1188         memset(&qp->rcv_wq[0], 0, NUM_MADS_RCV_WQES * sizeof(qp->rcv_wq[0]));
1189         nds = sizeof(qp->rcv_wq[0].wqe) >> 4;
1190         /* iterrate through the list */
1191         for (j = 0, i = 0, next_i = 1;
1192              j < NUM_MADS_RCV_WQES;
1193              MOD_INC(i, NUM_MADS_RCV_WQES), MOD_INC(next_i, NUM_MADS_RCV_WQES),
1194              ++j) {
1195
1196                 qp->rcv_bufs[i] = ib_buffers.rcv_mad_buf[i];
1197                 /* link the WQE to the next one */
1198                 bus_addr = virt_to_bus(&qp->rcv_wq[next_i].wqe);
1199                 ptr = qp->rcv_wq[i].wqe.control +
1200                     MT_BYTE_OFFSET(arbelprm_wqe_segment_ctrl_recv_st,
1201                                    wqe_segment_next);
1202                 INS_FLD(bus_addr >> 6, ptr, arbelprm_recv_wqe_segment_next_st,
1203                         nda_31_6);
1204                 INS_FLD(nds, ptr, arbelprm_recv_wqe_segment_next_st, nds);
1205
1206                 /* set the allocated buffers */
1207                 qp->rcv_bufs[i] = ib_buffers.rcv_mad_buf[i];
1208                 bus_addr = virt_to_bus(qp->rcv_bufs[i]);
1209                 qp->rcv_wq[i].wqe.mpointer[0].local_addr_l = bus_addr;
1210                 qp->rcv_wq[i].wqe.mpointer[0].byte_count = GRH_SIZE;
1211                 bus_addr = virt_to_bus(qp->rcv_bufs[i] + GRH_SIZE);
1212                 qp->rcv_wq[i].wqe.mpointer[1].local_addr_l = bus_addr;
1213                 qp->rcv_wq[i].wqe.mpointer[1].byte_count = MAD_BUF_SZ;
1214
1215                 for (k = 0; k < (((sizeof(qp->rcv_wq[i])) >> 4) - 1); ++k) {
1216                         qp->rcv_wq[i].wqe.mpointer[k].lkey = INVALID_WQE_LKEY;
1217                 }
1218         }
1219         cpu_to_be_buf(&qp->rcv_wq[0],
1220                       NUM_MADS_RCV_WQES * sizeof(qp->rcv_wq[0]));
1221
1222         for (i = 0; i < qp->max_recv_wqes; ++i) {
1223                 qp->rcv_wq[i].wqe_cont.qp = qp;
1224         }
1225
1226         /* set the pointer to the send WQEs buffer */
1227         qp->snd_wq = dev_buffers_p->mads_qp_snd_queue;
1228
1229         qp->snd_wqe_alloc_idx = 0;
1230         qp->max_snd_wqes = NUM_MADS_SND_WQES;
1231         qp->snd_wqe_cur_free = NUM_MADS_SND_WQES;
1232
1233         memset(&qp->snd_wq[0], 0, NUM_MADS_SND_WQES * sizeof(qp->snd_wq[i]));
1234         /* iterrate through the list */
1235         for (j = 0, i = 0, next_i = 1;
1236              j < NUM_MADS_RCV_WQES;
1237              MOD_INC(i, NUM_MADS_SND_WQES), MOD_INC(next_i, NUM_MADS_SND_WQES),
1238              ++j) {
1239
1240                 /* link the WQE to the next one */
1241                 bus_addr = virt_to_bus(&qp->snd_wq[next_i].wqe_cont.wqe);
1242                 INS_FLD(bus_addr >> 6, &qp->snd_wq[i].wqe_cont.wqe.next.next,
1243                         arbelprm_wqe_segment_next_st, nda_31_6);
1244
1245                 /* set the allocated buffers */
1246                 qp->snd_bufs[i] = ib_buffers.send_mad_buf[i];
1247                 bus_addr = virt_to_bus(qp->snd_bufs[i]);
1248                 qp->snd_wq[i].wqe_cont.wqe.mpointer[0].local_addr_l = bus_addr;
1249                 qp->snd_wq[i].wqe_cont.wqe.mpointer[0].lkey = dev_ib_data.mkey;
1250                 qp->snd_wq[i].wqe_cont.wqe.mpointer[0].byte_count =
1251                     qp->send_buf_sz;
1252
1253         }
1254
1255         cpu_to_be_buf(&qp->snd_wq[0],
1256                       NUM_MADS_SND_WQES * sizeof(qp->snd_wq[i]));
1257
1258         for (i = 0; i < qp->max_snd_wqes; ++i) {
1259                 qp->snd_wq[i].wqe_cont.qp = qp;
1260         }
1261
1262         /* qp number and cq numbers are already set up */
1263         qp->snd_cq.cq_buf = dev_buffers_p->mads_snd_cq_buf;
1264         qp->rcv_cq.cq_buf = dev_buffers_p->mads_rcv_cq_buf;
1265         qp->snd_cq.num_cqes = NUM_MADS_SND_CQES;
1266         qp->rcv_cq.num_cqes = NUM_MADS_RCV_CQES;
1267         qp->snd_cq.arm_db_ctx_idx = MADS_SND_CQ_ARM_DB_IDX;
1268         qp->snd_cq.ci_db_ctx_idx = MADS_SND_CQ_CI_DB_IDX;
1269         qp->rcv_cq.arm_db_ctx_idx = MADS_RCV_CQ_ARM_DB_IDX;
1270         qp->rcv_cq.ci_db_ctx_idx = MADS_RCV_CQ_CI_DB_IDX;
1271         qp->rcv_db_record_index = MADS_RCV_QP_DB_IDX;
1272         qp->snd_db_record_index = MADS_SND_QP_DB_IDX;
1273         qp->qkey = GLOBAL_QKEY;
1274         rc = create_udqp(qp);
1275         if (!rc) {
1276                 *qp_pp = qp;
1277                 *snd_cq_pp = &qp->snd_cq;
1278                 *rcv_cq_pp = &qp->rcv_cq;
1279         }
1280
1281         return rc;
1282 }
1283
1284 static int create_ipoib_qp(void **qp_pp,
1285                            void **snd_cq_pp, void **rcv_cq_pp, __u32 qkey)
1286 {
1287         __u8 i, next_i, j, k;
1288         int rc;
1289         struct udqp_st *qp;
1290         __u32 bus_addr;
1291         __u8 nds;
1292         void *ptr;
1293
1294         qp = &dev_ib_data.ipoib_qp;
1295
1296         /* set the pointer to the receive WQEs buffer */
1297         qp->rcv_wq = dev_buffers_p->ipoib_qp_rcv_queue;
1298
1299         qp->send_buf_sz = IPOIB_SND_BUF_SZ;
1300         qp->rcv_buf_sz = IPOIB_RCV_BUF_SZ;
1301
1302         qp->max_recv_wqes = NUM_IPOIB_RCV_WQES;
1303         qp->recv_wqe_cur_free = NUM_IPOIB_RCV_WQES;
1304
1305         qp->rcv_uar_context =
1306             dev_ib_data.uar_context_base + 8 * IPOIB_RCV_QP_DB_IDX;
1307         qp->send_uar_context =
1308             dev_ib_data.uar_context_base + 8 * IPOIB_SND_QP_DB_IDX;
1309
1310         memset(&qp->rcv_wq[0], 0, NUM_IPOIB_RCV_WQES * sizeof(qp->rcv_wq[0]));
1311         nds = sizeof(qp->rcv_wq[0].wqe) >> 4;
1312         /* iterrate through the list */
1313         for (j = 0, i = 0, next_i = 1;
1314              j < NUM_IPOIB_RCV_WQES;
1315              MOD_INC(i, NUM_IPOIB_RCV_WQES), MOD_INC(next_i,
1316                                                      NUM_IPOIB_RCV_WQES), ++j) {
1317
1318                 /* link the WQE to the next one */
1319                 bus_addr = virt_to_bus(&qp->rcv_wq[next_i].wqe);
1320                 ptr = qp->rcv_wq[i].wqe.control +
1321                     MT_BYTE_OFFSET(arbelprm_wqe_segment_ctrl_recv_st,
1322                                    wqe_segment_next);
1323                 INS_FLD(bus_addr >> 6, ptr, arbelprm_recv_wqe_segment_next_st,
1324                         nda_31_6);
1325                 INS_FLD(nds, ptr, arbelprm_recv_wqe_segment_next_st, nds);
1326
1327                 /* set the allocated buffers */
1328                 qp->rcv_bufs[i] = ib_buffers.ipoib_rcv_buf[i];
1329                 bus_addr = virt_to_bus(qp->rcv_bufs[i]);
1330                 qp->rcv_wq[i].wqe.mpointer[0].local_addr_l = bus_addr;
1331                 qp->rcv_wq[i].wqe.mpointer[0].byte_count = GRH_SIZE;
1332                 bus_addr = virt_to_bus(qp->rcv_bufs[i] + GRH_SIZE);
1333                 qp->rcv_wq[i].wqe.mpointer[1].local_addr_l = bus_addr;
1334                 qp->rcv_wq[i].wqe.mpointer[1].byte_count = IPOIB_RCV_BUF_SZ;
1335
1336                 for (k = 0; k < (((sizeof(qp->rcv_wq[i].wqe)) >> 4) - 1); ++k) {
1337                         qp->rcv_wq[i].wqe.mpointer[k].lkey = INVALID_WQE_LKEY;
1338                 }
1339         }
1340         cpu_to_be_buf(&qp->rcv_wq[0],
1341                       NUM_IPOIB_RCV_WQES * sizeof(qp->rcv_wq[0]));
1342
1343         for (i = 0; i < qp->max_recv_wqes; ++i) {
1344                 qp->rcv_wq[i].wqe_cont.qp = qp;
1345         }
1346
1347         /* set the pointer to the send WQEs buffer */
1348         qp->snd_wq = dev_buffers_p->ipoib_qp_snd_queue;
1349
1350         qp->snd_wqe_alloc_idx = 0;
1351         qp->max_snd_wqes = NUM_IPOIB_SND_WQES;
1352         qp->snd_wqe_cur_free = NUM_IPOIB_SND_WQES;
1353
1354         memset(&qp->snd_wq[0], 0, NUM_IPOIB_SND_WQES * sizeof(qp->snd_wq[i]));
1355         /* iterrate through the list */
1356         for (j = 0, i = 0, next_i = 1;
1357              j < NUM_IPOIB_RCV_WQES;
1358              MOD_INC(i, NUM_IPOIB_SND_WQES), MOD_INC(next_i,
1359                                                      NUM_IPOIB_SND_WQES), ++j) {
1360
1361                 /* link the WQE to the next one */
1362                 bus_addr = virt_to_bus(&qp->snd_wq[next_i].wqe_cont.wqe);
1363                 INS_FLD(bus_addr >> 6, &qp->snd_wq[i].wqe_cont.wqe.next.next,
1364                         arbelprm_wqe_segment_next_st, nda_31_6);
1365
1366                 /* set the allocated buffers */
1367                 qp->snd_bufs[i] = ib_buffers.send_ipoib_buf[i];
1368                 bus_addr = virt_to_bus(qp->snd_bufs[i]);
1369                 qp->snd_wq[i].wqe_cont.wqe.mpointer[0].local_addr_l = bus_addr;
1370                 qp->snd_wq[i].wqe_cont.wqe.mpointer[0].lkey = dev_ib_data.mkey;
1371
1372         }
1373         cpu_to_be_buf(&qp->snd_wq[0],
1374                       NUM_IPOIB_SND_WQES * sizeof(qp->snd_wq[i]));
1375
1376         for (i = 0; i < qp->max_snd_wqes; ++i) {
1377                 qp->snd_wq[i].wqe_cont.qp = qp;
1378         }
1379
1380         /* qp number and cq numbers are already set up */
1381         qp->snd_cq.cq_buf = dev_buffers_p->ipoib_snd_cq_buf;
1382         qp->rcv_cq.cq_buf = dev_buffers_p->ipoib_rcv_cq_buf;
1383         qp->snd_cq.num_cqes = NUM_IPOIB_SND_CQES;
1384         qp->rcv_cq.num_cqes = NUM_IPOIB_RCV_CQES;
1385         qp->snd_cq.arm_db_ctx_idx = IPOIB_SND_CQ_ARM_DB_IDX;
1386         qp->snd_cq.ci_db_ctx_idx = IPOIB_SND_CQ_CI_DB_IDX;
1387         qp->rcv_cq.arm_db_ctx_idx = IPOIB_RCV_CQ_ARM_DB_IDX;
1388         qp->rcv_cq.ci_db_ctx_idx = IPOIB_RCV_CQ_CI_DB_IDX;
1389         qp->rcv_db_record_index = IPOIB_RCV_QP_DB_IDX;
1390         qp->snd_db_record_index = IPOIB_SND_QP_DB_IDX;
1391         qp->qkey = qkey;
1392         rc = create_udqp(qp);
1393         if (!rc) {
1394                 *qp_pp = qp;
1395                 *snd_cq_pp = &qp->snd_cq;
1396                 *rcv_cq_pp = &qp->rcv_cq;
1397         }
1398
1399         return rc;
1400 }
1401
1402 static int create_udqp(struct udqp_st *qp)
1403 {
1404         int rc, ret = 0;
1405         void *inprm;
1406         struct recv_wqe_st *rcv_wqe;
1407
1408         inprm = dev_buffers_p->inprm_buf;
1409
1410         qp->rcv_cq.arm_db_ctx_pointer =
1411             dev_ib_data.uar_context_base + 8 * qp->rcv_cq.arm_db_ctx_idx;
1412         qp->rcv_cq.ci_db_ctx_pointer =
1413             dev_ib_data.uar_context_base + 8 * qp->rcv_cq.ci_db_ctx_idx;
1414         qp->snd_cq.arm_db_ctx_pointer =
1415             dev_ib_data.uar_context_base + 8 * qp->snd_cq.arm_db_ctx_idx;
1416         qp->snd_cq.ci_db_ctx_pointer =
1417             dev_ib_data.uar_context_base + 8 * qp->snd_cq.ci_db_ctx_idx;
1418
1419         /* create send CQ */
1420         init_cq_buf(qp->snd_cq.cq_buf, qp->snd_cq.num_cqes);
1421         qp->snd_cq.cons_counter = 0;
1422         prep_sw2hw_cq_buf(inprm,
1423                           dev_ib_data.eq.eqn,
1424                           qp->snd_cq.cqn,
1425                           qp->snd_cq.cq_buf,
1426                           qp->snd_cq.ci_db_ctx_idx, qp->snd_cq.arm_db_ctx_idx);
1427
1428         rc = cmd_sw2hw_cq(qp->snd_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ);
1429         if (rc) {
1430                 ret = -1;
1431                 eprintf("");
1432                 goto exit;
1433         }
1434
1435         /* create receive CQ */
1436         init_cq_buf(qp->rcv_cq.cq_buf, qp->rcv_cq.num_cqes);
1437         qp->rcv_cq.cons_counter = 0;
1438         memset(inprm, 0, SW2HW_CQ_IBUF_SZ);
1439         prep_sw2hw_cq_buf(inprm,
1440                           dev_ib_data.eq.eqn,
1441                           qp->rcv_cq.cqn,
1442                           qp->rcv_cq.cq_buf,
1443                           qp->rcv_cq.ci_db_ctx_idx, qp->rcv_cq.arm_db_ctx_idx);
1444
1445         rc = cmd_sw2hw_cq(qp->rcv_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ);
1446         if (rc) {
1447                 ret = -1;
1448                 eprintf("");
1449                 goto undo_snd_cq;
1450         }
1451
1452         prep_rst2init_qpee_buf(inprm,
1453                                qp->snd_cq.cqn,
1454                                qp->rcv_cq.cqn,
1455                                qp->qkey,
1456                                my_log2(qp->max_recv_wqes),
1457                                my_log2(sizeof(qp->rcv_wq[0])) - 4,
1458                                my_log2(qp->max_snd_wqes),
1459                                my_log2(sizeof(qp->snd_wq[0])) - 4,
1460                                virt_to_bus(qp->snd_wq),
1461                                qp->snd_db_record_index,
1462                                virt_to_bus(qp->rcv_wq),
1463                                qp->rcv_db_record_index);
1464
1465         rc = cmd_rst2init_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ);
1466         if (rc) {
1467                 ret = -1;
1468                 eprintf("");
1469                 goto undo_rcv_cq;
1470         }
1471
1472         qp->last_posted_rcv_wqe = NULL;
1473         qp->last_posted_snd_wqe = NULL;
1474
1475         /* post all the buffers to the receive queue */
1476         while (1) {
1477                 /* allocate wqe */
1478                 rcv_wqe = alloc_rcv_wqe(qp);
1479                 if (!rcv_wqe)
1480                         break;
1481
1482                 /* post the buffer */
1483                 rc = post_rcv_buf(qp, rcv_wqe);
1484                 if (rc) {
1485                         ret = -1;
1486                         eprintf("");
1487                         goto undo_rcv_cq;
1488                 }
1489         }
1490
1491         prep_init2rtr_qpee_buf(inprm);
1492         rc = cmd_init2rtr_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ);
1493         if (rc) {
1494                 ret = -1;
1495                 eprintf("");
1496                 goto undo_rcv_cq;
1497         }
1498
1499         memset(inprm, 0, QPCTX_IBUF_SZ);
1500         rc = cmd_rtr2rts_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ);
1501         if (rc) {
1502                 ret = -1;
1503                 eprintf("");
1504                 goto undo_rcv_cq;
1505         }
1506
1507         goto exit;
1508
1509       undo_rcv_cq:
1510         rc = cmd_hw2sw_cq(qp->rcv_cq.cqn);
1511         if (rc)
1512                 eprintf("");
1513
1514       undo_snd_cq:
1515         rc = cmd_hw2sw_cq(qp->snd_cq.cqn);
1516         if (rc)
1517                 eprintf("");
1518
1519       exit:
1520         return ret;
1521 }
1522
1523 static int destroy_udqp(struct udqp_st *qp)
1524 {
1525         int rc;
1526
1527         rc = cmd_2err_qpee(qp->qpn);
1528         if (rc) {
1529                 eprintf("");
1530                 return rc;
1531         }
1532         tprintf("cmd_2err_qpee(0x%lx) success", qp->qpn);
1533
1534         rc = cmd_2rst_qpee(qp->qpn);
1535         if (rc) {
1536                 eprintf("");
1537                 return rc;
1538         }
1539         tprintf("cmd_2rst_qpee(0x%lx) success", qp->qpn);
1540
1541         rc = cmd_hw2sw_cq(qp->rcv_cq.cqn);
1542         if (rc) {
1543                 eprintf("");
1544                 return rc;
1545         }
1546         tprintf("cmd_hw2sw_cq(0x%lx) success", qp->snd_cq.cqn);
1547
1548         rc = cmd_hw2sw_cq(qp->snd_cq.cqn);
1549         if (rc) {
1550                 eprintf("");
1551                 return rc;
1552         }
1553         tprintf("cmd_hw2sw_cq(0x%lx) success", qp->rcv_cq.cqn);
1554
1555         return rc;
1556 }
1557
1558 static void prep_send_wqe_buf(void *qph,
1559                               void *avh,
1560                               void *wqeh,
1561                               const void *buf,
1562                               unsigned int offset, __u16 len, __u8 e)
1563 {
1564         struct ud_send_wqe_st *snd_wqe = wqeh;
1565         struct ud_av_st *av = avh;
1566
1567         if (qph) {
1568         }
1569         /* suppress warnings */
1570         INS_FLD_TO_BE(e, &snd_wqe->next.control,
1571                       arbelprm_wqe_segment_ctrl_send_st, e);
1572         INS_FLD_TO_BE(1, &snd_wqe->next.control,
1573                       arbelprm_wqe_segment_ctrl_send_st, always1);
1574         INS_FLD_TO_BE(1, &snd_wqe->next.next, arbelprm_wqe_segment_next_st,
1575                       always1);
1576         memcpy(&snd_wqe->udseg, &av->av, sizeof av->av);
1577         INS_FLD_TO_BE(av->dest_qp, snd_wqe->udseg.av,
1578                       arbelprm_wqe_segment_ud_st, destination_qp);
1579         INS_FLD_TO_BE(av->qkey, snd_wqe->udseg.av, arbelprm_wqe_segment_ud_st,
1580                       q_key);
1581
1582         if (buf) {
1583                 memcpy(bus_to_virt
1584                        (be32_to_cpu(snd_wqe->mpointer[0].local_addr_l)) +
1585                        offset, buf, len);
1586                 len += offset;
1587         }
1588         snd_wqe->mpointer[0].byte_count = cpu_to_be32(len);
1589
1590         DBG ( "prep_send_wqe_buf()\n" );
1591         DBG ( "snd_wqe:\n" );
1592         DBG_HD ( snd_wqe, sizeof ( *snd_wqe ) );
1593         DBG ( "packet:\n" );
1594         DBG_HD ( bus_to_virt(be32_to_cpu(snd_wqe->mpointer[0].local_addr_l)),
1595                  len );
1596 }
1597
1598 static void *alloc_ud_av(void)
1599 {
1600         u8 next_free;
1601
1602         if (dev_ib_data.udav.udav_next_free == FL_EOL) {
1603                 return NULL;
1604         }
1605
1606         next_free = dev_ib_data.udav.udav_next_free;
1607         dev_ib_data.udav.udav_next_free =
1608             dev_buffers_p->av_array[next_free].ud_av.next_free;
1609         tprintf("allocated udav %d", next_free);
1610         return &dev_buffers_p->av_array[next_free].ud_av;
1611 }
1612
1613 static void free_ud_av(void *avh)
1614 {
1615         union ud_av_u *avu;
1616         __u8 idx, old_idx;
1617         struct ud_av_st *av = avh;
1618
1619         avu = (union ud_av_u *)av;
1620
1621         idx = avu - dev_buffers_p->av_array;
1622         tprintf("freeing udav idx=%d", idx);
1623         old_idx = dev_ib_data.udav.udav_next_free;
1624         dev_ib_data.udav.udav_next_free = idx;
1625         avu->ud_av.next_free = old_idx;
1626 }
1627
1628 static int update_cq_cons_idx(struct cq_st *cq)
1629 {
1630         /* write doorbell record */
1631         WRITE_DWORD_VOL(cq->ci_db_ctx_pointer, 0, htonl(cq->cons_counter));
1632
1633         /*
1634            INS_FLD_TO_BE(cq->cons_counter,
1635            cq->ci_db_ctx_pointer,
1636            arbelprm_cq_arm_db_record_st,
1637            counter);
1638
1639            INS_FLD_TO_BE(cq->cqn,
1640            cq->ci_db_ctx_pointer,
1641            arbelprm_cq_arm_db_record_st,
1642            cq_number);
1643
1644            INS_FLD_TO_BE(1,
1645            cq->ci_db_ctx_pointer,
1646            arbelprm_cq_arm_db_record_st,
1647            res); */
1648
1649         return 0;
1650 }
1651
1652 static int poll_cq(void *cqh, union cqe_st *cqe_p, u8 * num_cqes)
1653 {
1654         union cqe_st cqe;
1655         int rc;
1656         u32 *ptr;
1657         struct cq_st *cq = cqh;
1658         __u32 cons_idx = cq->cons_counter & (cq->num_cqes - 1);
1659
1660         ptr = (u32 *) (&(cq->cq_buf[cons_idx]));
1661         barrier();
1662         if ((ptr[7] & 0x80000000) == 0) {
1663                 cqe = cq->cq_buf[cons_idx];
1664                 be_to_cpu_buf(&cqe, sizeof(cqe));
1665                 *cqe_p = cqe;
1666                 ptr[7] = 0x80000000;
1667                 barrier();
1668                 cq->cons_counter++;
1669                 rc = update_cq_cons_idx(cq);
1670                 if (rc) {
1671                         return rc;
1672                 }
1673                 *num_cqes = 1;
1674         } else
1675                 *num_cqes = 0;
1676
1677         return 0;
1678 }
1679
1680 static void dev2ib_cqe(struct ib_cqe_st *ib_cqe_p, union cqe_st *cqe_p)
1681 {
1682         __u8 opcode;
1683         __u32 wqe_addr_ba;
1684
1685         opcode =
1686             EX_FLD(cqe_p->good_cqe, arbelprm_completion_queue_entry_st, opcode);
1687         if (opcode >= CQE_ERROR_OPCODE)
1688                 ib_cqe_p->is_error = 1;
1689         else
1690                 ib_cqe_p->is_error = 0;
1691
1692         ib_cqe_p->is_send =
1693             EX_FLD(cqe_p->good_cqe, arbelprm_completion_queue_entry_st, s);
1694         wqe_addr_ba =
1695             EX_FLD(cqe_p->good_cqe, arbelprm_completion_queue_entry_st,
1696                    wqe_adr) << 6;
1697         ib_cqe_p->wqe = bus_to_virt(wqe_addr_ba);
1698
1699         ib_cqe_p->count =
1700             EX_FLD(cqe_p->good_cqe, arbelprm_completion_queue_entry_st,
1701                    byte_cnt);
1702 }
1703
1704 static int ib_poll_cq(void *cqh, struct ib_cqe_st *ib_cqe_p, u8 * num_cqes)
1705 {
1706         int rc;
1707         union cqe_st cqe;
1708         struct cq_st *cq = cqh;
1709         __u8 opcode;
1710
1711         rc = poll_cq(cq, &cqe, num_cqes);
1712         if (rc || ((*num_cqes) == 0)) {
1713                 return rc;
1714         }
1715
1716         dev2ib_cqe(ib_cqe_p, &cqe);
1717
1718         opcode =
1719             EX_FLD(cqe.good_cqe, arbelprm_completion_queue_entry_st, opcode);
1720         if (opcode >= CQE_ERROR_OPCODE) {
1721                 struct ud_send_wqe_st *wqe_p, wqe;
1722                 __u32 *ptr;
1723                 unsigned int i;
1724
1725                 wqe_p =
1726                     bus_to_virt(EX_FLD
1727                                 (cqe.error_cqe,
1728                                  arbelprm_completion_with_error_st,
1729                                  wqe_addr) << 6);
1730                 eprintf("syndrome=0x%lx",
1731                         EX_FLD(cqe.error_cqe, arbelprm_completion_with_error_st,
1732                                syndrome));
1733                 eprintf("vendor_syndrome=0x%lx",
1734                         EX_FLD(cqe.error_cqe, arbelprm_completion_with_error_st,
1735                                vendor_code));
1736                 eprintf("wqe_addr=%p", wqe_p);
1737                 eprintf("myqpn=0x%lx",
1738                         EX_FLD(cqe.error_cqe, arbelprm_completion_with_error_st,
1739                                myqpn));
1740                 memcpy(&wqe, wqe_p, sizeof wqe);
1741                 be_to_cpu_buf(&wqe, sizeof wqe);
1742
1743                 eprintf("dumping wqe...");
1744                 ptr = (__u32 *) (&wqe);
1745                 for (i = 0; i < sizeof wqe; i += 4) {
1746                         printf("%lx : ", ptr[i >> 2]);
1747                 }
1748
1749         }
1750
1751         return rc;
1752 }
1753
1754 /* always work on ipoib qp */
1755 static int add_qp_to_mcast_group(union ib_gid_u mcast_gid, __u8 add)
1756 {
1757         void *mg;
1758         __u8 *tmp;
1759         int rc;
1760         __u16 mgid_hash;
1761         void *mgmqp_p;
1762
1763         tmp = dev_buffers_p->inprm_buf;
1764         memcpy(tmp, mcast_gid.raw, 16);
1765         be_to_cpu_buf(tmp, 16);
1766         rc = cmd_mgid_hash(tmp, &mgid_hash);
1767         if (!rc) {
1768                 mg = (void *)dev_buffers_p->inprm_buf;
1769                 memset(mg, 0, MT_STRUCT_SIZE(arbelprm_mgm_entry_st));
1770                 INS_FLD(mcast_gid.as_u32.dw[0], mg, arbelprm_mgm_entry_st,
1771                         mgid_128_96);
1772                 INS_FLD(mcast_gid.as_u32.dw[1], mg, arbelprm_mgm_entry_st,
1773                         mgid_95_64);
1774                 INS_FLD(mcast_gid.as_u32.dw[2], mg, arbelprm_mgm_entry_st,
1775                         mgid_63_32);
1776                 INS_FLD(mcast_gid.as_u32.dw[3], mg, arbelprm_mgm_entry_st,
1777                         mgid_31_0);
1778                 be_to_cpu_buf(mg +
1779                               MT_BYTE_OFFSET(arbelprm_mgm_entry_st,
1780                                              mgid_128_96), 16);
1781                 mgmqp_p = mg + MT_BYTE_OFFSET(arbelprm_mgm_entry_st, mgmqp_0);
1782                 INS_FLD(dev_ib_data.ipoib_qp.qpn, mgmqp_p, arbelprm_mgmqp_st,
1783                         qpn_i);
1784                 INS_FLD(add, mgmqp_p, arbelprm_mgmqp_st, qi);
1785                 rc = cmd_write_mgm(mg, mgid_hash);
1786         }
1787         return rc;
1788 }
1789
1790 static int clear_interrupt(void)
1791 {
1792         writel(dev_ib_data.clr_int_data, dev_ib_data.clr_int_addr);
1793         return 0;
1794 }
1795
1796 static struct ud_send_wqe_st *alloc_send_wqe(udqp_t qph)
1797 {
1798         struct udqp_st *qp = qph;
1799         __u32 idx;
1800
1801         if (qp->snd_wqe_cur_free) {
1802                 qp->snd_wqe_cur_free--;
1803                 idx = qp->snd_wqe_alloc_idx;
1804                 qp->snd_wqe_alloc_idx =
1805                     (qp->snd_wqe_alloc_idx + 1) & (qp->max_snd_wqes - 1);
1806                 return &qp->snd_wq[idx].wqe_cont.wqe;
1807         }
1808
1809         return NULL;
1810 }
1811
1812 static struct recv_wqe_st *alloc_rcv_wqe(struct udqp_st *qp)
1813 {
1814         __u32 idx;
1815
1816         if (qp->recv_wqe_cur_free) {
1817                 qp->recv_wqe_cur_free--;
1818                 idx = qp->recv_wqe_alloc_idx;
1819                 qp->recv_wqe_alloc_idx =
1820                     (qp->recv_wqe_alloc_idx + 1) & (qp->max_recv_wqes - 1);
1821                 return &qp->rcv_wq[idx].wqe_cont.wqe;
1822         }
1823
1824         return NULL;
1825 }
1826
1827 static int free_send_wqe(struct ud_send_wqe_st *wqe)
1828 {
1829         struct udqp_st *qp = ((struct ude_send_wqe_cont_st *)wqe)->qp;
1830         qp->snd_wqe_cur_free++;
1831
1832         return 0;
1833 }
1834
1835 static int free_rcv_wqe(struct recv_wqe_st *wqe)
1836 {
1837         struct udqp_st *qp = ((struct recv_wqe_cont_st *)wqe)->qp;
1838         qp->recv_wqe_cur_free++;
1839
1840         return 0;
1841 }
1842
1843 static int free_wqe(void *wqe)
1844 {
1845         int rc = 0;
1846         struct recv_wqe_st *rcv_wqe;
1847
1848 //      tprintf("free wqe= 0x%x", wqe);
1849         if ((wqe >= (void *)(dev_ib_data.ipoib_qp.rcv_wq)) &&
1850             (wqe <
1851              (void *)(&dev_ib_data.ipoib_qp.rcv_wq[NUM_IPOIB_RCV_WQES]))) {
1852                 /* ipoib receive wqe */
1853                 free_rcv_wqe(wqe);
1854                 rcv_wqe = alloc_rcv_wqe(&dev_ib_data.ipoib_qp);
1855                 if (rcv_wqe) {
1856                         rc = post_rcv_buf(&dev_ib_data.ipoib_qp, rcv_wqe);
1857                         if (rc) {
1858                                 eprintf("");
1859                         }
1860                 }
1861         } else if (wqe >= (void *)(dev_ib_data.ipoib_qp.snd_wq) &&
1862                    wqe <
1863                    (void *)(&dev_ib_data.ipoib_qp.snd_wq[NUM_IPOIB_SND_WQES])) {
1864                 /* ipoib send wqe */
1865                 free_send_wqe(wqe);
1866         } else if (wqe >= (void *)(dev_ib_data.mads_qp.rcv_wq) &&
1867                    wqe <
1868                    (void *)(&dev_ib_data.mads_qp.rcv_wq[NUM_MADS_RCV_WQES])) {
1869                 /* mads receive wqe */
1870                 free_rcv_wqe(wqe);
1871                 rcv_wqe = alloc_rcv_wqe(&dev_ib_data.mads_qp);
1872                 if (rcv_wqe) {
1873                         rc = post_rcv_buf(&dev_ib_data.mads_qp, rcv_wqe);
1874                         if (rc) {
1875                                 eprintf("");
1876                         }
1877                 }
1878         } else if (wqe >= (void *)(dev_ib_data.mads_qp.snd_wq) &&
1879                    wqe <
1880                    (void *)(&dev_ib_data.mads_qp.snd_wq[NUM_MADS_SND_WQES])) {
1881                 /* mads send wqe */
1882                 free_send_wqe(wqe);
1883         } else {
1884                 rc = -1;
1885                 eprintf("");
1886         }
1887
1888         return rc;
1889 }
1890
1891 static int update_eq_cons_idx(struct eq_st *eq)
1892 {
1893         writel(eq->cons_counter, eq->ci_base_base_addr);
1894         return 0;
1895 }
1896
1897 static void dev2ib_eqe(struct ib_eqe_st *ib_eqe_p, struct eqe_t *eqe_p)
1898 {
1899         void *tmp;
1900
1901         ib_eqe_p->event_type =
1902             EX_FLD(eqe_p, arbelprm_event_queue_entry_st, event_type);
1903
1904         tmp = eqe_p + MT_BYTE_OFFSET(arbelprm_event_queue_entry_st, event_data);
1905         ib_eqe_p->cqn = EX_FLD(tmp, arbelprm_completion_event_st, cqn);
1906 }
1907
1908 static int poll_eq(struct ib_eqe_st *ib_eqe_p, __u8 * num_eqes)
1909 {
1910         struct eqe_t eqe;
1911         u8 owner;
1912         int rc;
1913         u32 *ptr;
1914         struct eq_st *eq = &dev_ib_data.eq;
1915         __u32 cons_idx = eq->cons_counter & (eq->eq_size - 1);
1916
1917         ptr = (u32 *) (&(eq->eq_buf[cons_idx]));
1918         owner = (ptr[7] & 0x80000000) ? OWNER_HW : OWNER_SW;
1919         if (owner == OWNER_SW) {
1920                 eqe = eq->eq_buf[cons_idx];
1921                 be_to_cpu_buf(&eqe, sizeof(eqe));
1922                 dev2ib_eqe(ib_eqe_p, &eqe);
1923                 ptr[7] |= 0x80000000;
1924                 eq->eq_buf[cons_idx] = eqe;
1925                 eq->cons_counter++;
1926                 rc = update_eq_cons_idx(eq);
1927                 if (rc) {
1928                         return -1;
1929                 }
1930                 *num_eqes = 1;
1931         } else {
1932                 *num_eqes = 0;
1933         }
1934         return 0;
1935 }
1936
1937 static int ib_device_close(void)
1938 {
1939         iounmap(memfree_pci_dev.uar);
1940         iounmap(memfree_pci_dev.cr_space);
1941         return 0;
1942 }
1943
1944 static __u32 dev_get_qpn(void *qph)
1945 {
1946         struct udqp_st *qp = qph;
1947
1948         return qp->qpn;
1949 }
1950
1951 static void dev_post_dbell(void *dbell, __u32 offset)
1952 {
1953         __u32 *ptr;
1954         unsigned long address;
1955
1956         ptr = dbell;
1957
1958         if (((ptr[0] >> 24) & 0xff) != 1) {
1959                 eprintf("");
1960         }
1961         tprintf("ptr[0]= 0x%lx", ptr[0]);
1962         tprintf("ptr[1]= 0x%lx", ptr[1]);
1963         address = (unsigned long)(memfree_pci_dev.uar) + offset;
1964         tprintf("va=0x%lx pa=0x%lx", address,
1965                 virt_to_bus((const void *)address));
1966         writel(htonl(ptr[0]), memfree_pci_dev.uar + offset);
1967         barrier();
1968         address += 4;
1969         tprintf("va=0x%lx pa=0x%lx", address,
1970                 virt_to_bus((const void *)address));
1971         writel(htonl(ptr[1]), address /*memfree_pci_dev.uar + offset + 4 */ );
1972 }