cm_req.init_depth = (uint8_t)ep_ptr->param.ep_attr.max_rdma_read_out;\r
\r
/* time wait before retrying a pkt after receiving a RNR NAK */\r
- cm_req.rnr_nak_timeout = 12; /* 163.84ms */\r
+ cm_req.rnr_nak_timeout = IB_RNR_NAK_TIMEOUT; /* dapl_ibal_util.h */\r
\r
/* \r
* number of time local QP should retry after receiving RNR NACK before\r
* reporting an error\r
*/\r
- cm_req.rnr_retry_cnt = 6; /* 7 is infinite */\r
+ cm_req.rnr_retry_cnt = IB_RNR_RETRY_CNT;\r
\r
cm_req.remote_resp_timeout = 16; /* 250ms */\r
cm_req.local_resp_timeout = 16; /* 250ms */\r
IN DAPL_EP *ep_ptr,\r
IN DAT_CLOSE_FLAGS disconnect_flags)\r
{\r
- DAPL_IA *ia_ptr;\r
- ib_api_status_t ib_status;\r
- ib_cm_dreq_t cm_dreq;\r
+ ib_api_status_t ib_status = IB_SUCCESS;\r
+ ib_cm_dreq_t cm_dreq;\r
\r
- UNUSED_PARAM( disconnect_flags );\r
-\r
- ia_ptr = ep_ptr->header.owner_ia;\r
- ib_status = IB_SUCCESS;\r
+ UNUSED_PARAM( disconnect_flags );\r
\r
dapl_os_memzero(&cm_dreq, sizeof(ib_cm_dreq_t));\r
\r
ep_ptr->sent_discreq = DAT_TRUE;\r
ib_status = ib_cm_dreq ( &cm_dreq );\r
\r
- dapl_dbg_log (DAPL_DBG_TYPE_CM,"--> DsD: EP=%p QP= %p DREQ SENT status =%s\n", \r
- ep_ptr, ep_ptr->qp_handle,ib_get_err_str(ib_status));\r
+ /* tolerate INVALID_STATE error as the other side can race ahead and\r
+ * generate a DREQ before we do.\r
+ */ \r
+ if ( ib_status == IB_INVALID_STATE )\r
+ ib_status = IB_SUCCESS;\r
+\r
+ if (ib_status )\r
+ {\r
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,\r
+ "--> DsD: EP=%p QP= %p DREQ SEND status %s\n", \r
+ ep_ptr, ep_ptr->qp_handle,ib_get_err_str(ib_status));\r
+ }\r
}\r
return dapl_ib_status_convert (ib_status);\r
}\r
cm_rep.flags = 0;\r
cm_rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED;\r
cm_rep.target_ack_delay = 14;\r
- cm_rep.rnr_nak_timeout = 12;\r
- cm_rep.rnr_retry_cnt = 6;\r
+ /* see dapl_ibal_util.h for IB_RNR_XXX */;\r
+ cm_rep.rnr_nak_timeout = IB_RNR_NAK_TIMEOUT;\r
+ cm_rep.rnr_retry_cnt = IB_RNR_RETRY_CNT;\r
cm_rep.p_recv_wr = NULL;\r
cm_rep.pp_recv_failure = NULL;\r
\r
*/\r
STATIC _INLINE_ DAT_RETURN \r
dapls_ib_post_recv (\r
- IN DAPL_EP *ep_ptr,\r
+ IN DAPL_EP *ep_ptr,\r
IN DAPL_COOKIE *cookie,\r
IN DAT_COUNT num_segments,\r
IN DAT_LMR_TRIPLET *local_iov)\r
{\r
ib_api_status_t ib_status;\r
- ib_recv_wr_t recv_wr, *failed_wr_p;\r
+ ib_recv_wr_t recv_wr, *failed_wr_p;\r
ib_local_ds_t ds_array[DAPL_DEFAULT_DS_ENTRIES], *ds_array_p;\r
DAT_COUNT i, total_len;\r
\r
{\r
ds_array_p = dapl_os_alloc( num_segments * sizeof(ib_local_ds_t) );\r
}\r
- recv_wr.ds_array = ds_array_p;\r
+ recv_wr.ds_array = ds_array_p;\r
\r
if (NULL == ds_array_p)\r
{\r
return (DAT_INSUFFICIENT_RESOURCES);\r
}\r
\r
- total_len = 0;\r
+ total_len = 0;\r
\r
for (i = 0; i < num_segments; i++, ds_array_p++)\r
{\r
- ds_array_p->length = (uint32_t)local_iov[i].segment_length;\r
- ds_array_p->lkey = local_iov[i].lmr_context;\r
- ds_array_p->vaddr = local_iov[i].virtual_address;\r
- total_len += ds_array_p->length;\r
+ ds_array_p->length = (uint32_t)local_iov[i].segment_length;\r
+ ds_array_p->lkey = local_iov[i].lmr_context;\r
+ ds_array_p->vaddr = local_iov[i].virtual_address;\r
+ total_len += ds_array_p->length;\r
}\r
\r
if (cookie != NULL)\r
{\r
- cookie->val.dto.size = total_len;\r
-\r
- dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: EP = %p QP = %p cookie= %p, num_seg= %d\n", \r
- ep_ptr, ep_ptr->qp_handle, cookie, num_segments);\r
+ cookie->val.dto.size = total_len;\r
+ dapl_dbg_log (DAPL_DBG_TYPE_EP,\r
+ "--> DsPR: EP = %p QP = %p cookie= %p, num_seg= %d\n", \r
+ ep_ptr, ep_ptr->qp_handle, cookie, num_segments);\r
}\r
-\r
recv_wr.p_next = NULL;\r
\r
ib_status = ib_post_recv( ep_ptr->qp_handle, &recv_wr, &failed_wr_p );\r
\r
if (IB_SUCCESS == ib_status)\r
{\r
- return DAT_SUCCESS;\r
+ return DAT_SUCCESS;\r
}\r
- else\r
- {\r
- dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: post_recv status = %s\n", \r
- ib_get_err_str(ib_status));\r
\r
- /*\r
- * Moving QP to error state; \r
- */\r
- ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);\r
- ep_ptr->qp_state = IB_QPS_ERROR;\r
+ dapl_dbg_log (DAPL_DBG_TYPE_DTO_COMP_ERR,\r
+ "--> DsPR: post_recv status %s\n",\r
+ ib_get_err_str(ib_status));\r
+ /*\r
+ * Moving QP to error state; \r
+ */\r
+ (void) dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);\r
+ ep_ptr->qp_state = IB_QPS_ERROR;\r
\r
return (dapl_ib_status_convert (ib_status));\r
- }\r
}\r
\r
\r
*/\r
STATIC _INLINE_ DAT_RETURN \r
dapls_ib_post_send (\r
- IN DAPL_EP *ep_ptr,\r
- IN ib_send_op_type_t op_type,\r
- IN DAPL_COOKIE *cookie,\r
- IN DAT_COUNT num_segments,\r
- IN DAT_LMR_TRIPLET *local_iov,\r
- IN const DAT_RMR_TRIPLET *remote_iov,\r
- IN DAT_COMPLETION_FLAGS completion_flags)\r
+ IN DAPL_EP *ep_ptr,\r
+ IN ib_send_op_type_t op_type,\r
+ IN DAPL_COOKIE *cookie,\r
+ IN DAT_COUNT num_segments,\r
+ IN DAT_LMR_TRIPLET *local_iov,\r
+ IN const DAT_RMR_TRIPLET *remote_iov,\r
+ IN DAT_COMPLETION_FLAGS completion_flags )\r
{\r
- ib_api_status_t ib_status;\r
+ ib_api_status_t ib_status;\r
ib_send_wr_t send_wr, *failed_wr_p;\r
ib_local_ds_t ds_array[DAPL_DEFAULT_DS_ENTRIES], *ds_array_p;\r
DAT_COUNT i, total_len;\r
if (ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED)\r
{\r
ib_qp_attr_t qp_attr;\r
+\r
ib_query_qp ( ep_ptr->qp_handle, &qp_attr );\r
\r
- dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsPS: !CONN EP (%p) ep_state=%d QP_state=%d\n", \r
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,\r
+ "--> DsPS: !CONN EP (%p) ep_state=%d QP_state=%d\n", \r
ep_ptr, ep_ptr->param.ep_state, qp_attr.state );\r
\r
- return ( DAT_ERROR(DAT_INVALID_STATE, DAT_INVALID_STATE_EP_DISCONNECTED ) );\r
+ return(DAT_ERROR(DAT_INVALID_STATE, DAT_INVALID_STATE_EP_DISCONNECTED));\r
}\r
dapl_os_memzero (&send_wr, sizeof(ib_send_wr_t));\r
+\r
send_wr.wr_type = op_type;\r
send_wr.num_ds = num_segments;\r
\r
{\r
ds_array_p = dapl_os_alloc( num_segments * sizeof(ib_local_ds_t) );\r
}\r
- send_wr.ds_array = ds_array_p;\r
+ send_wr.ds_array = ds_array_p;\r
\r
if (NULL == ds_array_p)\r
{\r
- return (DAT_INSUFFICIENT_RESOURCES);\r
+ return (DAT_INSUFFICIENT_RESOURCES);\r
}\r
\r
- total_len = 0;\r
+ total_len = 0;\r
\r
for (i = 0; i < num_segments; i++, ds_array_p++)\r
{\r
- ds_array_p->length = (uint32_t)local_iov[i].segment_length;\r
- ds_array_p->lkey = local_iov[i].lmr_context;\r
- ds_array_p->vaddr = local_iov[i].virtual_address;\r
- total_len += ds_array_p->length;\r
+ ds_array_p->length = (uint32_t)local_iov[i].segment_length;\r
+ ds_array_p->lkey = local_iov[i].lmr_context;\r
+ ds_array_p->vaddr = local_iov[i].virtual_address;\r
+ total_len += ds_array_p->length;\r
}\r
\r
+ /* NULL cookie is OK if no Completion event is requested */\r
if (cookie != NULL)\r
{\r
- cookie->val.dto.size = total_len;\r
+ dapl_os_assert( cookie->ep == ep_ptr );\r
+ cookie->val.dto.size = total_len;\r
\r
- dapl_dbg_log (DAPL_DBG_TYPE_EP,\r
- "--> DsPS: EVD=%p EP=%p QP=%p type=%d, sg=%d ln=%d, ck=%p 0x" F64x "\n", \r
+ dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPS: EVD=%p EP=%p QP=%p type=%d "\r
+ "sg=%d ln=%d ck=%p 0x" F64x "\n", \r
ep_ptr->param.request_evd_handle, ep_ptr, ep_ptr->qp_handle, \r
op_type, num_segments, total_len,\r
cookie, cookie->val.dto.cookie.as_64 );\r
}\r
-\r
- send_wr.wr_id = (DAT_UINT64)cookie;\r
+ send_wr.wr_id = (DAT_UINT64)cookie;\r
\r
/* RC for now */\r
if (total_len > 0)\r
send_wr.remote_ops.rkey = remote_iov->rmr_context;\r
}\r
\r
- send_wr.send_opt = 0;\r
+ send_wr.send_opt = 0;\r
\r
- send_wr.send_opt |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & \r
- completion_flags) ? IB_SEND_OPT_FENCE : 0;\r
- send_wr.send_opt |= (DAT_COMPLETION_SUPPRESS_FLAG & \r
- completion_flags) ? 0 : IB_SEND_OPT_SIGNALED;\r
- send_wr.send_opt |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & \r
- completion_flags) ? IB_SEND_OPT_SOLICITED : 0;\r
+ send_wr.send_opt |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & \r
+ completion_flags) ? IB_SEND_OPT_FENCE : 0;\r
+ send_wr.send_opt |= (DAT_COMPLETION_SUPPRESS_FLAG & \r
+ completion_flags) ? 0 : IB_SEND_OPT_SIGNALED;\r
+ send_wr.send_opt |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & \r
+ completion_flags) ? IB_SEND_OPT_SOLICITED : 0;\r
\r
dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPS: EP=%p QP=%p send_opt=0x%x,"\r
"rem_addr=%p, rem_rkey=0x%x completion_flags=0x%x\n",\r
\r
if (IB_SUCCESS == ib_status)\r
{\r
- return DAT_SUCCESS;\r
+ return DAT_SUCCESS;\r
}\r
- else\r
- {\r
- dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPS: EP=%p post_send status = %s\n", \r
- ep_ptr, ib_get_err_str(ib_status));\r
\r
- /*\r
- * Moving QP to error state; \r
- */\r
- ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);\r
- ep_ptr->qp_state = IB_QPS_ERROR;\r
+ dapl_dbg_log (DAPL_DBG_TYPE_DTO_COMP_ERR,\r
+ "--> DsPS: EP=%p post_send status %s\n", \r
+ ep_ptr, ib_get_err_str(ib_status));\r
+ /*\r
+ * Moving QP to error state; \r
+ */\r
+ (void) dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);\r
+ ep_ptr->qp_state = IB_QPS_ERROR;\r
\r
return (dapl_ib_status_convert (ib_status));\r
- }\r
}\r
\r
/*\r