[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.2b3-247-ge72d9b2
Service Account
noreply at mpich.org
Wed Jul 15 22:23:12 CDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via e72d9b2cc827de86ae50c797f65c91eaf2b206cc (commit)
via 473f4433ab72351dd643b004bb211eec835a7e84 (commit)
via 995420f0b93af5bfe8e955fbd606092adc44dae8 (commit)
via 7b522671dfe2aa84a6797cbdc4434c6dee8b117e (commit)
via 65c58bad76aebaa9b9b2b51a9c13b922a0f35bb0 (commit)
via 8474718d84de3c81d4f56d49b25c2b13300cdc7b (commit)
via dfb295b2a13d7aa85b99db975dbffade8c18915c (commit)
via e33f33e387b938ded92e436178b33dc459dc0db3 (commit)
via 658ad69657546b8b4e1104aea6aa7182df804165 (commit)
via 563122c3088a3fa10e93175170ec99f58dea8b89 (commit)
via 9e8299df6905a2c894d8ebc60a43b40a3a5545a7 (commit)
via d7f7edb4511a9f82be10935b63b288903dd1c903 (commit)
from 113ef90ed25eafdce0ae38f7e3c8eb437301bed3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/e72d9b2cc827de86ae50c797f65c91eaf2b206cc
commit e72d9b2cc827de86ae50c797f65c91eaf2b206cc
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Jul 15 20:45:22 2015 -0500
Code format cleanup.
No reviewer.
diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c
index 4fee88f..b155b17 100644
--- a/src/mpid/ch3/src/ch3u_win_fns.c
+++ b/src/mpid/ch3/src/ch3u_win_fns.c
@@ -298,11 +298,11 @@ int MPIDI_CH3U_Win_shared_query(MPID_Win * win_ptr, int target_rank, MPI_Aint *
*size = win_ptr->size;
*disp_unit = win_ptr->disp_unit;
- fn_exit:
+ fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_SHARED_QUERY);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
- fn_fail:
+ fn_fail:
goto fn_exit;
/* --END ERROR HANDLING-- */
}
http://git.mpich.org/mpich.git/commitdiff/473f4433ab72351dd643b004bb211eec835a7e84
commit 473f4433ab72351dd643b004bb211eec835a7e84
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Jul 15 16:55:13 2015 -0500
Rename MPI_RC to MPIDI_CH3I_NM_OFI_RC
MPI_RC is not an MPICH function and should be named
as an internal function.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
index be7b0d9..095e991 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
@@ -124,7 +124,7 @@ static inline int MPID_nem_ofi_conn_req_callback(cq_tagged_entry_t * wc, MPID_Re
MPIU_Assertp(vc);
MPIDI_VC_Init(vc, NULL, 0);
- MPI_RC(MPIDI_GetTagFromPort(bc, &vc->port_name_tag));
+ MPIDI_CH3I_NM_OFI_RC(MPIDI_GetTagFromPort(bc, &vc->port_name_tag));
ret = MPIU_Str_get_binary_arg(bc, "OFI", addr, gl_data.bound_addrlen, &len);
MPIU_ERR_CHKANDJUMP((ret != MPIU_STR_SUCCESS && ret != MPIU_STR_NOMEM) ||
(size_t) len != gl_data.bound_addrlen,
@@ -168,10 +168,10 @@ static inline int MPID_nem_ofi_handle_packet(cq_tagged_entry_t * wc ATTRIBUTE((u
if (MPID_cc_get(rreq->cc) == 1) {
vc = REQ_OFI(rreq)->vc;
MPIU_Assert(vc);
- MPI_RC(MPID_nem_handle_pkt(vc, REQ_OFI(rreq)->pack_buffer, REQ_OFI(rreq)->pack_buffer_size));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_handle_pkt(vc, REQ_OFI(rreq)->pack_buffer, REQ_OFI(rreq)->pack_buffer_size));
MPIU_Free(REQ_OFI(rreq)->pack_buffer);
}
- MPI_RC(MPID_Request_complete(rreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(rreq));
END_FUNC_RC(FCNAME);
}
@@ -186,8 +186,8 @@ static inline int MPID_nem_ofi_cts_send_callback(cq_tagged_entry_t * wc, MPID_Re
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- MPI_RC(MPID_nem_ofi_handle_packet(wc, REQ_OFI(sreq)->parent));
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_handle_packet(wc, REQ_OFI(sreq)->parent));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
END_FUNC_RC(FCNAME);
}
@@ -280,7 +280,7 @@ int MPID_nem_ofi_connect_to_root_callback(cq_tagged_entry_t * wc ATTRIBUTE((unus
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
END_FUNC_RC(FCNAME);
}
@@ -303,12 +303,12 @@ int MPID_nem_ofi_cm_init(MPIDI_PG_t * pg_p, int pg_rank ATTRIBUTE((unused)))
/* Set up CH3 and netmod data structures */
/* ------------------------------------- */
if (gl_data.api_set == API_SET_1) {
- MPI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted,
+ MPIDI_CH3I_NM_OFI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted,
MPID_nem_ofi_anysource_matched));
MPIDI_Anysource_iprobe_fn = MPID_nem_ofi_anysource_iprobe;
MPIDI_Anysource_improbe_fn = MPID_nem_ofi_anysource_improbe;
} else {
- MPI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted_2,
+ MPIDI_CH3I_NM_OFI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted_2,
MPID_nem_ofi_anysource_matched));
MPIDI_Anysource_iprobe_fn = MPID_nem_ofi_anysource_iprobe_2;
MPIDI_Anysource_improbe_fn = MPID_nem_ofi_anysource_improbe_2;
@@ -375,13 +375,13 @@ int MPID_nem_ofi_cm_finalize()
&(REQ_OFI(gl_data.persistent_req)->ofi_context)), cancel);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.persistent_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.persistent_req->status, 0);
- MPI_RC(MPID_Request_complete(gl_data.persistent_req));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(gl_data.persistent_req));
FI_RC(fi_cancel((fid_t) gl_data.endpoint, &(REQ_OFI(gl_data.conn_req)->ofi_context)), cancel);
MPIU_Free(gl_data.conn_req->dev.user_buf);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.conn_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.conn_req->status, 0);
- MPI_RC(MPID_Request_complete(gl_data.conn_req));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(gl_data.conn_req));
END_FUNC_RC(FCNAME);
}
@@ -408,7 +408,7 @@ int MPID_nem_ofi_vc_connect(MPIDI_VC_t * vc)
goto fn_exit;
}
- MPI_RC(vc->pg->getConnInfo(vc->pg_rank, bc, OFI_KVSAPPSTRLEN, vc->pg));
+ MPIDI_CH3I_NM_OFI_RC(vc->pg->getConnInfo(vc->pg_rank, bc, OFI_KVSAPPSTRLEN, vc->pg));
ret = MPIU_Str_get_binary_arg(bc, "OFI", addr, gl_data.bound_addrlen, &len);
MPIU_ERR_CHKANDJUMP((ret != MPIU_STR_SUCCESS && ret != MPIU_STR_NOMEM) ||
(size_t) len != gl_data.bound_addrlen,
@@ -496,7 +496,7 @@ int MPID_nem_ofi_vc_terminate(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- MPI_RC(MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED));
+ MPIDI_CH3I_NM_OFI_RC(MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED));
VC_OFI(vc)->ready = 0;
END_FUNC_RC(FCNAME);
}
@@ -537,7 +537,7 @@ int MPID_nem_ofi_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc)
mpi_errno = MPI_ERR_OTHER;
goto fn_fail;
}
- MPI_RC(MPIDI_GetTagFromPort(business_card, &new_vc->port_name_tag));
+ MPIDI_CH3I_NM_OFI_RC(MPIDI_GetTagFromPort(business_card, &new_vc->port_name_tag));
ret = MPIU_Str_get_binary_arg(business_card, "OFI", addr, gl_data.bound_addrlen, &len);
MPIU_ERR_CHKANDJUMP((ret != MPIU_STR_SUCCESS && ret != MPIU_STR_NOMEM) ||
(size_t) len != gl_data.bound_addrlen,
@@ -547,7 +547,7 @@ int MPID_nem_ofi_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc)
VC_OFI(new_vc)->ready = 1;
str_errno = MPIU_Str_add_int_arg(&bc, &my_bc_len, "tag", new_vc->port_name_tag);
MPIU_ERR_CHKANDJUMP(str_errno, mpi_errno, MPI_ERR_OTHER, "**argstr_port_name_tag");
- MPI_RC(MPID_nem_ofi_get_business_card(MPIR_Process.comm_world->rank, &bc, &my_bc_len));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_get_business_card(MPIR_Process.comm_world->rank, &bc, &my_bc_len));
my_bc_len = OFI_KVSAPPSTRLEN - my_bc_len;
MPID_nem_ofi_create_req(&sreq, 1);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
index ba8fcf8..d0a3169 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
@@ -201,7 +201,7 @@ fn_fail: \
#STR); \
} while (0)
-#define MPI_RC(FUNC) \
+#define MPIDI_CH3I_NM_OFI_RC(FUNC) \
do \
{ \
mpi_errno = FUNC; \
@@ -211,7 +211,7 @@ fn_fail: \
#define VC_READY_CHECK(vc) \
({ \
if (1 != VC_OFI(vc)->ready) { \
- MPI_RC(MPID_nem_ofi_vc_connect(vc)); \
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_vc_connect(vc)); \
} \
})
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
index 5c4c3d9..bf1291f 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
@@ -214,7 +214,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* Get our business card */
/* -------------------------------- */
my_bc = *bc_val_p;
- MPI_RC(MPID_nem_ofi_get_business_card(pg_rank, bc_val_p, val_max_sz_p));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_get_business_card(pg_rank, bc_val_p, val_max_sz_p));
/* -------------------------------- */
/* Publish the business card */
@@ -289,7 +289,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* required, like connection management and */
/* startcontig messages */
/* --------------------------------------------- */
- MPI_RC(MPID_nem_ofi_cm_init(pg_p, pg_rank));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_cm_init(pg_p, pg_rank));
fn_exit:
if (fi_addrs)
MPIU_Free(fi_addrs);
@@ -316,7 +316,7 @@ int MPID_nem_ofi_finalize(void)
/* Cancels any persistent/global requests and */
/* frees any resources from cm_init() */
/* --------------------------------------------- */
- MPI_RC(MPID_nem_ofi_cm_finalize());
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_cm_finalize());
FI_RC(fi_close((fid_t) gl_data.endpoint), epclose);
FI_RC(fi_close((fid_t) gl_data.av), avclose);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index a29f6f7..f6b7177 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -146,7 +146,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
msg.data = 0ULL;
FI_RC_RETRY(fi_tsendmsg(gl_data.endpoint,&msg,0ULL),tsend);
}
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
break;
case MPID_MSG_CTS | MPID_MSG_RTS | MPID_MSG_DATA:
@@ -158,16 +158,16 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
}
else {
vc = REQ_OFI(sreq)->vc;
- MPI_RC(reqFn(vc, sreq, &complete));
+ MPIDI_CH3I_NM_OFI_RC(reqFn(vc, sreq, &complete));
}
gl_data.rts_cts_in_flight--;
break;
case MPID_MSG_RTS:
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
break;
}
END_FUNC_RC(FCNAME);
@@ -183,8 +183,8 @@ static int MPID_nem_ofi_cts_recv_callback(cq_tagged_entry_t * wc, MPID_Request *
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- MPI_RC(MPID_nem_ofi_data_callback(wc, REQ_OFI(rreq)->parent));
- MPI_RC(MPID_Request_complete(rreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_data_callback(wc, REQ_OFI(rreq)->parent));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(rreq));
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
index 30baab3..a067d27 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
@@ -59,7 +59,7 @@ int MPID_nem_ofi_poll(int in_blocking_poll)
if (NULL != wc.op_context) {
req = context_to_req(wc.op_context);
if (REQ_OFI(req)->event_callback) {
- MPI_RC(REQ_OFI(req)->event_callback(&wc, req));
+ MPIDI_CH3I_NM_OFI_RC(REQ_OFI(req)->event_callback(&wc, req));
continue;
}
reqFn = req->dev.OnDataAvail;
@@ -70,7 +70,7 @@ int MPID_nem_ofi_poll(int in_blocking_poll)
vc = REQ_OFI(req)->vc;
complete = 0;
- MPI_RC(reqFn(vc, req, &complete));
+ MPIDI_CH3I_NM_OFI_RC(reqFn(vc, req, &complete));
continue;
}
else {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
index 2c09e89..fe62b02 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
@@ -41,8 +41,8 @@ static inline int MPID_nem_ofi_sync_recv_callback(cq_tagged_entry_t * wc ATTRIBU
BEGIN_FUNC(FCNAME);
MPIDI_CH3U_Recvq_DP(REQ_OFI(rreq)->parent);
- MPI_RC(MPID_Request_complete(REQ_OFI(rreq)->parent));
- MPI_RC(MPID_Request_complete(rreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(REQ_OFI(rreq)->parent));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(rreq));
END_FUNC_RC(FCNAME);
}
@@ -60,7 +60,7 @@ static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((u
BEGIN_FUNC(FCNAME);
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPI_RC(MPID_Request_complete(sreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(sreq));
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
index 863800f..4ab75d2 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
@@ -90,7 +90,7 @@ int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request
/* request object */
/* ---------------------------------------------------- */
MPIDI_CH3U_Recvq_DP(rreq);
- MPI_RC(MPID_Request_complete(rreq));
+ MPIDI_CH3I_NM_OFI_RC(MPID_Request_complete(rreq));
}
END_FUNC_RC(FCNAME);
}
http://git.mpich.org/mpich.git/commitdiff/995420f0b93af5bfe8e955fbd606092adc44dae8
commit 995420f0b93af5bfe8e955fbd606092adc44dae8
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Jul 12 22:18:06 2015 -0500
Adding a request callback function for ibarrier in Win_fence.
In Win_fence implementation, when we enter the second Win_fence
and request of the ibarrier in the first Win_fence has not been
completed yet, we do not need to wait for that request's
completion and do not need to store that request on window
anymore. However, we do need to keep track of number of such
incomplete but 'out-of-date' requests on the window in order
to clean them up in Win_free.
In this patch, we achieve this by adding a counter for incomplete
but 'out-of-date' requests and a callback function to those requests.
We increment the counter when we meet one such request, and
decrement the counter in the callback function when that request
is got completed. By doing so, Win_free can just block waiting for
that counter to become 0 to clean up all pending requests.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 001b131..9011a61 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1854,6 +1854,9 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *, MPID_Request *,
/* Request-based operation handler */
int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request *);
+/* RMA Synchronization request handler */
+int MPIDI_CH3_Req_handler_rma_sync_complete(MPID_Request *);
+
/* Thread Support */
#ifdef MPICH_IS_THREADED
#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h
index c5d8c47..c916223 100644
--- a/src/mpid/ch3/include/mpidpre.h
+++ b/src/mpid/ch3/include/mpidpre.h
@@ -343,6 +343,15 @@ typedef struct MPIDI_Win_basic_info {
struct MPIDI_RMA_Target_lock_entry *target_lock_entry_pool_head; \
struct MPIDI_RMA_Target_lock_entry *target_lock_entry_pool_tail; \
int current_target_lock_data_bytes; \
+ int dangling_request_cnt; /* This counter tracks number of \
+ ibarrier requests (used in \
+ Win_fence) that are not \
+ completed but are out-of-date \
+ for the current RMA epoch. \
+ we do not need to wait for \
+ their completion during RMA \
+ epoch, but we need to clean \
+ them up in Win_free. */ \
#ifdef MPIDI_CH3_WIN_DECL
#define MPID_DEV_WIN_DECL \
diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c
index 431f7e2..822cea4 100644
--- a/src/mpid/ch3/src/ch3u_handle_op_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_op_req.c
@@ -39,3 +39,30 @@ int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request * sreq)
fn_fail:
goto fn_exit;
}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_Req_handler_rma_sync_complete
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_Req_handler_rma_sync_complete(MPID_Request * sreq)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPID_Win *win_ptr = NULL;
+
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_SYNC_COMPLETE);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_SYNC_COMPLETE);
+
+ MPID_Win_get_ptr(sreq->dev.source_win_handle, win_ptr);
+ MPIU_Assert(win_ptr != NULL);
+
+ /* decrement pending incomplete ibarrier request counter */
+ win_ptr->dangling_request_cnt--;
+ MPIU_Assert(win_ptr->dangling_request_cnt >= 0);
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_SYNC_COMPLETE);
+ return mpi_errno;
+
+ fn_fail:
+ goto fn_exit;
+}
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index 5d15c47..e124931 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -515,6 +515,13 @@ int MPID_Win_fence(int assert, MPID_Win * win_ptr)
if (win_ptr->fence_sync_req != MPI_REQUEST_NULL) {
MPID_Request *req_ptr;
MPID_Request_get_ptr(win_ptr->fence_sync_req, req_ptr);
+
+ if (!MPID_Request_is_complete(req_ptr)) {
+ req_ptr->dev.source_win_handle = win_ptr->handle;
+ req_ptr->request_completed_cb = MPIDI_CH3_Req_handler_rma_sync_complete;
+ win_ptr->dangling_request_cnt++;
+ }
+
MPID_Request_release(req_ptr);
win_ptr->fence_sync_req = MPI_REQUEST_NULL;
win_ptr->states.access_state = MPIDI_RMA_NONE;
@@ -556,6 +563,13 @@ int MPID_Win_fence(int assert, MPID_Win * win_ptr)
if (win_ptr->fence_sync_req != MPI_REQUEST_NULL) {
MPID_Request *req_ptr;
MPID_Request_get_ptr(win_ptr->fence_sync_req, req_ptr);
+
+ if (!MPID_Request_is_complete(req_ptr)) {
+ req_ptr->dev.source_win_handle = win_ptr->handle;
+ req_ptr->request_completed_cb = MPIDI_CH3_Req_handler_rma_sync_complete;
+ win_ptr->dangling_request_cnt++;
+ }
+
MPID_Request_release(req_ptr);
win_ptr->fence_sync_req = MPI_REQUEST_NULL;
MPIDI_CH3I_num_active_issued_win--;
diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c
index d43852f..65dce1b 100644
--- a/src/mpid/ch3/src/mpid_rma.c
+++ b/src/mpid/ch3/src/mpid_rma.c
@@ -300,6 +300,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
(*win_ptr)->lock_epoch_count = 0;
(*win_ptr)->outstanding_locks = 0;
(*win_ptr)->current_target_lock_data_bytes = 0;
+ (*win_ptr)->dangling_request_cnt = 0;
/* Initialize the info flags */
(*win_ptr)->info_args.no_locks = 0;
diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c
index f105e97..3182cfe 100644
--- a/src/mpid/ch3/src/mpidi_rma.c
+++ b/src/mpid/ch3/src/mpidi_rma.c
@@ -158,26 +158,20 @@ int MPID_Win_free(MPID_Win ** win_ptr)
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPID_WIN_FREE);
- /* it is possible that there is a IBARRIER in MPI_WIN_FENCE with
- * MODE_NOPRECEDE not being completed, we let the progress engine
- * to delete its request when it is completed. */
+ MPIU_ERR_CHKANDJUMP(((*win_ptr)->states.access_state != MPIDI_RMA_NONE &&
+ (*win_ptr)->states.access_state != MPIDI_RMA_FENCE_ISSUED &&
+ (*win_ptr)->states.access_state != MPIDI_RMA_FENCE_GRANTED) ||
+ ((*win_ptr)->states.exposure_state != MPIDI_RMA_NONE),
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
+
if ((*win_ptr)->fence_sync_req != MPI_REQUEST_NULL) {
- MPID_Request *req_ptr;
- MPID_Request_get_ptr((*win_ptr)->fence_sync_req, req_ptr);
- MPID_Request_release(req_ptr);
- (*win_ptr)->fence_sync_req = MPI_REQUEST_NULL;
- (*win_ptr)->states.access_state = MPIDI_RMA_NONE;
- MPIDI_CH3I_num_active_issued_win--;
- MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
+ /* If the ibarrier for previous FENCE is not completed,
+ * set the window state to FENCE_ISSUED here, so that
+ * wait_progress_engine() will enter RMA progress and
+ * try to swtich the window state when ibarrier is completed. */
+ (*win_ptr)->states.access_state = MPIDI_RMA_FENCE_ISSUED;
}
- if ((*win_ptr)->states.access_state == MPIDI_RMA_FENCE_GRANTED)
- (*win_ptr)->states.access_state = MPIDI_RMA_NONE;
-
- MPIU_ERR_CHKANDJUMP((*win_ptr)->states.access_state != MPIDI_RMA_NONE ||
- (*win_ptr)->states.exposure_state != MPIDI_RMA_NONE,
- mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
-
/* 1. Here we must wait until all passive locks are released on this target,
* because for some UNLOCK messages, we do not send ACK back to origin,
* we must wait until lock is released so that we can free window.
@@ -191,7 +185,9 @@ int MPID_Win_free(MPID_Win ** win_ptr)
while ((*win_ptr)->current_lock_type != MPID_LOCK_NONE ||
(*win_ptr)->at_completion_counter != 0 ||
(*win_ptr)->target_lock_queue_head != NULL ||
- (*win_ptr)->current_target_lock_data_bytes != 0) {
+ (*win_ptr)->current_target_lock_data_bytes != 0 ||
+ (*win_ptr)->fence_sync_req != MPI_REQUEST_NULL ||
+ (*win_ptr)->dangling_request_cnt != 0) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
http://git.mpich.org/mpich.git/commitdiff/7b522671dfe2aa84a6797cbdc4434c6dee8b117e
commit 7b522671dfe2aa84a6797cbdc4434c6dee8b117e
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Jul 12 22:10:38 2015 -0500
Use request_completed_cb to complete user request in req-based RMA operation.
Originally we use OnDataAvail function pointer in the request
to complete the user request when the operation is completed,
which is not a good choice since the function pointer should
be used in send/recv when the next data chunk is available.
Here we use request_completed_cb function pointer to complete
the user request.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h
index dd03b16..c2f8834 100644
--- a/src/mpid/ch3/include/mpid_rma_issue.h
+++ b/src/mpid/ch3/include/mpid_rma_issue.h
@@ -1296,20 +1296,8 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op)
* - progress engine: complete PUT/ACC req.
* - GET/GET_ACC packet handler: complete GET/GET_ACC reqs.
*
- * We always set OnFinal which should be called when sending or
- * receiving the last segment. However, short put/acc ops are
- * issued in one packet and the lower layer only check OnDataAvail
- * so we have to set OnDataAvail as well.
- *
- * Note that a noncontig send also uses OnDataAvail to loop all
- * segments but it must be changed to OnFinal when sending the
- * last segment, so it is also correct for us.
- *
* TODO: implement stack for overriding functions*/
- if (req_ptr[i]->dev.OnDataAvail == NULL) {
- req_ptr[i]->dev.OnDataAvail = MPIDI_CH3_ReqHandler_ReqOpsComplete;
- }
- req_ptr[i]->dev.OnFinal = MPIDI_CH3_ReqHandler_ReqOpsComplete;
+ req_ptr[i]->request_completed_cb = MPIDI_CH3_ReqHandler_ReqOpsComplete;
} /* end of for loop */
if (incomplete_req_cnt) {
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index a08be2c..001b131 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1852,8 +1852,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *, MPID_Request *,
int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *, MPID_Request *,
int * );
/* Request-based operation handler */
-int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t *, MPID_Request *,
- int *);
+int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request *);
/* Thread Support */
#ifdef MPICH_IS_THREADED
diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c
index d0ca7c9..431f7e2 100644
--- a/src/mpid/ch3/src/ch3u_handle_op_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_op_req.c
@@ -14,7 +14,7 @@
#define FUNCNAME MPIDI_CH3_ReqHandler_ReqOpsComplete
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t * vc, MPID_Request * sreq, int *complete)
+int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request * sreq)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *ureq = NULL;
@@ -32,13 +32,6 @@ int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t * vc, MPID_Request * sreq, in
MPIU_ERR_POP(mpi_errno);
}
- mpi_errno = MPID_Request_complete(sreq);
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- *complete = TRUE;
-
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_REQOPSCOMPLETE);
return mpi_errno;
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 3d818f2..91b8eec 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1715,16 +1715,7 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
}
if (complete) {
- /* Request-based RMA defines final actions for completing user request. */
- int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
- reqFn = req->dev.OnFinal;
-
- if (reqFn) {
- mpi_errno = reqFn(vc, req, &complete);
- }
- else {
- mpi_errno = MPID_Request_complete(req);
- }
+ mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1867,16 +1858,7 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
}
if (complete) {
- /* Request-based RMA defines final actions for completing user request. */
- int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
- reqFn = req->dev.OnFinal;
-
- if (reqFn) {
- mpi_errno = reqFn(vc, req, &complete);
- }
- else {
- mpi_errno = MPID_Request_complete(req);
- }
+ mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
http://git.mpich.org/mpich.git/commitdiff/65c58bad76aebaa9b9b2b51a9c13b922a0f35bb0
commit 65c58bad76aebaa9b9b2b51a9c13b922a0f35bb0
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Jul 15 17:07:44 2015 -0500
Add request completed callback functions.
Here we add a function pointer in request struct: request
completed callback. It is triggered in MPID_Request_complete
routine when CC is set to 0.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index e7b60cc..e539ce7 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -1509,6 +1509,11 @@ typedef struct MPID_Request {
/* Errflag for NBC requests. Not used by other requests. */
mpir_errflag_t errflag;
+ /* request_completed_cb: the callback function triggered when this request
+ * is completed, i.e. CC is set to 0. Inside request_completed_cb, progress
+ * engine should not be called. */
+ int (*request_completed_cb)(struct MPID_Request *);
+
/* Other, device-specific information */
#ifdef MPID_DEV_REQUEST_DECL
MPID_DEV_REQUEST_DECL
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 889bc18..251024e 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -74,6 +74,7 @@ MPID_Request * MPID_Request_create(void)
req->comm = NULL;
req->greq_fns = NULL;
req->errflag = MPIR_ERR_NONE;
+ req->request_completed_cb = NULL;
req->dev.datatype_ptr = NULL;
req->dev.segment_ptr = NULL;
/* Masks and flags for channel device state in an MPID_Request */
@@ -613,6 +614,14 @@ int MPID_Request_complete(MPID_Request *req)
MPIDI_CH3U_Request_decrement_cc(req, &incomplete);
if (!incomplete) {
+ /* trigger request_completed callback function */
+ if (req->request_completed_cb != NULL) {
+ mpi_errno = req->request_completed_cb(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ }
+
MPID_Request_release(req);
MPIDI_CH3_Progress_signal_completion();
}
http://git.mpich.org/mpich.git/commitdiff/8474718d84de3c81d4f56d49b25c2b13300cdc7b
commit 8474718d84de3c81d4f56d49b25c2b13300cdc7b
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Jul 15 17:06:46 2015 -0500
Make MPID_Request_complete to return mpi_errno.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index 547ebc5..e7b60cc 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -3532,7 +3532,7 @@ void MPID_Request_release(MPID_Request *);
Module:
Request
@*/
-void MPID_Request_complete(MPID_Request *);
+int MPID_Request_complete(MPID_Request *);
typedef struct MPID_Grequest_class {
MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
index e25a81c..e4b02e5 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
@@ -134,7 +134,10 @@ static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid)
if (reqFn == 0) {
MPIU_Assert(reqtype != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ r_mpi_errno = MPID_Request_complete(sreq);
+ if (r_mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(r_mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
index 7c40109..74cf26d 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
@@ -155,7 +155,10 @@ int MPID_nem_mxm_improbe(MPIDI_VC_t * vc, int source, int tag, MPID_Comm * comm,
err = mxm_message_recv(&mxm_req, mxm_msg);
_mxm_req_wait(&mxm_req.base);
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*message = req;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
index 04039bd..be7b0d9 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
@@ -171,7 +171,7 @@ static inline int MPID_nem_ofi_handle_packet(cq_tagged_entry_t * wc ATTRIBUTE((u
MPI_RC(MPID_nem_handle_pkt(vc, REQ_OFI(rreq)->pack_buffer, REQ_OFI(rreq)->pack_buffer_size));
MPIU_Free(REQ_OFI(rreq)->pack_buffer);
}
- MPID_Request_complete(rreq);
+ MPI_RC(MPID_Request_complete(rreq));
END_FUNC_RC(FCNAME);
}
@@ -187,7 +187,7 @@ static inline int MPID_nem_ofi_cts_send_callback(cq_tagged_entry_t * wc, MPID_Re
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
MPI_RC(MPID_nem_ofi_handle_packet(wc, REQ_OFI(sreq)->parent));
- MPID_Request_complete(sreq);
+ MPI_RC(MPID_Request_complete(sreq));
END_FUNC_RC(FCNAME);
}
@@ -279,10 +279,10 @@ int MPID_nem_ofi_connect_to_root_callback(cq_tagged_entry_t * wc ATTRIBUTE((unus
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPID_Request_complete(sreq);
- END_FUNC(FCNAME);
- return mpi_errno;
+ MPI_RC(MPID_Request_complete(sreq));
+
+ END_FUNC_RC(FCNAME);
}
/* ------------------------------------------------------------------------ */
@@ -375,18 +375,14 @@ int MPID_nem_ofi_cm_finalize()
&(REQ_OFI(gl_data.persistent_req)->ofi_context)), cancel);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.persistent_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.persistent_req->status, 0);
- MPID_Request_complete(gl_data.persistent_req);
+ MPI_RC(MPID_Request_complete(gl_data.persistent_req));
FI_RC(fi_cancel((fid_t) gl_data.endpoint, &(REQ_OFI(gl_data.conn_req)->ofi_context)), cancel);
MPIU_Free(gl_data.conn_req->dev.user_buf);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.conn_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.conn_req->status, 0);
- MPID_Request_complete(gl_data.conn_req);
- fn_exit:
- END_FUNC(FCNAME);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
+ MPI_RC(MPID_Request_complete(gl_data.conn_req));
+ END_FUNC_RC(FCNAME);
}
/* ------------------------------------------------------------------------ */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index c13268b..a29f6f7 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -146,7 +146,8 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
msg.data = 0ULL;
FI_RC_RETRY(fi_tsendmsg(gl_data.endpoint,&msg,0ULL),tsend);
}
- MPID_Request_complete(sreq);
+ MPI_RC(MPID_Request_complete(sreq));
+
break;
case MPID_MSG_CTS | MPID_MSG_RTS | MPID_MSG_DATA:
if (REQ_OFI(sreq)->pack_buffer)
@@ -157,7 +158,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
- MPID_Request_complete(sreq);
+ MPI_RC(MPID_Request_complete(sreq));
}
else {
vc = REQ_OFI(sreq)->vc;
@@ -166,7 +167,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
gl_data.rts_cts_in_flight--;
break;
case MPID_MSG_RTS:
- MPID_Request_complete(sreq);
+ MPI_RC(MPID_Request_complete(sreq));
break;
}
END_FUNC_RC(FCNAME);
@@ -183,7 +184,8 @@ static int MPID_nem_ofi_cts_recv_callback(cq_tagged_entry_t * wc, MPID_Request *
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
MPI_RC(MPID_nem_ofi_data_callback(wc, REQ_OFI(rreq)->parent));
- MPID_Request_complete(rreq);
+ MPI_RC(MPID_Request_complete(rreq));
+
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
index c69b6d3..2c09e89 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
@@ -41,11 +41,10 @@ static inline int MPID_nem_ofi_sync_recv_callback(cq_tagged_entry_t * wc ATTRIBU
BEGIN_FUNC(FCNAME);
MPIDI_CH3U_Recvq_DP(REQ_OFI(rreq)->parent);
- MPID_Request_complete(REQ_OFI(rreq)->parent);
- MPID_Request_complete(rreq);
+ MPI_RC(MPID_Request_complete(REQ_OFI(rreq)->parent));
+ MPI_RC(MPID_Request_complete(rreq));
- END_FUNC(FCNAME);
- return mpi_errno;
+ END_FUNC_RC(FCNAME);
}
/* ------------------------------------------------------------------------ */
@@ -61,9 +60,8 @@ static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((u
BEGIN_FUNC(FCNAME);
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPID_Request_complete(sreq);
- END_FUNC(FCNAME);
- return mpi_errno;
+ MPI_RC(MPID_Request_complete(sreq));
+ END_FUNC_RC(FCNAME);
}
#define DO_CANCEL(req) \
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
index 4942f53..863800f 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
@@ -90,7 +90,7 @@ int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request
/* request object */
/* ---------------------------------------------------- */
MPIDI_CH3U_Recvq_DP(rreq);
- MPID_Request_complete(rreq);
+ MPI_RC(MPID_Request_complete(rreq));
}
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
index c298460..8d87025 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
@@ -409,11 +409,12 @@ int MPID_nem_ptl_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPID
#define FUNCNAME on_data_avail
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
-static inline void on_data_avail(MPID_Request * req)
+static inline int on_data_avail(MPID_Request * req)
{
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_VC_t *vc = req->ch.vc;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
+ int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_ON_DATA_AVAIL);
@@ -421,7 +422,11 @@ static inline void on_data_avail(MPID_Request * req)
reqFn = req->dev.OnDataAvail;
if (!reqFn) {
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
@@ -435,7 +440,11 @@ static inline void on_data_avail(MPID_Request * req)
if (vc->state == MPIDI_VC_STATE_CLOSED && vc_ptl->num_queued_sends == 0)
MPID_nem_ptl_vc_terminated(vc);
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_ON_DATA_AVAIL);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#undef FUNCNAME
@@ -542,8 +551,12 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e)
MPID_Request_get_ptr(handle, req);
if (--REQ_PTL(req)->num_gets == 0) {
MPIU_Free(TMPBUF(req));
- if (REQ_PTL(req)->put_done)
- on_data_avail(req); /* Otherwise we'll do it on the SEND */
+ if (REQ_PTL(req)->put_done) {
+ mpi_errno = on_data_avail(req); /* Otherwise we'll do it on the SEND */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ }
}
}
break;
@@ -554,8 +567,12 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e)
MPIU_Free(SENDBUF(req));
REQ_PTL(req)->put_done = 1;
- if (REQ_PTL(req)->num_gets == 0) /* Otherwise GET will do it */
- on_data_avail(req);
+ if (REQ_PTL(req)->num_gets == 0) { /* Otherwise GET will do it */
+ mpi_errno = on_data_avail(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ }
}
break;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
index 7f45331..f4eb6c5 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
@@ -20,7 +20,7 @@ static int handle_probe(const ptl_event_t *e)
if (e->ni_fail_type == PTL_NI_NO_MATCH) {
REQ_PTL(req)->found = FALSE;
- goto fn_exit;
+ goto finish_probe;
}
REQ_PTL(req)->found = TRUE;
@@ -28,8 +28,13 @@ static int handle_probe(const ptl_event_t *e)
req->status.MPI_TAG = NPTL_MATCH_GET_TAG(e->match_bits);
MPIR_STATUS_SET_COUNT(req->status, NPTL_HEADER_GET_LENGTH(e->hdr_data));
- fn_exit:
- MPID_Request_complete(req);
+ finish_probe:
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLE_PROBE);
return mpi_errno;
fn_fail:
@@ -47,7 +52,7 @@ static int handle_mprobe(const ptl_event_t *e)
if (e->ni_fail_type == PTL_NI_NO_MATCH) {
REQ_PTL(req)->found = FALSE;
- goto fn_exit;
+ goto finish_mprobe;
}
REQ_PTL(req)->found = TRUE;
@@ -76,9 +81,14 @@ static int handle_mprobe(const ptl_event_t *e)
REQ_PTL(req)->put_me = PTL_INVALID_HANDLE;
req->dev.recv_pending_count = 1;
+ finish_mprobe:
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
fn_exit:
MPIU_CHKPMEM_COMMIT();
- MPID_Request_complete(req);
MPIDI_FUNC_EXIT(MPID_STATE_HANDLE_PROBE);
return mpi_errno;
fn_fail:
@@ -389,7 +399,10 @@ int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *p
for (i = 0; i < REQ_PTL(sreq)->num_gets; i++) {
ret = PtlMEUnlink(REQ_PTL(sreq)->get_me_p[i]);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
@@ -400,7 +413,10 @@ int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *p
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
index 11e5a0d..3133ebb 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
@@ -70,7 +70,10 @@ static int handler_recv_complete(const ptl_event_t *e)
if (REQ_PTL(rreq)->chunk_buffer[i])
MPIU_Free(REQ_PTL(rreq)->chunk_buffer[i]);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_COMPLETE);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
index b982b28..8fd649c 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
@@ -86,7 +86,10 @@ static int handler_send(const ptl_event_t *e)
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
}
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_SEND);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
index 1311afe..5639213 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
@@ -1619,7 +1619,10 @@ static int MPID_nem_tcp_recv_handler(sockconn_t *const sc)
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "...complete");
sc_vc_ch->recv_active = NULL;
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
index 744fca7..f590442 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
@@ -149,7 +149,10 @@ int MPID_nem_tcp_send_queued(MPIDI_VC_t *vc, MPIDI_nem_tcp_request_queue_t *send
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
@@ -575,7 +578,10 @@ int MPID_nem_tcp_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPID
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -799,7 +805,10 @@ int MPID_nem_tcp_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn)
{
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -870,7 +879,10 @@ int MPID_nem_tcp_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno)
MPIDI_CH3I_Sendq_dequeue(&vc_tcp->send_queue, &req);
req->status.MPI_ERROR = req_errno;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
/* paused send queue */
@@ -878,7 +890,10 @@ int MPID_nem_tcp_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno)
MPIDI_CH3I_Sendq_dequeue(&vc_tcp->paused_send_queue, &req);
req->status.MPI_ERROR = req_errno;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
fn_exit:
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_isend.c b/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
index 2bf0863..267d4ed 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
@@ -67,7 +67,10 @@ int MPIDI_CH3_iSend (MPIDI_VC_t *vc, MPID_Request *sreq, void * hdr, MPIDI_msg_s
if (!reqFn)
{
MPIU_Assert (MPIDI_Request_get_type (sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete (sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else
{
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c b/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
index d7832f4..765b9ed 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
@@ -126,7 +126,10 @@ int MPIDI_CH3_iSendv (MPIDI_VC_t *vc, MPID_Request *sreq, MPID_IOV *iov, int n_i
if (!reqFn)
{
MPIU_Assert (MPIDI_Request_get_type (sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete (sreq);
+ mpi_errno = MPID_Request_complete (sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, ".... complete");
}
else
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
index 9e07339..37184ea 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
@@ -252,7 +252,10 @@ int MPIDI_CH3I_Shm_send_progress(void)
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
#endif
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* MT - clear the current active send before dequeuing/destroying the current request */
MPIDI_CH3I_shm_active_send = NULL;
@@ -825,7 +828,11 @@ int MPID_nem_handle_pkt(MPIDI_VC_t *vc, char *buf, MPIDI_msg_sz_t buflen)
#if !(defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT))
MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
#endif
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
complete = TRUE;
}
else
@@ -1095,7 +1102,10 @@ int MPIDI_CH3I_Complete_sendq_with_error(MPIDI_VC_t * vc)
MPIU_ERR_SET1(req->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
MPID_Request_release(req); /* ref count was incremented when added to queue */
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
req = next;
} else {
prev = req;
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c b/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
index 6950e7b..0729e00 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
@@ -79,7 +79,10 @@ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header,
if (!sreq->dev.OnDataAvail)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
else
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
index d3535d9..2c4cc03 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
@@ -305,7 +305,10 @@ static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *com
}
else {
*complete = 1;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
fn_fail:
@@ -478,7 +481,10 @@ int MPID_nem_lmt_dma_done_send(MPIDI_VC_t *vc, MPID_Request *sreq)
reason. */
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -611,7 +617,10 @@ int MPID_nem_lmt_dma_progress(void)
cur->req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(cur->req->status.MPI_ERROR, MPI_ERR_OTHER, "**recv_status", "**recv_status %d", *cur->status_p);
- MPID_Request_complete(cur->req);
+ mpi_errno = MPID_Request_complete(cur->req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
if (cur == outstanding_head) {
outstanding_head = cur->next;
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
index 1182ea2..5a232fd 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
@@ -504,7 +504,10 @@ static int lmt_shm_send_progress(MPIDI_VC_t *vc, MPID_Request *req, int *done)
while (last < data_sz);
*done = TRUE;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "completed req local_req=%d", req->handle);
@@ -512,6 +515,8 @@ static int lmt_shm_send_progress(MPIDI_VC_t *vc, MPID_Request *req, int *done)
copy_buf->sender_present.val = FALSE;
MPIDI_FUNC_EXIT(MPID_STATE_LMT_SHM_SEND_PROGRESS);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
/* Continued from note for lmt_shm_send_progress() above. The
@@ -643,12 +648,17 @@ static int lmt_shm_recv_progress(MPIDI_VC_t *vc, MPID_Request *req, int *done)
OPA_store_int(©_buf->owner_info.val.rank, NO_OWNER);
*done = TRUE;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
fn_exit:
copy_buf->receiver_present.val = FALSE;
MPIDI_FUNC_EXIT(MPID_STATE_LMT_SHM_RECV_PROGRESS);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#undef FUNCNAME
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
index 29be00f..8c33ddf 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
@@ -75,7 +75,10 @@ static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *com
}
else {
*complete = 1;
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
fn_fail:
diff --git a/src/mpid/ch3/channels/sock/src/ch3_isend.c b/src/mpid/ch3/channels/sock/src/ch3_isend.c
index afcfa87..a4f885d 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_isend.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_isend.c
@@ -74,7 +74,10 @@ int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPID_Request * sreq, void * hdr,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else {
int complete;
diff --git a/src/mpid/ch3/channels/sock/src/ch3_isendv.c b/src/mpid/ch3/channels/sock/src/ch3_isendv.c
index 8dd760c..6041627 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_isendv.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_isendv.c
@@ -139,7 +139,10 @@ int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else {
int complete;
diff --git a/src/mpid/ch3/channels/sock/src/ch3_progress.c b/src/mpid/ch3/channels/sock/src/ch3_progress.c
index a8f22cf..e6f81fb 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_progress.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_progress.c
@@ -508,7 +508,10 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
complete = TRUE;
}
else {
@@ -567,7 +570,11 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
complete = TRUE;
}
else {
@@ -611,7 +618,10 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
complete = TRUE;
}
else {
@@ -885,7 +895,10 @@ static int ReadMoreData( MPIDI_CH3I_Connection_t * conn, MPID_Request *rreq )
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
complete = TRUE;
}
else {
diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h
index cc364ab..dd03b16 100644
--- a/src/mpid/ch3/include/mpid_rma_issue.h
+++ b/src/mpid/ch3/include/mpid_rma_issue.h
@@ -1264,7 +1264,10 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op)
* and release ch3 ref. */
/* Complete user request and release the ch3 ref */
- MPID_Request_complete(op->ureq);
+ mpi_errno = MPID_Request_complete(op->ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else {
MPID_Request **req_ptr = NULL;
@@ -1316,7 +1319,10 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op)
else {
/* all requests are completed */
/* Complete user request and release ch3 ref */
- MPID_Request_complete(op->ureq);
+ mpi_errno = MPID_Request_complete(op->ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
op->ureq = NULL;
}
}
diff --git a/src/mpid/ch3/src/ch3u_buffer.c b/src/mpid/ch3/src/ch3u_buffer.c
index c2ef71b..e852ace 100644
--- a/src/mpid/ch3/src/ch3u_buffer.c
+++ b/src/mpid/ch3/src/ch3u_buffer.c
@@ -230,6 +230,7 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
MPI_Datatype datatype )
{
MPID_Request * const sreq = rreq->partner_request;
+ int mpi_errno = MPI_SUCCESS;
if (sreq != NULL)
{
@@ -240,7 +241,10 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
buf, count, datatype, &data_sz,
&rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else
{
@@ -251,7 +255,13 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
/* no other thread can possibly be waiting on rreq, so it is safe to
reset ref_count and cc */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
- return MPI_SUCCESS;
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
diff --git a/src/mpid/ch3/src/ch3u_eager.c b/src/mpid/ch3/src/ch3u_eager.c
index 515529e..843db2f 100644
--- a/src/mpid/ch3/src/ch3u_eager.c
+++ b/src/mpid/ch3/src/ch3u_eager.c
@@ -499,7 +499,10 @@ int MPIDI_CH3_PktHandler_EagerShortSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
* all rreq field modifications must be complete at this point. This macro
* also kicks the progress engine, which was previously done here via
* MPIDI_CH3_Progress_signal_completion(). */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
fn_fail:
/* MT note: it may be possible to narrow this CS after careful
@@ -643,7 +646,10 @@ int MPIDI_CH3_PktHandler_EagerSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
/* return the number of bytes processed in this function */
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else {
@@ -666,7 +672,10 @@ int MPIDI_CH3_PktHandler_EagerSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else
@@ -728,7 +737,10 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
/* return the number of bytes processed in this function */
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;;
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else {
@@ -746,7 +758,10 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else
@@ -788,7 +803,10 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
else
{
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
/* we didn't process anything but the header in this case */
diff --git a/src/mpid/ch3/src/ch3u_eagersync.c b/src/mpid/ch3/src/ch3u_eagersync.c
index 877699b..32d3877 100644
--- a/src/mpid/ch3/src/ch3u_eagersync.c
+++ b/src/mpid/ch3/src/ch3u_eagersync.c
@@ -257,7 +257,10 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else {
@@ -272,7 +275,10 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else
@@ -300,7 +306,10 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
{
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else {
@@ -315,7 +324,10 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else
@@ -338,6 +350,7 @@ int MPIDI_CH3_PktHandler_EagerSyncAck( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
{
MPIDI_CH3_Pkt_eager_sync_ack_t * esa_pkt = &pkt->eager_sync_ack;
MPID_Request * sreq;
+ int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"received eager sync ack pkt, sreq=0x%08x", esa_pkt->sender_req_id);
@@ -347,11 +360,18 @@ int MPIDI_CH3_PktHandler_EagerSyncAck( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
transfer could still be in progress) */
/* FIXME: This sometimes segfaults */
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
- return MPI_SUCCESS;
+
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#ifdef MPICH_DBG_OUTPUT
diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c
index ac83287..d0ca7c9 100644
--- a/src/mpid/ch3/src/ch3u_handle_op_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_op_req.c
@@ -27,9 +27,16 @@ int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t * vc, MPID_Request * sreq, in
/* Complete user request and release ref of completion handler.
* Note that ch3 ref is released by later clean_up call. */
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
- MPID_Request_complete(sreq);
*complete = TRUE;
fn_exit:
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 272b428..a1f13e3 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -29,7 +29,10 @@ int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq, int *comple
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*complete = TRUE;
}
else {
@@ -37,8 +40,12 @@ int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq, int *comple
}
in_routine = FALSE;
+
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
/* ----------------------------------------------------------------------- */
@@ -63,10 +70,20 @@ int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq, int *comple
int MPIDI_CH3_ReqHandler_RecvComplete(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPID_Request * rreq, int *complete)
{
+ int mpi_errno = MPI_SUCCESS;
+
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
- return MPI_SUCCESS;
+
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#undef FUNCNAME
@@ -104,7 +121,10 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -119,7 +139,7 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTRECVCOMPLETE);
- return MPI_SUCCESS;
+ return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
@@ -200,7 +220,10 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq,
MPIDI_CH3U_SRBuf_free(rreq);
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -215,7 +238,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq,
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_ACCUMRECVCOMPLETE);
- return MPI_SUCCESS;
+ return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
@@ -378,12 +401,16 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
}
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
fn_exit:
MPIU_CHKPMEM_COMMIT();
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE);
- return MPI_SUCCESS;
+ return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
@@ -509,13 +536,17 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
}
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
fn_exit:
MPIU_CHKPMEM_COMMIT();
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE);
- return MPI_SUCCESS;
+ return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
@@ -896,12 +927,18 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete(MPIDI_VC_t * vc,
/* mark receive data transfer as complete and decrement CC in receive
* request */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
- fn_fail:
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GETDERIVEDDTRECVCOMPLETE);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
@@ -913,6 +950,7 @@ int MPIDI_CH3_ReqHandler_UnpackUEBufComplete(MPIDI_VC_t * vc ATTRIBUTE((unused))
MPID_Request * rreq, int *complete)
{
int recv_pending;
+ int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
@@ -931,11 +969,18 @@ int MPIDI_CH3_ReqHandler_UnpackUEBufComplete(MPIDI_VC_t * vc ATTRIBUTE((unused))
}
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
- return MPI_SUCCESS;
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#undef FUNCNAME
@@ -965,12 +1010,19 @@ int MPIDI_CH3_ReqHandler_UnpackSRBufComplete(MPIDI_VC_t * vc, MPID_Request * rre
}
else {
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
}
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFCOMPLETE);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
#undef FUNCNAME
@@ -2020,10 +2072,16 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete(MPIDI_VC_t * vc,
/* mark receive data transfer as complete and decrement CC in receive
* request */
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*complete = TRUE;
- fn_fail:
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_PIGGYBACKLOCKOPRECVCOMPLETE);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c
index 9807f77..47cfb3c 100644
--- a/src/mpid/ch3/src/ch3u_handle_send_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_send_req.c
@@ -24,15 +24,21 @@ int MPIDI_CH3U_Handle_send_req(MPIDI_VC_t * vc, MPID_Request * sreq, int *comple
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
*complete = 1;
}
else {
mpi_errno = reqFn(vc, sreq, complete);
}
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
/* ----------------------------------------------------------------------- */
@@ -75,7 +81,10 @@ int MPIDI_CH3_ReqHandler_GetSendComplete(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_Assert(win_ptr->at_completion_counter >= 0);
/* mark data transfer as complete and decrement CC */
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -138,7 +147,10 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -204,7 +216,10 @@ int MPIDI_CH3_ReqHandler_CASSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -269,7 +284,10 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index f1bde26..889bc18 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -606,15 +606,21 @@ int MPIDI_CH3U_Request_unpack_uebuf(MPID_Request * rreq)
return mpi_errno;
}
-void MPID_Request_complete(MPID_Request *req)
+int MPID_Request_complete(MPID_Request *req)
{
int incomplete;
+ int mpi_errno = MPI_SUCCESS;
MPIDI_CH3U_Request_decrement_cc(req, &incomplete);
if (!incomplete) {
MPID_Request_release(req);
MPIDI_CH3_Progress_signal_completion();
}
+
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
void MPID_Request_release(MPID_Request *req)
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index cca6f95..2d2ea4b 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -94,7 +94,10 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
}
else {
@@ -274,7 +277,10 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
}
else {
@@ -446,7 +452,10 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
}
else {
@@ -663,7 +672,10 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
}
else {
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 284db47..3d818f2 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1297,7 +1297,11 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len);
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
@@ -1570,7 +1574,10 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
}
if (complete) {
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
@@ -1716,8 +1723,12 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
mpi_errno = reqFn(vc, req, &complete);
}
else {
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
}
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*rreqp = NULL;
}
@@ -1864,7 +1875,10 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
mpi_errno = reqFn(vc, req, &complete);
}
else {
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ }
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
}
*rreqp = NULL;
diff --git a/src/mpid/ch3/src/ch3u_rma_reqops.c b/src/mpid/ch3/src/ch3u_rma_reqops.c
index 56a596c..606533f 100644
--- a/src/mpid/ch3/src/ch3u_rma_reqops.c
+++ b/src/mpid/ch3/src/ch3u_rma_reqops.c
@@ -55,7 +55,10 @@ int MPID_Rput(const void *origin_addr, int origin_count,
}
}
else {
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
*request = ureq;
@@ -115,7 +118,10 @@ int MPID_Rget(void *origin_addr, int origin_count,
}
}
else {
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
*request = ureq;
@@ -175,7 +181,10 @@ int MPID_Raccumulate(const void *origin_addr, int origin_count,
}
}
else {
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
*request = ureq;
@@ -239,7 +248,10 @@ int MPID_Rget_accumulate(const void *origin_addr, int origin_count,
}
}
else {
- MPID_Request_complete(ureq);
+ mpi_errno = MPID_Request_complete(ureq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
*request = ureq;
diff --git a/src/mpid/ch3/src/ch3u_rndv.c b/src/mpid/ch3/src/ch3u_rndv.c
index 557f597..249b063 100644
--- a/src/mpid/ch3/src/ch3u_rndv.c
+++ b/src/mpid/ch3/src/ch3u_rndv.c
@@ -300,7 +300,10 @@ int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (req->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else {
@@ -315,7 +318,10 @@ int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPID_Request_complete(req);
+ mpi_errno = MPID_Request_complete(req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
*rreqp = NULL;
}
else
diff --git a/src/mpid/ch3/src/mpid_cancel_recv.c b/src/mpid/ch3/src/mpid_cancel_recv.c
index 9ac32bd..e46d4cb 100644
--- a/src/mpid/ch3/src/mpid_cancel_recv.c
+++ b/src/mpid/ch3/src/mpid_cancel_recv.c
@@ -13,6 +13,7 @@
int MPID_Cancel_recv(MPID_Request * rreq)
{
int netmod_cancelled = TRUE;
+ int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_CANCEL_RECV);
@@ -39,7 +40,10 @@ int MPID_Cancel_recv(MPID_Request * rreq)
"request 0x%08x cancelled", rreq->handle);
MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
MPIR_STATUS_SET_COUNT(rreq->status, 0);
- MPID_Request_complete(rreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else
{
@@ -47,6 +51,9 @@ int MPID_Cancel_recv(MPID_Request * rreq)
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
+ fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_CANCEL_RECV);
- return MPI_SUCCESS;
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
diff --git a/src/mpid/ch3/src/mpid_cancel_send.c b/src/mpid/ch3/src/mpid_cancel_send.c
index 772b98a..1609c21 100644
--- a/src/mpid/ch3/src/mpid_cancel_send.c
+++ b/src/mpid/ch3/src/mpid_cancel_send.c
@@ -76,7 +76,10 @@ int MPID_Cancel_send(MPID_Request * sreq)
MPID_Request_release(rreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else
{
@@ -292,6 +295,7 @@ int MPIDI_CH3_PktHandler_CancelSendResp( MPIDI_VC_t *vc ATTRIBUTE((unused)),
{
MPIDI_CH3_Pkt_cancel_send_resp_t * resp_pkt = &pkt->cancel_send_resp;
MPID_Request * sreq;
+ int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"received cancel send resp pkt, sreq=0x%08x, ack=%d",
@@ -323,11 +327,17 @@ int MPIDI_CH3_PktHandler_CancelSendResp( MPIDI_VC_t *vc ATTRIBUTE((unused)),
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
- MPID_Request_complete(sreq);
-
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
*rreqp = NULL;
- return MPI_SUCCESS;
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
/*
diff --git a/src/mpid/ch3/src/mpidi_isend_self.c b/src/mpid/ch3/src/mpidi_isend_self.c
index 9e9a5d1..93e7aa4 100644
--- a/src/mpid/ch3/src/mpidi_isend_self.c
+++ b/src/mpid/ch3/src/mpidi_isend_self.c
@@ -93,8 +93,15 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
MPIDI_CH3U_Buffer_copy(buf, count, datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_Request_complete(rreq);
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(rreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
}
else
{
@@ -130,7 +137,10 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
MPIR_STATUS_SET_COUNT(rreq->status, 0);
/* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
- MPID_Request_complete(sreq);
+ mpi_errno = MPID_Request_complete(sreq);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
/* --END ERROR HANDLING-- */
}
@@ -148,4 +158,6 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
*request = sreq;
return mpi_errno;
+ fn_fail:
+ goto fn_exit;
}
diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c
index a3933c1..80ddfd9 100644
--- a/src/mpid/common/sched/mpid_sched.c
+++ b/src/mpid/common/sched/mpid_sched.c
@@ -911,7 +911,11 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made
break;
}
- MPID_Request_complete(s->req);
+ mpi_errno = MPID_Request_complete(s->req);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
s->req = NULL;
MPIU_Free(s->entries);
MPIU_Free(s);
http://git.mpich.org/mpich.git/commitdiff/dfb295b2a13d7aa85b99db975dbffade8c18915c
commit dfb295b2a13d7aa85b99db975dbffade8c18915c
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Thu Jul 2 23:41:26 2015 -0500
Code-refactoring: use MPID_Request_create routine when creating send/recv requests
Most of work in macros of creating send/recv requests are the
same with MPID_Request_create routine. Here we directly call
MPID_Request_create from those macros to make the code simpler.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
index 2d661c8..b0203dc 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
@@ -76,25 +76,12 @@ static inline MPID_nem_ptl_req_area * REQ_PTL(MPID_Request *req) {
} while (0)
#define MPID_nem_ptl_request_create_sreq(sreq_, errno_, comm_) do { \
- (sreq_) = MPIU_Handle_obj_alloc(&MPID_Request_mem); \
- MPIU_ERR_CHKANDJUMP1((sreq_) == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "request"); \
+ (sreq_) = MPID_Request_create(); \
MPIU_Object_set_ref((sreq_), 2); \
(sreq_)->kind = MPID_REQUEST_SEND; \
MPIR_Comm_add_ref(comm_); \
(sreq_)->comm = comm_; \
- (sreq_)->greq_fns = NULL; \
- MPID_cc_set(&(sreq_)->cc, 1); \
- (sreq_)->cc_ptr = &(sreq_)->cc; \
(sreq_)->status.MPI_ERROR = MPI_SUCCESS; \
- MPIR_STATUS_SET_CANCEL_BIT((sreq_)->status, FALSE); \
- (sreq_)->dev.cancel_pending = FALSE; \
- (sreq_)->dev.state = 0; \
- (sreq_)->dev.datatype_ptr = NULL; \
- (sreq_)->dev.segment_ptr = NULL; \
- (sreq_)->dev.tmpbuf = NULL; \
- (sreq_)->dev.ext_hdr_ptr = NULL; \
- (sreq_)->dev.ext_hdr_sz = 0; \
- \
MPID_nem_ptl_init_req(sreq_); \
} while (0)
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 9d0f943..a08be2c 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -313,81 +313,29 @@ extern MPIDI_Process_t MPIDI_Process;
*/
#define MPIDI_Request_create_sreq(sreq_, mpi_errno_, FAIL_) \
{ \
- (sreq_) = MPIU_Handle_obj_alloc(&MPID_Request_mem); \
- if ((sreq_) == NULL) \
- { \
- MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"unable to allocate a request");\
- (mpi_errno_) = MPIR_ERR_MEMALLOCFAILED; \
- FAIL_; \
- } \
- MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE, \
- "allocated request, handle=0x%08x", (sreq_)->handle);\
- \
+ (sreq_) = MPID_Request_create(); \
MPIU_Object_set_ref((sreq_), 2); \
(sreq_)->kind = MPID_REQUEST_SEND; \
(sreq_)->comm = comm; \
- (sreq_)->greq_fns = NULL; \
- MPID_cc_set(&(sreq_)->cc, 1); \
- (sreq_)->cc_ptr = &(sreq_)->cc; \
(sreq_)->partner_request = NULL; \
MPIR_Comm_add_ref(comm); \
- (sreq_)->status.MPI_ERROR = MPI_SUCCESS; \
- MPIR_STATUS_SET_CANCEL_BIT((sreq_)->status, FALSE); \
- (sreq_)->dev.state = 0; \
- (sreq_)->dev.cancel_pending = FALSE; \
(sreq_)->dev.match.parts.rank = rank; \
(sreq_)->dev.match.parts.tag = tag; \
(sreq_)->dev.match.parts.context_id = comm->context_id + context_offset; \
(sreq_)->dev.user_buf = (void *) buf; \
(sreq_)->dev.user_count = count; \
- (sreq_)->dev.drop_data = FALSE; \
(sreq_)->dev.datatype = datatype; \
- (sreq_)->dev.datatype_ptr = NULL; \
- (sreq_)->dev.segment_ptr = NULL; \
- (sreq_)->dev.OnDataAvail = NULL; \
- (sreq_)->dev.OnFinal = NULL; \
(sreq_)->dev.iov_count = 0; \
- (sreq_)->dev.iov_offset = 0; \
- (sreq_)->dev.tmpbuf = NULL; \
- (sreq_)->dev.ext_hdr_ptr = NULL; \
- (sreq_)->dev.ext_hdr_sz = 0; \
MPIDI_Request_clear_dbg(sreq_); \
}
/* This is the receive request version of MPIDI_Request_create_sreq */
#define MPIDI_Request_create_rreq(rreq_, mpi_errno_, FAIL_) \
{ \
- (rreq_) = MPIU_Handle_obj_alloc(&MPID_Request_mem); \
- if ((rreq_) == NULL) \
- { \
- MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"unable to allocate a request");\
- (mpi_errno_) = MPIR_ERR_MEMALLOCFAILED; \
- FAIL_; \
- } \
- MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE, \
- "allocated request, handle=0x%08x", (rreq_)->handle);\
- \
+ (rreq_) = MPID_Request_create(); \
MPIU_Object_set_ref((rreq_), 2); \
(rreq_)->kind = MPID_REQUEST_RECV; \
- (rreq_)->comm = NULL; \
- (rreq_)->greq_fns = NULL; \
- MPID_cc_set(&(rreq_)->cc, 1); \
- (rreq_)->cc_ptr = &(rreq_)->cc; \
- (rreq_)->status.MPI_ERROR = MPI_SUCCESS; \
- MPIR_STATUS_SET_CANCEL_BIT((rreq_)->status, FALSE); \
(rreq_)->partner_request = NULL; \
- (rreq_)->dev.state = 0; \
- (rreq_)->dev.cancel_pending = FALSE; \
- (rreq_)->dev.datatype_ptr = NULL; \
- (rreq_)->dev.segment_ptr = NULL; \
- (rreq_)->dev.iov_offset = 0; \
- (rreq_)->dev.OnDataAvail = NULL; \
- (rreq_)->dev.OnFinal = NULL; \
- (rreq_)->dev.drop_data = FALSE; \
- (rreq_)->dev.tmpbuf = NULL; \
- (rreq_)->dev.ext_hdr_ptr = NULL; \
- (rreq_)->dev.ext_hdr_sz = 0; \
- MPIDI_CH3_REQUEST_INIT(rreq_);\
}
/* creates a new, trivially complete recv request that is suitable for
http://git.mpich.org/mpich.git/commitdiff/e33f33e387b938ded92e436178b33dc459dc0db3
commit e33f33e387b938ded92e436178b33dc459dc0db3
Author: Pavan Balaji <balaji at anl.gov>
Date: Sun Jul 12 16:12:55 2015 -0500
Avoid destroying request directly.
Use MPID_Request_release everywhere, and we internally check to see if
the reference count has touched zero, before destroying the request.
In some cases, we explicitly release the reference twice, where we
want both the progress engine and the user request to be freed at
once. This is still better than force setting the reference to 0.
We also completely remove the MPID_Request_destroy interface. It
should never be used. Inside the MPID_Request_cleanup routine, we
check for the reference count and destroy it if the reference count
falls to zero.
Signed-off-by: Xin Zhao <xinzhao3 at illinois.edu>
diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h
index ea80607..628a11f 100644
--- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h
+++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h
@@ -156,8 +156,10 @@ typedef union MPIDI_CH3_nem_pkt
if (mpi_errno != MPI_SUCCESS) \
{ \
if (NULL != _rts_req) { \
- MPIU_Object_set_ref(_rts_req, 0); \
- MPIDI_CH3_Request_destroy(_rts_req); \
+ /* error case: drop both the ch3 and nemesis \
+ * references, so the request can be cleanly freed */ \
+ MPID_Request_release(_rts_req); \
+ MPID_Request_release(_rts_req); \
} \
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rtspkt"); \
} \
@@ -167,8 +169,10 @@ typedef union MPIDI_CH3_nem_pkt
if (_rts_req->status.MPI_ERROR != MPI_SUCCESS) \
{ \
mpi_errno = _rts_req->status.MPI_ERROR; \
- MPIU_Object_set_ref(_rts_req, 0); \
- MPIDI_CH3_Request_destroy(_rts_req); \
+ /* error case: drop both the ch3 and nemesis \
+ * references, so the request can be cleanly freed */ \
+ MPID_Request_release(_rts_req); \
+ MPID_Request_release(_rts_req); \
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rtspkt"); \
} \
MPID_Request_release(_rts_req); \
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
index 58a619a..73f4b7d 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
@@ -86,7 +86,7 @@ int ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(struct MPIDI_VC *vc,
ret = fi_trecvmsg(gl_data.endpoint,&msg,msgflags);
if(ret == -ENOMSG) {
if (rreq_ptr) {
- MPIDI_CH3_Request_destroy(rreq);
+ MPID_Request_release(rreq);
*rreq_ptr = NULL;
*flag = 0;
}
diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h
index 2d38376..cc364ab 100644
--- a/src/mpid/ch3/include/mpid_rma_issue.h
+++ b/src/mpid/ch3/include/mpid_rma_issue.h
@@ -873,13 +873,22 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
/* --BEGIN ERROR HANDLING-- */
fn_fail:
if (rma_op->reqs_size == 1) {
- MPIDI_CH3_Request_destroy(rma_op->single_req);
+ /* error case: drop both our reference to the request and the
+ * progress engine's reference to it, since the progress
+ * engine didn't get a chance to see it yet. */
+ MPID_Request_release(rma_op->single_req);
+ MPID_Request_release(rma_op->single_req);
rma_op->single_req = NULL;
}
else if (rma_op->reqs_size > 1) {
for (i = 0; i < rma_op->reqs_size; i++) {
if (rma_op->multi_reqs[i] != NULL) {
- MPIDI_CH3_Request_destroy(rma_op->multi_reqs[i]);
+ /* error case: drop both our reference to the request
+ * and the progress engine's reference to it, since
+ * the progress engine didn't get a chance to see it
+ * yet. */
+ MPID_Request_release(rma_op->multi_reqs[i]);
+ MPID_Request_release(rma_op->multi_reqs[i]);
}
}
MPIU_Free(rma_op->multi_reqs);
diff --git a/src/mpid/ch3/include/mpidpost.h b/src/mpid/ch3/include/mpidpost.h
index aa0a079..25f84c1 100644
--- a/src/mpid/ch3/include/mpidpost.h
+++ b/src/mpid/ch3/include/mpidpost.h
@@ -17,20 +17,6 @@
*/
/*@
- MPIDI_CH3_Request_destroy - Release resources in use by an existing request
- object.
-
- Input Parameters:
-. req - pointer to the request object
-
- IMPLEMENTORS:
- MPIDI_CH3_Request_destroy() must call MPIDI_CH3U_Request_destroy() before
- request object is freed.
-@*/
-void MPIDI_CH3_Request_destroy(MPID_Request * req);
-
-
-/*@
MPIDI_CH3_Progress_start - Mark the beginning of a progress epoch.
Input Parameters:
@@ -147,18 +133,6 @@ int MPIDI_CH3_Comm_connect(char * port_name, int root, MPID_Comm * comm_ptr,
MPID_Comm ** newcomm);
-/*@
- MPIDI_CH3U_Request_destroy - Free resources associated with the channel
- device (ch3) component of a request.
-
- Input Parameters:
-. req - pointer to the request object
-
- IMPLEMENTORS:
- This routine must be called by MPIDI_CH3_Request_destroy().
-@*/
-void MPIDI_CH3U_Request_destroy(MPID_Request * req);
-
/* Include definitions from the channel which require items defined by this
file (mpidimpl.h) or the file it includes
(mpiimpl.h). */
@@ -188,21 +162,6 @@ void MPIDI_CH3U_Request_destroy(MPID_Request * req);
* Device level request management macros
*/
-/* We only export release and set completed on requests, since
- * other uses (such as incrementing the ref count) are done solely
- * by the device */
-
-#define MPID_Request_release(req_) \
-{ \
- int inuse_; \
- \
- MPIR_Request_release_ref((req_), &inuse_); \
- if (inuse_ == 0) \
- { \
- MPIDI_CH3_Request_destroy(req_); \
- } \
-}
-
/*
* Device level progress engine macros
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index b11e196..a05b2ff 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -690,7 +690,7 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr,
* operation. */
if (op->reqs_size == 1) {
MPIU_Assert(op->single_req != NULL);
- MPIDI_CH3_Request_destroy(op->single_req);
+ MPID_Request_release(op->single_req);
op->single_req = NULL;
win_ptr->active_req_cnt--;
op->reqs_size = 0;
@@ -699,7 +699,7 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr,
MPIU_Assert(op->multi_reqs != NULL);
for (i = 0; i < op->reqs_size; i++) {
if (op->multi_reqs[i] != NULL) {
- MPIDI_CH3_Request_destroy(op->multi_reqs[i]);
+ MPID_Request_release(op->multi_reqs[i]);
op->multi_reqs[i] = NULL;
win_ptr->active_req_cnt--;
}
diff --git a/src/mpid/ch3/src/ch3u_eagersync.c b/src/mpid/ch3/src/ch3u_eagersync.c
index aa8efb3..877699b 100644
--- a/src/mpid/ch3/src/ch3u_eagersync.c
+++ b/src/mpid/ch3/src/ch3u_eagersync.c
@@ -81,8 +81,7 @@ int MPIDI_CH3_EagerSyncNoncontigSend( MPID_Request **sreq_p,
{
/* Make sure to destroy the request before setting the pointer to
* NULL, otherwise we lose the handle on the request */
- MPIU_Object_set_ref(sreq, 0);
- MPIDI_CH3_Request_destroy(sreq);
+ MPID_Request_release(sreq);
*sreq_p = NULL;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 8aa9af1..f1bde26 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -110,76 +110,6 @@ MPID_Request * MPID_Request_create(void)
return req;
}
-/* FIXME: We need a lighter-weight version of this to avoid all of the
- extra checks. One posibility would be a single, no special case (no
- comm, datatype, or srbuf to check) and a more general (check everything)
- version. */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_Request_destroy
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-void MPIDI_CH3_Request_destroy(MPID_Request * req)
-{
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQUEST_DESTROY);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQUEST_DESTROY);
- MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE,
- "freeing request, handle=0x%08x", req->handle);
-
-#ifdef MPICH_DBG_OUTPUT
- /*MPIU_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPID_REQUEST);*/
- if (HANDLE_GET_MPI_KIND(req->handle) != MPID_REQUEST)
- {
- int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
- FCNAME, __LINE__, MPI_ERR_OTHER,
- "**invalid_handle", "**invalid_handle %d", req->handle);
- MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
- }
- /* XXX DJG FIXME should we be checking this? */
- /*MPIU_Assert(req->ref_count == 0);*/
- if (req->ref_count != 0)
- {
- int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
- FCNAME, __LINE__, MPI_ERR_OTHER,
- "**invalid_refcount", "**invalid_refcount %d", req->ref_count);
- MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
- }
-#endif
-
- /* FIXME: We need a better way to handle these so that we
- do not always need to initialize these fields and check them
- when we destroy a request */
- /* FIXME: We need a way to call these routines ONLY when the
- related ref count has become zero. */
- if (req->comm != NULL) {
- MPIR_Comm_release(req->comm, 0);
- }
-
- if (req->greq_fns != NULL) {
- MPIU_Free(req->greq_fns);
- }
-
- if (req->dev.datatype_ptr != NULL) {
- MPID_Datatype_release(req->dev.datatype_ptr);
- }
-
- if (req->dev.segment_ptr != NULL) {
- MPID_Segment_free(req->dev.segment_ptr);
- }
-
- if (MPIDI_Request_get_srbuf_flag(req)) {
- MPIDI_CH3U_SRBuf_free(req);
- }
-
- if (req->dev.ext_hdr_ptr != NULL) {
- MPIU_Free(req->dev.ext_hdr_ptr);
- }
-
- MPIU_Handle_obj_free(&MPID_Request_mem, req);
-
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQUEST_DESTROY);
-}
-
/* ------------------------------------------------------------------------- */
/* Here are the routines to manipulate the iovs in the requests */
@@ -686,3 +616,65 @@ void MPID_Request_complete(MPID_Request *req)
MPIDI_CH3_Progress_signal_completion();
}
}
+
+void MPID_Request_release(MPID_Request *req)
+{
+ int inuse;
+
+ MPIR_Request_release_ref(req, &inuse);
+ if (inuse == 0) {
+ MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE,
+ "freeing request, handle=0x%08x", req->handle);
+
+#ifdef MPICH_DBG_OUTPUT
+ /*MPIU_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPID_REQUEST);*/
+ if (HANDLE_GET_MPI_KIND(req->handle) != MPID_REQUEST)
+ {
+ int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
+ FCNAME, __LINE__, MPI_ERR_OTHER,
+ "**invalid_handle", "**invalid_handle %d", req->handle);
+ MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
+ }
+ /* XXX DJG FIXME should we be checking this? */
+ /*MPIU_Assert(req->ref_count == 0);*/
+ if (req->ref_count != 0)
+ {
+ int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
+ FCNAME, __LINE__, MPI_ERR_OTHER,
+ "**invalid_refcount", "**invalid_refcount %d", req->ref_count);
+ MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
+ }
+#endif
+
+ /* FIXME: We need a better way to handle these so that we do
+ not always need to initialize these fields and check them
+ when we destroy a request */
+ /* FIXME: We need a way to call these routines ONLY when the
+ related ref count has become zero. */
+ if (req->comm != NULL) {
+ MPIR_Comm_release(req->comm, 0);
+ }
+
+ if (req->greq_fns != NULL) {
+ MPIU_Free(req->greq_fns);
+ }
+
+ if (req->dev.datatype_ptr != NULL) {
+ MPID_Datatype_release(req->dev.datatype_ptr);
+ }
+
+ if (req->dev.segment_ptr != NULL) {
+ MPID_Segment_free(req->dev.segment_ptr);
+ }
+
+ if (MPIDI_Request_get_srbuf_flag(req)) {
+ MPIDI_CH3U_SRBuf_free(req);
+ }
+
+ if (req->dev.ext_hdr_ptr != NULL) {
+ MPIU_Free(req->dev.ext_hdr_ptr);
+ }
+
+ MPIU_Handle_obj_free(&MPID_Request_mem, req);
+ }
+}
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 5695c50..284db47 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -530,8 +530,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
- MPIU_Object_set_ref(req, 0);
- MPIDI_CH3_Request_destroy(req);
+ MPID_Request_release(req);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
@@ -548,8 +547,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
- MPIU_Object_set_ref(req, 0);
- MPIDI_CH3_Request_destroy(req);
+ MPID_Request_release(req);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
@@ -970,8 +968,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
- MPIU_Object_set_ref(resp_req, 0);
- MPIDI_CH3_Request_destroy(resp_req);
+ MPID_Request_release(resp_req);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
diff --git a/src/mpid/ch3/src/mpid_cancel_send.c b/src/mpid/ch3/src/mpid_cancel_send.c
index 51d39b6..772b98a 100644
--- a/src/mpid/ch3/src/mpid_cancel_send.c
+++ b/src/mpid/ch3/src/mpid_cancel_send.c
@@ -67,17 +67,16 @@ int MPID_Cancel_send(MPID_Request * sreq)
"send-to-self cancellation successful, sreq=0x%08x, rreq=0x%08x",
sreq->handle, rreq->handle));
- /* Pull the message out of the unexpected queue since it's being
- * cancelled */
- MPIU_Object_set_ref(rreq, 0);
- MPIDI_CH3_Request_destroy(rreq);
+ /* Pull the message out of the unexpected queue since it's
+ * being cancelled. The below request release drops one
+ * reference. We explicitly drop a second reference,
+ * because the receive request will never be visible to
+ * the user. */
+ MPID_Request_release(rreq);
+ MPID_Request_release(rreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
- /* no other thread should be waiting on sreq, so it is safe to
- reset ref_count and cc */
- MPID_cc_set(&sreq->cc, 0);
- /* FIXME should be a decr and assert, not a set */
- MPIU_Object_set_ref(sreq, 1);
+ MPID_Request_complete(sreq);
}
else
{
diff --git a/src/mpid/ch3/src/mpidi_isend_self.c b/src/mpid/ch3/src/mpidi_isend_self.c
index 5136105..9e9a5d1 100644
--- a/src/mpid/ch3/src/mpidi_isend_self.c
+++ b/src/mpid/ch3/src/mpidi_isend_self.c
@@ -45,10 +45,12 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
/* --BEGIN ERROR HANDLING-- */
if (rreq == NULL)
{
- /* Set the refcount to 0 since the user will never have a chance to
- * release their reference */
- MPIU_Object_set_ref(sreq, 0);
- MPIDI_CH3_Request_destroy(sreq);
+ /* We release the send request twice, once to release the
+ * progress engine reference and the second to release the
+ * user reference since the user will never have a chance to
+ * release their reference. */
+ MPID_Request_release(sreq);
+ MPID_Request_release(sreq);
sreq = NULL;
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**nomem",
"**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
@@ -60,8 +62,12 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
* which this message is being sent has been revoked and we shouldn't
* bother finishing this. */
if (!found && MPID_cc_get(rreq->cc) == 0) {
- MPIU_Object_set_ref(sreq, 0);
- MPIDI_CH3_Request_destroy(sreq);
+ /* We release the send request twice, once to release the
+ * progress engine reference and the second to release the
+ * user reference since the user will never have a chance to
+ * release their reference. */
+ MPID_Request_release(sreq);
+ MPID_Request_release(sreq);
sreq = NULL;
goto fn_exit;
}
@@ -124,8 +130,7 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
MPIR_STATUS_SET_COUNT(rreq->status, 0);
/* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
- MPIU_Object_set_ref(sreq, 1);
- MPID_cc_set(&sreq->cc, 0);
+ MPID_Request_complete(sreq);
/* --END ERROR HANDLING-- */
}
diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c
index 8ffc5e9..a3933c1 100644
--- a/src/mpid/common/sched/mpid_sched.c
+++ b/src/mpid/common/sched/mpid_sched.c
@@ -424,12 +424,7 @@ fn_fail:
if (*req)
*req = NULL;
if (r) {
- /* overly complicated, but we can't just destroy the req directly
- * because we aren't truly inside the device and don't have access to
- * MPIDI_CH3_Request_destroy (we may not even be CH3) */
- int inuse = TRUE;
- MPIR_Request_release_ref(r, &inuse); /* the schedule's ref */
- MPIU_Assert(inuse);
+ MPID_Request_release(r); /* the schedule's ref */
MPID_Request_release(r); /* the user's ref */
}
http://git.mpich.org/mpich.git/commitdiff/658ad69657546b8b4e1104aea6aa7182df804165
commit 658ad69657546b8b4e1104aea6aa7182df804165
Author: Pavan Balaji <balaji at anl.gov>
Date: Mon Jul 6 12:55:49 2015 -0500
Use better variable name in macro.
Signed-off-by: Xin Zhao <xinzhao3 at illinois.edu>
diff --git a/src/mpid/common/datatype/mpid_datatype.h b/src/mpid/common/datatype/mpid_datatype.h
index 7fd50be..073fd32 100644
--- a/src/mpid/common/datatype/mpid_datatype.h
+++ b/src/mpid/common/datatype/mpid_datatype.h
@@ -58,10 +58,10 @@
* structures.
*/
#define MPID_Datatype_release(datatype_ptr) do { \
- int inuse; \
+ int inuse_; \
\
- MPIU_Object_release_ref((datatype_ptr),&inuse); \
- if (!inuse) { \
+ MPIU_Object_release_ref((datatype_ptr),&inuse_); \
+ if (!inuse_) { \
int lmpi_errno = MPI_SUCCESS; \
if (MPIR_Process.attr_free && datatype_ptr->attributes) { \
lmpi_errno = MPIR_Process.attr_free( datatype_ptr->handle, \
http://git.mpich.org/mpich.git/commitdiff/563122c3088a3fa10e93175170ec99f58dea8b89
commit 563122c3088a3fa10e93175170ec99f58dea8b89
Author: Pavan Balaji <balaji at anl.gov>
Date: Sat Jul 4 20:04:38 2015 -0500
Change MPID_Request_set_completed to MPID_Request_complete.
We should never blindly set the completion counter to zero. We should
always decrement it. If we think that the counter should be 1 before
decrementing, we can assert that it is zero after the decrement.
Signed-off-by: Xin Zhao <xinzhao3 at illinois.edu>
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index 9bf3442..547ebc5 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -3494,13 +3494,13 @@ int MPID_Progress_poke(void);
This routine is intended for use by 'MPI_Grequest_start' only. Note that
once a request is created with this routine, any progress engine must assume
that an outside function can complete a request with
- 'MPID_Request_set_completed'.
+ 'MPID_Request_complete'.
The request object returned by this routine should be initialized such that
ref_count is one and handle contains a valid handle referring to the object.
@*/
MPID_Request * MPID_Request_create(void);
-void MPID_Request_set_completed(MPID_Request *);
+
/*@
MPID_Request_release - Release a request
@@ -3517,6 +3517,23 @@ void MPID_Request_set_completed(MPID_Request *);
@*/
void MPID_Request_release(MPID_Request *);
+/*@
+ MPID_Request_complete - Complete a request
+
+ Input Parameter:
+. request - request to complete
+
+ Notes:
+ This routine is called to decrement the completion count of a
+ request object. If the completion count of the request object has
+ reached zero, the reference count for the object will be
+ decremented.
+
+ Module:
+ Request
+@*/
+void MPID_Request_complete(MPID_Request *);
+
typedef struct MPID_Grequest_class {
MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
MPI_Grequest_query_function *query_fn;
diff --git a/src/mpi/pt2pt/greq_complete.c b/src/mpi/pt2pt/greq_complete.c
index eba269c..8c70240 100644
--- a/src/mpi/pt2pt/greq_complete.c
+++ b/src/mpi/pt2pt/greq_complete.c
@@ -37,7 +37,7 @@ void MPIR_Grequest_complete_impl(MPID_Request *request_ptr)
{
/* Set the request as completed. This does not change the
reference count on the generalized request */
- MPID_Request_set_completed( request_ptr );
+ MPID_Request_complete( request_ptr );
/* The request release comes with the wait/test, not this complete
routine, so we don't call the MPID_Request_release routine */
diff --git a/src/mpi/pt2pt/greq_start.c b/src/mpi/pt2pt/greq_start.c
index 8137024..23da864 100644
--- a/src/mpi/pt2pt/greq_start.c
+++ b/src/mpi/pt2pt/greq_start.c
@@ -102,6 +102,11 @@ int MPIR_Grequest_start_impl(MPI_Grequest_query_function *query_fn,
(*request_ptr)->greq_fns->grequest_extra_state = extra_state;
(*request_ptr)->greq_fns->greq_lang = MPID_LANG_C;
+ /* Add an additional reference to the greq. One of them will be
+ * released when we complete the request, and the second one, when
+ * we test or wait on it. */
+ MPIR_Request_add_ref((*request_ptr));
+
MPIU_CHKPMEM_COMMIT();
fn_exit:
return mpi_errno;
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 25ae448..9d0f943 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1242,9 +1242,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
This code is designed to support one particular model of thread-safety.
It is common to many of the channels and was moved into this file because
the MPIDI_CH3_Progress_signal_completion reference is used by the
- macro the implements MPID_Request_set_completed. Note that there is
- a function version of MPID_Request_set_completed for use by greq_complete.c
-
+ function the implements MPID_Request_complete.
@*/
/*
diff --git a/src/mpid/ch3/src/ch3u_buffer.c b/src/mpid/ch3/src/ch3u_buffer.c
index 52d5dbd..c2ef71b 100644
--- a/src/mpid/ch3/src/ch3u_buffer.c
+++ b/src/mpid/ch3/src/ch3u_buffer.c
@@ -240,8 +240,7 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
buf, count, datatype, &data_sz,
&rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_Request_set_completed(sreq);
- MPID_Request_release(sreq);
+ MPID_Request_complete(sreq);
}
else
{
@@ -252,9 +251,7 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
/* no other thread can possibly be waiting on rreq, so it is safe to
reset ref_count and cc */
- /* FIXME DJG really? shouldn't we at least decr+assert? */
- MPID_cc_set(rreq->cc_ptr, 0);
- MPIU_Object_set_ref(rreq, 1);
+ MPID_Request_complete(rreq);
return MPI_SUCCESS;
}
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 68b8da0..8aa9af1 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -676,45 +676,6 @@ int MPIDI_CH3U_Request_unpack_uebuf(MPID_Request * rreq)
return mpi_errno;
}
-/* MT note: The following order of operations is _essential_ for correct
- * operation of the fine-grained multithreading code. Assume that
- * _signal_completion() acquires and releases a mutex in order to update the
- * global completion counter (it does for fine-grained ch3:nemesis). Further,
- * assume the following standard pattern is used by the request consumer to wait
- * for completion:
- *
- * if (req is not complete (req->cc!=0)) {
- * // progress_enter:
- * acquire mutex;
- * my_count = global_count;
- * release mutex;
- *
- * while (req is not complete (req->cc!=0)) {
- * progress_wait(&my_count);
- * }
- * }
- *
- * Where progress_wait will attempt to make progress forever as long as
- * (my_count==global_count). If it is possible for the consumer to see the
- * global completion count before seeing the request's completion counter drop
- * to zero, the consumer could spin in progress_wait forever without a chance to
- * retest the request.
- *
- * If the mutex approach is dropped in favor of atomic access, additional memory
- * barriers must be inserted. The mutex acquire/release currently enforces
- * sufficient ordering constraints provided the statement order below is not
- * accidentally inverted.
- *
- * See also the note above the MSGQUEUE CS macros and request completion in
- * mpiimplthread.h.
- */
-void MPID_Request_set_completed(MPID_Request *req)
-{
- MPID_cc_set(req->cc_ptr, 0);
- /* MT do not reorder! see note above*/
- MPIDI_CH3_Progress_signal_completion();
-}
-
void MPID_Request_complete(MPID_Request *req)
{
int incomplete;
diff --git a/src/mpid/ch3/src/ch3u_rma_reqops.c b/src/mpid/ch3/src/ch3u_rma_reqops.c
index 0a870ce..56a596c 100644
--- a/src/mpid/ch3/src/ch3u_rma_reqops.c
+++ b/src/mpid/ch3/src/ch3u_rma_reqops.c
@@ -41,12 +41,11 @@ int MPID_Rput(const void *origin_addr, int origin_count,
MPIU_ERR_CHKANDJUMP(ureq == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
ureq->kind = MPID_WIN_REQUEST;
+ /* This request is referenced by user and ch3 by default. */
+ MPIU_Object_set_ref(ureq, 2);
+
/* Enqueue or perform the RMA operation */
if (target_rank != MPI_PROC_NULL && data_sz != 0) {
-
- /* This request is referenced by user and ch3 by default. */
- MPIU_Object_set_ref(ureq, 2);
-
mpi_errno = MPIDI_CH3I_Put(origin_addr, origin_count,
origin_datatype, target_rank,
target_disp, target_count, target_datatype, win_ptr, ureq);
@@ -56,8 +55,7 @@ int MPID_Rput(const void *origin_addr, int origin_count,
}
}
else {
- /* set cc=0 if it is not a valid operation. */
- MPID_Request_set_completed(ureq);
+ MPID_Request_complete(ureq);
}
*request = ureq;
@@ -103,12 +101,11 @@ int MPID_Rget(void *origin_addr, int origin_count,
MPIU_ERR_CHKANDJUMP(ureq == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
ureq->kind = MPID_WIN_REQUEST;
+ /* This request is referenced by user and ch3 by default. */
+ MPIU_Object_set_ref(ureq, 2);
+
/* Enqueue or perform the RMA operation */
if (target_rank != MPI_PROC_NULL && data_sz != 0) {
-
- /* This request is referenced by user and ch3 by default. */
- MPIU_Object_set_ref(ureq, 2);
-
mpi_errno = MPIDI_CH3I_Get(origin_addr, origin_count,
origin_datatype, target_rank,
target_disp, target_count, target_datatype, win_ptr, ureq);
@@ -118,8 +115,7 @@ int MPID_Rget(void *origin_addr, int origin_count,
}
}
else {
- /* set cc=0 if it is not a valid operation. */
- MPID_Request_set_completed(ureq);
+ MPID_Request_complete(ureq);
}
*request = ureq;
@@ -163,14 +159,13 @@ int MPID_Raccumulate(const void *origin_addr, int origin_count,
MPIU_ERR_CHKANDJUMP(ureq == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
ureq->kind = MPID_WIN_REQUEST;
+ /* This request is referenced by user and ch3 by default. */
+ MPIU_Object_set_ref(ureq, 2);
+
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
/* Enqueue or perform the RMA operation */
if (target_rank != MPI_PROC_NULL && data_sz != 0) {
-
- /* This request is referenced by user and ch3 by default. */
- MPIU_Object_set_ref(ureq, 2);
-
mpi_errno = MPIDI_CH3I_Accumulate(origin_addr, origin_count,
origin_datatype, target_rank,
target_disp, target_count,
@@ -180,8 +175,7 @@ int MPID_Raccumulate(const void *origin_addr, int origin_count,
}
}
else {
- /* set cc=0 if it is not a valid operation. */
- MPID_Request_set_completed(ureq);
+ MPID_Request_complete(ureq);
}
*request = ureq;
@@ -226,16 +220,15 @@ int MPID_Rget_accumulate(const void *origin_addr, int origin_count,
MPIU_ERR_CHKANDJUMP(ureq == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
ureq->kind = MPID_WIN_REQUEST;
+ /* This request is referenced by user and ch3 by default. */
+ MPIU_Object_set_ref(ureq, 2);
+
/* Note that GACC is only a no-op if no data goes in both directions */
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, trg_data_sz, dtp, dt_true_lb);
/* Enqueue or perform the RMA operation */
if (target_rank != MPI_PROC_NULL && (data_sz != 0 || trg_data_sz != 0)) {
-
- /* This request is referenced by user and ch3 by default. */
- MPIU_Object_set_ref(ureq, 2);
-
mpi_errno = MPIDI_CH3I_Get_accumulate(origin_addr, origin_count,
origin_datatype, result_addr,
result_count, result_datatype,
@@ -246,8 +239,7 @@ int MPID_Rget_accumulate(const void *origin_addr, int origin_count,
}
}
else {
- /* set cc=0 if it is not a valid operation. */
- MPID_Request_set_completed(ureq);
+ MPID_Request_complete(ureq);
}
*request = ureq;
diff --git a/src/mpid/ch3/src/mpid_cancel_recv.c b/src/mpid/ch3/src/mpid_cancel_recv.c
index 8563a52..9ac32bd 100644
--- a/src/mpid/ch3/src/mpid_cancel_recv.c
+++ b/src/mpid/ch3/src/mpid_cancel_recv.c
@@ -39,8 +39,7 @@ int MPID_Cancel_recv(MPID_Request * rreq)
"request 0x%08x cancelled", rreq->handle);
MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
MPIR_STATUS_SET_COUNT(rreq->status, 0);
- MPID_Request_set_completed(rreq);
- MPID_Request_release(rreq);
+ MPID_Request_complete(rreq);
}
else
{
diff --git a/src/mpid/ch3/src/mpidi_isend_self.c b/src/mpid/ch3/src/mpidi_isend_self.c
index 85f9893..5136105 100644
--- a/src/mpid/ch3/src/mpidi_isend_self.c
+++ b/src/mpid/ch3/src/mpidi_isend_self.c
@@ -87,11 +87,8 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
MPIDI_CH3U_Buffer_copy(buf, count, datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_Request_set_completed(rreq);
- MPID_Request_release(rreq);
- /* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
- MPIU_Object_set_ref(sreq, 1);
- MPID_cc_set(&sreq->cc, 0);
+ MPID_Request_complete(rreq);
+ MPID_Request_complete(sreq);
}
else
{
diff --git a/src/mpid/common/hcoll/hcoll_rte.c b/src/mpid/common/hcoll/hcoll_rte.c
index b6a3ea7..12d0c2b 100644
--- a/src/mpid/common/hcoll/hcoll_rte.c
+++ b/src/mpid/common/hcoll/hcoll_rte.c
@@ -420,7 +420,7 @@ static void coll_handle_complete(void *handle)
MPID_Request *req;
if (NULL != handle) {
req = (MPID_Request *) handle;
- MPID_Request_set_completed(req);
+ MPID_Request_complete(req);
}
}
diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c
index 2227a30..8ffc5e9 100644
--- a/src/mpid/common/sched/mpid_sched.c
+++ b/src/mpid/common/sched/mpid_sched.c
@@ -916,8 +916,7 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made
break;
}
- MPID_Request_set_completed(s->req);
- MPID_Request_release(s->req);
+ MPID_Request_complete(s->req);
s->req = NULL;
MPIU_Free(s->entries);
MPIU_Free(s);
http://git.mpich.org/mpich.git/commitdiff/9e8299df6905a2c894d8ebc60a43b40a3a5545a7
commit 9e8299df6905a2c894d8ebc60a43b40a3a5545a7
Author: Pavan Balaji <balaji at anl.gov>
Date: Sat Jul 4 20:07:06 2015 -0500
Replace MPIDI_CH3U_Request_complete with MPID_Request_complete.
We upgraded request completion to an MPID_ function, so higher level
functions (particularly generalized request functions) can use them.
Without this, we were force setting the completion count to 0 in those
functions, which is unsafe.
Signed-off-by: Xin Zhao <xinzhao3 at illinois.edu>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
index f14d22d..e25a81c 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_poll.c
@@ -134,7 +134,7 @@ static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid)
if (reqFn == 0) {
MPIU_Assert(reqtype != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_probe.c b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_probe.c
index a060353..9100a1a 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_probe.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_probe.c
@@ -213,7 +213,7 @@ int MPID_nem_llc_improbe(MPIDI_VC_t * vc, int source, int tag, MPID_Comm * comm,
mpi_errno = MPID_nem_llc_poll(0);
} while (!MPID_Request_is_complete(req));
-// MPIDI_CH3U_Request_complete(req); // This operation is done in llc_poll.
+// MPID_Request_complete(req); // This operation is done in llc_poll.
*message = req;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_send.c b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_send.c
index e0e0c04..a7d93ce 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/llc/llc_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/llc/llc_send.c
@@ -451,7 +451,7 @@ int MPID_nem_llc_send_queued(MPIDI_VC_t * vc, rque_t * send_queue)
sreq->status.MPI_ERROR = mpi_errno;
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "OnDataAvail = %p", sreq->dev.OnDataAvail);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
continue;
}
if (!MPIDI_nem_llc_Rqst_iov_update(sreq, ret)) {
@@ -881,7 +881,7 @@ int llc_poll(int in_blocking_poll, llc_send_f sfnc, llc_recv_f rfnc)
}
/* Mark completion on rreq */
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
llc_errno = LLC_cmd_free(lcmd, 1);
MPIU_ERR_CHKANDJUMP(llc_errno, mpi_errno, MPI_ERR_OTHER, "**LLC_cmd_free");
@@ -905,7 +905,7 @@ int llc_poll(int in_blocking_poll, llc_send_f sfnc, llc_recv_f rfnc)
req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET(req->status.MPI_ERROR, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
if (lcmd->iov_local[0].addr != 0) {
MPIU_Free((void *) lcmd->iov_local[0].addr);
@@ -916,7 +916,7 @@ int llc_poll(int in_blocking_poll, llc_send_f sfnc, llc_recv_f rfnc)
req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET(req->status.MPI_ERROR, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
else if (lcmd->opcode == LLC_OPCODE_RECV) {
/* Probably ch3 dequeued and completed this request. */
@@ -924,7 +924,7 @@ int llc_poll(int in_blocking_poll, llc_send_f sfnc, llc_recv_f rfnc)
MPIDI_CH3U_Recvq_DP(req);
req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET(req->status.MPI_ERROR, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
#endif
}
else {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
index 602373d..7c40109 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c
@@ -155,7 +155,7 @@ int MPID_nem_mxm_improbe(MPIDI_VC_t * vc, int source, int tag, MPID_Comm * comm,
err = mxm_message_recv(&mxm_req, mxm_msg);
_mxm_req_wait(&mxm_req.base);
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
*message = req;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
index f6ac722..04039bd 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
@@ -171,7 +171,7 @@ static inline int MPID_nem_ofi_handle_packet(cq_tagged_entry_t * wc ATTRIBUTE((u
MPI_RC(MPID_nem_handle_pkt(vc, REQ_OFI(rreq)->pack_buffer, REQ_OFI(rreq)->pack_buffer_size));
MPIU_Free(REQ_OFI(rreq)->pack_buffer);
}
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
END_FUNC_RC(FCNAME);
}
@@ -187,7 +187,7 @@ static inline int MPID_nem_ofi_cts_send_callback(cq_tagged_entry_t * wc, MPID_Re
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
MPI_RC(MPID_nem_ofi_handle_packet(wc, REQ_OFI(sreq)->parent));
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
END_FUNC_RC(FCNAME);
}
@@ -279,7 +279,7 @@ int MPID_nem_ofi_connect_to_root_callback(cq_tagged_entry_t * wc ATTRIBUTE((unus
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
END_FUNC(FCNAME);
return mpi_errno;
@@ -375,13 +375,13 @@ int MPID_nem_ofi_cm_finalize()
&(REQ_OFI(gl_data.persistent_req)->ofi_context)), cancel);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.persistent_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.persistent_req->status, 0);
- MPIDI_CH3U_Request_complete(gl_data.persistent_req);
+ MPID_Request_complete(gl_data.persistent_req);
FI_RC(fi_cancel((fid_t) gl_data.endpoint, &(REQ_OFI(gl_data.conn_req)->ofi_context)), cancel);
MPIU_Free(gl_data.conn_req->dev.user_buf);
MPIR_STATUS_SET_CANCEL_BIT(gl_data.conn_req->status, TRUE);
MPIR_STATUS_SET_COUNT(gl_data.conn_req->status, 0);
- MPIDI_CH3U_Request_complete(gl_data.conn_req);
+ MPID_Request_complete(gl_data.conn_req);
fn_exit:
END_FUNC(FCNAME);
return mpi_errno;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index e7b042a..c13268b 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -146,7 +146,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
msg.data = 0ULL;
FI_RC_RETRY(fi_tsendmsg(gl_data.endpoint,&msg,0ULL),tsend);
}
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
break;
case MPID_MSG_CTS | MPID_MSG_RTS | MPID_MSG_DATA:
if (REQ_OFI(sreq)->pack_buffer)
@@ -157,7 +157,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
else {
vc = REQ_OFI(sreq)->vc;
@@ -166,7 +166,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
gl_data.rts_cts_in_flight--;
break;
case MPID_MSG_RTS:
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
break;
}
END_FUNC_RC(FCNAME);
@@ -183,7 +183,7 @@ static int MPID_nem_ofi_cts_recv_callback(cq_tagged_entry_t * wc, MPID_Request *
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
MPI_RC(MPID_nem_ofi_data_callback(wc, REQ_OFI(rreq)->parent));
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
index 875d6d7..c69b6d3 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
@@ -41,8 +41,8 @@ static inline int MPID_nem_ofi_sync_recv_callback(cq_tagged_entry_t * wc ATTRIBU
BEGIN_FUNC(FCNAME);
MPIDI_CH3U_Recvq_DP(REQ_OFI(rreq)->parent);
- MPIDI_CH3U_Request_complete(REQ_OFI(rreq)->parent);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(REQ_OFI(rreq)->parent);
+ MPID_Request_complete(rreq);
END_FUNC(FCNAME);
return mpi_errno;
@@ -61,7 +61,7 @@ static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((u
BEGIN_FUNC(FCNAME);
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
END_FUNC(FCNAME);
return mpi_errno;
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
index 28d33c7..4942f53 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
@@ -90,7 +90,7 @@ int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request
/* request object */
/* ---------------------------------------------------- */
MPIDI_CH3U_Recvq_DP(rreq);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
}
END_FUNC_RC(FCNAME);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
index 2097003..c298460 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c
@@ -421,7 +421,7 @@ static inline void on_data_avail(MPID_Request * req)
reqFn = req->dev.OnDataAvail;
if (!reqFn) {
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
index d04ba04..7f45331 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_probe.c
@@ -29,7 +29,7 @@ static int handle_probe(const ptl_event_t *e)
MPIR_STATUS_SET_COUNT(req->status, NPTL_HEADER_GET_LENGTH(e->hdr_data));
fn_exit:
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
MPIDI_FUNC_EXIT(MPID_STATE_HANDLE_PROBE);
return mpi_errno;
fn_fail:
@@ -78,7 +78,7 @@ static int handle_mprobe(const ptl_event_t *e)
fn_exit:
MPIU_CHKPMEM_COMMIT();
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
MPIDI_FUNC_EXIT(MPID_STATE_HANDLE_PROBE);
return mpi_errno;
fn_fail:
@@ -389,7 +389,7 @@ int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *p
for (i = 0; i < REQ_PTL(sreq)->num_gets; i++) {
ret = PtlMEUnlink(REQ_PTL(sreq)->get_me_p[i]);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
@@ -400,7 +400,7 @@ int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *p
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
*rreqp = NULL;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
index 357630c..11e5a0d 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
@@ -70,7 +70,7 @@ static int handler_recv_complete(const ptl_event_t *e)
if (REQ_PTL(rreq)->chunk_buffer[i])
MPIU_Free(REQ_PTL(rreq)->chunk_buffer[i]);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_COMPLETE);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
index 31ad075..b982b28 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
@@ -86,7 +86,7 @@ static int handler_send(const ptl_event_t *e)
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
}
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_SEND);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
index 1e6bc28..1311afe 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c
@@ -1619,7 +1619,7 @@ static int MPID_nem_tcp_recv_handler(sockconn_t *const sc)
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "...complete");
sc_vc_ch->recv_active = NULL;
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
index e140c01..744fca7 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c
@@ -149,7 +149,7 @@ int MPID_nem_tcp_send_queued(MPIDI_VC_t *vc, MPIDI_nem_tcp_request_queue_t *send
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
@@ -575,7 +575,7 @@ int MPID_nem_tcp_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPID
if (!reqFn)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -799,7 +799,7 @@ int MPID_nem_tcp_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn)
{
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -870,7 +870,7 @@ int MPID_nem_tcp_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno)
MPIDI_CH3I_Sendq_dequeue(&vc_tcp->send_queue, &req);
req->status.MPI_ERROR = req_errno;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
/* paused send queue */
@@ -878,7 +878,7 @@ int MPID_nem_tcp_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno)
MPIDI_CH3I_Sendq_dequeue(&vc_tcp->paused_send_queue, &req);
req->status.MPI_ERROR = req_errno;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
fn_exit:
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_isend.c b/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
index 91594e0..2bf0863 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_isend.c
@@ -27,7 +27,7 @@ int MPIDI_CH3_iSend (MPIDI_VC_t *vc, MPID_Request *sreq, void * hdr, MPIDI_msg_s
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
sreq->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(sreq->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
goto fn_fail;
}
@@ -67,7 +67,7 @@ int MPIDI_CH3_iSend (MPIDI_VC_t *vc, MPID_Request *sreq, void * hdr, MPIDI_msg_s
if (!reqFn)
{
MPIU_Assert (MPIDI_Request_get_type (sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete (sreq);
+ MPID_Request_complete (sreq);
}
else
{
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c b/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
index b98455e..d7832f4 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_isendv.c
@@ -32,7 +32,7 @@ int MPIDI_CH3_iSendv (MPIDI_VC_t *vc, MPID_Request *sreq, MPID_IOV *iov, int n_i
sreq->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(sreq->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
MPIU_ERR_SET1(mpi_errno, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
goto fn_fail;
}
@@ -126,7 +126,7 @@ int MPIDI_CH3_iSendv (MPIDI_VC_t *vc, MPID_Request *sreq, MPID_IOV *iov, int n_i
if (!reqFn)
{
MPIU_Assert (MPIDI_Request_get_type (sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete (sreq);
+ MPID_Request_complete (sreq);
MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, ".... complete");
}
else
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
index 367bf40..9e07339 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
@@ -252,7 +252,7 @@ int MPIDI_CH3I_Shm_send_progress(void)
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
#endif
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
/* MT - clear the current active send before dequeuing/destroying the current request */
MPIDI_CH3I_shm_active_send = NULL;
@@ -825,7 +825,7 @@ int MPID_nem_handle_pkt(MPIDI_VC_t *vc, char *buf, MPIDI_msg_sz_t buflen)
#if !(defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT))
MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
#endif
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
complete = TRUE;
}
else
@@ -1095,7 +1095,7 @@ int MPIDI_CH3I_Complete_sendq_with_error(MPIDI_VC_t * vc)
MPIU_ERR_SET1(req->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
MPID_Request_release(req); /* ref count was incremented when added to queue */
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
req = next;
} else {
prev = req;
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c b/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
index 76814d5..6950e7b 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3i_eagernoncontig.c
@@ -79,7 +79,7 @@ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header,
if (!sreq->dev.OnDataAvail)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
else
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
index 4b98c1a..d3535d9 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
@@ -305,7 +305,7 @@ static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *com
}
else {
*complete = 1;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
fn_fail:
@@ -478,7 +478,7 @@ int MPID_nem_lmt_dma_done_send(MPIDI_VC_t *vc, MPID_Request *sreq)
reason. */
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
@@ -611,7 +611,7 @@ int MPID_nem_lmt_dma_progress(void)
cur->req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(cur->req->status.MPI_ERROR, MPI_ERR_OTHER, "**recv_status", "**recv_status %d", *cur->status_p);
- MPIDI_CH3U_Request_complete(cur->req);
+ MPID_Request_complete(cur->req);
if (cur == outstanding_head) {
outstanding_head = cur->next;
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
index 84e77bc..1182ea2 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c
@@ -504,7 +504,7 @@ static int lmt_shm_send_progress(MPIDI_VC_t *vc, MPID_Request *req, int *done)
while (last < data_sz);
*done = TRUE;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "completed req local_req=%d", req->handle);
@@ -643,7 +643,7 @@ static int lmt_shm_recv_progress(MPIDI_VC_t *vc, MPID_Request *req, int *done)
OPA_store_int(©_buf->owner_info.val.rank, NO_OWNER);
*done = TRUE;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
fn_exit:
copy_buf->receiver_present.val = FALSE;
@@ -818,7 +818,7 @@ int MPID_nem_lmt_shm_vc_terminated(MPIDI_VC_t *vc)
while (!MPID_nem_lmt_rtsq_empty(vc_ch->lmt_rts_queue)) {
MPID_nem_lmt_rtsq_dequeue(&vc_ch->lmt_rts_queue, &req);
req->status.MPI_ERROR = req_errno;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
MPIU_THREAD_CS_EXIT(LMT,);
}
@@ -830,7 +830,7 @@ int MPID_nem_lmt_shm_vc_terminated(MPIDI_VC_t *vc)
if (vc_ch->lmt_active_lmt) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Clearing active LMT");
vc_ch->lmt_active_lmt->req->status.MPI_ERROR = req_errno;
- MPIDI_CH3U_Request_complete(vc_ch->lmt_active_lmt->req);
+ MPID_Request_complete(vc_ch->lmt_active_lmt->req);
MPIU_Free(vc_ch->lmt_active_lmt);
vc_ch->lmt_active_lmt = NULL;
}
@@ -842,7 +842,7 @@ int MPID_nem_lmt_shm_vc_terminated(MPIDI_VC_t *vc)
while (!LMT_SHM_Q_EMPTY(vc_ch->lmt_queue)) {
LMT_SHM_Q_DEQUEUE(&vc_ch->lmt_queue, &we);
we->req->status.MPI_ERROR = req_errno;
- MPIDI_CH3U_Request_complete(we->req);
+ MPID_Request_complete(we->req);
MPIU_Free(we);
}
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
index 96155fb..29be00f 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_vmsplice.c
@@ -75,7 +75,7 @@ static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *com
}
else {
*complete = 1;
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
fn_fail:
diff --git a/src/mpid/ch3/channels/sock/src/ch3_isend.c b/src/mpid/ch3/channels/sock/src/ch3_isend.c
index ae9340f..afcfa87 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_isend.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_isend.c
@@ -74,7 +74,7 @@ int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPID_Request * sreq, void * hdr,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
else {
int complete;
@@ -144,7 +144,7 @@ int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPID_Request * sreq, void * hdr,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT -CH3U_Request_complete() performs write barrier */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
@@ -189,7 +189,7 @@ int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPID_Request * sreq, void * hdr,
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete() performs write barrier */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
diff --git a/src/mpid/ch3/channels/sock/src/ch3_isendv.c b/src/mpid/ch3/channels/sock/src/ch3_isendv.c
index 302060c..8dd760c 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_isendv.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_isendv.c
@@ -139,7 +139,7 @@ int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
else {
int complete;
@@ -187,7 +187,7 @@ int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT - CH3U_Request_complete performs write barrier */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
/* Return error to calling routine */
mpi_errno = sreq->status.MPI_ERROR;
}
@@ -233,7 +233,7 @@ int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete performs write barrier */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
diff --git a/src/mpid/ch3/channels/sock/src/ch3_progress.c b/src/mpid/ch3/channels/sock/src/ch3_progress.c
index 88bb493..a8f22cf 100644
--- a/src/mpid/ch3/channels/sock/src/ch3_progress.c
+++ b/src/mpid/ch3/channels/sock/src/ch3_progress.c
@@ -508,7 +508,7 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
complete = TRUE;
}
else {
@@ -567,7 +567,7 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
complete = TRUE;
}
else {
@@ -611,7 +611,7 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
complete = TRUE;
}
else {
@@ -885,7 +885,7 @@ static int ReadMoreData( MPIDI_CH3I_Connection_t * conn, MPID_Request *rreq )
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
complete = TRUE;
}
else {
diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h
index e347170..2d38376 100644
--- a/src/mpid/ch3/include/mpid_rma_issue.h
+++ b/src/mpid/ch3/include/mpid_rma_issue.h
@@ -1255,7 +1255,7 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op)
* and release ch3 ref. */
/* Complete user request and release the ch3 ref */
- MPIDI_CH3U_Request_complete(op->ureq);
+ MPID_Request_complete(op->ureq);
}
else {
MPID_Request **req_ptr = NULL;
@@ -1307,7 +1307,7 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op)
else {
/* all requests are completed */
/* Complete user request and release ch3 ref */
- MPIDI_CH3U_Request_complete(op->ureq);
+ MPID_Request_complete(op->ureq);
op->ureq = NULL;
}
}
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 2fef156..25ae448 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -285,18 +285,6 @@ extern MPIDI_Process_t MPIDI_Process;
# error MPIU_HANDLE_ALLOCATION_METHOD not defined
#endif
-#define MPIDI_CH3U_Request_complete(req_) \
-{ \
- int incomplete__; \
- \
- MPIDI_CH3U_Request_decrement_cc((req_), &incomplete__); \
- if (!incomplete__) \
- { \
- MPID_Request_release(req_); \
- MPIDI_CH3_Progress_signal_completion(); \
- } \
-}
-
/* If the channel doesn't initialize anything in the request,
provide a dummy */
diff --git a/src/mpid/ch3/src/ch3u_eager.c b/src/mpid/ch3/src/ch3u_eager.c
index 683d481..515529e 100644
--- a/src/mpid/ch3/src/ch3u_eager.c
+++ b/src/mpid/ch3/src/ch3u_eager.c
@@ -499,7 +499,7 @@ int MPIDI_CH3_PktHandler_EagerShortSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
* all rreq field modifications must be complete at this point. This macro
* also kicks the progress engine, which was previously done here via
* MPIDI_CH3_Progress_signal_completion(). */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
fn_fail:
/* MT note: it may be possible to narrow this CS after careful
@@ -643,7 +643,7 @@ int MPIDI_CH3_PktHandler_EagerSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
/* return the number of bytes processed in this function */
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else {
@@ -666,7 +666,7 @@ int MPIDI_CH3_PktHandler_EagerSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else
@@ -728,7 +728,7 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
/* return the number of bytes processed in this function */
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;;
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else {
@@ -746,7 +746,7 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else
@@ -788,7 +788,7 @@ int MPIDI_CH3_PktHandler_ReadySend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
else
{
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
/* we didn't process anything but the header in this case */
diff --git a/src/mpid/ch3/src/ch3u_eagersync.c b/src/mpid/ch3/src/ch3u_eagersync.c
index f90dac6..aa8efb3 100644
--- a/src/mpid/ch3/src/ch3u_eagersync.c
+++ b/src/mpid/ch3/src/ch3u_eagersync.c
@@ -258,7 +258,7 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else {
@@ -273,7 +273,7 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else
@@ -301,7 +301,7 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
{
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else {
@@ -316,7 +316,7 @@ int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*rreqp = NULL;
}
else
@@ -348,7 +348,7 @@ int MPIDI_CH3_PktHandler_EagerSyncAck( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
transfer could still be in progress) */
/* FIXME: This sometimes segfaults */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c
index cad0d1c..ac83287 100644
--- a/src/mpid/ch3/src/ch3u_handle_op_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_op_req.c
@@ -27,9 +27,9 @@ int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t * vc, MPID_Request * sreq, in
/* Complete user request and release ref of completion handler.
* Note that ch3 ref is released by later clean_up call. */
- MPIDI_CH3U_Request_complete(ureq);
+ MPID_Request_complete(ureq);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
*complete = TRUE;
fn_exit:
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index f45f9c9..272b428 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -29,7 +29,7 @@ int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq, int *comple
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
}
else {
@@ -64,7 +64,7 @@ int MPIDI_CH3_ReqHandler_RecvComplete(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPID_Request * rreq, int *complete)
{
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
return MPI_SUCCESS;
}
@@ -104,7 +104,7 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -200,7 +200,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq,
MPIDI_CH3U_SRBuf_free(rreq);
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -378,7 +378,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
}
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
fn_exit:
MPIU_CHKPMEM_COMMIT();
@@ -509,7 +509,7 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
}
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
fn_exit:
@@ -896,7 +896,7 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete(MPIDI_VC_t * vc,
/* mark receive data transfer as complete and decrement CC in receive
* request */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
fn_fail:
@@ -931,7 +931,7 @@ int MPIDI_CH3_ReqHandler_UnpackUEBufComplete(MPIDI_VC_t * vc ATTRIBUTE((unused))
}
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
@@ -965,7 +965,7 @@ int MPIDI_CH3_ReqHandler_UnpackSRBufComplete(MPIDI_VC_t * vc, MPID_Request * rre
}
else {
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
}
@@ -2020,7 +2020,7 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete(MPIDI_VC_t * vc,
/* mark receive data transfer as complete and decrement CC in receive
* request */
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
*complete = TRUE;
fn_fail:
diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c
index 00910e9..9807f77 100644
--- a/src/mpid/ch3/src/ch3u_handle_send_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_send_req.c
@@ -24,7 +24,7 @@ int MPIDI_CH3U_Handle_send_req(MPIDI_VC_t * vc, MPID_Request * sreq, int *comple
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
*complete = 1;
}
else {
@@ -75,7 +75,7 @@ int MPIDI_CH3_ReqHandler_GetSendComplete(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_Assert(win_ptr->at_completion_counter >= 0);
/* mark data transfer as complete and decrement CC */
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -138,7 +138,7 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -204,7 +204,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
@@ -269,7 +269,7 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
/* NOTE: finish_op_on_target() must be called after we complete this request,
* because inside finish_op_on_target() we may call this request handler
diff --git a/src/mpid/ch3/src/ch3u_recvq.c b/src/mpid/ch3/src/ch3u_recvq.c
index fe16e57..3f59267 100644
--- a/src/mpid/ch3/src/ch3u_recvq.c
+++ b/src/mpid/ch3/src/ch3u_recvq.c
@@ -636,7 +636,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
- MPIDI_CH3U_Request_complete(rreq);
+ MPID_Request_complete(rreq);
goto lock_exit;
}
}
@@ -902,7 +902,7 @@ static inline void dequeue_and_set_error(MPID_Request **req, MPID_Request *prev
/* set error and complete */
(*req)->status.MPI_ERROR = *error;
- MPIDI_CH3U_Request_complete(*req);
+ MPID_Request_complete(*req);
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE,
(MPIU_DBG_FDEST, "set error of req %p (%#08x) to %#x and completing.",
*req, (*req)->handle, *error));
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 30a130e..68b8da0 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -714,3 +714,14 @@ void MPID_Request_set_completed(MPID_Request *req)
/* MT do not reorder! see note above*/
MPIDI_CH3_Progress_signal_completion();
}
+
+void MPID_Request_complete(MPID_Request *req)
+{
+ int incomplete;
+
+ MPIDI_CH3U_Request_decrement_cc(req, &incomplete);
+ if (!incomplete) {
+ MPID_Request_release(req);
+ MPIDI_CH3_Progress_signal_completion();
+ }
+}
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index 357db59..cca6f95 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -94,7 +94,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPIDI_CH3U_Request_complete(ureq);
+ MPID_Request_complete(ureq);
}
}
else {
@@ -274,7 +274,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPIDI_CH3U_Request_complete(ureq);
+ MPID_Request_complete(ureq);
}
}
else {
@@ -446,7 +446,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPIDI_CH3U_Request_complete(ureq);
+ MPID_Request_complete(ureq);
}
}
else {
@@ -663,7 +663,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
if (ureq) {
/* Complete user request and release the ch3 ref */
- MPIDI_CH3U_Request_complete(ureq);
+ MPID_Request_complete(ureq);
}
}
else {
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 2fac191..5695c50 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1300,7 +1300,7 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len);
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
@@ -1573,7 +1573,7 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
}
if (complete) {
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
*rreqp = NULL;
}
@@ -1719,7 +1719,7 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
mpi_errno = reqFn(vc, req, &complete);
}
else {
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
*rreqp = NULL;
}
@@ -1867,7 +1867,7 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
mpi_errno = reqFn(vc, req, &complete);
}
else {
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
}
*rreqp = NULL;
diff --git a/src/mpid/ch3/src/ch3u_rndv.c b/src/mpid/ch3/src/ch3u_rndv.c
index 3bb42b2..557f597 100644
--- a/src/mpid/ch3/src/ch3u_rndv.c
+++ b/src/mpid/ch3/src/ch3u_rndv.c
@@ -300,7 +300,7 @@ int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (req->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
*rreqp = NULL;
}
else {
@@ -315,7 +315,7 @@ int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
if (complete)
{
- MPIDI_CH3U_Request_complete(req);
+ MPID_Request_complete(req);
*rreqp = NULL;
}
else
diff --git a/src/mpid/ch3/src/mpid_cancel_send.c b/src/mpid/ch3/src/mpid_cancel_send.c
index 2b626d7..51d39b6 100644
--- a/src/mpid/ch3/src/mpid_cancel_send.c
+++ b/src/mpid/ch3/src/mpid_cancel_send.c
@@ -324,7 +324,7 @@ int MPIDI_CH3_PktHandler_CancelSendResp( MPIDI_VC_t *vc ATTRIBUTE((unused)),
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
- MPIDI_CH3U_Request_complete(sreq);
+ MPID_Request_complete(sreq);
*rreqp = NULL;
http://git.mpich.org/mpich.git/commitdiff/d7f7edb4511a9f82be10935b63b288903dd1c903
commit d7f7edb4511a9f82be10935b63b288903dd1c903
Author: Pavan Balaji <balaji at anl.gov>
Date: Sat Jul 4 19:27:00 2015 -0500
Deleted MPID_REQUEST_SET_COMPLETED macro.
We had a macro and a function that pretty much did the same thing.
Get rid of one of them.
Signed-off-by: Xin Zhao <xinzhao3 at illinois.edu>
diff --git a/src/mpid/ch3/include/mpidpost.h b/src/mpid/ch3/include/mpidpost.h
index 20c3060..aa0a079 100644
--- a/src/mpid/ch3/include/mpidpost.h
+++ b/src/mpid/ch3/include/mpidpost.h
@@ -203,46 +203,6 @@ void MPIDI_CH3U_Request_destroy(MPID_Request * req);
} \
}
-/* MT note: The following order of operations is _essential_ for correct
- * operation of the fine-grained multithreading code. Assume that
- * _signal_completion() acquires and releases a mutex in order to update the
- * global completion counter (it does for fine-grained ch3:nemesis). Further,
- * assume the following standard pattern is used by the request consumer to wait
- * for completion:
- *
- * if (req is not complete (req->cc!=0)) {
- * // progress_enter:
- * acquire mutex;
- * my_count = global_count;
- * release mutex;
- *
- * while (req is not complete (req->cc!=0)) {
- * progress_wait(&my_count);
- * }
- * }
- *
- * Where progress_wait will attempt to make progress forever as long as
- * (my_count==global_count). If it is possible for the consumer to see the
- * global completion count before seeing the request's completion counter drop
- * to zero, the consumer could spin in progress_wait forever without a chance to
- * retest the request.
- *
- * If the mutex approach is dropped in favor of atomic access, additional memory
- * barriers must be inserted. The mutex acquire/release currently enforces
- * sufficient ordering constraints provided the statement order below is not
- * accidentally inverted.
- *
- * See also the note above the MSGQUEUE CS macros and request completion in
- * mpiimplthread.h.
- */
-/* MPID_Request_set_completed (the function) is defined in ch3u_request.c */
-#define MPID_REQUEST_SET_COMPLETED(req_) \
-{ \
- MPID_cc_set((req_)->cc_ptr, 0); \
- /* MT do not reorder! see note above*/ \
- MPIDI_CH3_Progress_signal_completion(); \
-}
-
/*
* Device level progress engine macros
diff --git a/src/mpid/ch3/src/ch3u_buffer.c b/src/mpid/ch3/src/ch3u_buffer.c
index b7dba61..52d5dbd 100644
--- a/src/mpid/ch3/src/ch3u_buffer.c
+++ b/src/mpid/ch3/src/ch3u_buffer.c
@@ -240,7 +240,7 @@ int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
buf, count, datatype, &data_sz,
&rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_REQUEST_SET_COMPLETED(sreq);
+ MPID_Request_set_completed(sreq);
MPID_Request_release(sreq);
}
else
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 8f949e4..30a130e 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -676,11 +676,41 @@ int MPIDI_CH3U_Request_unpack_uebuf(MPID_Request * rreq)
return mpi_errno;
}
-/*
- * Export the function to set a request as completed for use by
- * the generalized request functions in mpich/src/pt2pt/greq_complete.c
+/* MT note: The following order of operations is _essential_ for correct
+ * operation of the fine-grained multithreading code. Assume that
+ * _signal_completion() acquires and releases a mutex in order to update the
+ * global completion counter (it does for fine-grained ch3:nemesis). Further,
+ * assume the following standard pattern is used by the request consumer to wait
+ * for completion:
+ *
+ * if (req is not complete (req->cc!=0)) {
+ * // progress_enter:
+ * acquire mutex;
+ * my_count = global_count;
+ * release mutex;
+ *
+ * while (req is not complete (req->cc!=0)) {
+ * progress_wait(&my_count);
+ * }
+ * }
+ *
+ * Where progress_wait will attempt to make progress forever as long as
+ * (my_count==global_count). If it is possible for the consumer to see the
+ * global completion count before seeing the request's completion counter drop
+ * to zero, the consumer could spin in progress_wait forever without a chance to
+ * retest the request.
+ *
+ * If the mutex approach is dropped in favor of atomic access, additional memory
+ * barriers must be inserted. The mutex acquire/release currently enforces
+ * sufficient ordering constraints provided the statement order below is not
+ * accidentally inverted.
+ *
+ * See also the note above the MSGQUEUE CS macros and request completion in
+ * mpiimplthread.h.
*/
-void MPID_Request_set_completed( MPID_Request *req )
+void MPID_Request_set_completed(MPID_Request *req)
{
- MPID_REQUEST_SET_COMPLETED(req);
+ MPID_cc_set(req->cc_ptr, 0);
+ /* MT do not reorder! see note above*/
+ MPIDI_CH3_Progress_signal_completion();
}
diff --git a/src/mpid/ch3/src/mpid_cancel_recv.c b/src/mpid/ch3/src/mpid_cancel_recv.c
index 054f8ce..8563a52 100644
--- a/src/mpid/ch3/src/mpid_cancel_recv.c
+++ b/src/mpid/ch3/src/mpid_cancel_recv.c
@@ -39,7 +39,7 @@ int MPID_Cancel_recv(MPID_Request * rreq)
"request 0x%08x cancelled", rreq->handle);
MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
MPIR_STATUS_SET_COUNT(rreq->status, 0);
- MPID_REQUEST_SET_COMPLETED(rreq);
+ MPID_Request_set_completed(rreq);
MPID_Request_release(rreq);
}
else
diff --git a/src/mpid/ch3/src/mpidi_isend_self.c b/src/mpid/ch3/src/mpidi_isend_self.c
index e239ba5..85f9893 100644
--- a/src/mpid/ch3/src/mpidi_isend_self.c
+++ b/src/mpid/ch3/src/mpidi_isend_self.c
@@ -87,7 +87,7 @@ int MPIDI_Isend_self(const void * buf, MPI_Aint count, MPI_Datatype datatype, in
MPIDI_CH3U_Buffer_copy(buf, count, datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
- MPID_REQUEST_SET_COMPLETED(rreq);
+ MPID_Request_set_completed(rreq);
MPID_Request_release(rreq);
/* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
MPIU_Object_set_ref(sreq, 1);
diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c
index 13c828e..2227a30 100644
--- a/src/mpid/common/sched/mpid_sched.c
+++ b/src/mpid/common/sched/mpid_sched.c
@@ -916,7 +916,7 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made
break;
}
- MPID_REQUEST_SET_COMPLETED(s->req);
+ MPID_Request_set_completed(s->req);
MPID_Request_release(s->req);
s->req = NULL;
MPIU_Free(s->entries);
-----------------------------------------------------------------------
Summary of changes:
src/include/mpiimpl.h | 26 +++-
src/mpi/pt2pt/greq_complete.c | 2 +-
src/mpi/pt2pt/greq_start.c | 5 +
.../ch3/channels/nemesis/include/mpid_nem_impl.h | 12 +-
.../ch3/channels/nemesis/netmod/llc/llc_poll.c | 5 +-
.../ch3/channels/nemesis/netmod/llc/llc_probe.c | 2 +-
.../ch3/channels/nemesis/netmod/llc/llc_send.c | 10 +-
.../ch3/channels/nemesis/netmod/mxm/mxm_probe.c | 5 +-
src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c | 38 ++---
.../ch3/channels/nemesis/netmod/ofi/ofi_impl.h | 4 +-
.../ch3/channels/nemesis/netmod/ofi/ofi_init.c | 6 +-
src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c | 14 +-
.../nemesis/netmod/ofi/ofi_probe_template.c | 2 +-
.../ch3/channels/nemesis/netmod/ofi/ofi_progress.c | 4 +-
.../ch3/channels/nemesis/netmod/ofi/ofi_tagged.c | 12 +-
.../nemesis/netmod/ofi/ofi_tagged_template.c | 2 +-
.../channels/nemesis/netmod/portals4/ptl_impl.h | 15 +--
.../ch3/channels/nemesis/netmod/portals4/ptl_nm.c | 29 +++-
.../channels/nemesis/netmod/portals4/ptl_probe.c | 30 +++-
.../channels/nemesis/netmod/portals4/ptl_recv.c | 5 +-
.../channels/nemesis/netmod/portals4/ptl_send.c | 5 +-
src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c | 5 +-
.../ch3/channels/nemesis/netmod/tcp/tcp_send.c | 25 +++-
src/mpid/ch3/channels/nemesis/src/ch3_isend.c | 7 +-
src/mpid/ch3/channels/nemesis/src/ch3_isendv.c | 7 +-
src/mpid/ch3/channels/nemesis/src/ch3_progress.c | 16 ++-
.../ch3/channels/nemesis/src/ch3i_eagernoncontig.c | 5 +-
.../ch3/channels/nemesis/src/mpid_nem_lmt_dma.c | 15 ++-
.../ch3/channels/nemesis/src/mpid_nem_lmt_shm.c | 20 ++-
.../channels/nemesis/src/mpid_nem_lmt_vmsplice.c | 5 +-
src/mpid/ch3/channels/sock/src/ch3_isend.c | 9 +-
src/mpid/ch3/channels/sock/src/ch3_isendv.c | 9 +-
src/mpid/ch3/channels/sock/src/ch3_progress.c | 21 ++-
src/mpid/ch3/include/mpid_rma_issue.h | 37 +++--
src/mpid/ch3/include/mpidimpl.h | 78 +---------
src/mpid/ch3/include/mpidpost.h | 81 ----------
src/mpid/ch3/include/mpidpre.h | 9 +
src/mpid/ch3/include/mpidrma.h | 4 +-
src/mpid/ch3/src/ch3u_buffer.c | 19 ++-
src/mpid/ch3/src/ch3u_eager.c | 30 +++-
src/mpid/ch3/src/ch3u_eagersync.c | 35 ++++-
src/mpid/ch3/src/ch3u_handle_op_req.c | 37 ++++-
src/mpid/ch3/src/ch3u_handle_recv_req.c | 94 +++++++++---
src/mpid/ch3/src/ch3u_handle_send_req.c | 28 +++-
src/mpid/ch3/src/ch3u_recvq.c | 4 +-
src/mpid/ch3/src/ch3u_request.c | 161 ++++++++++---------
src/mpid/ch3/src/ch3u_rma_ops.c | 20 ++-
src/mpid/ch3/src/ch3u_rma_pkthandler.c | 45 +++---
src/mpid/ch3/src/ch3u_rma_reqops.c | 52 ++++---
src/mpid/ch3/src/ch3u_rma_sync.c | 14 ++
src/mpid/ch3/src/ch3u_rndv.c | 10 +-
src/mpid/ch3/src/ch3u_win_fns.c | 4 +-
src/mpid/ch3/src/mpid_cancel_recv.c | 12 +-
src/mpid/ch3/src/mpid_cancel_send.c | 33 +++--
src/mpid/ch3/src/mpid_rma.c | 1 +
src/mpid/ch3/src/mpidi_isend_self.c | 40 ++++--
src/mpid/ch3/src/mpidi_rma.c | 32 ++---
src/mpid/common/datatype/mpid_datatype.h | 6 +-
src/mpid/common/hcoll/hcoll_rte.c | 2 +-
src/mpid/common/sched/mpid_sched.c | 14 +-
60 files changed, 746 insertions(+), 533 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list