[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1.3-218-g88d3409
Service Account
noreply at mpich.org
Thu Nov 13 21:38:32 CST 2014
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via 88d34091a9e6cff45dbec9bdb80208fedb4d4231 (commit)
via 8b1a69b92ab944c726ca1156b84c4ee9ff144b2b (commit)
via 8b277d9f5c029a661eb52adfe73443e7c74b2112 (commit)
via 9bc09a30a47fb411bf3f63f4c29a694fe779118e (commit)
via a9d968cc474bc369be2c514b7384944e6baefd18 (commit)
from c26861ba3ff025bb62351acf1d74945581f5c59c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/88d34091a9e6cff45dbec9bdb80208fedb4d4231
commit 88d34091a9e6cff45dbec9bdb80208fedb4d4231
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Nov 12 17:06:43 2014 -0600
Split shared request handler.
ReqHandler_GaccumLikeSendComplete is used for GACC-like operations,
including GACC, CAS and FOP. Here we split it into following three
functions:
ReqHandler_GaccumSendComplete
ReqHandler_CASSendComplete
ReqHandler_FOPSendComplete
It is convenient for us to add different actions in future for those
three kinds of operations.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index b5f47ed..a64d32f 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1923,8 +1923,12 @@ int MPIDI_CH3_ReqHandler_SendReloadIOV( MPIDI_VC_t *vc, MPID_Request *sreq,
int *complete );
int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *, MPID_Request *,
int * );
-int MPIDI_CH3_ReqHandler_GaccumLikeSendComplete( MPIDI_VC_t *, MPID_Request *,
- int * );
+int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *, MPID_Request *,
+ int * );
+int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *, MPID_Request *,
+ int * );
+int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *, MPID_Request *,
+ int * );
/* Request-based operation handler */
int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPIDI_VC_t *, MPID_Request *,
int *);
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 4f9e906..fffce27 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -214,8 +214,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc,
MPID_Segment_free(seg);
}
- resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumLikeSendComplete;
- resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumLikeSendComplete;
+ resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete;
+ resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete;
resp_req->dev.target_win_handle = rreq->dev.target_win_handle;
resp_req->dev.flags = rreq->dev.flags;
diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c
index e50f2dc..ab7d728 100644
--- a/src/mpid/ch3/src/ch3u_handle_send_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_send_req.c
@@ -72,18 +72,18 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)),
}
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_ReqHandler_GaccumLikeSendComplete
+#define FUNCNAME MPIDI_CH3_ReqHandler_GaccumSendComplete
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_ReqHandler_GaccumLikeSendComplete( MPIDI_VC_t *vc,
+int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc,
MPID_Request *rreq,
int *complete )
{
int mpi_errno = MPI_SUCCESS;
MPID_Win *win_ptr;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMLIKESENDCOMPLETE);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMSENDCOMPLETE);
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMLIKESENDCOMPLETE);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMSENDCOMPLETE);
/* This function is triggered when sending back process of GACC/FOP/CAS
is finished. Only GACC used user_buf. FOP and CAS can fit all data
in response packet. */
@@ -101,12 +101,90 @@ int MPIDI_CH3_ReqHandler_GaccumLikeSendComplete( MPIDI_VC_t *vc,
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
+ MPIDI_CH3U_Request_complete(rreq);
+ *complete = TRUE;
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMSENDCOMPLETE);
+ return mpi_errno;
+
+ fn_fail:
+ goto fn_exit;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_ReqHandler_CASSendComplete
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc,
+ MPID_Request *rreq,
+ int *complete )
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPID_Win *win_ptr;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_CASSENDCOMPLETE);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_CASSENDCOMPLETE);
+ /* This function is triggered when sending back process of GACC/FOP/CAS
+ is finished. Only GACC used user_buf. FOP and CAS can fit all data
+ in response packet. */
+ if (rreq->dev.user_buf != NULL)
+ MPIU_Free(rreq->dev.user_buf);
+
+ MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
+
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS,
+ rreq->dev.flags, rreq->dev.source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+
+ /* here we decrement the Active Target counter to guarantee the GET-like
+ operation are completed when counter reaches zero. */
+ win_ptr->at_completion_counter--;
+ MPIU_Assert(win_ptr->at_completion_counter >= 0);
+
+ MPIDI_CH3U_Request_complete(rreq);
+ *complete = TRUE;
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_CASSENDCOMPLETE);
+ return mpi_errno;
+
+ fn_fail:
+ goto fn_exit;
+}
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_ReqHandler_FOPSendComplete
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc,
+ MPID_Request *rreq,
+ int *complete )
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPID_Win *win_ptr;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPSENDCOMPLETE);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPSENDCOMPLETE);
+ /* This function is triggered when sending back process of GACC/FOP/CAS
+ is finished. Only GACC used user_buf. FOP and CAS can fit all data
+ in response packet. */
+ if (rreq->dev.user_buf != NULL)
+ MPIU_Free(rreq->dev.user_buf);
+
+ MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
+
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP,
+ rreq->dev.flags, rreq->dev.source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+
+ /* here we decrement the Active Target counter to guarantee the GET-like
+ operation are completed when counter reaches zero. */
+ win_ptr->at_completion_counter--;
+ MPIU_Assert(win_ptr->at_completion_counter >= 0);
MPIDI_CH3U_Request_complete(rreq);
*complete = TRUE;
fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMLIKESENDCOMPLETE);
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPSENDCOMPLETE);
return mpi_errno;
fn_fail:
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 751f31d..b95eacd 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -923,7 +923,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
(it is initialized to NULL by lower layer) */
req->dev.target_win_handle = cas_pkt->target_win_handle;
req->dev.flags = cas_pkt->flags;
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumLikeSendComplete;
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_CASSendComplete;
/* here we increment the Active Target counter to guarantee the GET-like
operation are completed when counter reaches zero. */
@@ -1081,7 +1081,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
(it is initialized to NULL by lower layer) */
resp_req->dev.target_win_handle = fop_pkt->target_win_handle;
resp_req->dev.flags = fop_pkt->flags;
- resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumLikeSendComplete;
+ resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPSendComplete;
/* here we increment the Active Target counter to guarantee the GET-like
operation are completed when counter reaches zero. */
http://git.mpich.org/mpich.git/commitdiff/8b1a69b92ab944c726ca1156b84c4ee9ff144b2b
commit 8b1a69b92ab944c726ca1156b84c4ee9ff144b2b
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Nov 12 17:04:21 2014 -0600
Code-refactoring: wrapping up action of finishing op on target.
Here we wrap up common action when one RMA op is finished on target
into a function to make code structure cleaner.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 8f1354b..4481c0e 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -476,6 +476,67 @@ static inline int check_piggyback_lock(MPID_Win *win_ptr, MPIDI_CH3_Pkt_t *pkt,
goto fn_exit;
}
+static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc,
+ MPIDI_CH3_Pkt_type_t type,
+ MPIDI_CH3_Pkt_flags_t flags,
+ MPI_Win source_win_handle) {
+ int mpi_errno = MPI_SUCCESS;
+
+ if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_ACCUMULATE) {
+ /* This is PUT or ACC */
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) {
+ if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) &&
+ !(flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) {
+ mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ }
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) {
+ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, flags,
+ source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
+ win_ptr->at_completion_counter--;
+ MPIU_Assert(win_ptr->at_completion_counter >= 0);
+ /* Signal the local process when the op counter reaches 0. */
+ if (win_ptr->at_completion_counter == 0)
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+ mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, flags,
+ source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ }
+ else {
+ /* This is GACC / GET / CAS / FOP */
+
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+ mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ MPIDI_CH3_Progress_signal_completion();
+ }
+
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
+ win_ptr->at_completion_counter--;
+ MPIU_Assert(win_ptr->at_completion_counter >= 0);
+ /* Signal the local process when the op counter reaches 0. */
+ if (win_ptr->at_completion_counter == 0)
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ }
+
+ fn_exit:
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
static inline int wait_progress_engine(void)
{
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 11c5c49..4f9e906 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -86,35 +86,9 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete( MPIDI_VC_t *vc,
MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) {
- if (!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) &&
- !(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) {
- mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, rreq->dev.source_win_handle);
- if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) {
- mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, rreq->dev.flags,
- rreq->dev.source_win_handle);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, rreq->dev.flags,
- rreq->dev.source_win_handle);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_PUT,
+ rreq->dev.flags, rreq->dev.source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* mark data transfer as complete and decrement CC */
MPIDI_CH3U_Request_complete(rreq);
@@ -164,34 +138,9 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc,
MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
MPIU_Free((char *) rreq->dev.final_user_buf + true_lb);
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) {
- if (!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) &&
- !(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) {
- mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, rreq->dev.source_win_handle);
- if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) {
- mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, rreq->dev.flags,
- rreq->dev.source_win_handle);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, rreq->dev.flags,
- rreq->dev.source_win_handle);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_ACCUMULATE,
+ rreq->dev.flags, rreq->dev.source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* mark data transfer as complete and decrement CC */
MPIDI_CH3U_Request_complete(rreq);
diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c
index 07ecad2..e50f2dc 100644
--- a/src/mpid/ch3/src/ch3u_handle_send_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_send_req.c
@@ -52,18 +52,9 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)),
MPID_Win_get_ptr(sreq->dev.target_win_handle, win_ptr);
- if (sreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- if (sreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET,
+ sreq->dev.flags, sreq->dev.source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* here we decrement the Active Target counter to guarantee the GET-like
operation are completed when counter reaches zero. */
@@ -101,24 +92,16 @@ int MPIDI_CH3_ReqHandler_GaccumLikeSendComplete( MPIDI_VC_t *vc,
MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET_ACCUM,
+ rreq->dev.flags, rreq->dev.source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+
/* here we decrement the Active Target counter to guarantee the GET-like
operation are completed when counter reaches zero. */
win_ptr->at_completion_counter--;
MPIU_Assert(win_ptr->at_completion_counter >= 0);
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
MPIDI_CH3U_Request_complete(rreq);
*complete = TRUE;
@@ -130,6 +113,7 @@ int MPIDI_CH3_ReqHandler_GaccumLikeSendComplete( MPIDI_VC_t *vc,
goto fn_exit;
}
+
int MPIDI_CH3_ReqHandler_SendReloadIOV( MPIDI_VC_t *vc ATTRIBUTE((unused)), MPID_Request *sreq,
int *complete )
{
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 757cee4..751f31d 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -936,18 +936,9 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPID_Request_release(req);
}
- if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
- if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS,
+ cas_pkt->flags, cas_pkt->source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_cas);
@@ -1104,18 +1095,9 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
}
- if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPIDI_CH3_Progress_signal_completion();
- }
-
- if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER) {
- win_ptr->at_completion_counter--;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
+ mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP,
+ fop_pkt->flags, fop_pkt->source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop);
http://git.mpich.org/mpich.git/commitdiff/8b277d9f5c029a661eb52adfe73443e7c74b2112
commit 8b277d9f5c029a661eb52adfe73443e7c74b2112
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Wed Nov 12 16:25:56 2014 -0600
Modify do_accumulate_op() to accept better arguments.
Originally do_accumulate_op() only accepts request pointer as
argument which is too restrict to be reused. Here we modify
it to access buffer address, count, datatype and op, so that
it can be reused in more general cases.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 64e8aac..8f1354b 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -342,48 +342,42 @@ static inline int MPIDI_CH3I_RMA_Handle_flush_ack(MPID_Win * win_ptr, int target
#define FUNCNAME do_accumulate_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int do_accumulate_op(MPID_Request *rreq)
+static inline int do_accumulate_op(void *source_buf, void *target_buf,
+ int acc_count, MPI_Datatype acc_dtp, MPI_Op acc_op)
{
int mpi_errno = MPI_SUCCESS;
- MPI_Aint true_lb, true_extent;
MPI_User_function *uop;
MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP);
MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP);
- MPIU_Assert(rreq->dev.final_user_buf != NULL);
-
- if (rreq->dev.op == MPI_REPLACE)
+ if (acc_op == MPI_REPLACE)
{
/* simply copy the data */
- mpi_errno = MPIR_Localcopy(rreq->dev.final_user_buf, rreq->dev.user_count,
- rreq->dev.datatype,
- rreq->dev.real_user_buf,
- rreq->dev.user_count,
- rreq->dev.datatype);
+ mpi_errno = MPIR_Localcopy(source_buf, acc_count, acc_dtp,
+ target_buf, acc_count, acc_dtp);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
goto fn_exit;
}
- if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN)
+ if (HANDLE_GET_KIND(acc_op) == HANDLE_KIND_BUILTIN)
{
/* get the function by indexing into the op table */
- uop = MPIR_OP_HDL_TO_FN(rreq->dev.op);
+ uop = MPIR_OP_HDL_TO_FN(acc_op);
}
else
{
/* --BEGIN ERROR HANDLING-- */
- mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op );
+ mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", acc_op );
return mpi_errno;
/* --END ERROR HANDLING-- */
}
- if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype))
+ if (MPIR_DATATYPE_IS_PREDEFINED(acc_dtp))
{
- (*uop)(rreq->dev.final_user_buf, rreq->dev.real_user_buf,
- &(rreq->dev.user_count), &(rreq->dev.datatype));
+ (*uop)(source_buf, target_buf, &acc_count, &acc_dtp);
}
else
{
@@ -405,13 +399,13 @@ static inline int do_accumulate_op(MPID_Request *rreq)
return mpi_errno;
}
/* --END ERROR HANDLING-- */
- MPID_Segment_init(NULL, rreq->dev.user_count,
- rreq->dev.datatype, segp, 0);
+ MPID_Segment_init(NULL, acc_count,
+ acc_dtp, segp, 0);
first = 0;
last = SEGMENT_IGNORE_LAST;
- MPID_Datatype_get_ptr(rreq->dev.datatype, dtp);
- vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1;
+ MPID_Datatype_get_ptr(acc_dtp, dtp);
+ vec_len = dtp->max_contig_blocks * acc_count + 1;
/* +1 needed because Rob says so */
dloop_vec = (DLOOP_VECTOR *)
MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR));
@@ -431,8 +425,8 @@ static inline int do_accumulate_op(MPID_Request *rreq)
for (i=0; i<vec_len; i++)
{
MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size, int);
- (*uop)((char *)rreq->dev.final_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
+ (*uop)((char *)source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
+ (char *)target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
}
@@ -441,10 +435,6 @@ static inline int do_accumulate_op(MPID_Request *rreq)
}
fn_exit:
- /* free the temporary buffer */
- MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
- MPIU_Free((char *) rreq->dev.final_user_buf + true_lb);
-
MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
return mpi_errno;
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 44c8a71..11c5c49 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -139,6 +139,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc,
int *complete )
{
int mpi_errno = MPI_SUCCESS;
+ MPI_Aint true_lb, true_extent;
MPID_Win *win_ptr;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_ACCUMRECVCOMPLETE);
@@ -151,13 +152,18 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc,
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
/* accumulate data from tmp_buf into user_buf */
- mpi_errno = do_accumulate_op(rreq);
+ mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf,
+ rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op);
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
+ /* free the temporary buffer */
+ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
+ MPIU_Free((char *) rreq->dev.final_user_buf + true_lb);
+
if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) {
if (!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) &&
!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) {
@@ -216,6 +222,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc,
MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp;
MPID_Request *resp_req;
MPID_IOV iov[MPID_IOV_LIMIT];
+ MPI_Aint true_lb, true_extent;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE);
@@ -287,12 +294,17 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc,
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
/* accumulate data from tmp_buf into user_buf */
- mpi_errno = do_accumulate_op(rreq);
+ mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf,
+ rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op);
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
+
+ /* free the temporary buffer */
+ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
+ MPIU_Free((char *) rreq->dev.final_user_buf + true_lb);
/* mark data transfer as complete and decrement CC */
MPIDI_CH3U_Request_complete(rreq);
http://git.mpich.org/mpich.git/commitdiff/9bc09a30a47fb411bf3f63f4c29a694fe779118e
commit 9bc09a30a47fb411bf3f63f4c29a694fe779118e
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Tue Nov 4 22:10:57 2014 -0600
Initialize flag in pkt when creating RMA operation.
Here we initalize packet flag as FLAG_NONE when creating
this packet, and add flags later when needed.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h
index c310329..08bad83 100644
--- a/src/mpid/ch3/include/mpid_rma_issue.h
+++ b/src/mpid/ch3/include/mpid_rma_issue.h
@@ -338,7 +338,7 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr,
rma_op->request = NULL;
- put_pkt->flags = flags;
+ put_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
put_pkt->lock_type = target_ptr->lock_type;
@@ -399,7 +399,7 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr,
rma_op->request = NULL;
- accum_pkt->flags = flags;
+ accum_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
accum_pkt->lock_type = target_ptr->lock_type;
@@ -484,7 +484,7 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr,
/* Note: Get_accumulate uses the same packet type as accumulate */
get_accum_pkt->request_handle = resp_req->handle;
- get_accum_pkt->flags = flags;
+ get_accum_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
get_accum_pkt->lock_type = target_ptr->lock_type;
@@ -599,7 +599,7 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
}
get_pkt->request_handle = rma_op->request->handle;
- get_pkt->flags = flags;
+ get_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
get_pkt->lock_type = target_ptr->lock_type;
@@ -699,7 +699,7 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op,
rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle;
cas_pkt->request_handle = rma_op->request->handle;
- cas_pkt->flags = flags;
+ cas_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
cas_pkt->lock_type = target_ptr->lock_type;
@@ -768,7 +768,7 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op,
fop_pkt->request_handle = resp_req->handle;
- fop_pkt->flags = flags;
+ fop_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
fop_pkt->lock_type = target_ptr->lock_type;
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index 4a6db9f..8ec172d 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -120,6 +120,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
put_pkt->source_win_handle = win_ptr->handle;
put_pkt->immed_len = 0;
put_pkt->origin_rank = rank;
+ put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->origin_addr = (void *) origin_addr;
@@ -303,6 +304,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
get_pkt->source_win_handle = win_ptr->handle;
get_pkt->origin_rank = rank;
+ get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->origin_addr = origin_addr;
@@ -468,6 +470,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
accum_pkt->source_win_handle = win_ptr->handle;
accum_pkt->immed_len = 0;
accum_pkt->origin_rank = rank;
+ accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = origin_count;
@@ -658,6 +661,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
get_pkt->source_win_handle = win_ptr->handle;
get_pkt->origin_rank = rank;
+ get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = result_addr;
new_ptr->origin_count = result_count;
@@ -693,6 +697,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
get_accum_pkt->source_win_handle = win_ptr->handle;
get_accum_pkt->immed_len = 0;
get_accum_pkt->origin_rank = rank;
+ get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = origin_count;
@@ -982,6 +987,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
cas_pkt->source_win_handle = win_ptr->handle;
cas_pkt->origin_rank = rank;
+ cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
@@ -1110,6 +1116,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
get_pkt->source_win_handle = win_ptr->handle;
get_pkt->origin_rank = rank;
+ get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = result_addr;
new_ptr->origin_count = 1;
@@ -1131,6 +1138,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
fop_pkt->immed_len = 0;
fop_pkt->origin_rank = rank;
+ fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
http://git.mpich.org/mpich.git/commitdiff/a9d968cc474bc369be2c514b7384944e6baefd18
commit a9d968cc474bc369be2c514b7384944e6baefd18
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Thu Nov 13 10:12:04 2014 -0600
Perf-tuning: issue FLUSH, FLUSH ACK, UNLOCK ACK messages only when needed.
When operation pending list and request lists are all empty, FLUSH message
needs to be sent by origin only when origin issued PUT/ACC operations since
the last synchronization calls, otherwise origin does not need to issue FLUSH
at all and does not need to wait for FLUSH ACK message.
Similiarly, origin waits for ACK of UNLOCK message only when origin issued
PUT/ACC operations since the last synchronization calls. However, UNLOCK
message always needs to be sent out because origin needs to unlock the
target process. This patch avoids issuing unnecessary
FLUSH / FLUSH ACK / UNLOCK ACK messages.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h
index 85b672a..e4d4a2e 100644
--- a/src/mpid/ch3/include/mpid_rma_oplist.h
+++ b/src/mpid/ch3/include/mpid_rma_oplist.h
@@ -118,6 +118,7 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr
e->outstanding_lock = 0;
e->disable_flush_local = 0;
e->win_complete_flag = 0;
+ e->put_acc_issued = 0;
e->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
e->sync.outstanding_acks = 0;
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index bdcc9a0..d182799 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -88,6 +88,8 @@ typedef struct MPIDI_RMA_Target {
int outstanding_lock;
int disable_flush_local;
int win_complete_flag;
+ int put_acc_issued; /* indicate if PUT/ACC is issued in this epoch
+ after the previous synchronization calls. */
/* The target structure is free to be cleaned up when all of the
* following conditions hold true:
diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h
index 25b890f..243c296 100644
--- a/src/mpid/ch3/include/mpidpkt.h
+++ b/src/mpid/ch3/include/mpidpkt.h
@@ -118,7 +118,8 @@ typedef enum {
MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE = 128,
MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 256,
MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_ACK = 512,
- MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 1024
+ MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 1024,
+ MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 2048
} MPIDI_CH3_Pkt_flags_t;
typedef struct MPIDI_CH3_Pkt_send {
@@ -551,6 +552,7 @@ typedef struct MPIDI_CH3_Pkt_unlock {
int target_rank; /* Used in unluck/flush response to look up the
* target state at the origin. */
int origin_rank;
+ MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_unlock_t;
typedef struct MPIDI_CH3_Pkt_flush {
diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h
index 7efc641..9e8b9f1 100644
--- a/src/mpid/ch3/include/mpidpre.h
+++ b/src/mpid/ch3/include/mpidpre.h
@@ -351,7 +351,6 @@ extern MPIDI_RMA_Pkt_orderings_t *MPIDI_RMA_Pkt_orderings;
int outstanding_locks; /* when issuing multiple lock requests in \
MPI_WIN_LOCK_ALL, this counter keeps track \
of number of locks not being granted yet. */ \
- int outstanding_unlocks; \
#ifdef MPIDI_CH3_WIN_DECL
#define MPID_DEV_WIN_DECL \
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index a939ed5..64e8aac 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -60,7 +60,8 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
#define FUNCNAME send_unlock_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int send_unlock_msg(int dest, MPID_Win * win_ptr)
+static inline int send_unlock_msg(int dest, MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
@@ -78,6 +79,7 @@ static inline int send_unlock_msg(int dest, MPID_Win * win_ptr)
MPIDI_Pkt_init(unlock_pkt, MPIDI_CH3_PKT_UNLOCK);
unlock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
unlock_pkt->source_win_handle = win_ptr->handle;
+ unlock_pkt->flags = flags;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
mpi_errno = MPIDI_CH3_iStartMsg(vc, unlock_pkt, sizeof(*unlock_pkt), &req);
@@ -324,14 +326,10 @@ static inline int MPIDI_CH3I_RMA_Handle_flush_ack(MPID_Win * win_ptr, int target
mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &t);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- if (t == NULL) {
- win_ptr->outstanding_unlocks--;
- MPIU_Assert(win_ptr->outstanding_unlocks >= 0);
- }
- else {
- t->sync.outstanding_acks--;
- MPIU_Assert(t->sync.outstanding_acks >= 0);
- }
+ t->sync.outstanding_acks--;
+ MPIU_Assert(t->sync.outstanding_acks >= 0);
+
+ t->put_acc_issued = 0; /* reset PUT_ACC_FLAG after FLUSH is completed */
fn_exit:
return mpi_errno;
diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c
index 5c06650..69e98d5 100644
--- a/src/mpid/ch3/src/ch3u_rma_oplist.c
+++ b/src/mpid/ch3/src/ch3u_rma_oplist.c
@@ -233,10 +233,21 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks == 0);
}
+ else if (target->read_op_list == NULL &&
+ target->write_op_list == NULL &&
+ target->dt_op_list == NULL &&
+ target->put_acc_issued == 0) {
+ /* both pending list and all waiting lists for
+ this target are empty, we do not need to send
+ FLUSH message then. */
+ target->sync.outstanding_acks--;
+ MPIU_Assert(target->sync.outstanding_acks >= 0);
+ }
else {
mpi_errno = send_flush_msg(target->target_rank, win_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
+
(*made_progress) = 1;
goto finish_issue;
}
@@ -247,10 +258,22 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks == 0);
}
+ else if (target->read_op_list == NULL &&
+ target->write_op_list == NULL &&
+ target->dt_op_list == NULL &&
+ target->put_acc_issued == 0) {
+ /* send message to unlock target, but don't need ACK */
+ mpi_errno = send_unlock_msg(target->target_rank, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
+
+ target->sync.outstanding_acks--;
+ MPIU_Assert(target->sync.outstanding_acks >= 0);
+ }
else {
- mpi_errno = send_unlock_msg(target->target_rank, win_ptr);
+ mpi_errno = send_unlock_msg(target->target_rank, win_ptr, MPIDI_CH3_PKT_FLAG_NONE);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
+
(*made_progress) = 1;
goto finish_issue;
}
@@ -327,6 +350,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe
curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) {
MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list),
&(target->write_op_list_tail), curr_op);
+ target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
+ PUT/ACC operation. */
}
else {
MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list),
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 6a0bbab..757cee4 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1519,9 +1519,11 @@ int MPIDI_CH3_PktHandler_Unlock(MPIDI_VC_t * vc ATTRIBUTE((unused)),
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
- mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_NONE,
- unlock_pkt->source_win_handle);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ if (!(unlock_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK)) {
+ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_NONE,
+ unlock_pkt->source_win_handle);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ }
MPIDI_CH3_Progress_signal_completion();
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index c1a2c01..5ecf2cd 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -1488,8 +1488,6 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr)
OPA_read_write_barrier();
}
- MPIU_Assert(win_ptr->outstanding_unlocks == 0);
-
/* Unlock MYSELF and processes on SHM. */
if (!(win_ptr->lock_all_assert & MPI_MODE_NOCHECK)) {
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
@@ -1503,8 +1501,7 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr)
if (i == rank) continue;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, i, &target_vc);
if (orig_vc->node_id == target_vc->node_id) {
- win_ptr->outstanding_unlocks++;
- mpi_errno = send_unlock_msg(i, win_ptr);
+ mpi_errno = send_unlock_msg(i, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
@@ -1561,8 +1558,7 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr)
continue;
}
- win_ptr->outstanding_unlocks++;
- mpi_errno = send_unlock_msg(i, win_ptr);
+ mpi_errno = send_unlock_msg(i, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
@@ -1579,12 +1575,12 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr)
mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
&remote_completed);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- if (!remote_completed || win_ptr->outstanding_unlocks) {
+ if (!remote_completed) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
- } while (!remote_completed || win_ptr->outstanding_unlocks);
+ } while (!remote_completed);
/* Cleanup all targets on this window. */
mpi_errno = MPIDI_CH3I_RMA_Cleanup_targets_win(win_ptr);
diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c
index 83a5b35..42b4521 100644
--- a/src/mpid/ch3/src/mpid_rma.c
+++ b/src/mpid/ch3/src/mpid_rma.c
@@ -336,7 +336,6 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
(*win_ptr)->lock_all_assert = 0;
(*win_ptr)->lock_epoch_count = 0;
(*win_ptr)->outstanding_locks = 0;
- (*win_ptr)->outstanding_unlocks = 0;
/* Initialize the info flags */
(*win_ptr)->info_args.no_locks = 0;
diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c
index 616f5ed..88ca0f8 100644
--- a/src/mpid/ch3/src/mpidi_rma.c
+++ b/src/mpid/ch3/src/mpidi_rma.c
@@ -169,6 +169,24 @@ int MPIDI_Win_free(MPID_Win ** win_ptr)
(*win_ptr)->states.exposure_state != MPIDI_RMA_NONE,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
+ /* 1. Here we must wait until all passive locks are released on this target,
+ because for some UNLOCK messages, we do not send ACK back to origin,
+ we must wait until lock is released so that we can free window.
+ 2. We also need to wait until AT completion counter being zero, because
+ this counter is increment everytime we meet a GET-like operation, it is
+ possible that when target entering Win_free, passive epoch is not finished
+ yet and there are still GETs doing on this target.
+ 3. We also need to wait until lock queue becomes empty. It is possible
+ that some lock requests is still waiting in the queue when target is
+ entering Win_free. */
+ while ((*win_ptr)->current_lock_type != MPID_LOCK_NONE ||
+ (*win_ptr)->at_completion_counter != 0 ||
+ (*win_ptr)->lock_queue != NULL) {
+ mpi_errno = wait_progress_engine();
+ if (mpi_errno != MPI_SUCCESS)
+ MPIU_ERR_POP(mpi_errno);
+ }
+
if (!(*win_ptr)->shm_allocated) {
/* when SHM is allocated, we already did a global barrier in
MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */
-----------------------------------------------------------------------
Summary of changes:
src/mpid/ch3/include/mpid_rma_issue.h | 12 ++--
src/mpid/ch3/include/mpid_rma_oplist.h | 1 +
src/mpid/ch3/include/mpid_rma_types.h | 2 +
src/mpid/ch3/include/mpidimpl.h | 8 ++-
src/mpid/ch3/include/mpidpkt.h | 4 +-
src/mpid/ch3/include/mpidpre.h | 1 -
src/mpid/ch3/include/mpidrma.h | 119 +++++++++++++++++++++---------
src/mpid/ch3/src/ch3u_handle_recv_req.c | 83 ++++++----------------
src/mpid/ch3/src/ch3u_handle_send_req.c | 120 +++++++++++++++++++++++--------
src/mpid/ch3/src/ch3u_rma_oplist.c | 27 +++++++-
src/mpid/ch3/src/ch3u_rma_ops.c | 8 ++
src/mpid/ch3/src/ch3u_rma_pkthandler.c | 42 ++++--------
src/mpid/ch3/src/ch3u_rma_sync.c | 12 +--
src/mpid/ch3/src/mpid_rma.c | 1 -
src/mpid/ch3/src/mpidi_rma.c | 18 +++++
15 files changed, 284 insertions(+), 174 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list