[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.2b4-90-gbead85d
Service Account
noreply at mpich.org
Sun Aug 9 14:01:22 CDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via bead85d38e01c7baf66c36aa6372a1272eab0176 (commit)
via 022928ff7314f8fddfd3c0e913b11310e2be547a (commit)
via 68b26e3c7d3ba336554f3bd8bf6cf138313180cb (commit)
from 18ddb4c38036d92a2f7181eccc6e403299f81efd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/bead85d38e01c7baf66c36aa6372a1272eab0176
commit bead85d38e01c7baf66c36aa6372a1272eab0176
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Fri Aug 7 12:58:45 2015 -0500
Use counter num_ops_flush_not_issued to control issuing of FLUSH messages.
Here we use counter 'num_ops_flush_not_issued' to count number of operations
that are issued out but no FLUSH message issued afterwards yet to detect
their remote completion. When we reach the tail of the pending list, if
this counter is non-zero, we will issue a FLUSH message and reset the
counter to zero; if the counter is zero, which means there is no operation
that needs detecting remote completion, then we do not need to issue FLUSH
message at all.
Originally we use a flag 'put_acc_issued' to achieve the similiar purpose,
but that is not a clean design and miss some cases. This counter is a better
design on saving FLUSH messages as many as possible.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h
index ab7aca9..4355d9d 100644
--- a/src/mpid/ch3/include/mpid_rma_oplist.h
+++ b/src/mpid/ch3/include/mpid_rma_oplist.h
@@ -36,6 +36,7 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
(target_)->num_pkts_wait_for_local_completion == 0) { \
local_completed_ = 1; \
if ((target_)->sync.sync_flag == MPIDI_RMA_SYNC_NONE && \
+ (target_)->num_ops_flush_not_issued == 0 && \
(target_)->sync.outstanding_acks == 0) \
remote_completed_ = 1; \
} \
@@ -265,12 +266,12 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr
e->lock_type = MPID_LOCK_NONE;
e->lock_mode = 0;
e->win_complete_flag = 0;
- e->put_acc_issued = 0;
e->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
e->sync.outstanding_acks = 0;
e->num_pkts_wait_for_local_completion = 0;
+ e->num_ops_flush_not_issued = 0;
return e;
}
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index 251fe11..4e3b9e4 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -74,8 +74,6 @@ typedef struct MPIDI_RMA_Target {
int lock_type; /* NONE, SHARED, EXCLUSIVE */
int lock_mode; /* e.g., MODE_NO_CHECK */
int win_complete_flag;
- int put_acc_issued; /* indicate if PUT/ACC is issued in this epoch
- * after the previous synchronization calls. */
/* The target structure is free to be cleaned up when all of the
* following conditions hold true:
@@ -96,6 +94,9 @@ typedef struct MPIDI_RMA_Target {
/* number of packets that are waiting for local completion */
int num_pkts_wait_for_local_completion;
+ /* number of operations that does not have a FLUSH issued afterwards */
+ int num_ops_flush_not_issued;
+
MPIDI_RMA_Pool_type_t pool_type;
} MPIDI_RMA_Target_t;
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index a879cda..6ffd034 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -751,7 +751,10 @@ static inline int handle_lock_ack_with_op(MPID_Win * win_ptr,
target->next_op_to_issue = op->next;
if (target->next_op_to_issue == NULL) {
- if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || op_flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+ if (((target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) &&
+ (op_flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)) ||
+ ((target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) &&
+ (op_flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))) {
/* We are done with ending sync, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
}
@@ -868,8 +871,6 @@ static inline int MPIDI_CH3I_RMA_Handle_ack(MPID_Win * win_ptr, int target_rank)
t->sync.outstanding_acks--;
MPIU_Assert(t->sync.outstanding_acks >= 0);
- t->put_acc_issued = 0; /* reset PUT_ACC_FLAG after FLUSH is completed */
-
fn_exit:
return mpi_errno;
fn_fail:
diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c
index 26d8cf4..f07bb1a 100644
--- a/src/mpid/ch3/src/ch3u_rma_progress.c
+++ b/src/mpid/ch3/src/ch3u_rma_progress.c
@@ -177,9 +177,11 @@ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Ta
if (target->win_complete_flag) {
if (target->pending_net_ops_list_head == NULL) {
MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE;
- if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH && target->put_acc_issued) {
+ if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH &&
+ target->num_ops_flush_not_issued > 0) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
target->sync.outstanding_acks++;
+ target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_decr_at_cnt_msg(target->target_rank, win_ptr, flags);
@@ -195,9 +197,10 @@ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Ta
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if (target->pending_net_ops_list_head == NULL) {
if (target->target_rank != rank) {
- if (target->put_acc_issued) {
+ if (target->num_ops_flush_not_issued > 0) {
target->sync.outstanding_acks++;
+ target->num_ops_flush_not_issued = 0;
mpi_errno = send_flush_msg(target->target_rank, win_ptr);
if (mpi_errno != MPI_SUCCESS)
@@ -220,15 +223,12 @@ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Ta
}
else {
MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
- if (!target->put_acc_issued) {
- /* We did not issue PUT/ACC since the last
- * synchronization call, therefore here we
- * don't need ACK back */
-
+ if (target->num_ops_flush_not_issued == 0) {
flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
}
else {
target->sync.outstanding_acks++;
+ target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
if (mpi_errno != MPI_SUCCESS)
@@ -295,6 +295,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
break;
}
+ target->num_ops_flush_not_issued++;
+
flags = MPIDI_CH3_PKT_FLAG_NONE;
if (first_op) {
@@ -312,9 +314,14 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
first_op = 0;
}
- /* piggyback FLUSH on every OP if ordered flush is not guaranteed. */
- if (!MPIDI_CH3U_Win_pkt_orderings.am_flush_ordered)
+ /* piggyback FLUSH on current OP if one of the following
+ * conditions meet:
+ * (1) ordered flush is not guaranteed;
+ * (2) operation is a READ op (GET, GACC, FOP, CAS) */
+ if ((!MPIDI_CH3U_Win_pkt_orderings.am_flush_ordered) ||
+ MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt)) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
+ }
if (curr_op->next == NULL) {
/* piggyback on last OP. */
@@ -335,8 +342,10 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
/* only increase ack counter when FLUSH or UNLOCK flag is set,
* but without LOCK piggyback. */
if (((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)
- || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)))
+ || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))) {
target->sync.outstanding_acks++;
+ target->num_ops_flush_not_issued = 0;
+ }
mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
if (mpi_errno != MPI_SUCCESS)
@@ -344,11 +353,6 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
(*made_progress) = 1;
- if (!MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt)) {
- target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
- * PUT/ACC operation. */
- }
-
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
/* If this operation is piggybacked with LOCK,
@@ -361,7 +365,10 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
target->next_op_to_issue = curr_op->next;
if (target->next_op_to_issue == NULL) {
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+ if (((target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) &&
+ (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)) ||
+ ((target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) &&
+ (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))) {
/* We are done with ending sync, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
}
http://git.mpich.org/mpich.git/commitdiff/022928ff7314f8fddfd3c0e913b11310e2be547a
commit 022928ff7314f8fddfd3c0e913b11310e2be547a
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Fri Aug 7 09:20:22 2015 -0500
Free RMA op object immediately after it is issued out.
Originally we maintain RMA operations after they are
issued out in order to detect the local completion of
those operations, since operation structure will be
freed in the request completion cb when operation is
locally completed. However, we do not need to maintain
operation structure at all after the operation is
issued out, instead, we can just use a counter to keep
track of number of operations that are waiting for
local completion, and decrement that counter in the
request completion cb.
In this patch, we delete all the code that related
to maintain ssued but incomplete operation structure.
Therefore, operation objects exist only when synchronization
is not finished and operations stay in the pending list.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h
index 4961571..ab7aca9 100644
--- a/src/mpid/ch3/include/mpid_rma_oplist.h
+++ b/src/mpid/ch3/include/mpid_rma_oplist.h
@@ -10,7 +10,6 @@
#include "mpl_utlist.h"
#include "mpid_rma_types.h"
-int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr);
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr);
int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target);
int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress);
@@ -21,28 +20,6 @@ extern MPIDI_RMA_Target_t *global_rma_target_pool_head, *global_rma_target_pool_
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
-#define MPIDI_CH3I_RMA_Get_issued_list_ptr(target_, op_, list_ptr_, mpi_errno_) \
- do { \
- MPI_Datatype target_datatype_; \
- int is_read_; \
- MPIDI_CH3I_RMA_PKT_IS_READ_OP((op_)->pkt, is_read_); \
- MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE((op_)->pkt, target_datatype_, mpi_errno_); \
- if ((target_datatype_ != MPI_DATATYPE_NULL && \
- !MPIR_DATATYPE_IS_PREDEFINED(target_datatype_)) || \
- ((op_)->origin_datatype != MPI_DATATYPE_NULL && \
- !MPIR_DATATYPE_IS_PREDEFINED((op_)->origin_datatype)) || \
- ((op_)->result_datatype != MPI_DATATYPE_NULL && \
- !MPIR_DATATYPE_IS_PREDEFINED((op_)->result_datatype))) { \
- list_ptr_ = &((target_)->issued_dt_op_list_head); \
- } \
- else if (!is_read_) { \
- list_ptr_ = &((target_)->issued_write_op_list_head); \
- } \
- else { \
- list_ptr_ = &((target_)->issued_read_op_list_head); \
- } \
- } while (0)
-
/* This macro returns two flags: local_completed and remote_completed,
* to indicate if the completion is reached on this target. */
#define MPIDI_CH3I_RMA_ops_completion(win_, target_, local_completed_, remote_completed_) \
@@ -56,23 +33,12 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
(target_)->access_state != MPIDI_RMA_LOCK_ISSUED && \
(target_)->pending_net_ops_list_head == NULL && \
(target_)->pending_user_ops_list_head == NULL && \
- (target_)->issued_read_op_list_head == NULL && \
- (target_)->issued_write_op_list_head == NULL && \
- (target_)->issued_dt_op_list_head == NULL) { \
+ (target_)->num_pkts_wait_for_local_completion == 0) { \
local_completed_ = 1; \
if ((target_)->sync.sync_flag == MPIDI_RMA_SYNC_NONE && \
(target_)->sync.outstanding_acks == 0) \
remote_completed_ = 1; \
} \
- \
- if (((target_)->sync.upgrade_flush_local && !remote_completed_) || \
- (!(target_)->sync.upgrade_flush_local && !local_completed_)) { \
- local_completed_ = 0; \
- } \
- else { \
- local_completed_ = 1; \
- } \
- \
} while (0)
@@ -117,19 +83,13 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
\
for (i_ = 0; i_ < (win_ptr_)->num_slots; i_++) { \
for (win_target_ = (win_ptr_)->slots[i_].target_list_head; win_target_;) { \
- int local_ = 0, remote_ = 0; \
+ int local_ = 0, remote_ ATTRIBUTE((unused)) = 0; \
\
- if (win_target_->sync.upgrade_flush_local) \
- total_remote_cnt_++; \
- else \
- total_local_cnt_++; \
+ total_local_cnt_++; \
\
MPIDI_CH3I_RMA_ops_completion((win_ptr_), win_target_, local_, remote_); \
\
- if (win_target_->sync.upgrade_flush_local) \
- remote_completed_targets_ += remote_; \
- else \
- local_completed_targets_ += local_; \
+ local_completed_targets_ += local_; \
\
win_target_ = win_target_->next; \
} \
@@ -242,8 +202,6 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr)
e->origin_datatype = MPI_DATATYPE_NULL;
e->result_datatype = MPI_DATATYPE_NULL;
- e->ref_cnt = 0;
-
return e;
}
@@ -298,9 +256,6 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr
MPL_DL_DELETE(win_ptr->target_pool_head, e);
}
- e->issued_read_op_list_head = NULL;
- e->issued_write_op_list_head = NULL;
- e->issued_dt_op_list_head = NULL;
e->pending_net_ops_list_head = NULL;
e->pending_user_ops_list_head = NULL;
e->next_op_to_issue = NULL;
@@ -314,7 +269,8 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr
e->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
e->sync.outstanding_acks = 0;
- e->sync.upgrade_flush_local = 0;
+
+ e->num_pkts_wait_for_local_completion = 0;
return e;
}
@@ -332,9 +288,6 @@ static inline int MPIDI_CH3I_Win_target_free(MPID_Win * win_ptr, MPIDI_RMA_Targe
/* We enqueue elements to the right pool, so when they get freed
* at window free time, they won't conflict with the global pool
* or other windows */
- MPIU_Assert(e->issued_read_op_list_head == NULL);
- MPIU_Assert(e->issued_write_op_list_head == NULL);
- MPIU_Assert(e->issued_dt_op_list_head == NULL);
MPIU_Assert(e->pending_net_ops_list_head == NULL);
MPIU_Assert(e->pending_user_ops_list_head == NULL);
@@ -569,16 +522,6 @@ static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t ** e)
if (new_ptr != NULL)
break;
- mpi_errno = MPIDI_CH3I_RMA_Free_ops_before_completion(win_ptr);
- if (mpi_errno != MPI_SUCCESS)
- MPIU_ERR_POP(mpi_errno);
-
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
- new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
- if (new_ptr != NULL)
- break;
-
mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_aggressive(win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index 3014c0a..251fe11 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -61,13 +61,9 @@ typedef struct MPIDI_RMA_Op {
MPID_Request *ureq;
- int ref_cnt;
} MPIDI_RMA_Op_t;
typedef struct MPIDI_RMA_Target {
- struct MPIDI_RMA_Op *issued_read_op_list_head;
- struct MPIDI_RMA_Op *issued_write_op_list_head;
- struct MPIDI_RMA_Op *issued_dt_op_list_head;
struct MPIDI_RMA_Op *pending_net_ops_list_head; /* pending operations that are waiting for network events */
struct MPIDI_RMA_Op *pending_user_ops_list_head; /* pending operations that are waiting for user events */
struct MPIDI_RMA_Op *next_op_to_issue;
@@ -95,10 +91,11 @@ typedef struct MPIDI_RMA_Target {
/* packets sent out that we are expecting an ack for */
int outstanding_acks;
- /* Marked when FLUSH_LOCAL is upgraded to FLUSH */
- int upgrade_flush_local;
} sync;
+ /* number of packets that are waiting for local completion */
+ int num_pkts_wait_for_local_completion;
+
MPIDI_RMA_Pool_type_t pool_type;
} MPIDI_RMA_Target_t;
diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h
index 6390bbc..45490c4 100644
--- a/src/mpid/ch3/include/mpidpre.h
+++ b/src/mpid/ch3/include/mpidpre.h
@@ -450,7 +450,7 @@ typedef struct MPIDI_Request {
* and freed when release request. */
MPIDI_msg_sz_t ext_hdr_sz;
- struct MPIDI_RMA_Op *rma_op_ptr;
+ struct MPIDI_RMA_Target *rma_target_ptr;
MPIDI_REQUEST_SEQNUM
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 4f5ea28..a879cda 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -644,6 +644,7 @@ static inline int check_and_set_req_completion(MPID_Win * win_ptr, MPIDI_RMA_Tar
MPIDI_RMA_Op_t * rma_op, int *op_completed)
{
int i, mpi_errno = MPI_SUCCESS;
+ int incomplete_req_cnt = 0;
MPID_Request **req = NULL;
MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SET_REQ_COMPLETION);
@@ -667,12 +668,12 @@ static inline int check_and_set_req_completion(MPID_Win * win_ptr, MPIDI_RMA_Tar
else {
(*req)->request_completed_cb = MPIDI_CH3_Req_handler_rma_op_complete;
(*req)->dev.source_win_handle = win_ptr->handle;
- (*req)->dev.rma_op_ptr = rma_op;
+ (*req)->dev.rma_target_ptr = target;
- rma_op->ref_cnt++;
+ incomplete_req_cnt++;
if (rma_op->ureq != NULL) {
- MPID_cc_set(&(rma_op->ureq->cc), rma_op->ref_cnt);
+ MPID_cc_set(&(rma_op->ureq->cc), incomplete_req_cnt);
(*req)->dev.request_handle = rma_op->ureq->handle;
}
@@ -680,31 +681,22 @@ static inline int check_and_set_req_completion(MPID_Win * win_ptr, MPIDI_RMA_Tar
}
}
- if (rma_op->ref_cnt == 0) {
+ if (incomplete_req_cnt == 0) {
if (rma_op->ureq != NULL) {
mpi_errno = MPID_Request_complete(rma_op->ureq);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
- MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_net_ops_list_head), rma_op);
-
(*op_completed) = TRUE;
}
else {
- MPIDI_RMA_Op_t **list_ptr = NULL;
-
- MPIDI_CH3I_RMA_Get_issued_list_ptr(target, rma_op, list_ptr, mpi_errno);
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- MPL_DL_DELETE(target->pending_net_ops_list_head, rma_op);
- MPL_DL_APPEND((*list_ptr), rma_op);
-
- win_ptr->active_req_cnt += rma_op->ref_cnt;
+ win_ptr->active_req_cnt += incomplete_req_cnt;
+ target->num_pkts_wait_for_local_completion += incomplete_req_cnt;
}
+ MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_net_ops_list_head), rma_op);
+
if (target->pending_net_ops_list_head == NULL) {
win_ptr->num_targets_with_pending_net_ops--;
MPIU_Assert(win_ptr->num_targets_with_pending_net_ops >= 0);
diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c
index 3288c57..269089f 100644
--- a/src/mpid/ch3/src/ch3u_handle_op_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_op_req.c
@@ -14,14 +14,17 @@
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIDI_CH3_Req_handler_rma_op_complete(MPID_Request * sreq)
{
- int i, mpi_errno = MPI_SUCCESS;
+ int mpi_errno = MPI_SUCCESS;
MPID_Request *ureq = NULL;
- MPIDI_RMA_Op_t *op = NULL;
MPID_Win *win_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE);
+ if (sreq->dev.rma_target_ptr != NULL) {
+ (sreq->dev.rma_target_ptr)->num_pkts_wait_for_local_completion--;
+ }
+
/* get window, decrement active request cnt on window */
MPID_Win_get_ptr(sreq->dev.source_win_handle, win_ptr);
MPIU_Assert(win_ptr != NULL);
@@ -36,50 +39,6 @@ int MPIDI_CH3_Req_handler_rma_op_complete(MPID_Request * sreq)
}
}
- op = sreq->dev.rma_op_ptr;
- /* Note: if rma_op_ptr is NULL, it means the operation
- * is freed before the completion. */
- if (op != NULL) {
- /* Mark this request as NULL in the operation struct,
- * so that when operation is freed before completion,
- * the operation do not need to try to notify this
- * request which is already completed. */
- if (op->reqs_size == 1) {
- MPIU_Assert(op->single_req->handle == sreq->handle);
- op->single_req = NULL;
- }
- else {
- for (i = 0; i < op->reqs_size; i++) {
- if (op->multi_reqs[i] == NULL)
- continue;
- else if (op->multi_reqs[i]->handle == sreq->handle) {
- op->multi_reqs[i] = NULL;
- break;
- }
- }
- }
-
- op->ref_cnt--;
- MPIU_Assert(op->ref_cnt >= 0);
- if (op->ref_cnt == 0) {
- MPIDI_RMA_Target_t *target = NULL;
- MPIDI_RMA_Op_t **list_ptr = NULL;
-
- mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, op->target_rank, &target);
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_POP(mpi_errno);
- }
- MPIU_Assert(target != NULL);
-
- MPIDI_CH3I_RMA_Get_issued_list_ptr(target, op, list_ptr, mpi_errno);
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list_ptr, op);
- }
- }
-
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE);
return mpi_errno;
diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c
index 94dbb91..fe7bfd3 100644
--- a/src/mpid/ch3/src/ch3u_request.c
+++ b/src/mpid/ch3/src/ch3u_request.c
@@ -100,7 +100,7 @@ MPID_Request * MPID_Request_create(void)
req->dev.tmpbuf = NULL;
req->dev.ext_hdr_ptr = NULL;
req->dev.ext_hdr_sz = 0;
- req->dev.rma_op_ptr = NULL;
+ req->dev.rma_target_ptr = NULL;
req->dev.request_handle = MPI_REQUEST_NULL;
#ifdef MPIDI_CH3_REQUEST_INIT
MPIDI_CH3_REQUEST_INIT(req);
diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c
index e344c3d..26d8cf4 100644
--- a/src/mpid/ch3/src/ch3u_rma_progress.c
+++ b/src/mpid/ch3/src/ch3u_rma_progress.c
@@ -447,105 +447,6 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress)
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Free_ops_before_completion
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr)
-{
- MPIDI_RMA_Op_t *curr_op = NULL;
- MPIDI_RMA_Target_t *curr_target = NULL;
- MPIDI_RMA_Op_t **op_list_head = NULL;
- int read_flag = 0;
- int i, made_progress = 0;
- int mpi_errno = MPI_SUCCESS;
-
- /* make nonblocking progress once */
- mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_ptr, &made_progress);
- if (mpi_errno != MPI_SUCCESS)
- MPIU_ERR_POP(mpi_errno);
-
- if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
- win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
- win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
- goto fn_exit;
-
- /* find targets that have operations */
- for (i = 0; i < win_ptr->num_slots; i++) {
- if (win_ptr->slots[i].target_list_head != NULL) {
- curr_target = win_ptr->slots[i].target_list_head;
- while (curr_target != NULL) {
- if (curr_target->issued_read_op_list_head != NULL ||
- curr_target->issued_write_op_list_head != NULL) {
- if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
- win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
- if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED)
- break;
- }
- else {
- break;
- }
- }
- curr_target = curr_target->next;
- }
- if (curr_target != NULL)
- break;
- }
- }
-
- if (curr_target == NULL)
- goto fn_exit;
-
- /* After we do this, all following Win_flush_local
- * must do a Win_flush instead. */
- curr_target->sync.upgrade_flush_local = 1;
-
- op_list_head = &curr_target->issued_read_op_list_head;
- read_flag = 1;
-
- curr_op = *op_list_head;
-
- /* free all ops in the list since we do not need to maintain them anymore */
- while (1) {
- if (curr_op != NULL) {
- /* Here we mark the operation pointer in the request
- * as NULL, so that when request is completed, it will
- * not try to free this operation. */
- if (curr_op->reqs_size == 1) {
- MPIU_Assert(curr_op->single_req != NULL);
- curr_op->single_req->dev.rma_op_ptr = NULL;
- }
- else {
- MPIU_Assert(curr_op->reqs_size > 1);
- MPIU_Assert(curr_op->multi_reqs != NULL);
- for (i = 0; i < curr_op->reqs_size; i++) {
- if (curr_op->multi_reqs[i] != NULL) {
- curr_op->multi_reqs[i]->dev.rma_op_ptr = NULL;
- }
- }
- }
- MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, op_list_head, curr_op);
- }
- else {
- if (read_flag == 1) {
- op_list_head = &curr_target->issued_write_op_list_head;
- read_flag = 0;
- }
- else {
- /* we reach the tail of write_op_list, break out. */
- break;
- }
- }
- curr_op = *op_list_head;
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_aggressive
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index 92fbccd..19dca35 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -343,15 +343,8 @@ static inline int flush_local_all(MPID_Win * win_ptr)
for (i = 0; i < win_ptr->num_slots; i++) {
curr_target = win_ptr->slots[i].target_list_head;
while (curr_target != NULL) {
- if (curr_target->sync.upgrade_flush_local) {
- if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) {
- curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
- }
- }
- else {
- if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL) {
- curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;
- }
+ if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL) {
+ curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;
}
curr_target = curr_target->next;
@@ -1418,11 +1411,6 @@ int MPID_Win_flush(int dest, MPID_Win * win_ptr)
MPIU_ERR_POP(mpi_errno);
}
- if (target != NULL && target->sync.upgrade_flush_local) {
- /* reset upgrade_flush_local flag in target to 0 */
- target->sync.upgrade_flush_local = 0;
- }
-
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPID_WIN_FLUSH);
return mpi_errno;
@@ -1460,35 +1448,28 @@ int MPID_Win_flush_local(int dest, MPID_Win * win_ptr)
}
if (dest == MPI_PROC_NULL)
- goto finish_flush_local;
+ goto fn_exit;
mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, dest, &target);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
if (target == NULL)
- goto finish_flush_local;
+ goto fn_exit;
if (rank == dest)
- goto finish_flush_local;
+ goto fn_exit;
if (win_ptr->shm_allocated) {
MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, dest, &target_vc);
MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
if (orig_vc->node_id == target_vc->node_id)
- goto finish_flush_local;
+ goto fn_exit;
}
/* Set sync_flag in sync struct. */
- if (target->sync.upgrade_flush_local) {
- if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) {
- target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
- }
- }
- else {
- if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
- target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;
- }
+ if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
+ target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;
/* Issue out all operations. */
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, dest, &made_progress);
@@ -1506,12 +1487,6 @@ int MPID_Win_flush_local(int dest, MPID_Win * win_ptr)
}
} while (!local_completed);
- finish_flush_local:
- if (target != NULL) {
- /* reset upgrade_flush_local flag in target to 0 */
- target->sync.upgrade_flush_local = 0;
- }
-
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPID_WIN_FLUSH_LOCAL);
return mpi_errno;
@@ -1743,8 +1718,6 @@ int MPID_Win_unlock_all(MPID_Win * win_ptr)
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_Win_flush_all(MPID_Win * win_ptr)
{
- int i;
- MPIDI_RMA_Target_t *curr_target = NULL;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPIDI_STATE_MPID_WIN_FLUSH_ALL);
@@ -1767,15 +1740,6 @@ int MPID_Win_flush_all(MPID_Win * win_ptr)
MPIU_Assert(win_ptr->active_req_cnt == 0);
- /* reset upgrade_flush_local flag in target to 0 */
- for (i = 0; i < win_ptr->num_slots; i++) {
- curr_target = win_ptr->slots[i].target_list_head;
- while (curr_target != NULL) {
- curr_target->sync.upgrade_flush_local = 0;
- curr_target = curr_target->next;
- }
- }
-
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPIDI_STATE_MPID_WIN_FLUSH_ALL);
return mpi_errno;
@@ -1792,8 +1756,6 @@ int MPID_Win_flush_all(MPID_Win * win_ptr)
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_Win_flush_local_all(MPID_Win * win_ptr)
{
- int i;
- MPIDI_RMA_Target_t *curr_target = NULL;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_WIN_FLUSH_LOCAL_ALL);
@@ -1816,15 +1778,6 @@ int MPID_Win_flush_local_all(MPID_Win * win_ptr)
MPIU_Assert(win_ptr->active_req_cnt == 0);
- /* reset upgrade_flush_local flag in target to 0 */
- for (i = 0; i < win_ptr->num_slots; i++) {
- curr_target = win_ptr->slots[i].target_list_head;
- while (curr_target != NULL) {
- curr_target->sync.upgrade_flush_local = 0;
- curr_target = curr_target->next;
- }
- }
-
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPID_WIN_FLUSH_LOCAL_ALL);
return mpi_errno;
http://git.mpich.org/mpich.git/commitdiff/68b26e3c7d3ba336554f3bd8bf6cf138313180cb
commit 68b26e3c7d3ba336554f3bd8bf6cf138313180cb
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Fri Aug 7 12:53:55 2015 -0500
Simplify macros for RMA op.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h
index 05780e4..5dc1001 100644
--- a/src/mpid/ch3/include/mpidpkt.h
+++ b/src/mpid/ch3/include/mpidpkt.h
@@ -613,36 +613,21 @@ MPIDI_CH3_PKT_DEFS
/* This macro judges if the RMA operation is a read operation,
* which means, it will triffer the issuing of response data from
* the target to the origin */
-#define MPIDI_CH3I_RMA_PKT_IS_READ_OP(pkt_, is_read_op_) \
- do { \
- if ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_GET_ACCUM || \
- (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_FOP || \
- (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_GET) { \
- is_read_op_ = TRUE; \
- } \
- else { \
- is_read_op_ = FALSE; \
- } \
- } while (0)
-
+#define MPIDI_CH3I_RMA_PKT_IS_READ_OP(pkt_) \
+ ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_GET_ACCUM || \
+ (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_FOP || \
+ (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_GET)
/* This macro judges if the RMA operation is a immed operation */
-#define MPIDI_CH3I_RMA_PKT_IS_IMMED_OP(pkt_, is_immed_op_) \
- do { \
- if ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_PUT_IMMED || \
- (pkt_).type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { \
- is_immed_op_ = TRUE; \
- } \
- else { \
- is_immed_op_ = FALSE; \
- } \
- } while (0)
+#define MPIDI_CH3I_RMA_PKT_IS_IMMED_OP(pkt_) \
+ ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_PUT_IMMED || \
+ (pkt_).type == MPIDI_CH3_PKT_ACCUMULATE_IMMED)
typedef struct MPIDI_CH3_Pkt_put {
MPIDI_CH3_Pkt_type_t type;
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 05d563a..4f5ea28 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -326,7 +326,6 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
MPIDI_CH3_Pkt_flags_t flag;
MPI_Win source_win_handle;
MPI_Request request_handle;
- int is_immed_op = FALSE;
int lock_discarded = 0, data_discarded = 0;
int mpi_errno = MPI_SUCCESS;
@@ -343,8 +342,8 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
lock_discarded = 1;
}
- MPIDI_CH3I_RMA_PKT_IS_IMMED_OP((*pkt), is_immed_op);
- if (is_immed_op || pkt->type == MPIDI_CH3_PKT_LOCK || pkt->type == MPIDI_CH3_PKT_GET) {
+ if (MPIDI_CH3I_RMA_PKT_IS_IMMED_OP(*pkt) || pkt->type == MPIDI_CH3_PKT_LOCK ||
+ pkt->type == MPIDI_CH3_PKT_GET) {
/* return bytes of data processed in this pkt handler */
(*buflen) = sizeof(MPIDI_CH3_Pkt_t);
@@ -425,12 +424,10 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
MPIDI_CH3_Pkt_t new_pkt;
MPIDI_CH3_Pkt_lock_t *lock_pkt = &new_pkt.lock;
MPI_Win target_win_handle;
- int is_read_op = FALSE;
- MPIDI_CH3I_RMA_PKT_IS_READ_OP((*pkt), is_read_op);
MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno);
- if (!is_read_op) {
+ if (!MPIDI_CH3I_RMA_PKT_IS_READ_OP(*pkt)) {
MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno);
request_handle = MPI_REQUEST_NULL;
}
@@ -525,8 +522,6 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
MPIU_ERR_POP(mpi_errno);
}
else {
- int is_read_op = FALSE;
-
if (lock_discarded)
flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED;
else if (data_discarded)
@@ -534,8 +529,7 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
else
flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED;
- MPIDI_CH3I_RMA_PKT_IS_READ_OP((*pkt), is_read_op);
- if (!is_read_op) {
+ if (!MPIDI_CH3I_RMA_PKT_IS_READ_OP(*pkt)) {
MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno);
request_handle = MPI_REQUEST_NULL;
}
diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c
index dc4324d..e344c3d 100644
--- a/src/mpid/ch3/src/ch3u_rma_progress.c
+++ b/src/mpid/ch3/src/ch3u_rma_progress.c
@@ -288,7 +288,6 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
curr_op = target->next_op_to_issue;
while (curr_op != NULL) {
int op_completed = FALSE;
- int is_read_op = FALSE;
if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
/* It is possible that the previous OP+LOCK changes
@@ -345,8 +344,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
(*made_progress) = 1;
- MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt, is_read_op);
- if (!is_read_op) {
+ if (!MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt)) {
target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
* PUT/ACC operation. */
}
-----------------------------------------------------------------------
Summary of changes:
src/mpid/ch3/include/mpid_rma_oplist.h | 72 ++---------------
src/mpid/ch3/include/mpid_rma_types.h | 14 ++--
src/mpid/ch3/include/mpidpkt.h | 41 +++-------
src/mpid/ch3/include/mpidpre.h | 2 +-
src/mpid/ch3/include/mpidrma.h | 47 ++++-------
src/mpid/ch3/src/ch3u_handle_op_req.c | 51 +-----------
src/mpid/ch3/src/ch3u_request.c | 2 +-
src/mpid/ch3/src/ch3u_rma_progress.c | 140 +++++--------------------------
src/mpid/ch3/src/ch3u_rma_sync.c | 63 ++-------------
9 files changed, 82 insertions(+), 350 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list