[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1.3-78-gfc7617f
Service Account
noreply at mpich.org
Mon Nov 3 11:09:04 CST 2014
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via fc7617f21a4247b75f7989ecd8063d2b3b29ae7e (commit)
via b1685139330580132c64e4b616ae91824b9bf5b8 (commit)
via ba1a400c4de2df8a662f0f9f856d824ed05ec221 (commit)
via 0eaf344b95ae93d0546c4a20f4c6bc0581a88e51 (commit)
via cb04acb30ecb4e244973fa7c8c6033f92c71c89d (commit)
via 61f952c7a7f9ec2c59e496bf8275b29f12132768 (commit)
via 5c51303260595939cb15d658e5f369bc148c9a4a (commit)
from 0086b7bf7f2ec4e4ef39bca1c153c1000e137a3c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/fc7617f21a4247b75f7989ecd8063d2b3b29ae7e
commit fc7617f21a4247b75f7989ecd8063d2b3b29ae7e
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 22:00:27 2014 -0600
Add global / local pools of RMA ops and related APIs.
Instead of allocating / deallocating RMA operations whenever
an RMA op is posted by user, we allocate fixed size operation
pools beforehand and take the op element from those pools
when an RMA op is posted.
With only a local (per-window) op pool, the number of ops
allocated can increase arbitrarily if many windows are created.
Alternatively, if we only use a global op pool, other windows
might use up all operations thus starving the window we are
working on.
In this patch we create two pools: a local (per-window) pool and a
global pool. Every window is guaranteed to have at least the number
of operations in the local pool. If we run out of these operations,
we check in the global pool to see if we have any operations left.
When an operation is released, it is added back to the same pool it
was allocated from.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h
index bc9fa21..bc64150 100644
--- a/src/mpid/ch3/include/mpid_rma_oplist.h
+++ b/src/mpid/ch3/include/mpid_rma_oplist.h
@@ -10,6 +10,66 @@
#include "mpl_utlist.h"
#include "mpid_rma_types.h"
+extern struct MPIDI_RMA_Op *global_rma_op_pool, *global_rma_op_pool_tail, *global_rma_op_pool_start;
+
+/* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and
+ * initialize it. If we cannot get one, return NULL. */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_Win_op_alloc
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr)
+{
+ MPIDI_RMA_Op_t *e;
+
+ if (win_ptr->op_pool == NULL) {
+ /* local pool is empty, try to find something in the global pool */
+ if (global_rma_op_pool == NULL)
+ return NULL;
+ else {
+ e = global_rma_op_pool;
+ MPL_LL_DELETE(global_rma_op_pool, global_rma_op_pool_tail, e);
+ }
+ }
+ else {
+ e = win_ptr->op_pool;
+ MPL_LL_DELETE(win_ptr->op_pool, win_ptr->op_pool_tail, e);
+ }
+
+ e->dataloop = NULL;
+ e->request = NULL;
+
+ return e;
+}
+
+/* MPIDI_CH3I_Win_op_free(): put an op element back to the op pool which
+ * it belongs to. */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_Win_op_free
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline int MPIDI_CH3I_Win_op_free(MPID_Win * win_ptr, MPIDI_RMA_Op_t * e)
+{
+ int mpi_errno = MPI_SUCCESS;
+
+ /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
+ if (e->dataloop != NULL) {
+ MPIU_Free(e->dataloop);
+ }
+
+ /* We enqueue elements to the right pool, so when they get freed
+ * at window free time, they won't conflict with the global pool
+ * or other windows */
+ /* use PREPEND when return objects back to the pool
+ in order to improve cache performance */
+ if (e->pool_type == MPIDI_RMA_POOL_WIN)
+ MPL_LL_PREPEND(win_ptr->op_pool, win_ptr->op_pool_tail, e);
+ else
+ MPL_LL_PREPEND(global_rma_op_pool, global_rma_op_pool_tail, e);
+
+ return mpi_errno;
+}
+
/* Return nonzero if the RMA operations list is empty.
*/
#undef FUNCNAME
@@ -40,9 +100,9 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * lis
#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list)
+static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list_tail)
{
- return (*list) ? (*list)->prev : NULL;
+ return (*list_tail);
}
@@ -55,9 +115,10 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * lis
#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
+static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Ops_list_t * list_tail,
+ MPIDI_RMA_Op_t * elem)
{
- MPL_DL_APPEND(*list, elem);
+ MPL_LL_APPEND(*list, *list_tail, elem);
}
@@ -71,30 +132,23 @@ static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_
#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list,
+static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Ops_list_t * list_tail,
MPIDI_RMA_Op_t ** new_elem)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_RMA_Op_t *tmp_ptr;
- MPIU_CHKPMEM_DECL(1);
- /* FIXME: We should use a pool allocator here */
- MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t),
- mpi_errno, "RMA operation entry");
+ tmp_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
+ MPIU_ERR_CHKANDJUMP(tmp_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem");
- tmp_ptr->next = NULL;
- tmp_ptr->dataloop = NULL;
- tmp_ptr->request = NULL;
-
- MPL_DL_APPEND(*list, tmp_ptr);
+ MPL_LL_APPEND(*list, *list_tail, tmp_ptr);
*new_elem = tmp_ptr;
fn_exit:
- MPIU_CHKPMEM_COMMIT();
return mpi_errno;
fn_fail:
- MPIU_CHKPMEM_REAP();
*new_elem = NULL;
goto fn_exit;
}
@@ -109,9 +163,10 @@ static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list,
#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
+static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Ops_list_t *list_tail,
+ MPIDI_RMA_Op_t * elem)
{
- MPL_DL_DELETE(*list, elem);
+ MPL_LL_DELETE(*list, *list_tail, elem);
}
@@ -124,19 +179,17 @@ static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list,
+static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Ops_list_t * list_tail,
MPIDI_RMA_Op_t * curr_ptr)
{
MPIDI_RMA_Op_t *tmp_ptr = curr_ptr;
MPIU_Assert(curr_ptr != NULL);
- MPL_DL_DELETE(*list, curr_ptr);
+ MPL_LL_DELETE(*list, *list_tail, curr_ptr);
- /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
- if (tmp_ptr->dataloop != NULL)
- MPIU_Free(tmp_ptr->dataloop);
- MPIU_Free(tmp_ptr);
+ MPIDI_CH3I_Win_op_free(win_ptr, tmp_ptr);
}
@@ -151,12 +204,13 @@ static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list,
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list,
+static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Ops_list_t * list_tail,
MPIDI_RMA_Op_t ** curr_ptr)
{
MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next;
- MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr);
+ MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list, list_tail, *curr_ptr);
*curr_ptr = next_ptr;
}
@@ -167,12 +221,13 @@ static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list,
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list)
+static inline void MPIDI_CH3I_RMA_Ops_free(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Ops_list_t * list_tail)
{
MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr;
- MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
- MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr);
+ MPL_LL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
+ MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list, list_tail, curr_ptr);
}
}
@@ -197,4 +252,19 @@ static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_p
}
}
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list_tail(MPID_Win * win_ptr, int target)
+{
+ if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE ||
+ win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) {
+ return &win_ptr->at_rma_ops_list_tail;
+ }
+ else {
+ return &win_ptr->targets[target].rma_ops_list_tail;
+ }
+}
+
#endif /* MPID_RMA_OPLIST_H_INCLUDED */
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index c95847c..0176c96 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -46,9 +46,13 @@ typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
int has_sticky_ub, has_sticky_lb;
} MPIDI_RMA_dtype_info;
+typedef enum MPIDI_RMA_Pool_type {
+ MPIDI_RMA_POOL_WIN = 6,
+ MPIDI_RMA_POOL_GLOBAL = 7
+} MPIDI_RMA_Pool_type_t;
+
/* for keeping track of RMA ops, which will be executed at the next sync call */
typedef struct MPIDI_RMA_Op {
- struct MPIDI_RMA_Op *prev; /* pointer to next element in list */
struct MPIDI_RMA_Op *next; /* pointer to next element in list */
void *origin_addr;
@@ -69,6 +73,7 @@ typedef struct MPIDI_RMA_Op {
int target_rank;
MPIDI_CH3_Pkt_t pkt;
+ MPIDI_RMA_Pool_type_t pool_type;
} MPIDI_RMA_Op_t;
typedef struct MPIDI_PT_single_op {
diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h
index 3d59636..a16f003 100644
--- a/src/mpid/ch3/include/mpidpre.h
+++ b/src/mpid/ch3/include/mpidpre.h
@@ -249,6 +249,7 @@ struct MPIDI_RMA_op; /* forward decl from mpidrma.h */
struct MPIDI_Win_target_state {
struct MPIDI_RMA_Op *rma_ops_list;
/* List of outstanding RMA operations */
+ struct MPIDI_RMA_Op *rma_ops_list_tail;
volatile enum MPIDI_CH3_Lock_states remote_lock_state;
/* Indicates the state of the target
process' "lock" for passive target
@@ -284,6 +285,7 @@ struct MPIDI_Win_target_state {
mode of operation */ \
struct MPIDI_RMA_Op *at_rma_ops_list; /* Ops list for active target \
mode of operation. */ \
+ struct MPIDI_RMA_Op *at_rma_ops_list_tail; \
enum MPIDI_Win_epoch_states epoch_state; \
int epoch_count; \
int fence_issued; /* Indicates if fence has been called, and if an \
@@ -296,6 +298,9 @@ struct MPIDI_Win_target_state {
int start_assert; /* assert passed to MPI_Win_start */ \
int shm_allocated; /* flag: TRUE iff this window has a shared memory \
region associated with it */ \
+ struct MPIDI_RMA_Op *op_pool_start; /* start pointer used for freeing */\
+ struct MPIDI_RMA_Op *op_pool; /* pool of operations */ \
+ struct MPIDI_RMA_Op *op_pool_tail; /* tail pointer to pool of operations. */ \
#ifdef MPIDI_CH3_WIN_DECL
#define MPID_DEV_WIN_DECL \
@@ -440,4 +445,8 @@ MPID_REQUEST_DECL
/* Tell the RMA code to use a table of RMA functions provided by the
ADI */
#define USE_MPID_RMA_TABLE
+
+int MPIDI_RMA_init(void);
+void MPIDI_RMA_finalize(void);
+
#endif /* !defined(MPICH_MPIDPRE_H_INCLUDED) */
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index f45a2b4..a9ee171 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -73,11 +73,12 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
MPIDI_CH3_Pkt_put_t *put_pkt = NULL;
/* queue it up */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -184,11 +185,12 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
MPIDI_CH3_Pkt_get_t *get_pkt = NULL;
/* queue it up */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -296,11 +298,12 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL;
/* queue it up */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -441,10 +444,11 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
/* Append the operation to the window's RMA ops queue */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -575,11 +579,12 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL;
/* Append this operation to the RMA ops queue */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -669,11 +674,12 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
}
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
MPIDI_CH3_Pkt_fop_t *fop_pkt = NULL;
/* Append this operation to the RMA ops queue */
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index 3629dd5..b63e956 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -349,9 +349,11 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
int *wait_for_rma_done_pkt, MPIDI_CH3_Pkt_flags_t sync_flags);
static int send_lock_put_or_acc(MPID_Win *, int);
static int send_lock_get(MPID_Win *, int);
-static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list);
+static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list,
+ MPIDI_RMA_Ops_list_t * ops_list_tail);
static inline int rma_list_gc(MPID_Win * win_ptr,
MPIDI_RMA_Ops_list_t * ops_list,
+ MPIDI_RMA_Ops_list_t * ops_list_tail,
MPIDI_RMA_Op_t * last_elm, int *nDone);
@@ -366,6 +368,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
int *rma_target_proc, *nops_to_proc, i, total_op_count, *curr_ops_cnt;
MPIDI_RMA_Op_t *curr_ptr;
MPIDI_RMA_Ops_list_t *ops_list;
+ MPIDI_RMA_Ops_list_t *ops_list_tail;
MPID_Comm *comm_ptr;
MPID_Progress_state progress_state;
int errflag = FALSE;
@@ -432,6 +435,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
/* Note, active target uses the following ops list, and passive
* target uses win_ptr->targets[..] */
ops_list = &win_ptr->at_rma_ops_list;
+ ops_list_tail = &win_ptr->at_rma_ops_list_tail;
/* set rma_target_proc[i] to 1 if rank i is a target of RMA
* ops from this process */
@@ -494,7 +498,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
- MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr);
+ MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr);
}
else {
nRequest++;
@@ -511,7 +515,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
- mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone);
+ mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, curr_ptr, &nDone);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
/* if (nDone > 0) printf("nDone = %d\n", nDone); */
@@ -533,7 +537,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
* needed while looping through the requests.
*/
if (total_op_count) {
- mpi_errno = rma_list_complete(win_ptr, ops_list);
+ mpi_errno = rma_list_complete(win_ptr, ops_list, ops_list_tail);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
@@ -927,6 +931,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
int i, j, dst, total_op_count, *curr_ops_cnt;
MPIDI_RMA_Op_t *curr_ptr;
MPIDI_RMA_Ops_list_t *ops_list;
+ MPIDI_RMA_Ops_list_t *ops_list_tail;
MPID_Comm *comm_ptr;
int start_grp_size, *ranks_in_win_grp, rank;
int nRequest = 0;
@@ -986,6 +991,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
/* Note, active target uses the following ops list, and passive
* target uses win_ptr->targets[..] */
ops_list = &win_ptr->at_rma_ops_list;
+ ops_list_tail = &win_ptr->at_rma_ops_list_tail;
MPIU_CHKLMEM_MALLOC(nops_to_proc, int *, comm_size * sizeof(int), mpi_errno, "nops_to_proc");
for (i = 0; i < comm_size; i++)
@@ -1026,7 +1032,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
- MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr);
+ MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr);
}
else {
nRequest++;
@@ -1037,7 +1043,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
mpi_errno = poke_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
- mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone);
+ mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, curr_ptr, &nDone);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
nRequest -= nDone;
@@ -1086,7 +1092,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
if (request) {
MPIDI_RMA_Op_t *new_ptr = NULL;
- mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1099,7 +1105,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
}
if (new_total_op_count) {
- mpi_errno = rma_list_complete(win_ptr, ops_list);
+ mpi_errno = rma_list_complete(win_ptr, ops_list, ops_list_tail);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
@@ -1921,7 +1927,7 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
*wait_for_rma_done_pkt = 0;
}
else {
- MPIDI_RMA_Op_t *tail = MPIDI_CH3I_RMA_Ops_tail(&win_ptr->targets[target_rank].rma_ops_list);
+ MPIDI_RMA_Op_t *tail = MPIDI_CH3I_RMA_Ops_tail(&win_ptr->targets[target_rank].rma_ops_list_tail);
/* Check if we can piggyback the RMA done acknowlegdement on the last
* operation in the epoch. */
@@ -1948,8 +1954,10 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
*wait_for_rma_done_pkt = 0;
MPIDI_CH3I_RMA_Ops_unlink(&win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail,
curr_ptr);
MPIDI_CH3I_RMA_Ops_append(&win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail,
curr_ptr);
break;
}
@@ -2023,7 +2031,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
- MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->targets[target_rank].rma_ops_list,
+ MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, &win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail,
&curr_ptr);
}
else {
@@ -2036,7 +2045,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
mpi_errno =
- rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, curr_ptr,
+ rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail, curr_ptr,
&nDone);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
@@ -2048,7 +2058,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
}
if (nops) {
- mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list);
+ mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
@@ -2337,7 +2348,8 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
/* Free MPIDI_RMA_Ops_list - the lock packet should still be in place, so
* we have to free two elements. */
- MPIDI_CH3I_RMA_Ops_free(&win_ptr->targets[target_rank].rma_ops_list);
+ MPIDI_CH3I_RMA_Ops_free(win_ptr, &win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail);
fn_fail:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_PUT_OR_ACC);
@@ -2450,7 +2462,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
/* Free MPIDI_RMA_Ops_list - the lock packet should still be in place, so
* we have to free two elements. */
- MPIDI_CH3I_RMA_Ops_free(&win_ptr->targets[target_rank].rma_ops_list);
+ MPIDI_CH3I_RMA_Ops_free(win_ptr, &win_ptr->targets[target_rank].rma_ops_list,
+ &win_ptr->targets[target_rank].rma_ops_list_tail);
fn_fail:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_GET);
@@ -2460,7 +2473,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
/* ------------------------------------------------------------------------ */
/* list_complete_timer/counter and list_block_timer defined above */
-static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list)
+static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list,
+ MPIDI_RMA_Ops_list_t *ops_list_tail)
{
int ntimes = 0, mpi_errno = 0;
MPIDI_RMA_Op_t *curr_ptr;
@@ -2470,7 +2484,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
/* Process all operations until they are complete */
while (!MPIDI_CH3I_RMA_Ops_isempty(ops_list)) {
int nDone = 0;
- mpi_errno = rma_list_gc(win_ptr, ops_list, NULL, &nDone);
+ mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, NULL, &nDone);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
ntimes++;
@@ -2510,6 +2524,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
*/
static inline int rma_list_gc(MPID_Win * win_ptr,
MPIDI_RMA_Ops_list_t * ops_list,
+ MPIDI_RMA_Ops_list_t * ops_list_tail,
MPIDI_RMA_Op_t * last_elm, int *nDone)
{
int mpi_errno = 0;
@@ -2538,7 +2553,7 @@ static inline int rma_list_gc(MPID_Win * win_ptr,
}
/* --END ERROR HANDLING-- */
MPID_Request_release(curr_ptr->request);
- MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr);
+ MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr);
nVisit++;
/* MT: avoid processing unissued operations enqueued by other
diff --git a/src/mpid/ch3/src/mpid_finalize.c b/src/mpid/ch3/src/mpid_finalize.c
index 5048ee7..1d60f85 100644
--- a/src/mpid/ch3/src/mpid_finalize.c
+++ b/src/mpid/ch3/src/mpid_finalize.c
@@ -144,6 +144,8 @@ int MPID_Finalize(void)
p = pNext;
}
}
+
+ MPIDI_RMA_finalize();
MPIU_Free(MPIDI_failed_procs_string);
diff --git a/src/mpid/ch3/src/mpid_init.c b/src/mpid/ch3/src/mpid_init.c
index e0a9a0f..6344f3b 100644
--- a/src/mpid/ch3/src/mpid_init.c
+++ b/src/mpid/ch3/src/mpid_init.c
@@ -338,6 +338,9 @@ int MPID_Init(int *argc, char ***argv, int requested, int *provided,
NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ mpi_errno = MPIDI_RMA_init();
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_INIT);
return mpi_errno;
diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c
index 139dd76..7ab57a1 100644
--- a/src/mpid/ch3/src/mpid_rma.c
+++ b/src/mpid/ch3/src/mpid_rma.c
@@ -257,7 +257,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
int mpi_errno = MPI_SUCCESS;
int i;
MPID_Comm *win_comm_ptr;
- MPIU_CHKPMEM_DECL(1);
+ MPIU_CHKPMEM_DECL(2);
MPIDI_STATE_DECL(MPID_STATE_WIN_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_WIN_INIT);
@@ -304,6 +304,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
(*win_ptr)->epoch_state = MPIDI_EPOCH_NONE;
(*win_ptr)->epoch_count = 0;
(*win_ptr)->at_rma_ops_list = NULL;
+ (*win_ptr)->at_rma_ops_list_tail = NULL;
(*win_ptr)->shm_allocated = FALSE;
/* Initialize the passive target lock state */
@@ -313,6 +314,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
for (i = 0; i < MPIR_Comm_size(win_comm_ptr); i++) {
(*win_ptr)->targets[i].rma_ops_list = NULL;
+ (*win_ptr)->targets[i].rma_ops_list_tail = NULL;
(*win_ptr)->targets[i].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
}
@@ -325,6 +327,16 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
(*win_ptr)->info_args.alloc_shared_noncontig = 0;
(*win_ptr)->info_args.alloc_shm = FALSE;
+ MPIU_CHKPMEM_MALLOC((*win_ptr)->op_pool_start, struct MPIDI_RMA_Op *,
+ sizeof(MPIDI_RMA_Op_t) * MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE, mpi_errno,
+ "RMA op pool");
+ (*win_ptr)->op_pool = NULL;
+ (*win_ptr)->op_pool_tail = NULL;
+ for (i = 0; i < MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE; i++) {
+ (*win_ptr)->op_pool_start[i].pool_type = MPIDI_RMA_POOL_WIN;
+ MPL_LL_APPEND((*win_ptr)->op_pool, (*win_ptr)->op_pool_tail, &((*win_ptr)->op_pool_start[i]));
+ }
+
MPID_WIN_FTABLE_SET_DEFAULTS(win_ptr);
fn_exit:
diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c
index 5a7221c..2cb9bd9 100644
--- a/src/mpid/ch3/src/mpidi_rma.c
+++ b/src/mpid/ch3/src/mpidi_rma.c
@@ -7,6 +7,87 @@
#include "mpidimpl.h"
#include "mpidrma.h"
+/*
+=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===
+
+cvars:
+ - name : MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE
+ category : CH3
+ type : int
+ default : 256
+ class : none
+ verbosity : MPI_T_VERBOSITY_USER_BASIC
+ scope : MPI_T_SCOPE_ALL_EQ
+ description : >-
+ Size of the window-private RMA operations pool (in number of
+ operations) that stores information about RMA operations that
+ could not be issued immediately. Requires a positive value.
+
+ - name : MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE
+ category : CH3
+ type : int
+ default : 16384
+ class : none
+ verbosity : MPI_T_VERBOSITY_USER_BASIC
+ scope : MPI_T_SCOPE_ALL_EQ
+ description : >-
+ Size of the Global RMA operations pool (in number of
+ operations) that stores information about RMA operations that
+ could not be issued immediatly. Requires a positive value.
+
+=== END_MPI_T_CVAR_INFO_BLOCK ===
+*/
+
+
+struct MPIDI_RMA_Op *global_rma_op_pool = NULL, *global_rma_op_pool_tail = NULL, *global_rma_op_pool_start = NULL;
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_RMA_init
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_RMA_init(void)
+{
+ int mpi_errno = MPI_SUCCESS;
+ int i;
+ MPIU_CHKPMEM_DECL(1);
+
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_RMA_INIT);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_RMA_INIT);
+
+ MPIU_CHKPMEM_MALLOC(global_rma_op_pool_start, struct MPIDI_RMA_Op *,
+ sizeof(struct MPIDI_RMA_Op) * MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE,
+ mpi_errno, "RMA op pool");
+ for (i = 0; i < MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE; i++) {
+ global_rma_op_pool_start[i].pool_type = MPIDI_RMA_POOL_GLOBAL;
+ MPL_LL_APPEND(global_rma_op_pool, global_rma_op_pool_tail, &(global_rma_op_pool_start[i]));
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RMA_INIT);
+ return mpi_errno;
+
+ fn_fail:
+ MPIU_CHKPMEM_REAP();
+ goto fn_fail;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_RMA_finalize
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+void MPIDI_RMA_finalize(void)
+{
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_RMA_FINALIZE);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_RMA_FINALIZE);
+
+ MPIU_Free(global_rma_op_pool_start);
+
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RMA_FINALIZE);
+}
+
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_free
@@ -43,6 +124,7 @@ int MPIDI_Win_free(MPID_Win ** win_ptr)
MPIU_Free((*win_ptr)->sizes);
MPIU_Free((*win_ptr)->disp_units);
MPIU_Free((*win_ptr)->all_win_handles);
+ MPIU_Free((*win_ptr)->op_pool_start);
/* Free the attached buffer for windows created with MPI_Win_allocate() */
if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE ||
http://git.mpich.org/mpich.git/commitdiff/b1685139330580132c64e4b616ae91824b9bf5b8
commit b1685139330580132c64e4b616ae91824b9bf5b8
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 21:54:13 2014 -0600
Embedding packet structure into RMA operation structure.
We were duplicating information in the operation structure and in the
packet structure when the message is actually issued. Since most of
the information is the same anyway, this patch just embeds a packet
structure into the operation structure, so that we eliminate unnessary
copy.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpi/errhan/errnames.txt b/src/mpi/errhan/errnames.txt
index 060b3b4..4007488 100644
--- a/src/mpi/errhan/errnames.txt
+++ b/src/mpi/errhan/errnames.txt
@@ -189,6 +189,9 @@ MPI_TYPECLASS_INTEGER, or MPI_TYPECLASS_COMPLEX
**memcpyalias:memcpy arguments alias each other
**memcpyalias %p %p %L:memcpy arguments alias each other, dst=%p src=%p len=%L
+**invalidpkt: Invalid packet type
+**invalidpkt %d: Invalid packet type (%d)
+
**rmatypenotatomic: Datatype not permitted for atomic operations
**rmatypenotatomic %D: Datatype (%D) not permitted for atomic operations
**winflavor: Window flavor is not compatible with the given operation
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index 234c3dc..c95847c 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -9,17 +9,6 @@
#include "mpidi_ch3_impl.h"
-typedef enum MPIDI_RMA_Op_type {
- MPIDI_RMA_PUT = 23,
- MPIDI_RMA_GET = 24,
- MPIDI_RMA_ACCUMULATE = 25,
- /* REMOVED: MPIDI_RMA_LOCK = 26, */
- MPIDI_RMA_ACC_CONTIG = 27,
- MPIDI_RMA_GET_ACCUMULATE = 28,
- MPIDI_RMA_COMPARE_AND_SWAP = 29,
- MPIDI_RMA_FETCH_AND_OP = 30
-} MPIDI_RMA_Op_type_t;
-
/* Special case RMA operations */
enum MPIDI_RMA_Datatype {
@@ -61,33 +50,29 @@ typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
typedef struct MPIDI_RMA_Op {
struct MPIDI_RMA_Op *prev; /* pointer to next element in list */
struct MPIDI_RMA_Op *next; /* pointer to next element in list */
- /* FIXME: It would be better to setup the packet that will be sent, at
- * least in most cases (if, as a result of the sync/ops/sync sequence,
- * a different packet type is needed, it can be extracted from the
- * information otherwise stored). */
- MPIDI_RMA_Op_type_t type;
+
void *origin_addr;
int origin_count;
MPI_Datatype origin_datatype;
- int target_rank;
- MPI_Aint target_disp;
- int target_count;
- MPI_Datatype target_datatype;
- MPI_Op op; /* for accumulate */
- /* Used to complete operations */
- struct MPID_Request *request;
- MPIDI_RMA_dtype_info dtype_info;
- void *dataloop;
+
+ void *compare_addr;
+ MPI_Datatype compare_datatype;
+
void *result_addr;
int result_count;
MPI_Datatype result_datatype;
- void *compare_addr;
- int compare_count;
- MPI_Datatype compare_datatype;
+
+ struct MPID_Request *request;
+ MPIDI_RMA_dtype_info dtype_info;
+ void *dataloop;
+
+ int target_rank;
+
+ MPIDI_CH3_Pkt_t pkt;
} MPIDI_RMA_Op_t;
typedef struct MPIDI_PT_single_op {
- int type; /* put, get, or accum. */
+ MPIDI_CH3_Pkt_type_t type; /* put, get, or accum. */
void *addr;
int count;
MPI_Datatype datatype;
diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h
index da2b500..caf1f47 100644
--- a/src/mpid/ch3/include/mpidpkt.h
+++ b/src/mpid/ch3/include/mpidpkt.h
@@ -193,6 +193,43 @@ MPIDI_CH3_PKT_DEFS
#endif
/* *INDENT-ON* */
+#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_) \
+ { \
+ err_ = MPI_SUCCESS; \
+ switch(pkt_.type) { \
+ case (MPIDI_CH3_PKT_PUT): \
+ datatype_ = pkt_.put.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_GET): \
+ datatype_ = pkt_.get.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_ACCUMULATE): \
+ case (MPIDI_CH3_PKT_GET_ACCUM): \
+ datatype_ = pkt_.accum.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_CAS): \
+ datatype_ = pkt_.cas.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_FOP): \
+ datatype_ = pkt_.fop.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_LOCK_PUT_UNLOCK): \
+ datatype_ = pkt_.lock_put_unlock.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_LOCK_GET_UNLOCK): \
+ datatype_ = pkt_.lock_get_unlock.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK): \
+ datatype_ = pkt_.lock_accum_unlock.datatype; \
+ break; \
+ case (MPIDI_CH3_PKT_ACCUM_IMMED): \
+ datatype_ = pkt_.accum_immed.datatype; \
+ break; \
+ default: \
+ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", pkt_.type); \
+ } \
+ }
+
typedef struct MPIDI_CH3_Pkt_put {
MPIDI_CH3_Pkt_type_t type;
MPIDI_CH3_Pkt_flags_t flags;
@@ -279,6 +316,7 @@ typedef struct MPIDI_CH3_Pkt_cas {
MPI_Datatype datatype;
void *addr;
MPI_Request request_handle;
+ MPI_Win source_win_handle;
MPI_Win target_win_handle; /* Used in the last RMA operation in each
* epoch for decrementing rma op counter in
* active target rma and for unlocking window
@@ -300,6 +338,7 @@ typedef struct MPIDI_CH3_Pkt_fop {
void *addr;
MPI_Op op;
MPI_Request request_handle;
+ MPI_Win source_win_handle;
MPI_Win target_win_handle; /* Used in the last RMA operation in each
* epoch for decrementing rma op counter in
* active target rma and for unlocking window
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index fd66526..00664ba 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -12,8 +12,7 @@
#include "mpid_rma_shm.h"
int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle,
- MPI_Win target_win_handle);
+ MPIDI_CH3_Pkt_flags_t flags);
#undef FUNCNAME
#define FUNCNAME send_lock_msg
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 5a8de15..d82fb53 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -905,7 +905,7 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
MPIDI_PT_single_op * single_op;
single_op = lock_queue->pt_single_op;
- if (single_op->type == MPIDI_RMA_PUT) {
+ if (single_op->type == MPIDI_CH3_PKT_LOCK_PUT_UNLOCK) {
mpi_errno = MPIR_Localcopy(single_op->data,
single_op->count,
single_op->datatype,
@@ -913,21 +913,21 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
single_op->count,
single_op->datatype);
}
- else if (single_op->type == MPIDI_RMA_ACCUMULATE) {
+ else if (single_op->type == MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK) {
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
mpi_errno = do_simple_accumulate(single_op);
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
}
- else if (single_op->type == MPIDI_RMA_GET) {
+ else if (single_op->type == MPIDI_CH3_PKT_LOCK_GET_UNLOCK) {
mpi_errno = do_simple_get(win_ptr, lock_queue);
}
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* if put or accumulate, send rma done packet and release lock. */
- if (single_op->type != MPIDI_RMA_GET) {
+ if (single_op->type != MPIDI_CH3_PKT_LOCK_GET_UNLOCK) {
/* NOTE: Only *queued* single_op operations are completed here.
Lock-op-unlock/single_op RMA ops can also be completed as
they arrive within various packet/request handlers via
diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c
index b8b113f..ad9307d 100644
--- a/src/mpid/ch3/src/ch3u_rma_oplist.c
+++ b/src/mpid/ch3/src/ch3u_rma_oplist.c
@@ -110,15 +110,10 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info,
#define FUNCNAME send_rma_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
+static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put;
- MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
+ MPIDI_CH3_Pkt_put_t *put_pkt = &rma_op->pkt.put;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum;
MPID_IOV iov[MPID_IOV_LIMIT];
int mpi_errno = MPI_SUCCESS;
int origin_dt_derived, target_dt_derived, iovcnt;
@@ -126,6 +121,7 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
MPIDI_VC_t *vc;
MPID_Comm *comm_ptr;
MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL;
+ MPI_Datatype target_datatype;
MPID_Request *resp_req = NULL;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG);
@@ -133,23 +129,15 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG);
- *request = NULL;
+ rma_op->request = NULL;
- if (rma_op->type == MPIDI_RMA_PUT) {
- MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
- put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) {
put_pkt->flags = flags;
- put_pkt->count = rma_op->target_count;
- put_pkt->datatype = rma_op->target_datatype;
- put_pkt->dataloop_size = 0;
- put_pkt->target_win_handle = target_win_handle;
- put_pkt->source_win_handle = source_win_handle;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt;
iov[0].MPID_IOV_LEN = sizeof(*put_pkt);
}
- else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) {
+ else if (rma_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) {
/* Create a request for the GACC response. Store the response buf, count, and
* datatype in it, and pass the request's handle in the GACC packet. When the
* response comes from the target, it will contain the request handle. */
@@ -161,8 +149,8 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
resp_req->dev.user_buf = rma_op->result_addr;
resp_req->dev.user_count = rma_op->result_count;
resp_req->dev.datatype = rma_op->result_datatype;
- resp_req->dev.target_win_handle = target_win_handle;
- resp_req->dev.source_win_handle = source_win_handle;
+ resp_req->dev.target_win_handle = accum_pkt->target_win_handle;
+ resp_req->dev.source_win_handle = accum_pkt->source_win_handle;
if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) {
MPID_Datatype *result_dtp = NULL;
@@ -173,32 +161,14 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
}
/* Note: Get_accumulate uses the same packet type as accumulate */
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
accum_pkt->request_handle = resp_req->handle;
+ accum_pkt->flags = flags;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
}
else {
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
@@ -220,9 +190,10 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
origin_dt_derived = 0;
}
- if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
+ MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(rma_op->pkt, target_datatype, mpi_errno);
+ if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
target_dt_derived = 1;
- MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp);
+ MPID_Datatype_get_ptr(target_datatype, target_dtp);
}
else {
target_dt_derived = 0;
@@ -230,31 +201,32 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
if (target_dt_derived) {
/* derived datatype on target. fill derived datatype info */
- dtype_info->is_contig = target_dtp->is_contig;
- dtype_info->max_contig_blocks = target_dtp->max_contig_blocks;
- dtype_info->size = target_dtp->size;
- dtype_info->extent = target_dtp->extent;
- dtype_info->dataloop_size = target_dtp->dataloop_size;
- dtype_info->dataloop_depth = target_dtp->dataloop_depth;
- dtype_info->eltype = target_dtp->eltype;
- dtype_info->dataloop = target_dtp->dataloop;
- dtype_info->ub = target_dtp->ub;
- dtype_info->lb = target_dtp->lb;
- dtype_info->true_ub = target_dtp->true_ub;
- dtype_info->true_lb = target_dtp->true_lb;
- dtype_info->has_sticky_ub = target_dtp->has_sticky_ub;
- dtype_info->has_sticky_lb = target_dtp->has_sticky_lb;
-
- MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop");
+ rma_op->dtype_info.is_contig = target_dtp->is_contig;
+ rma_op->dtype_info.max_contig_blocks = target_dtp->max_contig_blocks;
+ rma_op->dtype_info.size = target_dtp->size;
+ rma_op->dtype_info.extent = target_dtp->extent;
+ rma_op->dtype_info.dataloop_size = target_dtp->dataloop_size;
+ rma_op->dtype_info.dataloop_depth = target_dtp->dataloop_depth;
+ rma_op->dtype_info.eltype = target_dtp->eltype;
+ rma_op->dtype_info.dataloop = target_dtp->dataloop;
+ rma_op->dtype_info.ub = target_dtp->ub;
+ rma_op->dtype_info.lb = target_dtp->lb;
+ rma_op->dtype_info.true_ub = target_dtp->true_ub;
+ rma_op->dtype_info.true_lb = target_dtp->true_lb;
+ rma_op->dtype_info.has_sticky_ub = target_dtp->has_sticky_ub;
+ rma_op->dtype_info.has_sticky_lb = target_dtp->has_sticky_lb;
+
+ MPIU_CHKPMEM_MALLOC(rma_op->dataloop, void *, target_dtp->dataloop_size,
+ mpi_errno, "dataloop");
MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
- MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size);
+ MPIU_Memcpy(rma_op->dataloop, target_dtp->dataloop, target_dtp->dataloop_size);
MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
/* the dataloop can have undefined padding sections, so we need to let
* valgrind know that it is OK to pass this data to writev later on */
- MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size);
+ MPL_VG_MAKE_MEM_DEFINED(rma_op->dataloop, target_dtp->dataloop_size);
- if (rma_op->type == MPIDI_RMA_PUT) {
+ if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) {
put_pkt->dataloop_size = target_dtp->dataloop_size;
}
else {
@@ -272,36 +244,36 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
iovcnt = 2;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
+ mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &rma_op->request);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
else {
/* derived datatype on origin */
- *request = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ rma_op->request = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(rma_op->request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- MPIU_Object_set_ref(*request, 2);
- (*request)->kind = MPID_REQUEST_SEND;
+ MPIU_Object_set_ref(rma_op->request, 2);
+ rma_op->request->kind = MPID_REQUEST_SEND;
- (*request)->dev.segment_ptr = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
+ rma_op->request->dev.segment_ptr = MPID_Segment_alloc();
+ MPIU_ERR_CHKANDJUMP1(rma_op->request->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
- (*request)->dev.datatype_ptr = origin_dtp;
+ rma_op->request->dev.datatype_ptr = origin_dtp;
/* this will cause the datatype to be freed when the request
* is freed. */
MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count,
- rma_op->origin_datatype, (*request)->dev.segment_ptr, 0);
- (*request)->dev.segment_first = 0;
- (*request)->dev.segment_size = rma_op->origin_count * origin_type_size;
+ rma_op->origin_datatype, rma_op->request->dev.segment_ptr, 0);
+ rma_op->request->dev.segment_first = 0;
+ rma_op->request->dev.segment_size = rma_op->origin_count * origin_type_size;
- (*request)->dev.OnFinal = 0;
- (*request)->dev.OnDataAvail = 0;
+ rma_op->request->dev.OnFinal = 0;
+ rma_op->request->dev.OnDataAvail = 0;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
mpi_errno =
- vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
+ vc->sendNoncontig_fn(vc, rma_op->request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
@@ -310,38 +282,40 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
/* derived datatype on target */
MPID_Datatype *combined_dtp = NULL;
- *request = MPID_Request_create();
- if (*request == NULL) {
+ rma_op->request = MPID_Request_create();
+ if (rma_op->request == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
}
- MPIU_Object_set_ref(*request, 2);
- (*request)->kind = MPID_REQUEST_SEND;
+ MPIU_Object_set_ref(rma_op->request, 2);
+ rma_op->request->kind = MPID_REQUEST_SEND;
- (*request)->dev.segment_ptr = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
+ rma_op->request->dev.segment_ptr = MPID_Segment_alloc();
+ MPIU_ERR_CHKANDJUMP1(rma_op->request->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
/* create a new datatype containing the dtype_info, dataloop, and origin data */
mpi_errno =
- create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr,
- rma_op->origin_count, rma_op->origin_datatype, &combined_dtp);
+ create_datatype(&rma_op->dtype_info, rma_op->dataloop, target_dtp->dataloop_size,
+ rma_op->origin_addr, rma_op->origin_count, rma_op->origin_datatype,
+ &combined_dtp);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
- (*request)->dev.datatype_ptr = combined_dtp;
+ rma_op->request->dev.datatype_ptr = combined_dtp;
/* combined_datatype will be freed when request is freed */
- MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0);
- (*request)->dev.segment_first = 0;
- (*request)->dev.segment_size = combined_dtp->size;
+ MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, rma_op->request->dev.segment_ptr, 0);
+ rma_op->request->dev.segment_first = 0;
+ rma_op->request->dev.segment_size = combined_dtp->size;
- (*request)->dev.OnFinal = 0;
- (*request)->dev.OnDataAvail = 0;
+ rma_op->request->dev.OnFinal = 0;
+ rma_op->request->dev.OnDataAvail = 0;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
+ mpi_errno =
+ vc->sendNoncontig_fn(vc, rma_op->request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
@@ -354,7 +328,7 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
/* This operation can generate two requests; one for inbound and one for
* outbound data. */
if (resp_req != NULL) {
- if (*request != NULL) {
+ if (rma_op->request != NULL) {
/* If we have both inbound and outbound requests (i.e. GACC
* operation), we need to ensure that the source buffer is
* available and that the response data has been received before
@@ -370,12 +344,12 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
* it will be completed by the progress engine.
*/
- MPID_Request_release(*request);
- *request = resp_req;
+ MPID_Request_release(rma_op->request);
+ rma_op->request = resp_req;
}
else {
- *request = resp_req;
+ rma_op->request = resp_req;
}
/* For error checking */
@@ -391,13 +365,13 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
if (resp_req) {
MPID_Request_release(resp_req);
}
- if (*request) {
+ if (rma_op->request) {
MPIU_CHKPMEM_REAP();
- if ((*request)->dev.datatype_ptr)
- MPID_Datatype_release((*request)->dev.datatype_ptr);
- MPID_Request_release(*request);
+ if (rma_op->request->dev.datatype_ptr)
+ MPID_Datatype_release(rma_op->request->dev.datatype_ptr);
+ MPID_Request_release(rma_op->request);
}
- *request = NULL;
+ rma_op->request = NULL;
goto fn_exit;
/* --END ERROR HANDLING-- */
}
@@ -410,13 +384,9 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
- MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle, MPID_Request ** request)
+ MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum;
MPID_IOV iov[MPID_IOV_LIMIT];
int mpi_errno = MPI_SUCCESS;
int iovcnt;
@@ -428,24 +398,16 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG);
- *request = NULL;
+ rma_op->request = NULL;
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
/* FIXME: Make this size check efficient and match the packet type */
MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) {
- MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed;
+ MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &rma_op->pkt.accum_immed;
void *dest = accumi_pkt->data, *src = rma_op->origin_addr;
- MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED);
- accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
accumi_pkt->flags = flags;
- accumi_pkt->count = rma_op->target_count;
- accumi_pkt->datatype = rma_op->target_datatype;
- accumi_pkt->op = rma_op->op;
- accumi_pkt->target_win_handle = target_win_handle;
- accumi_pkt->source_win_handle = source_win_handle;
switch (len) {
case 1:
@@ -466,22 +428,13 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
comm_ptr = win_ptr->comm_ptr;
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), &rma_op->request);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
goto fn_exit;
}
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
@@ -505,7 +458,7 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
iovcnt = 2;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
+ mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &rma_op->request);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
@@ -514,10 +467,10 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
- if (*request) {
- MPID_Request_release(*request);
+ if (rma_op->request) {
+ MPID_Request_release(rma_op->request);
}
- *request = NULL;
+ rma_op->request = NULL;
goto fn_exit;
/* --END ERROR HANDLING-- */
}
@@ -531,13 +484,10 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
- MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle, MPID_Request ** request)
+ MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
{
int mpi_errno = MPI_SUCCESS;
- MPID_Request *rmw_req = NULL, *resp_req = NULL;
+ MPID_Request *rmw_req = NULL;
MPIDI_VC_t *vc;
MPID_Comm *comm_ptr;
MPI_Aint len;
@@ -545,44 +495,35 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG);
- *request = NULL;
-
/* Create a request for the RMW response. Store the origin buf, count, and
* datatype in it, and pass the request's handle RMW packet. When the
* response comes from the target, it will contain the request handle. */
- resp_req = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- *request = resp_req;
+ rma_op->request = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(rma_op->request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
/* Set refs on the request to 2: one for the response message, and one for
* the partial completion handler */
- MPIU_Object_set_ref(resp_req, 2);
+ MPIU_Object_set_ref(rma_op->request, 2);
- resp_req->dev.user_buf = rma_op->result_addr;
- resp_req->dev.user_count = rma_op->result_count;
- resp_req->dev.datatype = rma_op->result_datatype;
- resp_req->dev.target_win_handle = target_win_handle;
- resp_req->dev.source_win_handle = source_win_handle;
+ rma_op->request->dev.user_buf = rma_op->result_addr;
+ rma_op->request->dev.user_count = rma_op->result_count;
+ rma_op->request->dev.datatype = rma_op->result_datatype;
/* REQUIRE: All datatype arguments must be of the same, builtin
* type and counts must be 1. */
MPID_Datatype_get_size_macro(rma_op->origin_datatype, len);
comm_ptr = win_ptr->comm_ptr;
- if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) {
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas;
+ if (rma_op->pkt.type == MPIDI_CH3_PKT_CAS) {
+ MPIDI_CH3_Pkt_cas_t *cas_pkt = &rma_op->pkt.cas;
MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u));
- MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS);
+ rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle;
+ rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle;
- cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ cas_pkt->request_handle = rma_op->request->handle;
cas_pkt->flags = flags;
- cas_pkt->datatype = rma_op->target_datatype;
- cas_pkt->target_win_handle = target_win_handle;
- cas_pkt->request_handle = resp_req->handle;
MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len);
MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len);
@@ -598,25 +539,20 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
}
}
- else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) {
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop;
+ else if (rma_op->pkt.type == MPIDI_CH3_PKT_FOP) {
+ MPIDI_CH3_Pkt_fop_t *fop_pkt = &rma_op->pkt.fop;
MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u));
- MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP);
+ rma_op->request->dev.target_win_handle = fop_pkt->target_win_handle;
+ rma_op->request->dev.source_win_handle = fop_pkt->source_win_handle;
- fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ fop_pkt->request_handle = rma_op->request->handle;
fop_pkt->flags = flags;
- fop_pkt->datatype = rma_op->target_datatype;
- fop_pkt->target_win_handle = target_win_handle;
- fop_pkt->request_handle = resp_req->handle;
- fop_pkt->op = rma_op->op;
- if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) {
+ if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) {
/* Embed FOP data in the packet header */
- if (rma_op->op != MPI_NO_OP) {
+ if (fop_pkt->op != MPI_NO_OP) {
MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len);
}
@@ -663,10 +599,10 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
- if (*request) {
- MPID_Request_release(*request);
+ if (rma_op->request) {
+ MPID_Request_release(rma_op->request);
}
- *request = NULL;
+ rma_op->request = NULL;
if (rmw_req) {
MPID_Request_release(rmw_req);
}
@@ -680,19 +616,15 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
#define FUNCNAME recv_rma_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
+static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get;
+ MPIDI_CH3_Pkt_get_t *get_pkt = &rma_op->pkt.get;
int mpi_errno = MPI_SUCCESS;
MPIDI_VC_t *vc;
MPID_Comm *comm_ptr;
- MPID_Request *req = NULL;
MPID_Datatype *dtp;
+ MPI_Datatype target_datatype;
+ MPID_Request *req = NULL;
MPID_IOV iov[MPID_IOV_LIMIT];
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG);
@@ -704,36 +636,27 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
* and pass a handle to it in the get packet. When the get
* response comes from the target, it will contain the request
* handle. */
- req = MPID_Request_create();
- if (req == NULL) {
+ rma_op->request = MPID_Request_create();
+ if (rma_op->request == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
}
- *request = req;
+ MPIU_Object_set_ref(rma_op->request, 2);
- MPIU_Object_set_ref(req, 2);
-
- req->dev.user_buf = rma_op->origin_addr;
- req->dev.user_count = rma_op->origin_count;
- req->dev.datatype = rma_op->origin_datatype;
- req->dev.target_win_handle = MPI_WIN_NULL;
- req->dev.source_win_handle = source_win_handle;
- if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) {
- MPID_Datatype_get_ptr(req->dev.datatype, dtp);
- req->dev.datatype_ptr = dtp;
+ rma_op->request->dev.user_buf = rma_op->origin_addr;
+ rma_op->request->dev.user_count = rma_op->origin_count;
+ rma_op->request->dev.datatype = rma_op->origin_datatype;
+ rma_op->request->dev.target_win_handle = MPI_WIN_NULL;
+ rma_op->request->dev.source_win_handle = get_pkt->source_win_handle;
+ if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->request->dev.datatype)) {
+ MPID_Datatype_get_ptr(rma_op->request->dev.datatype, dtp);
+ rma_op->request->dev.datatype_ptr = dtp;
/* this will cause the datatype to be freed when the
* request is freed. */
}
- MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
- get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ get_pkt->request_handle = rma_op->request->handle;
get_pkt->flags = flags;
- get_pkt->count = rma_op->target_count;
- get_pkt->datatype = rma_op->target_datatype;
- get_pkt->request_handle = req->handle;
- get_pkt->target_win_handle = target_win_handle;
- get_pkt->source_win_handle = source_win_handle;
/* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
@@ -743,7 +666,8 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
comm_ptr = win_ptr->comm_ptr;
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
- if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
+ MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(rma_op->pkt, target_datatype, mpi_errno);
+ if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
/* basic datatype on target. simply send the get_pkt. */
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req);
@@ -753,39 +677,39 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
/* derived datatype on target. fill derived datatype info and
* send it along with get_pkt. */
- MPID_Datatype_get_ptr(rma_op->target_datatype, dtp);
- dtype_info->is_contig = dtp->is_contig;
- dtype_info->max_contig_blocks = dtp->max_contig_blocks;
- dtype_info->size = dtp->size;
- dtype_info->extent = dtp->extent;
- dtype_info->dataloop_size = dtp->dataloop_size;
- dtype_info->dataloop_depth = dtp->dataloop_depth;
- dtype_info->eltype = dtp->eltype;
- dtype_info->dataloop = dtp->dataloop;
- dtype_info->ub = dtp->ub;
- dtype_info->lb = dtp->lb;
- dtype_info->true_ub = dtp->true_ub;
- dtype_info->true_lb = dtp->true_lb;
- dtype_info->has_sticky_ub = dtp->has_sticky_ub;
- dtype_info->has_sticky_lb = dtp->has_sticky_lb;
-
- MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop");
+ MPID_Datatype_get_ptr(target_datatype, dtp);
+ rma_op->dtype_info.is_contig = dtp->is_contig;
+ rma_op->dtype_info.max_contig_blocks = dtp->max_contig_blocks;
+ rma_op->dtype_info.size = dtp->size;
+ rma_op->dtype_info.extent = dtp->extent;
+ rma_op->dtype_info.dataloop_size = dtp->dataloop_size;
+ rma_op->dtype_info.dataloop_depth = dtp->dataloop_depth;
+ rma_op->dtype_info.eltype = dtp->eltype;
+ rma_op->dtype_info.dataloop = dtp->dataloop;
+ rma_op->dtype_info.ub = dtp->ub;
+ rma_op->dtype_info.lb = dtp->lb;
+ rma_op->dtype_info.true_ub = dtp->true_ub;
+ rma_op->dtype_info.true_lb = dtp->true_lb;
+ rma_op->dtype_info.has_sticky_ub = dtp->has_sticky_ub;
+ rma_op->dtype_info.has_sticky_lb = dtp->has_sticky_lb;
+
+ MPIU_CHKPMEM_MALLOC(rma_op->dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop");
MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
- MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size);
+ MPIU_Memcpy(rma_op->dataloop, dtp->dataloop, dtp->dataloop_size);
MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
/* the dataloop can have undefined padding sections, so we need to let
* valgrind know that it is OK to pass this data to writev later on */
- MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size);
+ MPL_VG_MAKE_MEM_DEFINED(rma_op->dataloop, dtp->dataloop_size);
get_pkt->dataloop_size = dtp->dataloop_size;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt;
iov[0].MPID_IOV_LEN = sizeof(*get_pkt);
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info;
- iov[1].MPID_IOV_LEN = sizeof(*dtype_info);
- iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop;
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & rma_op->dtype_info;
+ iov[1].MPID_IOV_LEN = sizeof(rma_op->dtype_info);
+ iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->dataloop;
iov[2].MPID_IOV_LEN = dtp->dataloop_size;
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
@@ -858,70 +782,36 @@ static int send_flush_msg(int dest, MPID_Win * win_ptr)
int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle,
- MPI_Win target_win_handle)
+ MPIDI_CH3_Pkt_flags_t flags)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
- switch (op_ptr->type) {
- case (MPIDI_RMA_PUT):
- case (MPIDI_RMA_ACCUMULATE):
- mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
- target_win_handle, &op_ptr->dtype_info,
- &op_ptr->dataloop, &op_ptr->request);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
+ switch (op_ptr->pkt.type) {
+ case (MPIDI_CH3_PKT_PUT):
+ case (MPIDI_CH3_PKT_ACCUMULATE):
+ case (MPIDI_CH3_PKT_GET_ACCUM):
+ mpi_errno = send_rma_msg(op_ptr, win_ptr, flags);
break;
- case (MPIDI_RMA_GET_ACCUMULATE):
- if (op_ptr->op == MPI_NO_OP) {
- /* Note: Origin arguments are ignored for NO_OP, so we don't
- * need to release a ref to the origin datatype. */
-
- /* Convert the GAcc to a Get */
- op_ptr->type = MPIDI_RMA_GET;
- op_ptr->origin_addr = op_ptr->result_addr;
- op_ptr->origin_count = op_ptr->result_count;
- op_ptr->origin_datatype = op_ptr->result_datatype;
-
- mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
- target_win_handle, &op_ptr->dtype_info,
- &op_ptr->dataloop, &op_ptr->request);
- }
- else {
- mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
- target_win_handle, &op_ptr->dtype_info,
- &op_ptr->dataloop, &op_ptr->request);
- }
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
+ case (MPIDI_CH3_PKT_ACCUM_IMMED):
+ mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags);
break;
- case (MPIDI_RMA_ACC_CONTIG):
- mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags,
- source_win_handle, target_win_handle, &op_ptr->request);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
+ case (MPIDI_CH3_PKT_GET):
+ mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags);
break;
- case (MPIDI_RMA_GET):
- mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags,
- source_win_handle, target_win_handle,
- &op_ptr->dtype_info, &op_ptr->dataloop, &op_ptr->request);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
- break;
- case (MPIDI_RMA_COMPARE_AND_SWAP):
- case (MPIDI_RMA_FETCH_AND_OP):
- mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags,
- source_win_handle, target_win_handle, &op_ptr->request);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
+ case (MPIDI_CH3_PKT_CAS):
+ case (MPIDI_CH3_PKT_FOP):
+ mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags);
break;
default:
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winInvalidOp");
}
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
return mpi_errno;
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index 9024726..f45a2b4 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -74,6 +74,7 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
+ MPIDI_CH3_Pkt_put_t *put_pkt = NULL;
/* queue it up */
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
@@ -81,18 +82,21 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
MPIU_ERR_POP(mpi_errno);
}
+ put_pkt = &(new_ptr->pkt.put);
+ MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
+ put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ put_pkt->count = target_count;
+ put_pkt->datatype = target_datatype;
+ put_pkt->dataloop_size = 0;
+ put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ put_pkt->source_win_handle = win_ptr->handle;
+
/* FIXME: For contig and very short operations, use a streamlined op */
- new_ptr->type = MPIDI_RMA_PUT;
- /* Cast away const'ness for the origin address, as the
- * MPIDI_RMA_Op_t structure is used for both PUT and GET like
- * operations */
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = origin_count;
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = target_count;
- new_ptr->target_datatype = target_datatype;
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -181,6 +185,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
+ MPIDI_CH3_Pkt_get_t *get_pkt = NULL;
/* queue it up */
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
@@ -188,15 +193,21 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
MPIU_ERR_POP(mpi_errno);
}
+ get_pkt = &(new_ptr->pkt.get);
+ MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
+ get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ get_pkt->count = target_count;
+ get_pkt->datatype = target_datatype;
+ get_pkt->dataloop_size = 0;
+ get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ get_pkt->source_win_handle = win_ptr->handle;
+
/* FIXME: For contig and very short operations, use a streamlined op */
- new_ptr->type = MPIDI_RMA_GET;
new_ptr->origin_addr = origin_addr;
new_ptr->origin_count = origin_count;
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = target_count;
- new_ptr->target_datatype = target_datatype;
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -286,6 +297,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL;
/* queue it up */
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
@@ -296,32 +308,48 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
/* If predefined and contiguous, use a simplified element */
if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && enableShortACC) {
- new_ptr->type = MPIDI_RMA_ACC_CONTIG;
- /* Only the information needed for the contig/predefined acc */
- /* Cast away const'ness for origin_address as
- * MPIDI_RMA_Op_t contain both PUT and GET like ops */
- new_ptr->origin_addr = (void *) origin_addr;
- new_ptr->origin_count = origin_count;
- new_ptr->origin_datatype = origin_datatype;
- new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = target_count;
- new_ptr->target_datatype = target_datatype;
- new_ptr->op = op;
- goto fn_exit;
+ MPI_Aint origin_type_size;
+ size_t len;
+
+ MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
+ MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);
+ if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) {
+ MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt;
+
+ accumi_pkt = &(new_ptr->pkt.accum_immed);
+ MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED);
+ accumi_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ accumi_pkt->count = target_count;
+ accumi_pkt->datatype = target_datatype;
+ accumi_pkt->op = op;
+ accumi_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ accumi_pkt->source_win_handle = win_ptr->handle;
+
+ new_ptr->origin_addr = (void *) origin_addr;
+ new_ptr->origin_count = origin_count;
+ new_ptr->origin_datatype = origin_datatype;
+ new_ptr->target_rank = target_rank;
+ goto fn_exit;
+ }
}
- new_ptr->type = MPIDI_RMA_ACCUMULATE;
- /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
- * contain both PUT and GET like ops */
+ accum_pkt = &(new_ptr->pkt.accum);
+
+ MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
+ accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ accum_pkt->count = target_count;
+ accum_pkt->datatype = target_datatype;
+ accum_pkt->dataloop_size = 0;
+ accum_pkt->op = op;
+ accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ accum_pkt->source_win_handle = win_ptr->handle;
+
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = origin_count;
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = target_count;
- new_ptr->target_datatype = target_datatype;
- new_ptr->op = op;
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -423,20 +451,44 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
/* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
- new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
- /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
- * contain both PUT and GET like ops */
- new_ptr->origin_addr = (void *) origin_addr;
- new_ptr->origin_count = origin_count;
- new_ptr->origin_datatype = origin_datatype;
- new_ptr->result_addr = result_addr;
- new_ptr->result_count = result_count;
- new_ptr->result_datatype = result_datatype;
- new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = target_count;
- new_ptr->target_datatype = target_datatype;
- new_ptr->op = op;
+ if (op == MPI_NO_OP) {
+ /* Convert GAcc to a Get */
+ MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get);
+ MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
+ get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ get_pkt->count = target_count;
+ get_pkt->datatype = target_datatype;
+ get_pkt->dataloop_size = 0;
+ get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ get_pkt->source_win_handle = win_ptr->handle;
+
+ new_ptr->origin_addr = result_addr;
+ new_ptr->origin_count = result_count;
+ new_ptr->origin_datatype = result_datatype;
+ new_ptr->target_rank = target_rank;
+ }
+
+ else {
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &(new_ptr->pkt.accum);
+ MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
+ accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ accum_pkt->count = target_count;
+ accum_pkt->datatype = target_datatype;
+ accum_pkt->dataloop_size = 0;
+ accum_pkt->op = op;
+ accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ accum_pkt->source_win_handle = win_ptr->handle;
+
+ new_ptr->origin_addr = (void *) origin_addr;
+ new_ptr->origin_count = origin_count;
+ new_ptr->origin_datatype = origin_datatype;
+ new_ptr->result_addr = result_addr;
+ new_ptr->result_count = result_count;
+ new_ptr->result_datatype = result_datatype;
+ new_ptr->target_rank = target_rank;
+ }
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -524,6 +576,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
+ MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL;
/* Append this operation to the RMA ops queue */
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
@@ -531,20 +584,22 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
MPIU_ERR_POP(mpi_errno);
}
- new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
+ cas_pkt = &(new_ptr->pkt.cas);
+ MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS);
+ cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ cas_pkt->datatype = datatype;
+ cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+ cas_pkt->source_win_handle = win_ptr->handle;
+
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
new_ptr->origin_datatype = datatype;
- new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = 1;
- new_ptr->target_datatype = datatype;
new_ptr->result_addr = result_addr;
- new_ptr->result_count = 1;
new_ptr->result_datatype = datatype;
new_ptr->compare_addr = (void *) compare_addr;
- new_ptr->compare_count = 1;
new_ptr->compare_datatype = datatype;
+ new_ptr->target_rank = target_rank;
}
fn_exit:
@@ -615,6 +670,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
else {
MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
MPIDI_RMA_Op_t *new_ptr = NULL;
+ MPIDI_CH3_Pkt_fop_t *fop_pkt = NULL;
/* Append this operation to the RMA ops queue */
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
@@ -622,18 +678,21 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
MPIU_ERR_POP(mpi_errno);
}
- new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
+ fop_pkt = &(new_ptr->pkt.fop);
+ MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP);
+ fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
+ win_ptr->disp_units[target_rank] * target_disp;
+ fop_pkt->datatype = datatype;
+ fop_pkt->op = op;
+ fop_pkt->source_win_handle = win_ptr->handle;
+ fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
+
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
new_ptr->origin_datatype = datatype;
- new_ptr->target_rank = target_rank;
- new_ptr->target_disp = target_disp;
- new_ptr->target_count = 1;
- new_ptr->target_datatype = datatype;
new_ptr->result_addr = result_addr;
- new_ptr->result_count = 1;
new_ptr->result_datatype = datatype;
- new_ptr->op = op;
+ new_ptr->target_rank = target_rank;
}
fn_exit:
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 9e88165..a0df5bb 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1003,7 +1003,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle;
new_ptr->origin_rank = lock_put_unlock_pkt->origin_rank;
- new_ptr->pt_single_op->type = MPIDI_RMA_PUT;
+ new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_PUT_UNLOCK;
new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags;
new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr;
new_ptr->pt_single_op->count = lock_put_unlock_pkt->count;
@@ -1151,7 +1151,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle;
new_ptr->origin_rank = lock_get_unlock_pkt->origin_rank;
- new_ptr->pt_single_op->type = MPIDI_RMA_GET;
+ new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_GET_UNLOCK;
new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags;
new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr;
new_ptr->pt_single_op->count = lock_get_unlock_pkt->count;
@@ -1242,7 +1242,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle;
new_ptr->origin_rank = lock_accum_unlock_pkt->origin_rank;
- new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE;
+ new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK;
new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags;
new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr;
new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count;
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index fde0a90..3629dd5 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -367,7 +367,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
MPIDI_RMA_Op_t *curr_ptr;
MPIDI_RMA_Ops_list_t *ops_list;
MPID_Comm *comm_ptr;
- MPI_Win source_win_handle, target_win_handle;
MPID_Progress_state progress_state;
int errflag = FALSE;
MPIU_CHKLMEM_DECL(3);
@@ -487,11 +486,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
flags = MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE;
}
- source_win_handle = win_ptr->handle;
- target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
-
- mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags,
- source_win_handle, target_win_handle);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
@@ -933,7 +928,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
MPIDI_RMA_Op_t *curr_ptr;
MPIDI_RMA_Ops_list_t *ops_list;
MPID_Comm *comm_ptr;
- MPI_Win source_win_handle, target_win_handle;
int start_grp_size, *ranks_in_win_grp, rank;
int nRequest = 0;
int nRequestNew = 0;
@@ -1024,11 +1018,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
flags = MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE;
}
- source_win_handle = win_ptr->handle;
- target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
-
- mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags,
- source_win_handle, target_win_handle);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
@@ -1410,8 +1400,8 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr)
if (MPIR_CVAR_CH3_RMA_MERGE_LOCK_OP_UNLOCK &&
win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED &&
rma_op && rma_op->next == NULL /* There is only one op */ &&
- rma_op->type != MPIDI_RMA_COMPARE_AND_SWAP &&
- rma_op->type != MPIDI_RMA_FETCH_AND_OP && rma_op->type != MPIDI_RMA_GET_ACCUMULATE) {
+ rma_op->pkt.type != MPIDI_CH3_PKT_CAS &&
+ rma_op->pkt.type != MPIDI_CH3_PKT_FOP && rma_op->pkt.type != MPIDI_CH3_PKT_GET_ACCUM) {
/* Single put, get, or accumulate between the lock and unlock. If it
* is of small size and predefined datatype at the target, we
* do an optimization where the lock and the RMA operation are
@@ -1420,18 +1410,20 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr)
MPI_Aint type_size;
MPIDI_VC_t *vc;
MPIDI_RMA_Op_t *curr_op = rma_op;
+ MPI_Datatype target_datatype;
MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
MPID_Datatype_get_size_macro(curr_op->origin_datatype, type_size);
/* msg_sz typically = 65480 */
- if (MPIR_DATATYPE_IS_PREDEFINED(curr_op->target_datatype) &&
+ MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno);
+ if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype) &&
(type_size * curr_op->origin_count <= vc->eager_max_msg_sz)) {
single_op_opt = 1;
/* Set the lock granted flag to 1 */
win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
- if (curr_op->type == MPIDI_RMA_GET) {
+ if (curr_op->pkt.type == MPIDI_CH3_PKT_GET) {
mpi_errno = send_lock_get(win_ptr, dest);
wait_for_rma_done_pkt = 0;
}
@@ -1912,7 +1904,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
{
int mpi_errno = MPI_SUCCESS, nops;
MPIDI_RMA_Op_t *curr_ptr;
- MPI_Win source_win_handle = MPI_WIN_NULL, target_win_handle = MPI_WIN_NULL;
int nRequest = 0, nRequestNew = 0;
MPIDI_STATE_DECL(MPID_STATE_DO_PASSIVE_TARGET_RMA);
@@ -1935,9 +1926,9 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
/* Check if we can piggyback the RMA done acknowlegdement on the last
* operation in the epoch. */
- if (tail->type == MPIDI_RMA_GET ||
- tail->type == MPIDI_RMA_COMPARE_AND_SWAP ||
- tail->type == MPIDI_RMA_FETCH_AND_OP || tail->type == MPIDI_RMA_GET_ACCUMULATE) {
+ if (tail->pkt.type == MPIDI_CH3_PKT_GET ||
+ tail->pkt.type == MPIDI_CH3_PKT_CAS ||
+ tail->pkt.type == MPIDI_CH3_PKT_FOP || tail->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) {
/* last operation sends a response message. no need to wait
* for an additional rma done pkt */
*wait_for_rma_done_pkt = 0;
@@ -1952,7 +1943,7 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list);
while (curr_ptr != NULL) {
- if (curr_ptr->type == MPIDI_RMA_GET) {
+ if (curr_ptr->pkt.type == MPIDI_CH3_PKT_GET) {
/* Found a GET, move it to the end */
*wait_for_rma_done_pkt = 0;
@@ -1979,10 +1970,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list);
- if (curr_ptr != NULL) {
- target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
- }
-
while (curr_ptr != NULL) {
MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE;
@@ -2028,12 +2015,9 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
if (*wait_for_rma_done_pkt) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK;
}
-
- source_win_handle = win_ptr->handle;
}
- mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, source_win_handle,
- target_win_handle);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
@@ -2185,6 +2169,9 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &upkt.lock_put_unlock;
MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &upkt.lock_accum_unlock;
+ MPIDI_CH3_Pkt_put_t *put_pkt;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt;
+ MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt;
MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_PUT_OR_ACC);
@@ -2194,7 +2181,9 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
rma_op = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list);
- if (rma_op->type == MPIDI_RMA_PUT) {
+ if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) {
+ put_pkt = &rma_op->pkt.put;
+
MPIDI_Pkt_init(lock_put_unlock_pkt, MPIDI_CH3_PKT_LOCK_PUT_UNLOCK);
lock_put_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK |
MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK;
@@ -2203,18 +2192,17 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_put_unlock_pkt->lock_type = lock_type;
lock_put_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
- lock_put_unlock_pkt->addr =
- (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
-
- lock_put_unlock_pkt->count = rma_op->target_count;
- lock_put_unlock_pkt->datatype = rma_op->target_datatype;
+ lock_put_unlock_pkt->addr = put_pkt->addr;
+ lock_put_unlock_pkt->count = put_pkt->count;
+ lock_put_unlock_pkt->datatype = put_pkt->datatype;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_put_unlock_pkt;
iov[0].MPID_IOV_LEN = sizeof(*lock_put_unlock_pkt);
}
- else if (rma_op->type == MPIDI_RMA_ACCUMULATE) {
+ else if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) {
+ accum_pkt = &rma_op->pkt.accum;
+
MPIDI_Pkt_init(lock_accum_unlock_pkt, MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK);
lock_accum_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK |
MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK;
@@ -2223,18 +2211,17 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->lock_type = lock_type;
lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
- lock_accum_unlock_pkt->addr =
- (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
-
- lock_accum_unlock_pkt->count = rma_op->target_count;
- lock_accum_unlock_pkt->datatype = rma_op->target_datatype;
- lock_accum_unlock_pkt->op = rma_op->op;
+ lock_accum_unlock_pkt->addr = accum_pkt->addr;
+ lock_accum_unlock_pkt->count = accum_pkt->count;
+ lock_accum_unlock_pkt->datatype = accum_pkt->datatype;
+ lock_accum_unlock_pkt->op = accum_pkt->op;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_accum_unlock_pkt;
iov[0].MPID_IOV_LEN = sizeof(*lock_accum_unlock_pkt);
}
- else if (rma_op->type == MPIDI_RMA_ACC_CONTIG) {
+ else if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUM_IMMED) {
+ accumi_pkt = &rma_op->pkt.accum_immed;
+
MPIDI_Pkt_init(lock_accum_unlock_pkt, MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK);
lock_accum_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK |
MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK;
@@ -2243,13 +2230,10 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->lock_type = lock_type;
lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
- lock_accum_unlock_pkt->addr =
- (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
-
- lock_accum_unlock_pkt->count = rma_op->target_count;
- lock_accum_unlock_pkt->datatype = rma_op->target_datatype;
- lock_accum_unlock_pkt->op = rma_op->op;
+ lock_accum_unlock_pkt->addr = accumi_pkt->addr;
+ lock_accum_unlock_pkt->count = accumi_pkt->count;
+ lock_accum_unlock_pkt->datatype = accumi_pkt->datatype;
+ lock_accum_unlock_pkt->op = accumi_pkt->op;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_accum_unlock_pkt;
iov[0].MPID_IOV_LEN = sizeof(*lock_accum_unlock_pkt);
@@ -2375,6 +2359,7 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
MPID_Datatype *dtp;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &upkt.lock_get_unlock;
+ MPIDI_CH3_Pkt_get_t *get_pkt;
MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_GET);
@@ -2408,6 +2393,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
* request is freed. */
}
+ get_pkt = &rma_op->pkt.get;
+
MPIDI_Pkt_init(lock_get_unlock_pkt, MPIDI_CH3_PKT_LOCK_GET_UNLOCK);
lock_get_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | MPIDI_CH3_PKT_FLAG_RMA_UNLOCK; /* FIXME | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; */
lock_get_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
@@ -2415,12 +2402,9 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
lock_get_unlock_pkt->lock_type = lock_type;
lock_get_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
- lock_get_unlock_pkt->addr =
- (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
-
- lock_get_unlock_pkt->count = rma_op->target_count;
- lock_get_unlock_pkt->datatype = rma_op->target_datatype;
+ lock_get_unlock_pkt->addr = get_pkt->addr;
+ lock_get_unlock_pkt->count = get_pkt->count;
+ lock_get_unlock_pkt->datatype = get_pkt->datatype;
lock_get_unlock_pkt->request_handle = rreq->handle;
comm_ptr = win_ptr->comm_ptr;
http://git.mpich.org/mpich.git/commitdiff/ba1a400c4de2df8a662f0f9f856d824ed05ec221
commit ba1a400c4de2df8a662f0f9f856d824ed05ec221
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 21:50:07 2014 -0600
Rename ACK packets in RMA.
The packet type MPIDI_CH3_PKT_PT_RMA_DONE is used for ACK
of FLUSH / UNLOCK packets. Here we rename it to
MPIDI_CH3_PKT_FLUSH_ACK and modify the related functions
and data structures.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 0374fe8..8f18aa3 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1809,7 +1809,7 @@ int MPIDI_CH3_PktHandler_Unlock( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *,
MPIDI_msg_sz_t *, MPID_Request ** );
int MPIDI_CH3_PktHandler_Flush( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *,
MPIDI_msg_sz_t *, MPID_Request ** );
-int MPIDI_CH3_PktHandler_PtRMADone( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *,
+int MPIDI_CH3_PktHandler_FlushAck( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *,
MPIDI_msg_sz_t *, MPID_Request ** );
int MPIDI_CH3_PktHandler_LockPutUnlock( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *,
MPIDI_msg_sz_t *, MPID_Request ** );
diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h
index 6d3bed1..da2b500 100644
--- a/src/mpid/ch3/include/mpidpkt.h
+++ b/src/mpid/ch3/include/mpidpkt.h
@@ -88,7 +88,7 @@ typedef enum {
MPIDI_CH3_PKT_LOCK_GRANTED,
MPIDI_CH3_PKT_UNLOCK,
MPIDI_CH3_PKT_FLUSH,
- MPIDI_CH3_PKT_PT_RMA_DONE,
+ MPIDI_CH3_PKT_FLUSH_ACK,
MPIDI_CH3_PKT_LOCK_PUT_UNLOCK, /* optimization for single puts */
MPIDI_CH3_PKT_LOCK_GET_UNLOCK, /* optimization for single gets */
MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK, /* optimization for single accumulates */
@@ -326,11 +326,11 @@ typedef struct MPIDI_CH3_Pkt_lock {
typedef struct MPIDI_CH3_Pkt_lock_granted {
MPIDI_CH3_Pkt_type_t type;
MPI_Win source_win_handle;
- int target_rank; /* Used in pt_rma_done response to look up the
+ int target_rank; /* Used in flush_ack response to look up the
* target state at the origin. */
} MPIDI_CH3_Pkt_lock_granted_t;
-typedef MPIDI_CH3_Pkt_lock_granted_t MPIDI_CH3_Pkt_pt_rma_done_t;
+typedef MPIDI_CH3_Pkt_lock_granted_t MPIDI_CH3_Pkt_flush_ack_t;
typedef MPIDI_CH3_Pkt_lock_t MPIDI_CH3_Pkt_unlock_t;
typedef MPIDI_CH3_Pkt_lock_t MPIDI_CH3_Pkt_flush_t;
@@ -406,7 +406,7 @@ typedef union MPIDI_CH3_Pkt {
MPIDI_CH3_Pkt_lock_granted_t lock_granted;
MPIDI_CH3_Pkt_unlock_t unlock;
MPIDI_CH3_Pkt_flush_t flush;
- MPIDI_CH3_Pkt_pt_rma_done_t pt_rma_done;
+ MPIDI_CH3_Pkt_flush_ack_t flush_ack;
MPIDI_CH3_Pkt_lock_put_unlock_t lock_put_unlock;
MPIDI_CH3_Pkt_lock_get_unlock_t lock_get_unlock;
MPIDI_CH3_Pkt_lock_accum_unlock_t lock_accum_unlock;
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index aedbaf3..fd66526 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -150,27 +150,27 @@ static inline int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * w
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt
+#define FUNCNAME MPIDI_CH3I_Send_flush_ack_pkt
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
+static inline int MPIDI_CH3I_Send_flush_ack_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
MPI_Win source_win_handle)
{
MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done;
+ MPIDI_CH3_Pkt_flush_ack_t *flush_ack_pkt = &upkt.flush_ack;
MPID_Request *req;
int mpi_errno=MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT);
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT);
- MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE);
- pt_rma_done_pkt->source_win_handle = source_win_handle;
- pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank;
+ MPIDI_Pkt_init(flush_ack_pkt, MPIDI_CH3_PKT_FLUSH_ACK);
+ flush_ack_pkt->source_win_handle = source_win_handle;
+ flush_ack_pkt->target_rank = win_ptr->comm_ptr->rank;
/* Because this is in a packet handler, it is already within a critical section */
/* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
- mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_ack_pkt, sizeof(*flush_ack_pkt), &req);
/* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
@@ -182,7 +182,7 @@ static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_
}
fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT);
return mpi_errno;
}
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
index f60ba73..755f5e2 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
@@ -586,8 +586,8 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[],
MPIDI_CH3_PktHandler_Unlock;
pktArray[MPIDI_CH3_PKT_FLUSH] =
MPIDI_CH3_PktHandler_Flush;
- pktArray[MPIDI_CH3_PKT_PT_RMA_DONE] =
- MPIDI_CH3_PktHandler_PtRMADone;
+ pktArray[MPIDI_CH3_PKT_FLUSH_ACK] =
+ MPIDI_CH3_PktHandler_FlushAck;
pktArray[MPIDI_CH3_PKT_LOCK_PUT_UNLOCK] =
MPIDI_CH3_PktHandler_LockPutUnlock;
pktArray[MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK] =
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 2ea1074..5a8de15 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -936,7 +936,7 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
mpi_errno =
- MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr,
+ MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr,
lock_queue->source_win_handle);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 1b930e9..9e88165 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -1363,37 +1363,38 @@ int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone
+#define FUNCNAME MPIDI_CH3_PktHandler_FlushAck
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+int MPIDI_CH3_PktHandler_FlushAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
{
- MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done;
+ MPIDI_CH3_Pkt_flush_ack_t *flush_ack_pkt = &pkt->flush_ack;
MPID_Win *win_ptr = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK);
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK);
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt");
*buflen = sizeof(MPIDI_CH3_Pkt_t);
- MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr);
- MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state !=
+ MPID_Win_get_ptr(flush_ack_pkt->source_win_handle, win_ptr);
+ MPIU_Assert(win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state !=
MPIDI_CH3_WIN_LOCK_NONE);
- if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state ==
+ if (win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state ==
MPIDI_CH3_WIN_LOCK_FLUSH)
- win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state =
+ win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state =
MPIDI_CH3_WIN_LOCK_GRANTED;
else
- win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
+ win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
*rreqp = NULL;
MPIDI_CH3_Progress_signal_completion();
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK);
+ fn_exit:
return MPI_SUCCESS;
}
@@ -1565,7 +1566,7 @@ int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_r
if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) {
MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL);
- mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle);
+ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, source_win_handle);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1740,9 +1741,9 @@ int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
return MPI_SUCCESS;
}
-int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+int MPIDI_CH3_PktPrint_FlushAck(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n"));
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_FLUSH_ACK\n"));
MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle));
return MPI_SUCCESS;
}
diff --git a/src/mpid/ch3/src/mpidi_printf.c b/src/mpid/ch3/src/mpidi_printf.c
index 1876071..8e7d69d 100644
--- a/src/mpid/ch3/src/mpidi_printf.c
+++ b/src/mpid/ch3/src/mpidi_printf.c
@@ -152,8 +152,8 @@ void MPIDI_DBG_Print_packet(MPIDI_CH3_Pkt_t *pkt)
case MPIDI_CH3_PKT_LOCK_GET_UNLOCK:
MPIDI_CH3_PktPrint_LockGetUnlock( stdout, pkt );
break;
- case MPIDI_CH3_PKT_PT_RMA_DONE:
- MPIDI_CH3_PktPrint_PtRMADone( stdout, pkt );
+ case MPIDI_CH3_PKT_FLUSH_ACK:
+ MPIDI_CH3_PktPrint_FlushAck( stdout, pkt );
break;
case MPIDI_CH3_PKT_LOCK_GRANTED:
MPIDI_CH3_PktPrint_LockGranted( stdout, pkt );
@@ -364,11 +364,11 @@ const char *MPIDI_Pkt_GetDescString( MPIDI_CH3_Pkt_t *pkt )
pkt->lock_get_unlock.target_win_handle,
pkt->lock_get_unlock.request_handle );
break;
- case MPIDI_CH3_PKT_PT_RMA_DONE:
+ case MPIDI_CH3_PKT_FLUSH_ACK:
/* There is no rma_done packet type */
MPIU_Snprintf( pktmsg, sizeof(pktmsg),
"RMA_DONE - 0x%08X",
- pkt->lock_accum_unlock.source_win_handle );
+ pkt->flush_ack.source_win_handle );
break;
case MPIDI_CH3_PKT_LOCK_GRANTED:
MPIU_Snprintf( pktmsg, sizeof(pktmsg),
http://git.mpich.org/mpich.git/commitdiff/0eaf344b95ae93d0546c4a20f4c6bc0581a88e51
commit 0eaf344b95ae93d0546c4a20f4c6bc0581a88e51
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Mon Jul 21 22:55:23 2014 -0500
Avoid using VC in RMA lock queue structure.
We were adding an unnecessary dependency on VC structure
declarations in the mpidpkt.h file. The required information
in RMA lock queue is only the rank, but not actual VC.
Here we replace VC with rank.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
index af29902..234c3dc 100644
--- a/src/mpid/ch3/include/mpid_rma_types.h
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -102,7 +102,7 @@ typedef struct MPIDI_Win_lock_queue {
struct MPIDI_Win_lock_queue *next;
int lock_type;
MPI_Win source_win_handle;
- MPIDI_VC_t *vc;
+ int origin_rank;
struct MPIDI_PT_single_op *pt_single_op; /* to store info for
* lock-put-unlock optimization */
} MPIDI_Win_lock_queue;
diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h
index 46f2769..6d3bed1 100644
--- a/src/mpid/ch3/include/mpidpkt.h
+++ b/src/mpid/ch3/include/mpidpkt.h
@@ -320,6 +320,7 @@ typedef struct MPIDI_CH3_Pkt_lock {
MPI_Win source_win_handle;
int target_rank; /* Used in unluck/flush response to look up the
* target state at the origin. */
+ int origin_rank;
} MPIDI_CH3_Pkt_lock_t;
typedef struct MPIDI_CH3_Pkt_lock_granted {
@@ -338,6 +339,7 @@ typedef struct MPIDI_CH3_Pkt_lock_put_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
+ int origin_rank;
int lock_type;
void *addr;
int count;
@@ -349,6 +351,7 @@ typedef struct MPIDI_CH3_Pkt_lock_get_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
+ int origin_rank;
int lock_type;
void *addr;
int count;
@@ -361,6 +364,7 @@ typedef struct MPIDI_CH3_Pkt_lock_accum_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
+ int origin_rank;
int lock_type;
void *addr;
int count;
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index ee17ad7..aedbaf3 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -37,6 +37,7 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
lock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
lock_pkt->source_win_handle = win_ptr->handle;
lock_pkt->lock_type = lock_type;
+ lock_pkt->origin_rank = win_ptr->comm_ptr->rank;
win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED;
win_ptr->targets[dest].remote_lock_mode = lock_type;
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index 5c1b952..2ea1074 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -933,9 +933,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
they arrive within various packet/request handlers via
MPIDI_CH3_Finish_rma_op_target(). That call cannot be used
here, because it would enter this function recursively. */
-
+ MPIDI_VC_t *vc;
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
mpi_errno =
- MPIDI_CH3I_Send_pt_rma_done_pkt(lock_queue->vc, win_ptr,
+ MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr,
lock_queue->source_win_handle);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
@@ -982,8 +983,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
else {
/* send lock granted packet. */
+ MPIDI_VC_t *vc;
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
mpi_errno =
- MPIDI_CH3I_Send_lock_granted_pkt(lock_queue->vc, win_ptr,
+ MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr,
lock_queue->source_win_handle);
/* dequeue entry from lock queue */
@@ -1077,6 +1080,7 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
MPID_IOV iov[MPID_IOV_LIMIT];
int mpi_errno=MPI_SUCCESS;
MPI_Aint type_size;
+ MPIDI_VC_t *vc;
MPIDI_STATE_DECL(MPID_STATE_DO_SIMPLE_GET);
MPIDI_FUNC_ENTER(MPID_STATE_DO_SIMPLE_GET);
@@ -1108,9 +1112,10 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
MPID_Datatype_get_size_macro(lock_queue->pt_single_op->datatype, type_size);
iov[1].MPID_IOV_LEN = lock_queue->pt_single_op->count * type_size;
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
/* Because this is in a packet handler, it is already within a critical section */
/* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
- mpi_errno = MPIDI_CH3_iSendv(lock_queue->vc, req, iov, 2);
+ mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
/* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
index 7cb36a5..1b930e9 100644
--- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -914,7 +914,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_pkt->lock_type;
new_ptr->source_win_handle = lock_pkt->source_win_handle;
- new_ptr->vc = vc;
+ new_ptr->origin_rank = lock_pkt->origin_rank;
new_ptr->pt_single_op = NULL;
}
@@ -1001,7 +1001,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_put_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
+ new_ptr->origin_rank = lock_put_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_PUT;
new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags;
@@ -1149,7 +1149,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_get_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
+ new_ptr->origin_rank = lock_get_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_GET;
new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags;
@@ -1240,7 +1240,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_accum_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
+ new_ptr->origin_rank = lock_accum_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE;
new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags;
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index aeec089..fde0a90 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -2201,6 +2201,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_put_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_put_unlock_pkt->source_win_handle = win_ptr->handle;
lock_put_unlock_pkt->lock_type = lock_type;
+ lock_put_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_put_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
@@ -2220,6 +2221,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_accum_unlock_pkt->source_win_handle = win_ptr->handle;
lock_accum_unlock_pkt->lock_type = lock_type;
+ lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_accum_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
@@ -2239,6 +2241,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_accum_unlock_pkt->source_win_handle = win_ptr->handle;
lock_accum_unlock_pkt->lock_type = lock_type;
+ lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_accum_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
@@ -2410,6 +2413,7 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
lock_get_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_get_unlock_pkt->source_win_handle = win_ptr->handle;
lock_get_unlock_pkt->lock_type = lock_type;
+ lock_get_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_get_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
http://git.mpich.org/mpich.git/commitdiff/cb04acb30ecb4e244973fa7c8c6033f92c71c89d
commit cb04acb30ecb4e244973fa7c8c6033f92c71c89d
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 21:47:22 2014 -0600
Inline static functions to remove compiler warnings.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 18c40cd..ee17ad7 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -19,7 +19,7 @@ int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr,
#define FUNCNAME send_lock_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
+static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
@@ -64,7 +64,7 @@ static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
#define FUNCNAME send_unlock_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_unlock_msg(int dest, MPID_Win * win_ptr)
+static inline int send_unlock_msg(int dest, MPID_Win * win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
@@ -111,7 +111,7 @@ static int send_unlock_msg(int dest, MPID_Win * win_ptr)
#define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle)
+static inline int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted;
@@ -152,7 +152,7 @@ static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr,
#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
+static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
MPI_Win source_win_handle)
{
MPIDI_CH3_Pkt_t upkt;
@@ -190,7 +190,7 @@ static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
#define FUNCNAME acquire_local_lock
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
+static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK);
@@ -230,7 +230,7 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
#define FUNCNAME do_accumulate_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int do_accumulate_op(MPID_Request *rreq)
+static inline int do_accumulate_op(MPID_Request *rreq)
{
int mpi_errno = MPI_SUCCESS;
MPI_Aint true_lb, true_extent;
http://git.mpich.org/mpich.git/commitdiff/61f952c7a7f9ec2c59e496bf8275b29f12132768
commit 61f952c7a7f9ec2c59e496bf8275b29f12132768
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 21:46:05 2014 -0600
Code refactoring to clean up the RMA code.
Split RMA functionality into smaller files, and move functions
to where they belong based on the file names.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h
new file mode 100644
index 0000000..bc9fa21
--- /dev/null
+++ b/src/mpid/ch3/include/mpid_rma_oplist.h
@@ -0,0 +1,200 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ * (C) 2001 by Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#if !defined(MPID_RMA_OPLIST_H_INCLUDED)
+#define MPID_RMA_OPLIST_H_INCLUDED
+
+#include "mpl_utlist.h"
+#include "mpid_rma_types.h"
+
+/* Return nonzero if the RMA operations list is empty.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list)
+{
+ return *list == NULL;
+}
+
+
+/* Return a pointer to the first element in the list.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_head
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list)
+{
+ return *list;
+}
+
+
+/* Return a pointer to the last element in the list.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list)
+{
+ return (*list) ? (*list)->prev : NULL;
+}
+
+
+/* Append an element to the tail of the RMA ops list
+ *
+ * @param IN list Pointer to the RMA ops list
+ * @param IN elem Pointer to the element to be appended
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
+{
+ MPL_DL_APPEND(*list, elem);
+}
+
+
+/* Allocate a new element on the tail of the RMA operations list.
+ *
+ * @param IN list Pointer to the RMA ops list
+ * @param OUT new_ptr Pointer to the element that was allocated
+ * @return MPI error class
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Op_t ** new_elem)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_RMA_Op_t *tmp_ptr;
+ MPIU_CHKPMEM_DECL(1);
+
+ /* FIXME: We should use a pool allocator here */
+ MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t),
+ mpi_errno, "RMA operation entry");
+
+ tmp_ptr->next = NULL;
+ tmp_ptr->dataloop = NULL;
+ tmp_ptr->request = NULL;
+
+ MPL_DL_APPEND(*list, tmp_ptr);
+
+ *new_elem = tmp_ptr;
+
+ fn_exit:
+ MPIU_CHKPMEM_COMMIT();
+ return mpi_errno;
+ fn_fail:
+ MPIU_CHKPMEM_REAP();
+ *new_elem = NULL;
+ goto fn_exit;
+}
+
+
+/* Unlink an element from the RMA ops list
+ *
+ * @param IN list Pointer to the RMA ops list
+ * @param IN elem Pointer to the element to be unlinked
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
+{
+ MPL_DL_DELETE(*list, elem);
+}
+
+
+/* Free an element in the RMA operations list.
+ *
+ * @param IN list Pointer to the RMA ops list
+ * @param IN curr_ptr Pointer to the element to be freed.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Op_t * curr_ptr)
+{
+ MPIDI_RMA_Op_t *tmp_ptr = curr_ptr;
+
+ MPIU_Assert(curr_ptr != NULL);
+
+ MPL_DL_DELETE(*list, curr_ptr);
+
+ /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
+ if (tmp_ptr->dataloop != NULL)
+ MPIU_Free(tmp_ptr->dataloop);
+ MPIU_Free(tmp_ptr);
+}
+
+
+/* Free an element in the RMA operations list.
+ *
+ * @param IN list Pointer to the RMA ops list
+ * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
+ * to point to the element following the element that
+ * was freed.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list,
+ MPIDI_RMA_Op_t ** curr_ptr)
+{
+ MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next;
+
+ MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr);
+ *curr_ptr = next_ptr;
+}
+
+
+/* Free the entire RMA operations list.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Ops_free
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list)
+{
+ MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr;
+
+ MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
+ MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr);
+ }
+}
+
+
+/* Retrieve the RMA ops list pointer from the window. This routine detects
+ * whether we are in an active or passive target epoch and returns the correct
+ * ops list; we use a shared list for active target and separate per-target
+ * lists for passive target.
+ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target)
+{
+ if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE ||
+ win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) {
+ return &win_ptr->at_rma_ops_list;
+ }
+ else {
+ return &win_ptr->targets[target].rma_ops_list;
+ }
+}
+
+#endif /* MPID_RMA_OPLIST_H_INCLUDED */
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpid_rma_shm.h
similarity index 72%
copy from src/mpid/ch3/include/mpidrma.h
copy to src/mpid/ch3/include/mpid_rma_shm.h
index 3710086..7be3520 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpid_rma_shm.h
@@ -3,310 +3,12 @@
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
-#if !defined(MPICH_MPIDRMA_H_INCLUDED)
-#define MPICH_MPIDRMA_H_INCLUDED
-#include "mpl_utlist.h"
-#include "mpidi_ch3_impl.h"
-
-typedef enum MPIDI_RMA_Op_type {
- MPIDI_RMA_PUT = 23,
- MPIDI_RMA_GET = 24,
- MPIDI_RMA_ACCUMULATE = 25,
- /* REMOVED: MPIDI_RMA_LOCK = 26, */
- MPIDI_RMA_ACC_CONTIG = 27,
- MPIDI_RMA_GET_ACCUMULATE = 28,
- MPIDI_RMA_COMPARE_AND_SWAP = 29,
- MPIDI_RMA_FETCH_AND_OP = 30
-} MPIDI_RMA_Op_type_t;
-
-/* Special case RMA operations */
-
-enum MPIDI_RMA_Datatype {
- MPIDI_RMA_DATATYPE_BASIC = 50,
- MPIDI_RMA_DATATYPE_DERIVED = 51
-};
-
-/* We start with an arbitrarily chosen number (42), to help with
- * debugging when a packet type is not initialized or wrongly
- * initialized. */
-enum MPID_Lock_state {
- MPID_LOCK_NONE = 42,
- MPID_LOCK_SHARED_ALL
-};
-
-/*
- * RMA Declarations. We should move these into something separate from
- * a Request.
- */
-
-/* to send derived datatype across in RMA ops */
-typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
- int is_contig;
- int max_contig_blocks;
- MPI_Aint size;
- MPI_Aint extent;
- int dataloop_size; /* not needed because this info is sent in
- * packet header. remove it after lock/unlock
- * is implemented in the device */
- void *dataloop; /* pointer needed to update pointers
- * within dataloop on remote side */
- int dataloop_depth;
- int eltype;
- MPI_Aint ub, lb, true_ub, true_lb;
- int has_sticky_ub, has_sticky_lb;
-} MPIDI_RMA_dtype_info;
-
-/* for keeping track of RMA ops, which will be executed at the next sync call */
-typedef struct MPIDI_RMA_Op {
- struct MPIDI_RMA_Op *prev; /* pointer to next element in list */
- struct MPIDI_RMA_Op *next; /* pointer to next element in list */
- /* FIXME: It would be better to setup the packet that will be sent, at
- * least in most cases (if, as a result of the sync/ops/sync sequence,
- * a different packet type is needed, it can be extracted from the
- * information otherwise stored). */
- MPIDI_RMA_Op_type_t type;
- void *origin_addr;
- int origin_count;
- MPI_Datatype origin_datatype;
- int target_rank;
- MPI_Aint target_disp;
- int target_count;
- MPI_Datatype target_datatype;
- MPI_Op op; /* for accumulate */
- /* Used to complete operations */
- struct MPID_Request *request;
- MPIDI_RMA_dtype_info dtype_info;
- void *dataloop;
- void *result_addr;
- int result_count;
- MPI_Datatype result_datatype;
- void *compare_addr;
- int compare_count;
- MPI_Datatype compare_datatype;
-} MPIDI_RMA_Op_t;
-
-typedef struct MPIDI_PT_single_op {
- int type; /* put, get, or accum. */
- void *addr;
- int count;
- MPI_Datatype datatype;
- MPI_Op op;
- void *data; /* for queued puts and accumulates, data is copied here */
- MPI_Request request_handle; /* for gets */
- int data_recd; /* to indicate if the data has been received */
- MPIDI_CH3_Pkt_flags_t flags;
-} MPIDI_PT_single_op;
-
-typedef struct MPIDI_Win_lock_queue {
- struct MPIDI_Win_lock_queue *next;
- int lock_type;
- MPI_Win source_win_handle;
- MPIDI_VC_t *vc;
- struct MPIDI_PT_single_op *pt_single_op; /* to store info for
- * lock-put-unlock optimization */
-} MPIDI_Win_lock_queue;
-
-/* Routine use to tune RMA optimizations */
-void MPIDI_CH3_RMA_SetAccImmed(int flag);
-
-/*** RMA OPS LIST HELPER ROUTINES ***/
-
-typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t;
-
-/* Return nonzero if the RMA operations list is empty.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list)
-{
- return *list == NULL;
-}
-
-
-/* Return a pointer to the first element in the list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_head
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list)
-{
- return *list;
-}
-
-
-/* Return a pointer to the last element in the list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list)
-{
- return (*list) ? (*list)->prev : NULL;
-}
-
-
-/* Append an element to the tail of the RMA ops list
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN elem Pointer to the element to be appended
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
-{
- MPL_DL_APPEND(*list, elem);
-}
-
-
-/* Allocate a new element on the tail of the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param OUT new_ptr Pointer to the element that was allocated
- * @return MPI error class
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t ** new_elem)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_RMA_Op_t *tmp_ptr;
- MPIU_CHKPMEM_DECL(1);
-
- /* FIXME: We should use a pool allocator here */
- MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t),
- mpi_errno, "RMA operation entry");
-
- tmp_ptr->next = NULL;
- tmp_ptr->dataloop = NULL;
- tmp_ptr->request = NULL;
-
- MPL_DL_APPEND(*list, tmp_ptr);
-
- *new_elem = tmp_ptr;
-
- fn_exit:
- MPIU_CHKPMEM_COMMIT();
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- *new_elem = NULL;
- goto fn_exit;
-}
-
-
-/* Unlink an element from the RMA ops list
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN elem Pointer to the element to be unlinked
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
-{
- MPL_DL_DELETE(*list, elem);
-}
-
-
-/* Free an element in the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN curr_ptr Pointer to the element to be freed.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t * curr_ptr)
-{
- MPIDI_RMA_Op_t *tmp_ptr = curr_ptr;
-
- MPIU_Assert(curr_ptr != NULL);
+#if !defined(MPID_RMA_SHM_H_INCLUDED)
+#define MPID_RMA_SHM_H_INCLUDED
- MPL_DL_DELETE(*list, curr_ptr);
-
- /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
- if (tmp_ptr->dataloop != NULL)
- MPIU_Free(tmp_ptr->dataloop);
- MPIU_Free(tmp_ptr);
-}
-
-
-/* Free an element in the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
- * to point to the element following the element that
- * was freed.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t ** curr_ptr)
-{
- MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next;
-
- MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr);
- *curr_ptr = next_ptr;
-}
-
-
-/* Free the entire RMA operations list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list)
-{
- MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr;
-
- MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
- MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr);
- }
-}
-
-
-/* Retrieve the RMA ops list pointer from the window. This routine detects
- * whether we are in an active or passive target epoch and returns the correct
- * ops list; we use a shared list for active target and separate per-target
- * lists for passive target.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target)
-{
- if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE ||
- win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) {
- return &win_ptr->at_rma_ops_list;
- }
- else {
- return &win_ptr->targets[target].rma_ops_list;
- }
-}
-
-
-/* ------------------------------------------------------------------------ */
-/*
- * Followings are new routines for origin completion for RMA operations.
- */
-/* ------------------------------------------------------------------------ */
+#include "mpl_utlist.h"
+#include "mpid_rma_types.h"
#define ASSIGN_COPY(src, dest, count, type) \
{ \
@@ -1060,7 +762,4 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad
}
-#undef FUNCNAME
-#undef FCNAME
-
-#endif
+#endif /* MPID_RMA_SHM_H_INCLUDED */
diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h
new file mode 100644
index 0000000..af29902
--- /dev/null
+++ b/src/mpid/ch3/include/mpid_rma_types.h
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ * (C) 2001 by Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#if !defined(MPID_RMA_TYPES_H_INCLUDED)
+#define MPID_RMA_TYPES_H_INCLUDED
+
+#include "mpidi_ch3_impl.h"
+
+typedef enum MPIDI_RMA_Op_type {
+ MPIDI_RMA_PUT = 23,
+ MPIDI_RMA_GET = 24,
+ MPIDI_RMA_ACCUMULATE = 25,
+ /* REMOVED: MPIDI_RMA_LOCK = 26, */
+ MPIDI_RMA_ACC_CONTIG = 27,
+ MPIDI_RMA_GET_ACCUMULATE = 28,
+ MPIDI_RMA_COMPARE_AND_SWAP = 29,
+ MPIDI_RMA_FETCH_AND_OP = 30
+} MPIDI_RMA_Op_type_t;
+
+/* Special case RMA operations */
+
+enum MPIDI_RMA_Datatype {
+ MPIDI_RMA_DATATYPE_BASIC = 50,
+ MPIDI_RMA_DATATYPE_DERIVED = 51
+};
+
+/* We start with an arbitrarily chosen number (42), to help with
+ * debugging when a packet type is not initialized or wrongly
+ * initialized. */
+enum MPID_Lock_state {
+ MPID_LOCK_NONE = 42,
+ MPID_LOCK_SHARED_ALL
+};
+
+/*
+ * RMA Declarations. We should move these into something separate from
+ * a Request.
+ */
+
+/* to send derived datatype across in RMA ops */
+typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
+ int is_contig;
+ int max_contig_blocks;
+ MPI_Aint size;
+ MPI_Aint extent;
+ int dataloop_size; /* not needed because this info is sent in
+ * packet header. remove it after lock/unlock
+ * is implemented in the device */
+ void *dataloop; /* pointer needed to update pointers
+ * within dataloop on remote side */
+ int dataloop_depth;
+ int eltype;
+ MPI_Aint ub, lb, true_ub, true_lb;
+ int has_sticky_ub, has_sticky_lb;
+} MPIDI_RMA_dtype_info;
+
+/* for keeping track of RMA ops, which will be executed at the next sync call */
+typedef struct MPIDI_RMA_Op {
+ struct MPIDI_RMA_Op *prev; /* pointer to next element in list */
+ struct MPIDI_RMA_Op *next; /* pointer to next element in list */
+ /* FIXME: It would be better to setup the packet that will be sent, at
+ * least in most cases (if, as a result of the sync/ops/sync sequence,
+ * a different packet type is needed, it can be extracted from the
+ * information otherwise stored). */
+ MPIDI_RMA_Op_type_t type;
+ void *origin_addr;
+ int origin_count;
+ MPI_Datatype origin_datatype;
+ int target_rank;
+ MPI_Aint target_disp;
+ int target_count;
+ MPI_Datatype target_datatype;
+ MPI_Op op; /* for accumulate */
+ /* Used to complete operations */
+ struct MPID_Request *request;
+ MPIDI_RMA_dtype_info dtype_info;
+ void *dataloop;
+ void *result_addr;
+ int result_count;
+ MPI_Datatype result_datatype;
+ void *compare_addr;
+ int compare_count;
+ MPI_Datatype compare_datatype;
+} MPIDI_RMA_Op_t;
+
+typedef struct MPIDI_PT_single_op {
+ int type; /* put, get, or accum. */
+ void *addr;
+ int count;
+ MPI_Datatype datatype;
+ MPI_Op op;
+ void *data; /* for queued puts and accumulates, data is copied here */
+ MPI_Request request_handle; /* for gets */
+ int data_recd; /* to indicate if the data has been received */
+ MPIDI_CH3_Pkt_flags_t flags;
+} MPIDI_PT_single_op;
+
+typedef struct MPIDI_Win_lock_queue {
+ struct MPIDI_Win_lock_queue *next;
+ int lock_type;
+ MPI_Win source_win_handle;
+ MPIDI_VC_t *vc;
+ struct MPIDI_PT_single_op *pt_single_op; /* to store info for
+ * lock-put-unlock optimization */
+} MPIDI_Win_lock_queue;
+
+typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t;
+
+#endif /* MPID_RMA_TYPES_H_INCLUDED */
diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h
index 6a9db2d..0374fe8 100644
--- a/src/mpid/ch3/include/mpidimpl.h
+++ b/src/mpid/ch3/include/mpidimpl.h
@@ -1228,8 +1228,6 @@ void MPIDI_CH3_RMA_Init_Pvars(void);
/* internal */
int MPIDI_CH3I_Release_lock(MPID_Win * win_ptr);
int MPIDI_CH3I_Try_acquire_win_lock(MPID_Win * win_ptr, int requested_lock);
-int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win *win_ptr, int source_win_hdl);
-int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t * vc, MPID_Win *win_ptr, int source_win_hdl);
int MPIDI_CH3_Start_rma_op_target(MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags);
int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t *vc, MPID_Win *win_ptr, int is_rma_update,
MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle);
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 3710086..18c40cd 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -3,537 +3,102 @@
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
-#if !defined(MPICH_MPIDRMA_H_INCLUDED)
-#define MPICH_MPIDRMA_H_INCLUDED
-#include "mpl_utlist.h"
-#include "mpidi_ch3_impl.h"
+#if !defined(MPID_RMA_H_INCLUDED)
+#define MPID_RMA_H_INCLUDED
-typedef enum MPIDI_RMA_Op_type {
- MPIDI_RMA_PUT = 23,
- MPIDI_RMA_GET = 24,
- MPIDI_RMA_ACCUMULATE = 25,
- /* REMOVED: MPIDI_RMA_LOCK = 26, */
- MPIDI_RMA_ACC_CONTIG = 27,
- MPIDI_RMA_GET_ACCUMULATE = 28,
- MPIDI_RMA_COMPARE_AND_SWAP = 29,
- MPIDI_RMA_FETCH_AND_OP = 30
-} MPIDI_RMA_Op_type_t;
+#include "mpid_rma_types.h"
+#include "mpid_rma_oplist.h"
+#include "mpid_rma_shm.h"
-/* Special case RMA operations */
+int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle,
+ MPI_Win target_win_handle);
-enum MPIDI_RMA_Datatype {
- MPIDI_RMA_DATATYPE_BASIC = 50,
- MPIDI_RMA_DATATYPE_DERIVED = 51
-};
-
-/* We start with an arbitrarily chosen number (42), to help with
- * debugging when a packet type is not initialized or wrongly
- * initialized. */
-enum MPID_Lock_state {
- MPID_LOCK_NONE = 42,
- MPID_LOCK_SHARED_ALL
-};
-
-/*
- * RMA Declarations. We should move these into something separate from
- * a Request.
- */
-
-/* to send derived datatype across in RMA ops */
-typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
- int is_contig;
- int max_contig_blocks;
- MPI_Aint size;
- MPI_Aint extent;
- int dataloop_size; /* not needed because this info is sent in
- * packet header. remove it after lock/unlock
- * is implemented in the device */
- void *dataloop; /* pointer needed to update pointers
- * within dataloop on remote side */
- int dataloop_depth;
- int eltype;
- MPI_Aint ub, lb, true_ub, true_lb;
- int has_sticky_ub, has_sticky_lb;
-} MPIDI_RMA_dtype_info;
-
-/* for keeping track of RMA ops, which will be executed at the next sync call */
-typedef struct MPIDI_RMA_Op {
- struct MPIDI_RMA_Op *prev; /* pointer to next element in list */
- struct MPIDI_RMA_Op *next; /* pointer to next element in list */
- /* FIXME: It would be better to setup the packet that will be sent, at
- * least in most cases (if, as a result of the sync/ops/sync sequence,
- * a different packet type is needed, it can be extracted from the
- * information otherwise stored). */
- MPIDI_RMA_Op_type_t type;
- void *origin_addr;
- int origin_count;
- MPI_Datatype origin_datatype;
- int target_rank;
- MPI_Aint target_disp;
- int target_count;
- MPI_Datatype target_datatype;
- MPI_Op op; /* for accumulate */
- /* Used to complete operations */
- struct MPID_Request *request;
- MPIDI_RMA_dtype_info dtype_info;
- void *dataloop;
- void *result_addr;
- int result_count;
- MPI_Datatype result_datatype;
- void *compare_addr;
- int compare_count;
- MPI_Datatype compare_datatype;
-} MPIDI_RMA_Op_t;
-
-typedef struct MPIDI_PT_single_op {
- int type; /* put, get, or accum. */
- void *addr;
- int count;
- MPI_Datatype datatype;
- MPI_Op op;
- void *data; /* for queued puts and accumulates, data is copied here */
- MPI_Request request_handle; /* for gets */
- int data_recd; /* to indicate if the data has been received */
- MPIDI_CH3_Pkt_flags_t flags;
-} MPIDI_PT_single_op;
-
-typedef struct MPIDI_Win_lock_queue {
- struct MPIDI_Win_lock_queue *next;
- int lock_type;
- MPI_Win source_win_handle;
- MPIDI_VC_t *vc;
- struct MPIDI_PT_single_op *pt_single_op; /* to store info for
- * lock-put-unlock optimization */
-} MPIDI_Win_lock_queue;
-
-/* Routine use to tune RMA optimizations */
-void MPIDI_CH3_RMA_SetAccImmed(int flag);
-
-/*** RMA OPS LIST HELPER ROUTINES ***/
-
-typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t;
-
-/* Return nonzero if the RMA operations list is empty.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list)
-{
- return *list == NULL;
-}
-
-
-/* Return a pointer to the first element in the list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_head
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list)
-{
- return *list;
-}
-
-
-/* Return a pointer to the last element in the list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list)
-{
- return (*list) ? (*list)->prev : NULL;
-}
-
-
-/* Append an element to the tail of the RMA ops list
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN elem Pointer to the element to be appended
- */
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
+#define FUNCNAME send_lock_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
+static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
{
- MPL_DL_APPEND(*list, elem);
-}
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_lock_t *lock_pkt = &upkt.lock;
+ MPID_Request *req = NULL;
+ MPIDI_VC_t *vc;
+ MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_MSG);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_LOCK_MSG);
+ MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED);
-/* Allocate a new element on the tail of the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param OUT new_ptr Pointer to the element that was allocated
- * @return MPI error class
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t ** new_elem)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_RMA_Op_t *tmp_ptr;
- MPIU_CHKPMEM_DECL(1);
+ MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
- /* FIXME: We should use a pool allocator here */
- MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t),
- mpi_errno, "RMA operation entry");
+ MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK);
+ lock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
+ lock_pkt->source_win_handle = win_ptr->handle;
+ lock_pkt->lock_type = lock_type;
- tmp_ptr->next = NULL;
- tmp_ptr->dataloop = NULL;
- tmp_ptr->request = NULL;
+ win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED;
+ win_ptr->targets[dest].remote_lock_mode = lock_type;
- MPL_DL_APPEND(*list, tmp_ptr);
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
- *new_elem = tmp_ptr;
+ /* release the request returned by iStartMsg */
+ if (req != NULL) {
+ MPID_Request_release(req);
+ }
fn_exit:
- MPIU_CHKPMEM_COMMIT();
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_MSG);
return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
fn_fail:
- MPIU_CHKPMEM_REAP();
- *new_elem = NULL;
goto fn_exit;
+ /* --END ERROR HANDLING-- */
}
-
-/* Unlink an element from the RMA ops list
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN elem Pointer to the element to be unlinked
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem)
-{
- MPL_DL_DELETE(*list, elem);
-}
-
-
-/* Free an element in the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param IN curr_ptr Pointer to the element to be freed.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t * curr_ptr)
-{
- MPIDI_RMA_Op_t *tmp_ptr = curr_ptr;
-
- MPIU_Assert(curr_ptr != NULL);
-
- MPL_DL_DELETE(*list, curr_ptr);
-
- /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
- if (tmp_ptr->dataloop != NULL)
- MPIU_Free(tmp_ptr->dataloop);
- MPIU_Free(tmp_ptr);
-}
-
-
-/* Free an element in the RMA operations list.
- *
- * @param IN list Pointer to the RMA ops list
- * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
- * to point to the element following the element that
- * was freed.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list,
- MPIDI_RMA_Op_t ** curr_ptr)
-{
- MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next;
-
- MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr);
- *curr_ptr = next_ptr;
-}
-
-
-/* Free the entire RMA operations list.
- */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Ops_free
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list)
-{
- MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr;
-
- MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
- MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr);
- }
-}
-
-
-/* Retrieve the RMA ops list pointer from the window. This routine detects
- * whether we are in an active or passive target epoch and returns the correct
- * ops list; we use a shared list for active target and separate per-target
- * lists for passive target.
- */
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list
+#define FUNCNAME send_unlock_msg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target)
-{
- if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE ||
- win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) {
- return &win_ptr->at_rma_ops_list;
- }
- else {
- return &win_ptr->targets[target].rma_ops_list;
- }
-}
-
-
-/* ------------------------------------------------------------------------ */
-/*
- * Followings are new routines for origin completion for RMA operations.
- */
-/* ------------------------------------------------------------------------ */
-
-#define ASSIGN_COPY(src, dest, count, type) \
- { \
- type *src_ = (type *) src; \
- type *dest_ = (type *) dest; \
- int i; \
- for (i = 0; i < count; i++) \
- dest_[i] = src_[i]; \
- goto fn_exit; \
- }
-
-static inline int shm_copy(const void *src, int scount, MPI_Datatype stype,
- void *dest, int dcount, MPI_Datatype dtype)
+static int send_unlock_msg(int dest, MPID_Win * win_ptr)
{
int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &upkt.unlock;
+ MPID_Request *req = NULL;
+ MPIDI_VC_t *vc;
+ MPIDI_STATE_DECL(MPID_STATE_SEND_UNLOCK_MSG);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_UNLOCK_MSG);
- /* We use a threshold of operations under which a for loop of assignments is
- * used. Even though this happens at smaller block lengths, making it
- * potentially inefficient, it can take advantage of some vectorization
- * available on most modern processors. */
-#define SHM_OPS_THRESHOLD (16)
-
- if (MPIR_DATATYPE_IS_PREDEFINED(stype) && MPIR_DATATYPE_IS_PREDEFINED(dtype) &&
- scount <= SHM_OPS_THRESHOLD) {
-
- /* FIXME: We currently only optimize a few predefined datatypes, which
- * have a direct C datatype mapping. */
-
- /* The below list of datatypes is based on those specified in the MPI-3
- * standard on page 665. */
- switch (stype) {
- case MPI_CHAR:
- ASSIGN_COPY(src, dest, scount, char);
-
- case MPI_SHORT:
- ASSIGN_COPY(src, dest, scount, signed short int);
-
- case MPI_INT:
- ASSIGN_COPY(src, dest, scount, signed int);
-
- case MPI_LONG:
- ASSIGN_COPY(src, dest, scount, signed long int);
-
- case MPI_LONG_LONG_INT: /* covers MPI_LONG_LONG too */
- ASSIGN_COPY(src, dest, scount, signed long long int);
-
- case MPI_SIGNED_CHAR:
- ASSIGN_COPY(src, dest, scount, signed char);
-
- case MPI_UNSIGNED_CHAR:
- ASSIGN_COPY(src, dest, scount, unsigned char);
-
- case MPI_UNSIGNED_SHORT:
- ASSIGN_COPY(src, dest, scount, unsigned short int);
-
- case MPI_UNSIGNED:
- ASSIGN_COPY(src, dest, scount, unsigned int);
-
- case MPI_UNSIGNED_LONG:
- ASSIGN_COPY(src, dest, scount, unsigned long int);
-
- case MPI_UNSIGNED_LONG_LONG:
- ASSIGN_COPY(src, dest, scount, unsigned long long int);
-
- case MPI_FLOAT:
- ASSIGN_COPY(src, dest, scount, float);
-
- case MPI_DOUBLE:
- ASSIGN_COPY(src, dest, scount, double);
-
- case MPI_LONG_DOUBLE:
- ASSIGN_COPY(src, dest, scount, long double);
-
-#if 0
- /* FIXME: we need a configure check to define HAVE_WCHAR_T before
- * this can be enabled */
- case MPI_WCHAR:
- ASSIGN_COPY(src, dest, scount, wchar_t);
-#endif
-
-#if 0
- /* FIXME: we need a configure check to define HAVE_C_BOOL before
- * this can be enabled */
- case MPI_C_BOOL:
- ASSIGN_COPY(src, dest, scount, _Bool);
-#endif
-
-#if HAVE_INT8_T
- case MPI_INT8_T:
- ASSIGN_COPY(src, dest, scount, int8_t);
-#endif /* HAVE_INT8_T */
-
-#if HAVE_INT16_T
- case MPI_INT16_T:
- ASSIGN_COPY(src, dest, scount, int16_t);
-#endif /* HAVE_INT16_T */
-
-#if HAVE_INT32_T
- case MPI_INT32_T:
- ASSIGN_COPY(src, dest, scount, int32_t);
-#endif /* HAVE_INT32_T */
-
-#if HAVE_INT64_T
- case MPI_INT64_T:
- ASSIGN_COPY(src, dest, scount, int64_t);
-#endif /* HAVE_INT64_T */
-
-#if HAVE_UINT8_T
- case MPI_UINT8_T:
- ASSIGN_COPY(src, dest, scount, uint8_t);
-#endif /* HAVE_UINT8_T */
-
-#if HAVE_UINT16_T
- case MPI_UINT16_T:
- ASSIGN_COPY(src, dest, scount, uint16_t);
-#endif /* HAVE_UINT16_T */
-
-#if HAVE_UINT32_T
- case MPI_UINT32_T:
- ASSIGN_COPY(src, dest, scount, uint32_t);
-#endif /* HAVE_UINT32_T */
-
-#if HAVE_UINT64_T
- case MPI_UINT64_T:
- ASSIGN_COPY(src, dest, scount, uint64_t);
-#endif /* HAVE_UINT64_T */
-
- case MPI_AINT:
- ASSIGN_COPY(src, dest, scount, MPI_Aint);
-
- case MPI_COUNT:
- ASSIGN_COPY(src, dest, scount, MPI_Count);
-
- case MPI_OFFSET:
- ASSIGN_COPY(src, dest, scount, MPI_Offset);
-
-#if 0
- /* FIXME: we need a configure check to define HAVE_C_COMPLEX before
- * this can be enabled */
- case MPI_C_COMPLEX: /* covers MPI_C_FLOAT_COMPLEX as well */
- ASSIGN_COPY(src, dest, scount, float _Complex);
-#endif
-
-#if 0
- /* FIXME: we need a configure check to define HAVE_C_DOUPLE_COMPLEX
- * before this can be enabled */
- case MPI_C_DOUBLE_COMPLEX:
- ASSIGN_COPY(src, dest, scount, double _Complex);
-#endif
-
-#if 0
- /* FIXME: we need a configure check to define
- * HAVE_C_LONG_DOUPLE_COMPLEX before this can be enabled */
- case MPI_C_LONG_DOUBLE_COMPLEX:
- ASSIGN_COPY(src, dest, scount, long double _Complex);
-#endif
-
-#if 0
- /* Types that don't have a direct equivalent */
- case MPI_BYTE:
- case MPI_PACKED:
-#endif
-
-#if 0 /* Fortran types */
- case MPI_INTEGER:
- case MPI_REAL:
- case MPI_DOUBLE_PRECISION:
- case MPI_COMPLEX:
- case MPI_LOGICAL:
- case MPI_CHARACTER:
-#endif
-
-#if 0 /* C++ types */
- case MPI_CXX_BOOL:
- case MPI_CXX_FLOAT_COMPLEX:
- case MPI_CXX_DOUBLE_COMPLEX:
- case MPI_CXX_LONG_DOUBLE_COMPLEX:
-#endif
+ MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED);
-#if 0 /* Optional Fortran types */
- case MPI_DOUBLE_COMPLEX:
- case MPI_INTEGER1:
- case MPI_INTEGER2:
- case MPI_INTEGER4:
- case MPI_INTEGER8:
- case MPI_INTEGER16:
- case MPI_REAL2:
- case MPI_REAL4:
- case MPI_REAL8:
- case MPI_REAL16:
- case MPI_COMPLEX4:
- case MPI_COMPLEX8:
- case MPI_COMPLEX16:
- case MPI_COMPLEX32:
-#endif
+ MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
-#if 0 /* C datatypes for reduction functions */
- case MPI_FLOAT_INT:
- case MPI_DOUBLE_INT:
- case MPI_LONG_INT:
- case MPI_2INT:
- case MPI_LONG_DOUBLE_INT:
-#endif
+ /* Send a lock packet over to the target. wait for the lock_granted
+ * reply. Then do all the RMA ops. */
-#if 0 /* Fortran datatypes for reduction functions */
- case MPI_2REAL:
- case MPI_2DOUBLE_PRECISION:
- case MPI_2INTEGER:
-#endif
+ MPIDI_Pkt_init(unlock_pkt, MPIDI_CH3_PKT_UNLOCK);
+ unlock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
-#if 0 /* Random types not present in the standard */
- case MPI_2COMPLEX:
- case MPI_2DOUBLE_COMPLEX:
-#endif
+ /* Reset the local state of the target to unlocked */
+ win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
- default:
- /* Just to make sure the switch statement is not empty */
- ;
- }
- }
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, unlock_pkt, sizeof(*unlock_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
- mpi_errno = MPIR_Localcopy(src, scount, stype, dest, dcount, dtype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
+ /* Release the request returned by iStartMsg */
+ if (req != NULL) {
+ MPID_Request_release(req);
}
fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_UNLOCK_MSG);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
@@ -541,526 +106,274 @@ static inline int shm_copy(const void *src, int scount, MPI_Datatype stype,
/* --END ERROR HANDLING-- */
}
+
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_put_op
+#define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_put_op(const void *origin_addr, int origin_count, MPI_Datatype
- origin_datatype, int target_rank, MPI_Aint target_disp,
- int target_count, MPI_Datatype target_datatype,
- MPID_Win * win_ptr)
+static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle)
{
- int mpi_errno = MPI_SUCCESS;
- void *base = NULL;
- int disp_unit;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP);
-
- if (win_ptr->shm_allocated == TRUE) {
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted;
+ MPID_Request *req = NULL;
+ int mpi_errno;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
+
+ /* send lock granted packet */
+ MPIDI_Pkt_init(lock_granted_pkt, MPIDI_CH3_PKT_LOCK_GRANTED);
+ lock_granted_pkt->source_win_handle = source_win_handle;
+ lock_granted_pkt->target_rank = win_ptr->comm_ptr->rank;
+
+ MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE,
+ (MPIU_DBG_FDEST, "sending lock granted pkt on vc=%p, source_win_handle=%#08x",
+ vc, lock_granted_pkt->source_win_handle));
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_granted_pkt, sizeof(*lock_granted_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ if (mpi_errno) {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
- mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype,
- (char *) base + disp_unit * target_disp, target_count, target_datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
+ if (req != NULL) {
+ MPID_Request_release(req);
}
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
+
+ return mpi_errno;
}
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_acc_op
+#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_count, MPI_Datatype
- origin_datatype, int target_rank, MPI_Aint target_disp,
- int target_count, MPI_Datatype target_datatype, MPI_Op op,
- MPID_Win * win_ptr)
+static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
+ MPI_Win source_win_handle)
{
- void *base = NULL;
- int disp_unit, shm_op = 0;
- MPI_User_function *uop = NULL;
- MPID_Datatype *dtp;
- int mpi_errno = MPI_SUCCESS;
- MPIU_CHKLMEM_DECL(2);
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP);
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done;
+ MPID_Request *req;
+ int mpi_errno=MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP);
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
- if (win_ptr->shm_allocated == TRUE) {
- shm_op = 1;
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
- }
+ MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE);
+ pt_rma_done_pkt->source_win_handle = source_win_handle;
+ pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank;
- if (op == MPI_REPLACE) {
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype,
- (char *) base + disp_unit * target_disp, target_count,
- target_datatype);
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- goto fn_exit;
+ /* Because this is in a packet handler, it is already within a critical section */
+ /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req);
+ /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
- MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN),
- mpi_errno, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", op);
-
- /* get the function by indexing into the op table */
- uop = MPIR_OP_HDL_TO_FN(op);
-
- if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) &&
- MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
- /* Cast away const'ness for origin_address in order to
- * avoid changing the prototype for MPI_User_function */
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- (*uop) ((void *) origin_addr, (char *) base + disp_unit * target_disp,
- &target_count, &target_datatype);
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
+ if (req != NULL)
+ {
+ MPID_Request_release(req);
}
- else {
- /* derived datatype */
- MPID_Segment *segp;
- DLOOP_VECTOR *dloop_vec;
- MPI_Aint first, last;
- int vec_len, i, type_size, count;
- MPI_Datatype type;
- MPI_Aint true_lb, true_extent, extent;
- void *tmp_buf = NULL, *target_buf;
- const void *source_buf;
-
- if (origin_datatype != target_datatype) {
- /* first copy the data into a temporary buffer with
- * the same datatype as the target. Then do the
- * accumulate operation. */
-
- MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent);
- MPID_Datatype_get_extent_macro(target_datatype, extent);
-
- MPIU_CHKLMEM_MALLOC(tmp_buf, void *,
- target_count * (MPIR_MAX(extent, true_extent)),
- mpi_errno, "temporary buffer");
- /* adjust for potential negative lower bound in datatype */
- tmp_buf = (void *) ((char *) tmp_buf - true_lb);
-
- mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
- origin_datatype, tmp_buf, target_count, target_datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- }
-
- if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
- /* target predefined type, origin derived datatype */
-
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- (*uop) (tmp_buf, (char *) base + disp_unit * target_disp,
- &target_count, &target_datatype);
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- }
- else {
-
- segp = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPID_Segment_init(NULL, target_count, target_datatype, segp, 0);
- first = 0;
- last = SEGMENT_IGNORE_LAST;
-
- MPID_Datatype_get_ptr(target_datatype, dtp);
- vec_len = dtp->max_contig_blocks * target_count + 1;
- /* +1 needed because Rob says so */
- MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *,
- vec_len * sizeof(DLOOP_VECTOR), mpi_errno, "dloop vector");
-
- MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
+ return mpi_errno;
+}
- source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr;
- target_buf = (char *) base + disp_unit * target_disp;
- type = dtp->eltype;
- type_size = MPID_Datatype_get_basic_size(type);
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- for (i = 0; i < vec_len; i++) {
- MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int);
- (*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- (char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- &count, &type);
+#undef FUNCNAME
+#define FUNCNAME acquire_local_lock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_ACQUIRE_LOCAL_LOCK);
+
+ /* poke the progress engine until the local lock is granted */
+ if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
+ MPID_Progress_state progress_state;
+
+ MPID_Progress_start(&progress_state);
+ while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
+ mpi_errno = MPID_Progress_wait(&progress_state);
+ /* --BEGIN ERROR HANDLING-- */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPID_Progress_end(&progress_state);
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress");
}
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
-
- MPID_Segment_free(segp);
+ /* --END ERROR HANDLING-- */
}
+ MPID_Progress_end(&progress_state);
}
+ win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
+ win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_mode = lock_type;
+
fn_exit:
- MPIU_CHKLMEM_FREEALL();
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP);
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_ACQUIRE_LOCAL_LOCK);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
- if (shm_op)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_get_acc_op
+#define FUNCNAME do_accumulate_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_count, MPI_Datatype
- origin_datatype, void *result_addr, int result_count,
- MPI_Datatype result_datatype, int target_rank, MPI_Aint
- target_disp, int target_count,
- MPI_Datatype target_datatype, MPI_Op op,
- MPID_Win * win_ptr)
+static int do_accumulate_op(MPID_Request *rreq)
{
- int disp_unit, shm_locked = 0;
- void *base = NULL;
- MPI_User_function *uop = NULL;
- MPID_Datatype *dtp;
int mpi_errno = MPI_SUCCESS;
- MPIU_CHKLMEM_DECL(2);
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP);
-
- if (win_ptr->shm_allocated == TRUE) {
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- shm_locked = 1;
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
- }
-
- /* Perform the local get first, then the accumulate */
- mpi_errno = shm_copy((char *) base + disp_unit * target_disp, target_count, target_datatype,
- result_addr, result_count, result_datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- /* NO_OP: Don't perform the accumulate */
- if (op == MPI_NO_OP) {
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- shm_locked = 0;
- }
-
- goto fn_exit;
- }
-
- if (op == MPI_REPLACE) {
- mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype,
- (char *) base + disp_unit * target_disp, target_count,
- target_datatype);
-
+ MPI_Aint true_lb, true_extent;
+ MPI_User_function *uop;
+ MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP);
+
+ if (rreq->dev.op == MPI_REPLACE)
+ {
+ /* simply copy the data */
+ mpi_errno = MPIR_Localcopy(rreq->dev.user_buf, rreq->dev.user_count,
+ rreq->dev.datatype,
+ rreq->dev.real_user_buf,
+ rreq->dev.user_count,
+ rreq->dev.datatype);
if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- shm_locked = 0;
- }
-
+ MPIU_ERR_POP(mpi_errno);
+ }
goto fn_exit;
}
- MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN),
- mpi_errno, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", op);
-
- /* get the function by indexing into the op table */
- uop = MPIR_OP_HDL_TO_FN(op);
-
- if ((op == MPI_NO_OP || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) &&
- MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
- /* Cast away const'ness for origin_address in order to
- * avoid changing the prototype for MPI_User_function */
- (*uop) ((void *) origin_addr, (char *) base + disp_unit * target_disp,
- &target_count, &target_datatype);
+ if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN)
+ {
+ /* get the function by indexing into the op table */
+ uop = MPIR_OP_HDL_TO_FN(rreq->dev.op);
+ }
+ else
+ {
+ /* --BEGIN ERROR HANDLING-- */
+ mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op );
+ return mpi_errno;
+ /* --END ERROR HANDLING-- */
}
- else {
- /* derived datatype */
+ if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype))
+ {
+ (*uop)(rreq->dev.user_buf, rreq->dev.real_user_buf,
+ &(rreq->dev.user_count), &(rreq->dev.datatype));
+ }
+ else
+ {
+ /* derived datatype */
MPID_Segment *segp;
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
- int vec_len, i, type_size, count;
+ int vec_len, i, count;
+ MPI_Aint type_size;
MPI_Datatype type;
- MPI_Aint true_lb, true_extent, extent;
- void *tmp_buf = NULL, *target_buf;
- const void *source_buf;
-
- if (origin_datatype != target_datatype) {
- /* first copy the data into a temporary buffer with
- * the same datatype as the target. Then do the
- * accumulate operation. */
-
- MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent);
- MPID_Datatype_get_extent_macro(target_datatype, extent);
-
- MPIU_CHKLMEM_MALLOC(tmp_buf, void *,
- target_count * (MPIR_MAX(extent, true_extent)),
- mpi_errno, "temporary buffer");
- /* adjust for potential negative lower bound in datatype */
- tmp_buf = (void *) ((char *) tmp_buf - true_lb);
-
- mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
- origin_datatype, tmp_buf, target_count, target_datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
+ MPID_Datatype *dtp;
+
+ segp = MPID_Segment_alloc();
+ /* --BEGIN ERROR HANDLING-- */
+ if (!segp)
+ {
+ mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 );
+ MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
+ return mpi_errno;
}
-
- if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
- /* target predefined type, origin derived datatype */
-
- (*uop) (tmp_buf, (char *) base + disp_unit * target_disp,
- &target_count, &target_datatype);
+ /* --END ERROR HANDLING-- */
+ MPID_Segment_init(NULL, rreq->dev.user_count,
+ rreq->dev.datatype, segp, 0);
+ first = 0;
+ last = SEGMENT_IGNORE_LAST;
+
+ MPID_Datatype_get_ptr(rreq->dev.datatype, dtp);
+ vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1;
+ /* +1 needed because Rob says so */
+ dloop_vec = (DLOOP_VECTOR *)
+ MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR));
+ /* --BEGIN ERROR HANDLING-- */
+ if (!dloop_vec)
+ {
+ mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 );
+ MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
+ return mpi_errno;
}
- else {
-
- segp = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPID_Segment_init(NULL, target_count, target_datatype, segp, 0);
- first = 0;
- last = SEGMENT_IGNORE_LAST;
-
- MPID_Datatype_get_ptr(target_datatype, dtp);
- vec_len = dtp->max_contig_blocks * target_count + 1;
- /* +1 needed because Rob says so */
- MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *,
- vec_len * sizeof(DLOOP_VECTOR), mpi_errno, "dloop vector");
-
- MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
-
- source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr;
- target_buf = (char *) base + disp_unit * target_disp;
- type = dtp->eltype;
- type_size = MPID_Datatype_get_basic_size(type);
-
- for (i = 0; i < vec_len; i++) {
- MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int);
- (*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- (char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- &count, &type);
- }
-
- MPID_Segment_free(segp);
+ /* --END ERROR HANDLING-- */
+
+ MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
+
+ type = dtp->eltype;
+ MPID_Datatype_get_size_macro(type, type_size);
+ for (i=0; i<vec_len; i++)
+ {
+ MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size, int);
+ (*uop)((char *)rreq->dev.user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
+ (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
+ &count, &type);
}
- }
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- shm_locked = 0;
+ MPID_Segment_free(segp);
+ MPIU_Free(dloop_vec);
}
- fn_exit:
- MPIU_CHKLMEM_FREEALL();
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- }
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
+ fn_exit:
+ /* free the temporary buffer */
+ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
+ MPIU_Free((char *) rreq->dev.user_buf + true_lb);
+ MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_get_op
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_get_op(void *origin_addr, int origin_count,
- MPI_Datatype origin_datatype, int target_rank,
- MPI_Aint target_disp, int target_count,
- MPI_Datatype target_datatype, MPID_Win * win_ptr)
-{
- void *base = NULL;
- int disp_unit;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_OP);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_OP);
-
- if (win_ptr->shm_allocated == TRUE) {
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
- }
-
- mpi_errno = shm_copy((char *) base + disp_unit * target_disp, target_count, target_datatype,
- origin_addr, origin_count, origin_datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_GET_OP);
return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
+ fn_fail:
goto fn_exit;
- /* --END ERROR HANDLING-- */
}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_cas_op
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_cas_op(const void *origin_addr, const void *compare_addr,
- void *result_addr, MPI_Datatype datatype, int target_rank,
- MPI_Aint target_disp, MPID_Win * win_ptr)
+static inline int wait_progress_engine(void)
{
- void *base = NULL, *dest_addr = NULL;
- int disp_unit;
- MPI_Aint len;
- int shm_locked = 0;
int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP);
+ MPID_Progress_state progress_state;
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP);
-
- if (win_ptr->shm_allocated == TRUE) {
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
-
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- shm_locked = 1;
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
- }
-
- dest_addr = (char *) base + disp_unit * target_disp;
-
- MPID_Datatype_get_size_macro(datatype, len);
- MPIU_Memcpy(result_addr, dest_addr, len);
-
- if (MPIR_Compare_equal(compare_addr, dest_addr, datatype)) {
- MPIU_Memcpy(dest_addr, origin_addr, len);
- }
-
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- shm_locked = 0;
+ MPID_Progress_start(&progress_state);
+ mpi_errno = MPID_Progress_wait(&progress_state);
+ /* --BEGIN ERROR HANDLING-- */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPID_Progress_end(&progress_state);
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress");
}
+ /* --END ERROR HANDLING-- */
+ MPID_Progress_end(&progress_state);
fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP);
return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
fn_fail:
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- }
goto fn_exit;
- /* --END ERROR HANDLING-- */
}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Shm_fop_op
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_addr,
- MPI_Datatype datatype, int target_rank,
- MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr)
+static inline int poke_progress_engine(void)
{
- void *base = NULL, *dest_addr = NULL;
- MPI_User_function *uop = NULL;
- int disp_unit;
- MPI_Aint len;
- int one, shm_locked = 0;
int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP);
+ MPID_Progress_state progress_state;
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP);
-
- if (win_ptr->shm_allocated == TRUE) {
- base = win_ptr->shm_base_addrs[target_rank];
- disp_unit = win_ptr->disp_units[target_rank];
-
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- shm_locked = 1;
- }
- else {
- base = win_ptr->base;
- disp_unit = win_ptr->disp_unit;
- }
-
- dest_addr = (char *) base + disp_unit * target_disp;
-
- MPID_Datatype_get_size_macro(datatype, len);
- MPIU_Memcpy(result_addr, dest_addr, len);
-
- uop = MPIR_OP_HDL_TO_FN(op);
- one = 1;
-
- (*uop) ((void *) origin_addr, dest_addr, &one, &datatype);
-
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- shm_locked = 0;
- }
+ MPID_Progress_start(&progress_state);
+ mpi_errno = MPID_Progress_poke();
+ if (mpi_errno != MPI_SUCCESS)
+ MPIU_ERR_POP(mpi_errno);
+ MPID_Progress_end(&progress_state);
fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP);
return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
fn_fail:
- if (shm_locked) {
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- }
goto fn_exit;
- /* --END ERROR HANDLING-- */
}
-
-#undef FUNCNAME
-#undef FCNAME
-
-#endif
+#endif /* MPID_RMA_H_INCLUDED */
diff --git a/src/mpid/ch3/src/Makefile.mk b/src/mpid/ch3/src/Makefile.mk
index 4272c26..c3b528e 100644
--- a/src/mpid/ch3/src/Makefile.mk
+++ b/src/mpid/ch3/src/Makefile.mk
@@ -17,10 +17,11 @@ mpi_core_sources += \
src/mpid/ch3/src/ch3u_port.c \
src/mpid/ch3/src/ch3u_recvq.c \
src/mpid/ch3/src/ch3u_request.c \
- src/mpid/ch3/src/ch3u_rma_acc_ops.c \
+ src/mpid/ch3/src/ch3u_rma_oplist.c \
src/mpid/ch3/src/ch3u_rma_ops.c \
src/mpid/ch3/src/ch3u_rma_reqops.c \
src/mpid/ch3/src/ch3u_rma_sync.c \
+ src/mpid/ch3/src/ch3u_rma_pkthandler.c \
src/mpid/ch3/src/ch3u_rndv.c \
src/mpid/ch3/src/ch3u_eager.c \
src/mpid/ch3/src/ch3u_eagersync.c \
@@ -56,6 +57,7 @@ mpi_core_sources += \
src/mpid/ch3/src/mpid_startall.c \
src/mpid/ch3/src/mpid_vc.c \
src/mpid/ch3/src/mpid_rma.c \
+ src/mpid/ch3/src/mpidi_rma.c \
src/mpid/ch3/src/mpidi_isend_self.c \
src/mpid/ch3/src/mpidi_pg.c \
src/mpid/ch3/src/mpidi_printf.c
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c
index db6760c..5c1b952 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_req.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c
@@ -8,7 +8,6 @@
#include "mpidrma.h"
static int create_derived_datatype(MPID_Request * rreq, MPID_Datatype ** dtp);
-static int do_accumulate_op(MPID_Request * rreq);
static int do_simple_accumulate(MPIDI_PT_single_op *single_op);
static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue);
@@ -827,119 +826,6 @@ static int create_derived_datatype(MPID_Request *req, MPID_Datatype **dtp)
return mpi_errno;
}
-
-#undef FUNCNAME
-#define FUNCNAME do_accumulate_op
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int do_accumulate_op(MPID_Request *rreq)
-{
- int mpi_errno = MPI_SUCCESS;
- MPI_Aint true_lb, true_extent;
- MPI_User_function *uop;
- MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP);
-
- if (rreq->dev.op == MPI_REPLACE)
- {
- /* simply copy the data */
- mpi_errno = MPIR_Localcopy(rreq->dev.user_buf, rreq->dev.user_count,
- rreq->dev.datatype,
- rreq->dev.real_user_buf,
- rreq->dev.user_count,
- rreq->dev.datatype);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- goto fn_exit;
- }
-
- if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN)
- {
- /* get the function by indexing into the op table */
- uop = MPIR_OP_HDL_TO_FN(rreq->dev.op);
- }
- else
- {
- /* --BEGIN ERROR HANDLING-- */
- mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op );
- return mpi_errno;
- /* --END ERROR HANDLING-- */
- }
-
- if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype))
- {
- (*uop)(rreq->dev.user_buf, rreq->dev.real_user_buf,
- &(rreq->dev.user_count), &(rreq->dev.datatype));
- }
- else
- {
- /* derived datatype */
- MPID_Segment *segp;
- DLOOP_VECTOR *dloop_vec;
- MPI_Aint first, last;
- int vec_len, i, count;
- MPI_Aint type_size;
- MPI_Datatype type;
- MPID_Datatype *dtp;
-
- segp = MPID_Segment_alloc();
- /* --BEGIN ERROR HANDLING-- */
- if (!segp)
- {
- mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 );
- MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
- return mpi_errno;
- }
- /* --END ERROR HANDLING-- */
- MPID_Segment_init(NULL, rreq->dev.user_count,
- rreq->dev.datatype, segp, 0);
- first = 0;
- last = SEGMENT_IGNORE_LAST;
-
- MPID_Datatype_get_ptr(rreq->dev.datatype, dtp);
- vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1;
- /* +1 needed because Rob says so */
- dloop_vec = (DLOOP_VECTOR *)
- MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR));
- /* --BEGIN ERROR HANDLING-- */
- if (!dloop_vec)
- {
- mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 );
- MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
- return mpi_errno;
- }
- /* --END ERROR HANDLING-- */
-
- MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
-
- type = dtp->eltype;
- MPID_Datatype_get_size_macro(type, type_size);
- for (i=0; i<vec_len; i++)
- {
- MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size, int);
- (*uop)((char *)rreq->dev.user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
- &count, &type);
- }
-
- MPID_Segment_free(segp);
- MPIU_Free(dloop_vec);
- }
-
- fn_exit:
- /* free the temporary buffer */
- MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent);
- MPIU_Free((char *) rreq->dev.user_buf + true_lb);
-
- MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
-
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
static int entered_flag = 0;
static int entered_count = 0;
@@ -1129,43 +1015,6 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
}
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr,
- MPI_Win source_win_handle)
-{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done;
- MPID_Request *req;
- int mpi_errno=MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
-
- MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE);
- pt_rma_done_pkt->source_win_handle = source_win_handle;
- pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank;
-
- /* Because this is in a packet handler, it is already within a critical section */
- /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
- mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req);
- /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
- }
-
- if (req != NULL)
- {
- MPID_Request_release(req);
- }
-
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
- return mpi_errno;
-}
-
#undef FUNCNAME
#define FUNCNAME do_simple_accumulate
diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c
new file mode 100644
index 0000000..b8b113f
--- /dev/null
+++ b/src/mpid/ch3/src/ch3u_rma_oplist.c
@@ -0,0 +1,932 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ * (C) 2001 by Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include "mpidimpl.h"
+#include "mpidrma.h"
+
+/*
+=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===
+
+cvars:
+ - name : MPIR_CVAR_CH3_RMA_ACC_IMMED
+ category : CH3
+ type : boolean
+ default : true
+ class : none
+ verbosity : MPI_T_VERBOSITY_USER_BASIC
+ scope : MPI_T_SCOPE_ALL_EQ
+ description : >-
+ Use the immediate accumulate optimization
+
+=== END_MPI_T_CVAR_INFO_BLOCK ===
+*/
+
+static int send_flush_msg(int dest, MPID_Win *win_ptr);
+static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags);
+static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
+ MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags);
+static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags);
+static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
+ MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags);
+
+/* create_datatype() creates a new struct datatype for the dtype_info
+ and the dataloop of the target datatype together with the user data */
+#undef FUNCNAME
+#define FUNCNAME create_datatype
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info,
+ const void *dataloop, MPI_Aint dataloop_sz,
+ const void *o_addr, int o_count, MPI_Datatype o_datatype,
+ MPID_Datatype ** combined_dtp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ /* datatype_set_contents wants an array 'ints' which is the
+ * blocklens array with count prepended to it. So blocklens
+ * points to the 2nd element of ints to avoid having to copy
+ * blocklens into ints later. */
+ int ints[4];
+ int *blocklens = &ints[1];
+ MPI_Aint displaces[3];
+ MPI_Datatype datatypes[3];
+ const int count = 3;
+ MPI_Datatype combined_datatype;
+ MPIDI_STATE_DECL(MPID_STATE_CREATE_DATATYPE);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_CREATE_DATATYPE);
+
+ /* create datatype */
+ displaces[0] = MPIU_PtrToAint(dtype_info);
+ blocklens[0] = sizeof(*dtype_info);
+ datatypes[0] = MPI_BYTE;
+
+ displaces[1] = MPIU_PtrToAint(dataloop);
+ MPIU_Assign_trunc(blocklens[1], dataloop_sz, int);
+ datatypes[1] = MPI_BYTE;
+
+ displaces[2] = MPIU_PtrToAint(o_addr);
+ blocklens[2] = o_count;
+ datatypes[2] = o_datatype;
+
+ mpi_errno = MPID_Type_struct(count, blocklens, displaces, datatypes, &combined_datatype);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+
+ ints[0] = count;
+
+ MPID_Datatype_get_ptr(combined_datatype, *combined_dtp);
+ mpi_errno = MPID_Datatype_set_contents(*combined_dtp, MPI_COMBINER_STRUCT, count + 1, /* ints (cnt,blklen) */
+ count, /* aints (disps) */
+ count, /* types */
+ ints, displaces, datatypes);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+
+ /* Commit datatype */
+
+ MPID_Dataloop_create(combined_datatype,
+ &(*combined_dtp)->dataloop,
+ &(*combined_dtp)->dataloop_size,
+ &(*combined_dtp)->dataloop_depth, MPID_DATALOOP_HOMOGENEOUS);
+
+ /* create heterogeneous dataloop */
+ MPID_Dataloop_create(combined_datatype,
+ &(*combined_dtp)->hetero_dloop,
+ &(*combined_dtp)->hetero_dloop_size,
+ &(*combined_dtp)->hetero_dloop_depth, MPID_DATALOOP_HETEROGENEOUS);
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_CREATE_DATATYPE);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME send_rma_msg
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags,
+ MPI_Win source_win_handle,
+ MPI_Win target_win_handle,
+ MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
+{
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
+ MPID_IOV iov[MPID_IOV_LIMIT];
+ int mpi_errno = MPI_SUCCESS;
+ int origin_dt_derived, target_dt_derived, iovcnt;
+ MPI_Aint origin_type_size;
+ MPIDI_VC_t *vc;
+ MPID_Comm *comm_ptr;
+ MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL;
+ MPID_Request *resp_req = NULL;
+ MPIU_CHKPMEM_DECL(1);
+ MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG);
+ MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG);
+
+ *request = NULL;
+
+ if (rma_op->type == MPIDI_RMA_PUT) {
+ MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
+ put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ put_pkt->flags = flags;
+ put_pkt->count = rma_op->target_count;
+ put_pkt->datatype = rma_op->target_datatype;
+ put_pkt->dataloop_size = 0;
+ put_pkt->target_win_handle = target_win_handle;
+ put_pkt->source_win_handle = source_win_handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*put_pkt);
+ }
+ else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) {
+ /* Create a request for the GACC response. Store the response buf, count, and
+ * datatype in it, and pass the request's handle in the GACC packet. When the
+ * response comes from the target, it will contain the request handle. */
+ resp_req = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+
+ MPIU_Object_set_ref(resp_req, 2);
+
+ resp_req->dev.user_buf = rma_op->result_addr;
+ resp_req->dev.user_count = rma_op->result_count;
+ resp_req->dev.datatype = rma_op->result_datatype;
+ resp_req->dev.target_win_handle = target_win_handle;
+ resp_req->dev.source_win_handle = source_win_handle;
+
+ if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) {
+ MPID_Datatype *result_dtp = NULL;
+ MPID_Datatype_get_ptr(resp_req->dev.datatype, result_dtp);
+ resp_req->dev.datatype_ptr = result_dtp;
+ /* this will cause the datatype to be freed when the
+ * request is freed. */
+ }
+
+ /* Note: Get_accumulate uses the same packet type as accumulate */
+ MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
+ accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ accum_pkt->flags = flags;
+ accum_pkt->count = rma_op->target_count;
+ accum_pkt->datatype = rma_op->target_datatype;
+ accum_pkt->dataloop_size = 0;
+ accum_pkt->op = rma_op->op;
+ accum_pkt->target_win_handle = target_win_handle;
+ accum_pkt->source_win_handle = source_win_handle;
+ accum_pkt->request_handle = resp_req->handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
+ }
+ else {
+ MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
+ accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ accum_pkt->flags = flags;
+ accum_pkt->count = rma_op->target_count;
+ accum_pkt->datatype = rma_op->target_datatype;
+ accum_pkt->dataloop_size = 0;
+ accum_pkt->op = rma_op->op;
+ accum_pkt->target_win_handle = target_win_handle;
+ accum_pkt->source_win_handle = source_win_handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
+ }
+
+ /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
+ * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
+ * fflush(stdout);
+ */
+
+ comm_ptr = win_ptr->comm_ptr;
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+
+ if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) {
+ origin_dt_derived = 1;
+ MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp);
+ }
+ else {
+ origin_dt_derived = 0;
+ }
+
+ if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
+ target_dt_derived = 1;
+ MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp);
+ }
+ else {
+ target_dt_derived = 0;
+ }
+
+ if (target_dt_derived) {
+ /* derived datatype on target. fill derived datatype info */
+ dtype_info->is_contig = target_dtp->is_contig;
+ dtype_info->max_contig_blocks = target_dtp->max_contig_blocks;
+ dtype_info->size = target_dtp->size;
+ dtype_info->extent = target_dtp->extent;
+ dtype_info->dataloop_size = target_dtp->dataloop_size;
+ dtype_info->dataloop_depth = target_dtp->dataloop_depth;
+ dtype_info->eltype = target_dtp->eltype;
+ dtype_info->dataloop = target_dtp->dataloop;
+ dtype_info->ub = target_dtp->ub;
+ dtype_info->lb = target_dtp->lb;
+ dtype_info->true_ub = target_dtp->true_ub;
+ dtype_info->true_lb = target_dtp->true_lb;
+ dtype_info->has_sticky_ub = target_dtp->has_sticky_ub;
+ dtype_info->has_sticky_lb = target_dtp->has_sticky_lb;
+
+ MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop");
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
+ MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size);
+ MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
+ /* the dataloop can have undefined padding sections, so we need to let
+ * valgrind know that it is OK to pass this data to writev later on */
+ MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size);
+
+ if (rma_op->type == MPIDI_RMA_PUT) {
+ put_pkt->dataloop_size = target_dtp->dataloop_size;
+ }
+ else {
+ accum_pkt->dataloop_size = target_dtp->dataloop_size;
+ }
+ }
+
+ MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
+
+ if (!target_dt_derived) {
+ /* basic datatype on target */
+ if (!origin_dt_derived) {
+ /* basic datatype on origin */
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
+ iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
+ iovcnt = 2;
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+ else {
+ /* derived datatype on origin */
+ *request = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+
+ MPIU_Object_set_ref(*request, 2);
+ (*request)->kind = MPID_REQUEST_SEND;
+
+ (*request)->dev.segment_ptr = MPID_Segment_alloc();
+ MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
+ "**nomem", "**nomem %s", "MPID_Segment_alloc");
+
+ (*request)->dev.datatype_ptr = origin_dtp;
+ /* this will cause the datatype to be freed when the request
+ * is freed. */
+ MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count,
+ rma_op->origin_datatype, (*request)->dev.segment_ptr, 0);
+ (*request)->dev.segment_first = 0;
+ (*request)->dev.segment_size = rma_op->origin_count * origin_type_size;
+
+ (*request)->dev.OnFinal = 0;
+ (*request)->dev.OnDataAvail = 0;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno =
+ vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+ }
+ else {
+ /* derived datatype on target */
+ MPID_Datatype *combined_dtp = NULL;
+
+ *request = MPID_Request_create();
+ if (*request == NULL) {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ }
+
+ MPIU_Object_set_ref(*request, 2);
+ (*request)->kind = MPID_REQUEST_SEND;
+
+ (*request)->dev.segment_ptr = MPID_Segment_alloc();
+ MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
+ "**nomem", "**nomem %s", "MPID_Segment_alloc");
+
+ /* create a new datatype containing the dtype_info, dataloop, and origin data */
+
+ mpi_errno =
+ create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr,
+ rma_op->origin_count, rma_op->origin_datatype, &combined_dtp);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+
+ (*request)->dev.datatype_ptr = combined_dtp;
+ /* combined_datatype will be freed when request is freed */
+
+ MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0);
+ (*request)->dev.segment_first = 0;
+ (*request)->dev.segment_size = combined_dtp->size;
+
+ (*request)->dev.OnFinal = 0;
+ (*request)->dev.OnDataAvail = 0;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+
+ /* we're done with the datatypes */
+ if (origin_dt_derived)
+ MPID_Datatype_release(origin_dtp);
+ MPID_Datatype_release(target_dtp);
+ }
+
+ /* This operation can generate two requests; one for inbound and one for
+ * outbound data. */
+ if (resp_req != NULL) {
+ if (*request != NULL) {
+ /* If we have both inbound and outbound requests (i.e. GACC
+ * operation), we need to ensure that the source buffer is
+ * available and that the response data has been received before
+ * informing the origin that this operation is complete. Because
+ * the update needs to be done atomically at the target, they will
+ * not send back data until it has been received. Therefore,
+ * completion of the response request implies that the send request
+ * has completed.
+ *
+ * Therefore: refs on the response request are set to two: one is
+ * held by the progress engine and the other by the RMA op
+ * completion code. Refs on the outbound request are set to one;
+ * it will be completed by the progress engine.
+ */
+
+ MPID_Request_release(*request);
+ *request = resp_req;
+
+ }
+ else {
+ *request = resp_req;
+ }
+
+ /* For error checking */
+ resp_req = NULL;
+ }
+
+ fn_exit:
+ MPIU_CHKPMEM_COMMIT();
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_RMA_MSG);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ if (resp_req) {
+ MPID_Request_release(resp_req);
+ }
+ if (*request) {
+ MPIU_CHKPMEM_REAP();
+ if ((*request)->dev.datatype_ptr)
+ MPID_Datatype_release((*request)->dev.datatype_ptr);
+ MPID_Request_release(*request);
+ }
+ *request = NULL;
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+/*
+ * Use this for contiguous accumulate operations
+ */
+#undef FUNCNAME
+#define FUNCNAME send_contig_acc_msg
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
+ MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags,
+ MPI_Win source_win_handle,
+ MPI_Win target_win_handle, MPID_Request ** request)
+{
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
+ MPID_IOV iov[MPID_IOV_LIMIT];
+ int mpi_errno = MPI_SUCCESS;
+ int iovcnt;
+ MPI_Aint origin_type_size;
+ MPIDI_VC_t *vc;
+ MPID_Comm *comm_ptr;
+ size_t len;
+ MPIDI_STATE_DECL(MPID_STATE_SEND_CONTIG_ACC_MSG);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG);
+
+ *request = NULL;
+
+ MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
+ /* FIXME: Make this size check efficient and match the packet type */
+ MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
+ if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) {
+ MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed;
+ void *dest = accumi_pkt->data, *src = rma_op->origin_addr;
+
+ MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED);
+ accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ accumi_pkt->flags = flags;
+ accumi_pkt->count = rma_op->target_count;
+ accumi_pkt->datatype = rma_op->target_datatype;
+ accumi_pkt->op = rma_op->op;
+ accumi_pkt->target_win_handle = target_win_handle;
+ accumi_pkt->source_win_handle = source_win_handle;
+
+ switch (len) {
+ case 1:
+ *(uint8_t *) dest = *(uint8_t *) src;
+ break;
+ case 2:
+ *(uint16_t *) dest = *(uint16_t *) src;
+ break;
+ case 4:
+ *(uint32_t *) dest = *(uint32_t *) src;
+ break;
+ case 8:
+ *(uint64_t *) dest = *(uint64_t *) src;
+ break;
+ default:
+ MPIU_Memcpy(accumi_pkt->data, (void *) rma_op->origin_addr, len);
+ }
+ comm_ptr = win_ptr->comm_ptr;
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ goto fn_exit;
+ }
+
+ MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
+ accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ accum_pkt->flags = flags;
+ accum_pkt->count = rma_op->target_count;
+ accum_pkt->datatype = rma_op->target_datatype;
+ accum_pkt->dataloop_size = 0;
+ accum_pkt->op = rma_op->op;
+ accum_pkt->target_win_handle = target_win_handle;
+ accum_pkt->source_win_handle = source_win_handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
+
+ /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
+ * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
+ * fflush(stdout);
+ */
+
+ comm_ptr = win_ptr->comm_ptr;
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+
+
+ /* basic datatype on target */
+ /* basic datatype on origin */
+ /* FIXME: This is still very heavyweight for a small message operation,
+ * such as a single word update */
+ /* One possibility is to use iStartMsg with a buffer that is just large
+ * enough, though note that nemesis has an optimization for this */
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
+ iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
+ iovcnt = 2;
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_CONTIG_ACC_MSG);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ if (*request) {
+ MPID_Request_release(*request);
+ }
+ *request = NULL;
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+/*
+ * Initiate an immediate RMW accumulate operation
+ */
+#undef FUNCNAME
+#define FUNCNAME send_immed_rmw_msg
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
+ MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags,
+ MPI_Win source_win_handle,
+ MPI_Win target_win_handle, MPID_Request ** request)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPID_Request *rmw_req = NULL, *resp_req = NULL;
+ MPIDI_VC_t *vc;
+ MPID_Comm *comm_ptr;
+ MPI_Aint len;
+ MPIDI_STATE_DECL(MPID_STATE_SEND_IMMED_RMW_MSG);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG);
+
+ *request = NULL;
+
+ /* Create a request for the RMW response. Store the origin buf, count, and
+ * datatype in it, and pass the request's handle RMW packet. When the
+ * response comes from the target, it will contain the request handle. */
+ resp_req = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ *request = resp_req;
+
+ /* Set refs on the request to 2: one for the response message, and one for
+ * the partial completion handler */
+ MPIU_Object_set_ref(resp_req, 2);
+
+ resp_req->dev.user_buf = rma_op->result_addr;
+ resp_req->dev.user_count = rma_op->result_count;
+ resp_req->dev.datatype = rma_op->result_datatype;
+ resp_req->dev.target_win_handle = target_win_handle;
+ resp_req->dev.source_win_handle = source_win_handle;
+
+ /* REQUIRE: All datatype arguments must be of the same, builtin
+ * type and counts must be 1. */
+ MPID_Datatype_get_size_macro(rma_op->origin_datatype, len);
+ comm_ptr = win_ptr->comm_ptr;
+
+ if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) {
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas;
+
+ MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u));
+
+ MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS);
+
+ cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ cas_pkt->flags = flags;
+ cas_pkt->datatype = rma_op->target_datatype;
+ cas_pkt->target_win_handle = target_win_handle;
+ cas_pkt->request_handle = resp_req->handle;
+
+ MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len);
+ MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len);
+
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+
+ if (rmw_req != NULL) {
+ MPID_Request_release(rmw_req);
+ }
+ }
+
+ else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) {
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop;
+
+ MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u));
+
+ MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP);
+
+ fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ fop_pkt->flags = flags;
+ fop_pkt->datatype = rma_op->target_datatype;
+ fop_pkt->target_win_handle = target_win_handle;
+ fop_pkt->request_handle = resp_req->handle;
+ fop_pkt->op = rma_op->op;
+
+ if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) {
+ /* Embed FOP data in the packet header */
+ if (rma_op->op != MPI_NO_OP) {
+ MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len);
+ }
+
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &rmw_req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+
+ if (rmw_req != NULL) {
+ MPID_Request_release(rmw_req);
+ }
+ }
+ else {
+ /* Data is too big to copy into the FOP header, use an IOV to send it */
+ MPID_IOV iov[MPID_IOV_LIMIT];
+
+ rmw_req = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(rmw_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ MPIU_Object_set_ref(rmw_req, 1);
+
+ rmw_req->dev.OnFinal = 0;
+ rmw_req->dev.OnDataAvail = 0;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*fop_pkt);
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
+ iov[1].MPID_IOV_LEN = len; /* count == 1 */
+
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iSendv(vc, rmw_req, iov, 2);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+ }
+ else {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_IMMED_RMW_MSG);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ if (*request) {
+ MPID_Request_release(*request);
+ }
+ *request = NULL;
+ if (rmw_req) {
+ MPID_Request_release(rmw_req);
+ }
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+
+#undef FUNCNAME
+#define FUNCNAME recv_rma_msg
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags,
+ MPI_Win source_win_handle,
+ MPI_Win target_win_handle,
+ MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
+{
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_VC_t *vc;
+ MPID_Comm *comm_ptr;
+ MPID_Request *req = NULL;
+ MPID_Datatype *dtp;
+ MPID_IOV iov[MPID_IOV_LIMIT];
+ MPIU_CHKPMEM_DECL(1);
+ MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG);
+ MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_RECV_RMA_MSG);
+
+ /* create a request, store the origin buf, cnt, datatype in it,
+ * and pass a handle to it in the get packet. When the get
+ * response comes from the target, it will contain the request
+ * handle. */
+ req = MPID_Request_create();
+ if (req == NULL) {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ }
+
+ *request = req;
+
+ MPIU_Object_set_ref(req, 2);
+
+ req->dev.user_buf = rma_op->origin_addr;
+ req->dev.user_count = rma_op->origin_count;
+ req->dev.datatype = rma_op->origin_datatype;
+ req->dev.target_win_handle = MPI_WIN_NULL;
+ req->dev.source_win_handle = source_win_handle;
+ if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) {
+ MPID_Datatype_get_ptr(req->dev.datatype, dtp);
+ req->dev.datatype_ptr = dtp;
+ /* this will cause the datatype to be freed when the
+ * request is freed. */
+ }
+
+ MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
+ get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
+ win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
+ get_pkt->flags = flags;
+ get_pkt->count = rma_op->target_count;
+ get_pkt->datatype = rma_op->target_datatype;
+ get_pkt->request_handle = req->handle;
+ get_pkt->target_win_handle = target_win_handle;
+ get_pkt->source_win_handle = source_win_handle;
+
+/* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
+ rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
+ fflush(stdout);
+*/
+
+ comm_ptr = win_ptr->comm_ptr;
+ MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
+
+ if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
+ /* basic datatype on target. simply send the get_pkt. */
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ }
+ else {
+ /* derived datatype on target. fill derived datatype info and
+ * send it along with get_pkt. */
+
+ MPID_Datatype_get_ptr(rma_op->target_datatype, dtp);
+ dtype_info->is_contig = dtp->is_contig;
+ dtype_info->max_contig_blocks = dtp->max_contig_blocks;
+ dtype_info->size = dtp->size;
+ dtype_info->extent = dtp->extent;
+ dtype_info->dataloop_size = dtp->dataloop_size;
+ dtype_info->dataloop_depth = dtp->dataloop_depth;
+ dtype_info->eltype = dtp->eltype;
+ dtype_info->dataloop = dtp->dataloop;
+ dtype_info->ub = dtp->ub;
+ dtype_info->lb = dtp->lb;
+ dtype_info->true_ub = dtp->true_ub;
+ dtype_info->true_lb = dtp->true_lb;
+ dtype_info->has_sticky_ub = dtp->has_sticky_ub;
+ dtype_info->has_sticky_lb = dtp->has_sticky_lb;
+
+ MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop");
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
+ MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size);
+ MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
+
+ /* the dataloop can have undefined padding sections, so we need to let
+ * valgrind know that it is OK to pass this data to writev later on */
+ MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size);
+
+ get_pkt->dataloop_size = dtp->dataloop_size;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*get_pkt);
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info;
+ iov[1].MPID_IOV_LEN = sizeof(*dtype_info);
+ iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop;
+ iov[2].MPID_IOV_LEN = dtp->dataloop_size;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, 3, &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+
+ /* release the target datatype */
+ MPID_Datatype_release(dtp);
+ }
+
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+
+ /* release the request returned by iStartMsg or iStartMsgv */
+ if (req != NULL) {
+ MPID_Request_release(req);
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_RECV_RMA_MSG);
+ return mpi_errno;
+
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ MPIU_CHKPMEM_REAP();
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME send_flush_msg
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+static int send_flush_msg(int dest, MPID_Win * win_ptr)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush;
+ MPID_Request *req = NULL;
+ MPIDI_VC_t *vc;
+ MPIDI_STATE_DECL(MPID_STATE_SEND_FLUSH_MSG);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_FLUSH_MSG);
+
+ MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
+
+ MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH);
+ flush_pkt->target_win_handle = win_ptr->all_win_handles[dest];
+ flush_pkt->source_win_handle = win_ptr->handle;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
+
+ /* Release the request returned by iStartMsg */
+ if (req != NULL) {
+ MPID_Request_release(req);
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_FLUSH_MSG);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr,
+ MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle,
+ MPI_Win target_win_handle)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
+
+ switch (op_ptr->type) {
+ case (MPIDI_RMA_PUT):
+ case (MPIDI_RMA_ACCUMULATE):
+ mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
+ target_win_handle, &op_ptr->dtype_info,
+ &op_ptr->dataloop, &op_ptr->request);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ break;
+ case (MPIDI_RMA_GET_ACCUMULATE):
+ if (op_ptr->op == MPI_NO_OP) {
+ /* Note: Origin arguments are ignored for NO_OP, so we don't
+ * need to release a ref to the origin datatype. */
+
+ /* Convert the GAcc to a Get */
+ op_ptr->type = MPIDI_RMA_GET;
+ op_ptr->origin_addr = op_ptr->result_addr;
+ op_ptr->origin_count = op_ptr->result_count;
+ op_ptr->origin_datatype = op_ptr->result_datatype;
+
+ mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
+ target_win_handle, &op_ptr->dtype_info,
+ &op_ptr->dataloop, &op_ptr->request);
+ }
+ else {
+ mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle,
+ target_win_handle, &op_ptr->dtype_info,
+ &op_ptr->dataloop, &op_ptr->request);
+ }
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ break;
+ case (MPIDI_RMA_ACC_CONTIG):
+ mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags,
+ source_win_handle, target_win_handle, &op_ptr->request);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ break;
+ case (MPIDI_RMA_GET):
+ mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags,
+ source_win_handle, target_win_handle,
+ &op_ptr->dtype_info, &op_ptr->dataloop, &op_ptr->request);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ break;
+ case (MPIDI_RMA_COMPARE_AND_SWAP):
+ case (MPIDI_RMA_FETCH_AND_OP):
+ mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags,
+ source_win_handle, target_win_handle, &op_ptr->request);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ break;
+ default:
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winInvalidOp");
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index d91a626..9024726 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -11,107 +11,6 @@ static int enableShortACC = 1;
#define MPIDI_PASSIVE_TARGET_DONE_TAG 348297
#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924
-/*
- * TODO:
- * Explore use of alternate allocation mechanisms for the RMA queue elements
- * (Because profiling has shown that queue element allocation/deallocation
- * can take a significant amount of time in the RMA operations).
- * 1: Current approach (uses perm memory malloc/free)
- * 2: Preallocate and maintain list (use perm memory malloc, but
- * free onto window; use first; free on window free)
- * 3: Preallocate and maintain list (use separate memory, but free to
- * thread/process; free in Finalize handler. Option to use for
- * single-threaded to avoid thread overheads)
- * Possible interface
- * int MPIDI_RMAListAlloc(MPIDI_RMA_Op_t **a,MPID_Win *win)
- * int MPIDI_RMAListFree(MPIDI_RMA_Op_t *a, MPID_Win *win)
- * return value is error code (e.g., allocation failure).
- */
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_Win_free
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_Win_free(MPID_Win ** win_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- int in_use;
- MPID_Comm *comm_ptr;
- int errflag = FALSE;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE);
-
- MPIU_ERR_CHKANDJUMP((*win_ptr)->epoch_state != MPIDI_EPOCH_NONE,
- mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
-
- if (!(*win_ptr)->shm_allocated) {
- /* when SHM is allocated, we already did a global barrier in
- MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */
- mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
- }
-
- comm_ptr = (*win_ptr)->comm_ptr;
- mpi_errno = MPIR_Comm_free_impl(comm_ptr);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
-
- MPIU_Free((*win_ptr)->targets);
- MPIU_Free((*win_ptr)->base_addrs);
- MPIU_Free((*win_ptr)->sizes);
- MPIU_Free((*win_ptr)->disp_units);
- MPIU_Free((*win_ptr)->all_win_handles);
-
- /* Free the attached buffer for windows created with MPI_Win_allocate() */
- if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE ||
- (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) {
- if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) {
- MPIU_Free((*win_ptr)->base);
- }
- }
-
- MPIU_Object_release_ref(*win_ptr, &in_use);
- /* MPI windows don't have reference count semantics, so this should always be true */
- MPIU_Assert(!in_use);
- MPIU_Handle_obj_free(&MPID_Win_mem, *win_ptr);
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FREE);
- return mpi_errno;
-
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_Win_shared_query
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_Win_shared_query(MPID_Win * win_ptr, int target_rank, MPI_Aint * size,
- int *disp_unit, void *baseptr)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
-
- *(void **) baseptr = win_ptr->base;
- *size = win_ptr->size;
- *disp_unit = win_ptr->disp_unit;
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Put
#undef FCNAME
@@ -448,36 +347,300 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
#undef FUNCNAME
-#define FUNCNAME MPIDI_Alloc_mem
+#define FUNCNAME MPIDI_Get_accumulate
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
+ MPI_Datatype origin_datatype, void *result_addr, int result_count,
+ MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
+ int target_count, MPI_Datatype target_datatype, MPI_Op op,
+ MPID_Win * win_ptr)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_msg_sz_t data_sz;
+ int rank;
+ int dt_contig ATTRIBUTE((unused));
+ MPI_Aint dt_true_lb ATTRIBUTE((unused));
+ MPID_Datatype *dtp;
+ MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);
+
+ if (target_rank == MPI_PROC_NULL) {
+ goto fn_exit;
+ }
+
+ if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
+ win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
+ }
+
+ MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
+
+ MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb);
+
+ if (data_sz == 0) {
+ goto fn_exit;
+ }
+
+ rank = win_ptr->comm_ptr->rank;
+
+ if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
+ win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
+ /* check if target is local and shared memory is allocated on window,
+ * if so, we directly perform this operation on shared memory region. */
+
+ /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
+ * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
+ * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
+ * which is only set to TRUE when SHM region is allocated in nemesis.
+ * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
+ */
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
+ }
+
+ /* Do =! rank first (most likely branch?) */
+ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
+ (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
+ mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
+ result_addr, result_count, result_datatype,
+ target_rank, target_disp, target_count,
+ target_datatype, op, win_ptr);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ }
+ else {
+ MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Op_t *new_ptr = NULL;
+
+ /* Append the operation to the window's RMA ops queue */
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
+
+ new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
+ /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
+ * contain both PUT and GET like ops */
+ new_ptr->origin_addr = (void *) origin_addr;
+ new_ptr->origin_count = origin_count;
+ new_ptr->origin_datatype = origin_datatype;
+ new_ptr->result_addr = result_addr;
+ new_ptr->result_count = result_count;
+ new_ptr->result_datatype = result_datatype;
+ new_ptr->target_rank = target_rank;
+ new_ptr->target_disp = target_disp;
+ new_ptr->target_count = target_count;
+ new_ptr->target_datatype = target_datatype;
+ new_ptr->op = op;
+
+ /* if source or target datatypes are derived, increment their
+ * reference counts */
+ if (op != MPI_NO_OP && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
+ MPID_Datatype_get_ptr(origin_datatype, dtp);
+ MPID_Datatype_add_ref(dtp);
+ }
+ if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
+ MPID_Datatype_get_ptr(result_datatype, dtp);
+ MPID_Datatype_add_ref(dtp);
+ }
+ if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
+ MPID_Datatype_get_ptr(target_datatype, dtp);
+ MPID_Datatype_add_ref(dtp);
+ }
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE);
+ return mpi_errno;
+
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_Compare_and_swap
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-void *MPIDI_Alloc_mem(size_t size, MPID_Info * info_ptr)
+int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
+ void *result_addr, MPI_Datatype datatype, int target_rank,
+ MPI_Aint target_disp, MPID_Win * win_ptr)
{
- void *ap;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_ALLOC_MEM);
+ int mpi_errno = MPI_SUCCESS;
+ int rank;
+ MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
+
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
+
+ if (target_rank == MPI_PROC_NULL) {
+ goto fn_exit;
+ }
+
+ if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
+ win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
+ }
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_ALLOC_MEM);
+ MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
- ap = MPIU_Malloc(size);
+ rank = win_ptr->comm_ptr->rank;
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_ALLOC_MEM);
- return ap;
+ if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
+ win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
+ /* check if target is local and shared memory is allocated on window,
+ * if so, we directly perform this operation on shared memory region. */
+
+ /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
+ * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
+ * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
+ * which is only set to TRUE when SHM region is allocated in nemesis.
+ * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
+ */
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
+ }
+
+ /* The datatype must be predefined, and one of: C integer, Fortran integer,
+ * Logical, Multi-language types, or Byte. This is checked above the ADI,
+ * so there's no need to check it again here. */
+
+ /* FIXME: For shared memory windows, we should provide an implementation
+ * that uses a processor atomic operation. */
+ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
+ (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
+ mpi_errno = MPIDI_CH3I_Shm_cas_op(origin_addr, compare_addr, result_addr,
+ datatype, target_rank, target_disp, win_ptr);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ }
+ else {
+ MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Op_t *new_ptr = NULL;
+
+ /* Append this operation to the RMA ops queue */
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
+ new_ptr->origin_addr = (void *) origin_addr;
+ new_ptr->origin_count = 1;
+ new_ptr->origin_datatype = datatype;
+ new_ptr->target_rank = target_rank;
+ new_ptr->target_disp = target_disp;
+ new_ptr->target_count = 1;
+ new_ptr->target_datatype = datatype;
+ new_ptr->result_addr = result_addr;
+ new_ptr->result_count = 1;
+ new_ptr->result_datatype = datatype;
+ new_ptr->compare_addr = (void *) compare_addr;
+ new_ptr->compare_count = 1;
+ new_ptr->compare_datatype = datatype;
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
}
#undef FUNCNAME
-#define FUNCNAME MPIDI_Free_mem
+#define FUNCNAME MPIDI_Fetch_and_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_Free_mem(void *ptr)
+int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
+ MPI_Datatype datatype, int target_rank,
+ MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr)
{
int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_FREE_MEM);
+ int rank;
+ MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
+
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_FETCH_AND_OP);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_FETCH_AND_OP);
+
+ if (target_rank == MPI_PROC_NULL) {
+ goto fn_exit;
+ }
+
+ if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
+ win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
+ }
+
+ MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
+
+ rank = win_ptr->comm_ptr->rank;
+
+ if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
+ win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
+ /* check if target is local and shared memory is allocated on window,
+ * if so, we directly perform this operation on shared memory region. */
+
+ /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
+ * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
+ * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
+ * which is only set to TRUE when SHM region is allocated in nemesis.
+ * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
+ */
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
+ MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
+ }
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_FREE_MEM);
+ /* The datatype and op must be predefined. This is checked above the ADI,
+ * so there's no need to check it again here. */
- MPIU_Free(ptr);
+ /* FIXME: For shared memory windows, we should provide an implementation
+ * that uses a processor atomic operation. */
+ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
+ (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
+ mpi_errno = MPIDI_CH3I_Shm_fop_op(origin_addr, result_addr, datatype,
+ target_rank, target_disp, op, win_ptr);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ }
+ else {
+ MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
+ MPIDI_RMA_Op_t *new_ptr = NULL;
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_FREE_MEM);
+ /* Append this operation to the RMA ops queue */
+ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
+ new_ptr->origin_addr = (void *) origin_addr;
+ new_ptr->origin_count = 1;
+ new_ptr->origin_datatype = datatype;
+ new_ptr->target_rank = target_rank;
+ new_ptr->target_disp = target_disp;
+ new_ptr->target_count = 1;
+ new_ptr->target_datatype = datatype;
+ new_ptr->result_addr = result_addr;
+ new_ptr->result_count = 1;
+ new_ptr->result_datatype = datatype;
+ new_ptr->op = op;
+ }
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_FETCH_AND_OP);
return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
}
diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
new file mode 100644
index 0000000..7cb36a5
--- /dev/null
+++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c
@@ -0,0 +1,1756 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ * (C) 2001 by Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include "mpidimpl.h"
+#include "mpidrma.h"
+
+
+/* ------------------------------------------------------------------------ */
+/*
+ * The following routines are the packet handlers for the packet types
+ * used above in the implementation of the RMA operations in terms
+ * of messages.
+ */
+/* ------------------------------------------------------------------------ */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Put
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_put_t *put_pkt = &pkt->put;
+ MPID_Request *req = NULL;
+ MPI_Aint type_size;
+ int complete = 0;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ MPID_Win *win_ptr;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received put pkt");
+
+ if (put_pkt->count == 0) {
+ /* it's a 0-byte message sent just to decrement the
+ * completion counter. This happens only in
+ * post/start/complete/wait sync model; therefore, no need
+ * to check lock queue. */
+ if (put_pkt->target_win_handle != MPI_WIN_NULL) {
+ MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr);
+ mpi_errno =
+ MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, put_pkt->flags, MPI_WIN_NULL);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ }
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+ goto fn_exit;
+ }
+
+ MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, put_pkt->flags);
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ req = MPID_Request_create();
+ MPIU_Object_set_ref(req, 1);
+
+ req->dev.user_buf = put_pkt->addr;
+ req->dev.user_count = put_pkt->count;
+ req->dev.target_win_handle = put_pkt->target_win_handle;
+ req->dev.source_win_handle = put_pkt->source_win_handle;
+ req->dev.flags = put_pkt->flags;
+
+ if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) {
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP);
+ req->dev.datatype = put_pkt->datatype;
+
+ MPID_Datatype_get_size_macro(put_pkt->datatype, type_size);
+ req->dev.recv_data_sz = type_size * put_pkt->count;
+
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
+ /* FIXME: Only change the handling of completion if
+ * post_data_receive reset the handler. There should
+ * be a cleaner way to do this */
+ if (!req->dev.OnDataAvail) {
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
+ }
+
+ /* return the number of bytes processed in this function */
+ *buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
+
+ if (complete) {
+ mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ if (complete) {
+ *rreqp = NULL;
+ goto fn_exit;
+ }
+ }
+ }
+ else {
+ /* derived datatype */
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT);
+ req->dev.datatype = MPI_DATATYPE_NULL;
+ req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
+
+ req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
+ MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
+ if (!req->dev.dtype_info) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_RMA_dtype_info");
+ }
+
+ req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size);
+ if (!req->dev.dataloop) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ put_pkt->dataloop_size);
+ }
+
+ /* if we received all of the dtype_info and dataloop, copy it
+ * now and call the handler, otherwise set the iov and let the
+ * channel copy it */
+ if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) {
+ /* copy all of dtype_info and dataloop */
+ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
+ MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
+ put_pkt->dataloop_size);
+
+ *buflen =
+ sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size;
+
+ /* All dtype data has been received, call req handler */
+ mpi_errno = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete(vc, req, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
+ if (complete) {
+ *rreqp = NULL;
+ goto fn_exit;
+ }
+ }
+ else {
+ req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info);
+ req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
+ req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
+ req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size;
+ req->dev.iov_count = 2;
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete;
+ }
+
+ }
+
+ *rreqp = req;
+
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
+ }
+
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Get
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get;
+ MPID_Request *req = NULL;
+ MPID_IOV iov[MPID_IOV_LIMIT];
+ int complete;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ MPID_Win *win_ptr;
+ int mpi_errno = MPI_SUCCESS;
+ MPI_Aint type_size;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get pkt");
+
+ MPIU_Assert(get_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(get_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, get_pkt->flags);
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ req = MPID_Request_create();
+ req->dev.target_win_handle = get_pkt->target_win_handle;
+ req->dev.source_win_handle = get_pkt->source_win_handle;
+ req->dev.flags = get_pkt->flags;
+
+ /* here we increment the Active Target counter to guarantee the GET-like
+ operation are completed when counter reaches zero. */
+ win_ptr->at_completion_counter++;
+
+ if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) {
+ /* basic datatype. send the data. */
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp;
+
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
+ req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete;
+ req->kind = MPID_REQUEST_SEND;
+
+ MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
+ get_resp_pkt->request_handle = get_pkt->request_handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
+
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt->addr;
+ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size);
+ iov[1].MPID_IOV_LEN = get_pkt->count * type_size;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ /* --BEGIN ERROR HANDLING-- */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_Object_set_ref(req, 0);
+ MPIDI_CH3_Request_destroy(req);
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+ /* --END ERROR HANDLING-- */
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+ }
+ else {
+ /* derived datatype. first get the dtype_info and dataloop. */
+
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP_DERIVED_DT);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete;
+ req->dev.OnFinal = 0;
+ req->dev.user_buf = get_pkt->addr;
+ req->dev.user_count = get_pkt->count;
+ req->dev.datatype = MPI_DATATYPE_NULL;
+ req->dev.request_handle = get_pkt->request_handle;
+
+ req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
+ MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
+ if (!req->dev.dtype_info) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_RMA_dtype_info");
+ }
+
+ req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size);
+ if (!req->dev.dataloop) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ get_pkt->dataloop_size);
+ }
+
+ /* if we received all of the dtype_info and dataloop, copy it
+ * now and call the handler, otherwise set the iov and let the
+ * channel copy it */
+ if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) {
+ /* copy all of dtype_info and dataloop */
+ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
+ MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
+ get_pkt->dataloop_size);
+
+ *buflen =
+ sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size;
+
+ /* All dtype data has been received, call req handler */
+ mpi_errno = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete(vc, req, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET");
+ if (complete)
+ *rreqp = NULL;
+ }
+ else {
+ req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info;
+ req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
+ req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
+ req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size;
+ req->dev.iov_count = 2;
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = req;
+ }
+
+ }
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
+ return mpi_errno;
+
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_accum_t *accum_pkt = &pkt->accum;
+ MPID_Request *req = NULL;
+ MPI_Aint true_lb, true_extent, extent;
+ void *tmp_buf = NULL;
+ int complete = 0;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ MPID_Win *win_ptr;
+ int mpi_errno = MPI_SUCCESS;
+ MPI_Aint type_size;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt");
+
+ MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ req = MPID_Request_create();
+ MPIU_Object_set_ref(req, 1);
+ *rreqp = req;
+
+ req->dev.user_count = accum_pkt->count;
+ req->dev.op = accum_pkt->op;
+ req->dev.real_user_buf = accum_pkt->addr;
+ req->dev.target_win_handle = accum_pkt->target_win_handle;
+ req->dev.source_win_handle = accum_pkt->source_win_handle;
+ req->dev.flags = accum_pkt->flags;
+
+ if (accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM) {
+ req->dev.resp_request_handle = accum_pkt->request_handle;
+ }
+ else {
+ req->dev.resp_request_handle = MPI_REQUEST_NULL;
+ }
+
+ if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) {
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP);
+ req->dev.datatype = accum_pkt->datatype;
+
+ MPIR_Type_get_true_extent_impl(accum_pkt->datatype, &true_lb, &true_extent);
+ MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent);
+
+ /* Predefined types should always have zero lb */
+ MPIU_Assert(true_lb == 0);
+
+ tmp_buf = MPIU_Malloc(accum_pkt->count * (MPIR_MAX(extent, true_extent)));
+ if (!tmp_buf) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ accum_pkt->count * MPIR_MAX(extent, true_extent));
+ }
+
+ req->dev.user_buf = tmp_buf;
+
+ MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size);
+ req->dev.recv_data_sz = type_size * accum_pkt->count;
+
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
+ /* FIXME: Only change the handling of completion if
+ * post_data_receive reset the handler. There should
+ * be a cleaner way to do this */
+ if (!req->dev.OnDataAvail) {
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
+ }
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+
+ if (complete) {
+ mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ if (complete) {
+ *rreqp = NULL;
+ goto fn_exit;
+ }
+ }
+ }
+ else {
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete;
+ req->dev.datatype = MPI_DATATYPE_NULL;
+ req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
+
+ req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
+ MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
+ if (!req->dev.dtype_info) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_RMA_dtype_info");
+ }
+
+ req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size);
+ if (!req->dev.dataloop) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ accum_pkt->dataloop_size);
+ }
+
+ if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) {
+ /* copy all of dtype_info and dataloop */
+ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
+ MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
+ accum_pkt->dataloop_size);
+
+ *buflen =
+ sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size;
+
+ /* All dtype data has been received, call req handler */
+ mpi_errno = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete(vc, req, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_ACCUMULATE");
+ if (complete) {
+ *rreqp = NULL;
+ goto fn_exit;
+ }
+ }
+ else {
+ req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info;
+ req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
+ req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
+ req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size;
+ req->dev.iov_count = 2;
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ }
+
+ }
+
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+
+}
+
+/* Special accumulate for short data items entirely within the packet */
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate_Immed
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_accum_immed_t *accum_pkt = &pkt->accum_immed;
+ MPID_Win *win_ptr;
+ MPI_Aint extent;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt");
+
+ MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
+
+ /* return the number of bytes processed in this function */
+ /* data_len == 0 (all within packet) */
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+
+ MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent);
+
+ if (win_ptr->shm_allocated == TRUE)
+ MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
+ /* Data is already present */
+ if (accum_pkt->op == MPI_REPLACE) {
+ /* no datatypes required */
+ int len;
+ MPIU_Assign_trunc(len, (accum_pkt->count * extent), int);
+ /* FIXME: use immediate copy because this is short */
+ MPIUI_Memcpy(accum_pkt->addr, accum_pkt->data, len);
+ }
+ else {
+ if (HANDLE_GET_KIND(accum_pkt->op) == HANDLE_KIND_BUILTIN) {
+ MPI_User_function *uop;
+ /* get the function by indexing into the op table */
+ uop = MPIR_OP_HDL_TO_FN(accum_pkt->op);
+ (*uop) (accum_pkt->data, accum_pkt->addr, &(accum_pkt->count), &(accum_pkt->datatype));
+ }
+ else {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OP, "**opnotpredefined",
+ "**opnotpredefined %d", accum_pkt->op);
+ }
+ }
+ if (win_ptr->shm_allocated == TRUE)
+ MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
+
+ /* There are additional steps to take if this is a passive
+ * target RMA or the last operation from the source */
+
+ /* Here is the code executed in PutAccumRespComplete after the
+ * accumulation operation */
+ MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
+
+ mpi_errno = MPIDI_CH3_Finish_rma_op_target(vc, win_ptr, TRUE,
+ accum_pkt->flags, accum_pkt->source_win_handle);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_CAS
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp;
+ MPIDI_CH3_Pkt_cas_t *cas_pkt = &pkt->cas;
+ MPID_Win *win_ptr;
+ MPID_Request *req;
+ MPI_Aint len;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt");
+
+ MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags);
+
+ /* return the number of bytes processed in this function */
+ /* data_len == 0 (all within packet) */
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+
+ MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP);
+ cas_resp_pkt->request_handle = cas_pkt->request_handle;
+
+ /* Copy old value into the response packet */
+ MPID_Datatype_get_size_macro(cas_pkt->datatype, len);
+ MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u));
+
+ if (win_ptr->shm_allocated == TRUE)
+ MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
+
+ MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len);
+
+ /* Compare and replace if equal */
+ if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) {
+ MPIU_Memcpy(cas_pkt->addr, &cas_pkt->origin_data, len);
+ }
+
+ if (win_ptr->shm_allocated == TRUE)
+ MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
+
+ /* Send the response packet */
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+
+ if (req != NULL) {
+ if (!MPID_Request_is_complete(req)) {
+ /* sending process is not completed, set proper OnDataAvail
+ (it is initialized to NULL by lower layer) */
+ req->dev.target_win_handle = cas_pkt->target_win_handle;
+ req->dev.flags = cas_pkt->flags;
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetAccumRespComplete;
+
+ /* here we increment the Active Target counter to guarantee the GET-like
+ operation are completed when counter reaches zero. */
+ win_ptr->at_completion_counter++;
+
+ MPID_Request_release(req);
+ goto fn_exit;
+ }
+ else
+ MPID_Request_release(req);
+ }
+
+
+ /* There are additional steps to take if this is a passive
+ * target RMA or the last operation from the source */
+
+ mpi_errno = MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, cas_pkt->flags, MPI_WIN_NULL);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_CASResp
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &pkt->cas_resp;
+ MPID_Request *req;
+ MPI_Aint len;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS response pkt");
+
+ MPID_Request_get_ptr(cas_resp_pkt->request_handle, req);
+ MPID_Datatype_get_size_macro(req->dev.datatype, len);
+
+ MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len);
+
+ MPIDI_CH3U_Request_complete(req);
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_FOP
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_fop_t *fop_pkt = &pkt->fop;
+ MPID_Request *req;
+ MPID_Win *win_ptr;
+ int data_complete = 0;
+ MPI_Aint len;
+ MPIU_CHKPMEM_DECL(1);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt");
+
+ MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL);
+ MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags);
+
+ req = MPID_Request_create();
+ MPIU_ERR_CHKANDJUMP(req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
+ MPIU_Object_set_ref(req, 1); /* Ref is held by progress engine */
+ *rreqp = NULL;
+
+ req->dev.user_buf = NULL; /* will be set later */
+ req->dev.user_count = 1;
+ req->dev.datatype = fop_pkt->datatype;
+ req->dev.op = fop_pkt->op;
+ req->dev.real_user_buf = fop_pkt->addr;
+ req->dev.target_win_handle = fop_pkt->target_win_handle;
+ req->dev.request_handle = fop_pkt->request_handle;
+ req->dev.flags = fop_pkt->flags;
+
+ MPID_Datatype_get_size_macro(req->dev.datatype, len);
+ MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u));
+
+ /* Set up the user buffer and receive data if needed */
+ if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) {
+ req->dev.user_buf = fop_pkt->origin_data;
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ data_complete = 1;
+ }
+ else {
+ /* Data won't fit in the header, allocate temp space and receive it */
+ MPIDI_msg_sz_t data_len;
+ void *data_buf;
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+ req->dev.recv_data_sz = len; /* count == 1 for FOP */
+
+ MPIU_CHKPMEM_MALLOC(req->dev.user_buf, void *, len, mpi_errno, "**nomemreq");
+
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &data_complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
+
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPComplete;
+
+ if (!data_complete) {
+ *rreqp = req;
+ }
+
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+ }
+
+ if (data_complete) {
+ int fop_complete = 0;
+ mpi_errno = MPIDI_CH3_ReqHandler_FOPComplete(vc, req, &fop_complete);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ *rreqp = NULL;
+ }
+
+ fn_exit:
+ MPIU_CHKPMEM_COMMIT();
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ MPIU_CHKPMEM_REAP();
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_FOPResp
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp;
+ MPID_Request *req;
+ int complete = 0;
+ MPI_Aint len;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP response pkt");
+
+ MPID_Request_get_ptr(fop_resp_pkt->request_handle, req);
+ MPID_Datatype_get_size_macro(req->dev.datatype, len);
+
+ if (len <= sizeof(fop_resp_pkt->data)) {
+ MPIU_Memcpy(req->dev.user_buf, (void *) fop_resp_pkt->data, len);
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ complete = 1;
+ }
+ else {
+ /* Data was too big to embed in the header */
+ MPIDI_msg_sz_t data_len;
+ void *data_buf;
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+ req->dev.recv_data_sz = len; /* count == 1 for FOP */
+ *rreqp = req;
+
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER,
+ "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP");
+
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+ }
+
+ if (complete) {
+ MPIDI_CH3U_Request_complete(req);
+ *rreqp = NULL;
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Get_AccumResp
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp;
+ MPID_Request *req;
+ int complete;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ int mpi_errno = MPI_SUCCESS;
+ MPI_Aint type_size;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt");
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req);
+
+ MPID_Datatype_get_size_macro(req->dev.datatype, type_size);
+ req->dev.recv_data_sz = type_size * req->dev.user_count;
+
+ *rreqp = req;
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP");
+ if (complete) {
+ MPIDI_CH3U_Request_complete(req);
+ *rreqp = NULL;
+ }
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Lock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock;
+ MPID_Win *win_ptr = NULL;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr);
+
+ if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) {
+ /* send lock granted packet. */
+ mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, lock_pkt->source_win_handle);
+ if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
+ }
+
+ else {
+ /* queue the lock information */
+ MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
+
+ /* Note: This code is reached by the fechandadd rma tests */
+ /* FIXME: MT: This may need to be done atomically. */
+
+ /* FIXME: Since we need to add to the tail of the list,
+ * we should maintain a tail pointer rather than traversing the
+ * list each time to find the tail. */
+ curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
+ prev_ptr = curr_ptr;
+ while (curr_ptr != NULL) {
+ prev_ptr = curr_ptr;
+ curr_ptr = curr_ptr->next;
+ }
+
+ new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
+ if (!new_ptr) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_Win_lock_queue");
+ }
+ if (prev_ptr != NULL)
+ prev_ptr->next = new_ptr;
+ else
+ win_ptr->lock_queue = new_ptr;
+
+ new_ptr->next = NULL;
+ new_ptr->lock_type = lock_pkt->lock_type;
+ new_ptr->source_win_handle = lock_pkt->source_win_handle;
+ new_ptr->vc = vc;
+ new_ptr->pt_single_op = NULL;
+ }
+
+ *rreqp = NULL;
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
+ return mpi_errno;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_LockPutUnlock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &pkt->lock_put_unlock;
+ MPID_Win *win_ptr = NULL;
+ MPID_Request *req = NULL;
+ MPI_Aint type_size;
+ int complete;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_put_unlock pkt");
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ req = MPID_Request_create();
+ MPIU_Object_set_ref(req, 1);
+
+ req->dev.datatype = lock_put_unlock_pkt->datatype;
+ MPID_Datatype_get_size_macro(lock_put_unlock_pkt->datatype, type_size);
+ req->dev.recv_data_sz = type_size * lock_put_unlock_pkt->count;
+ req->dev.user_count = lock_put_unlock_pkt->count;
+ req->dev.target_win_handle = lock_put_unlock_pkt->target_win_handle;
+
+ MPID_Win_get_ptr(lock_put_unlock_pkt->target_win_handle, win_ptr);
+
+ if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_put_unlock_pkt->lock_type) == 1) {
+ /* do the put. for this optimization, only basic datatypes supported. */
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
+ req->dev.user_buf = lock_put_unlock_pkt->addr;
+ req->dev.source_win_handle = lock_put_unlock_pkt->source_win_handle;
+ req->dev.flags = lock_put_unlock_pkt->flags;
+ }
+
+ else {
+ /* queue the information */
+ MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
+
+ new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
+ if (!new_ptr) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_Win_lock_queue");
+ }
+
+ new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
+ if (new_ptr->pt_single_op == NULL) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_PT_single_op");
+ }
+
+ /* FIXME: MT: The queuing may need to be done atomically. */
+
+ curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
+ prev_ptr = curr_ptr;
+ while (curr_ptr != NULL) {
+ prev_ptr = curr_ptr;
+ curr_ptr = curr_ptr->next;
+ }
+
+ if (prev_ptr != NULL)
+ prev_ptr->next = new_ptr;
+ else
+ win_ptr->lock_queue = new_ptr;
+
+ new_ptr->next = NULL;
+ new_ptr->lock_type = lock_put_unlock_pkt->lock_type;
+ new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle;
+ new_ptr->vc = vc;
+
+ new_ptr->pt_single_op->type = MPIDI_RMA_PUT;
+ new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags;
+ new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr;
+ new_ptr->pt_single_op->count = lock_put_unlock_pkt->count;
+ new_ptr->pt_single_op->datatype = lock_put_unlock_pkt->datatype;
+ /* allocate memory to receive the data */
+ new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz);
+ if (new_ptr->pt_single_op->data == NULL) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ req->dev.recv_data_sz);
+ }
+
+ new_ptr->pt_single_op->data_recd = 0;
+
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_PUT);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete;
+ req->dev.user_buf = new_ptr->pt_single_op->data;
+ req->dev.lock_queue_entry = new_ptr;
+ }
+
+ int (*fcn) (MPIDI_VC_t *, struct MPID_Request *, int *);
+ fcn = req->dev.OnDataAvail;
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER,
+ "**ch3|postrecv", "**ch3|postrecv %s",
+ "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK");
+ }
+ req->dev.OnDataAvail = fcn;
+ *rreqp = req;
+
+ if (complete) {
+ mpi_errno = fcn(vc, req, &complete);
+ if (complete) {
+ *rreqp = NULL;
+ }
+ }
+
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER,
+ "**ch3|postrecv", "**ch3|postrecv %s",
+ "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK");
+ }
+
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
+ return mpi_errno;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_LockGetUnlock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &pkt->lock_get_unlock;
+ MPID_Win *win_ptr = NULL;
+ MPI_Aint type_size;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_get_unlock pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Win_get_ptr(lock_get_unlock_pkt->target_win_handle, win_ptr);
+
+ if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_get_unlock_pkt->lock_type) == 1) {
+ /* do the get. for this optimization, only basic datatypes supported. */
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp;
+ MPID_Request *req;
+ MPID_IOV iov[MPID_IOV_LIMIT];
+
+ req = MPID_Request_create();
+ req->dev.target_win_handle = lock_get_unlock_pkt->target_win_handle;
+ req->dev.source_win_handle = lock_get_unlock_pkt->source_win_handle;
+ req->dev.flags = lock_get_unlock_pkt->flags;
+
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP);
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
+ req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete;
+ req->kind = MPID_REQUEST_SEND;
+
+ /* here we increment the Active Target counter to guarantee the GET-like
+ operation are completed when counter reaches zero. */
+ win_ptr->at_completion_counter++;
+
+ MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
+ get_resp_pkt->request_handle = lock_get_unlock_pkt->request_handle;
+
+ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
+ iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
+
+ iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_get_unlock_pkt->addr;
+ MPID_Datatype_get_size_macro(lock_get_unlock_pkt->datatype, type_size);
+ iov[1].MPID_IOV_LEN = lock_get_unlock_pkt->count * type_size;
+
+ mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
+ /* --BEGIN ERROR HANDLING-- */
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_Object_set_ref(req, 0);
+ MPIDI_CH3_Request_destroy(req);
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
+ }
+ /* --END ERROR HANDLING-- */
+ }
+
+ else {
+ /* queue the information */
+ MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
+
+ /* FIXME: MT: This may need to be done atomically. */
+
+ curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
+ prev_ptr = curr_ptr;
+ while (curr_ptr != NULL) {
+ prev_ptr = curr_ptr;
+ curr_ptr = curr_ptr->next;
+ }
+
+ new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
+ if (!new_ptr) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_Win_lock_queue");
+ }
+ new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
+ if (new_ptr->pt_single_op == NULL) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_PT_Single_op");
+ }
+
+ if (prev_ptr != NULL)
+ prev_ptr->next = new_ptr;
+ else
+ win_ptr->lock_queue = new_ptr;
+
+ new_ptr->next = NULL;
+ new_ptr->lock_type = lock_get_unlock_pkt->lock_type;
+ new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle;
+ new_ptr->vc = vc;
+
+ new_ptr->pt_single_op->type = MPIDI_RMA_GET;
+ new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags;
+ new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr;
+ new_ptr->pt_single_op->count = lock_get_unlock_pkt->count;
+ new_ptr->pt_single_op->datatype = lock_get_unlock_pkt->datatype;
+ new_ptr->pt_single_op->data = NULL;
+ new_ptr->pt_single_op->request_handle = lock_get_unlock_pkt->request_handle;
+ new_ptr->pt_single_op->data_recd = 1;
+ }
+
+ *rreqp = NULL;
+
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
+ return mpi_errno;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_LockAccumUnlock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &pkt->lock_accum_unlock;
+ MPID_Request *req = NULL;
+ MPID_Win *win_ptr = NULL;
+ MPIDI_Win_lock_queue *curr_ptr = NULL, *prev_ptr = NULL, *new_ptr = NULL;
+ MPI_Aint type_size;
+ int complete;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_accum_unlock pkt");
+
+ /* no need to acquire the lock here because we need to receive the
+ * data into a temporary buffer first */
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ req = MPID_Request_create();
+ MPIU_Object_set_ref(req, 1);
+
+ req->dev.datatype = lock_accum_unlock_pkt->datatype;
+ MPID_Datatype_get_size_macro(lock_accum_unlock_pkt->datatype, type_size);
+ req->dev.recv_data_sz = type_size * lock_accum_unlock_pkt->count;
+ req->dev.user_count = lock_accum_unlock_pkt->count;
+ req->dev.target_win_handle = lock_accum_unlock_pkt->target_win_handle;
+ req->dev.flags = lock_accum_unlock_pkt->flags;
+
+ /* queue the information */
+
+ new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
+ if (!new_ptr) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_Win_lock_queue");
+ }
+
+ new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
+ if (new_ptr->pt_single_op == NULL) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
+ "MPIDI_PT_single_op");
+ }
+
+ MPID_Win_get_ptr(lock_accum_unlock_pkt->target_win_handle, win_ptr);
+
+ /* FIXME: MT: The queuing may need to be done atomically. */
+
+ curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
+ prev_ptr = curr_ptr;
+ while (curr_ptr != NULL) {
+ prev_ptr = curr_ptr;
+ curr_ptr = curr_ptr->next;
+ }
+
+ if (prev_ptr != NULL)
+ prev_ptr->next = new_ptr;
+ else
+ win_ptr->lock_queue = new_ptr;
+
+ new_ptr->next = NULL;
+ new_ptr->lock_type = lock_accum_unlock_pkt->lock_type;
+ new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle;
+ new_ptr->vc = vc;
+
+ new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE;
+ new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags;
+ new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr;
+ new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count;
+ new_ptr->pt_single_op->datatype = lock_accum_unlock_pkt->datatype;
+ new_ptr->pt_single_op->op = lock_accum_unlock_pkt->op;
+ /* allocate memory to receive the data */
+ new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz);
+ if (new_ptr->pt_single_op->data == NULL) {
+ MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
+ req->dev.recv_data_sz);
+ }
+
+ new_ptr->pt_single_op->data_recd = 0;
+
+ MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_ACCUM);
+ req->dev.user_buf = new_ptr->pt_single_op->data;
+ req->dev.lock_queue_entry = new_ptr;
+
+ *rreqp = req;
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ /* FIXME: Only change the handling of completion if
+ * post_data_receive reset the handler. There should
+ * be a cleaner way to do this */
+ if (!req->dev.OnDataAvail) {
+ req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete;
+ }
+ if (mpi_errno != MPI_SUCCESS) {
+ MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
+ "**ch3|postrecv %s", "MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK");
+ }
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+
+ if (complete) {
+ mpi_errno = MPIDI_CH3_ReqHandler_SinglePutAccumComplete(vc, req, &complete);
+ if (complete) {
+ *rreqp = NULL;
+ }
+ }
+ fn_fail:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
+ return mpi_errno;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_GetResp
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp;
+ MPID_Request *req;
+ int complete;
+ char *data_buf = NULL;
+ MPIDI_msg_sz_t data_len;
+ int mpi_errno = MPI_SUCCESS;
+ MPI_Aint type_size;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get response pkt");
+
+ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
+ data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Request_get_ptr(get_resp_pkt->request_handle, req);
+
+ MPID_Datatype_get_size_macro(req->dev.datatype, type_size);
+ req->dev.recv_data_sz = type_size * req->dev.user_count;
+
+ *rreqp = req;
+ mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
+ MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s",
+ "MPIDI_CH3_PKT_GET_RESP");
+ if (complete) {
+ MPIDI_CH3U_Request_complete(req);
+ *rreqp = NULL;
+ }
+ /* return the number of bytes processed in this function */
+ *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_LockGranted
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &pkt->lock_granted;
+ MPID_Win *win_ptr = NULL;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock granted pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Win_get_ptr(lock_granted_pkt->source_win_handle, win_ptr);
+ /* set the remote_lock_state flag in the window */
+ win_ptr->targets[lock_granted_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
+
+ *rreqp = NULL;
+ MPIDI_CH3_Progress_signal_completion();
+
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
+ return MPI_SUCCESS;
+}
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done;
+ MPID_Win *win_ptr = NULL;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+
+ MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr);
+ MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state !=
+ MPIDI_CH3_WIN_LOCK_NONE);
+
+ if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state ==
+ MPIDI_CH3_WIN_LOCK_FLUSH)
+ win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state =
+ MPIDI_CH3_WIN_LOCK_GRANTED;
+ else
+ win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
+
+ *rreqp = NULL;
+ MPIDI_CH3_Progress_signal_completion();
+
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
+ return MPI_SUCCESS;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Unlock
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Unlock(MPIDI_VC_t * vc ATTRIBUTE((unused)),
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &pkt->unlock;
+ MPID_Win *win_ptr = NULL;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received unlock pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+
+ MPID_Win_get_ptr(unlock_pkt->target_win_handle, win_ptr);
+ mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
+
+ MPIDI_CH3_Progress_signal_completion();
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_PktHandler_Flush
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_PktHandler_Flush(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_CH3_Pkt_flush_t *flush_pkt = &pkt->flush;
+ MPID_Win *win_ptr = NULL;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received flush pkt");
+
+ *buflen = sizeof(MPIDI_CH3_Pkt_t);
+ *rreqp = NULL;
+
+ /* This is a flush request packet */
+ if (flush_pkt->target_win_handle != MPI_WIN_NULL) {
+ MPID_Request *req = NULL;
+
+ MPID_Win_get_ptr(flush_pkt->target_win_handle, win_ptr);
+
+ flush_pkt->target_win_handle = MPI_WIN_NULL;
+ flush_pkt->target_rank = win_ptr->comm_ptr->rank;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
+
+ /* Release the request returned by iStartMsg */
+ if (req != NULL) {
+ MPID_Request_release(req);
+ }
+ }
+
+ /* This is a flush response packet */
+ else {
+ MPID_Win_get_ptr(flush_pkt->source_win_handle, win_ptr);
+ win_ptr->targets[flush_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
+ MPIDI_CH3_Progress_signal_completion();
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_Start_rma_op_target
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_Start_rma_op_target(MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
+
+ /* Lock with NOCHECK is piggybacked on this message. We should be able to
+ * immediately grab the lock. Otherwise, there is a synchronization error. */
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK && flags & MPIDI_CH3_PKT_FLAG_RMA_NOCHECK) {
+ int lock_acquired;
+ int lock_mode;
+
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_SHARED) {
+ lock_mode = MPI_LOCK_SHARED;
+ }
+ else if (flags & MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE) {
+ lock_mode = MPI_LOCK_EXCLUSIVE;
+ }
+ else {
+ MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|rma_flags");
+ }
+
+ lock_acquired = MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_mode);
+ MPIU_ERR_CHKANDJUMP(!lock_acquired, mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|nocheck_invalid");
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3_Finish_rma_op_target
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_rma_update,
+ MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
+
+ /* This function should be called by the target process after each RMA
+ * operation is completed, to update synchronization state. */
+
+ /* Last RMA operation from source. If active target RMA, decrement window
+ * counter. */
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE) {
+ MPIU_Assert(win_ptr->current_lock_type == MPID_LOCK_NONE);
+
+ win_ptr->at_completion_counter -= 1;
+ MPIU_Assert(win_ptr->at_completion_counter >= 0);
+
+ /* Signal the local process when the op counter reaches 0. */
+ if (win_ptr->at_completion_counter == 0)
+ MPIDI_CH3_Progress_signal_completion();
+ }
+
+ /* If passive target RMA, release lock on window and grant next lock in the
+ * lock queue if there is any. If requested by the origin, send an ack back
+ * to indicate completion at the target. */
+ else if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+ MPIU_Assert(win_ptr->current_lock_type != MPID_LOCK_NONE);
+
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) {
+ MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL);
+ mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+ }
+
+ mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
+ if (mpi_errno) {
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ /* The local process may be waiting for the lock. Signal completion to
+ * wake it up, so it can attempt to grab the lock. */
+ MPIDI_CH3_Progress_signal_completion();
+ }
+ else if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) {
+ /* Ensure store instructions have been performed before flush call is
+ * finished on origin process. */
+ OPA_read_write_barrier();
+
+ if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) {
+ MPIDI_CH3_Pkt_t upkt;
+ MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush;
+ MPID_Request *req = NULL;
+
+ MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received piggybacked flush request");
+ MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL);
+
+ MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH);
+ flush_pkt->source_win_handle = source_win_handle;
+ flush_pkt->target_win_handle = MPI_WIN_NULL;
+ flush_pkt->target_rank = win_ptr->comm_ptr->rank;
+
+ MPIU_THREAD_CS_ENTER(CH3COMM, vc);
+ mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
+ MPIU_THREAD_CS_EXIT(CH3COMM, vc);
+ MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER,
+ "**ch3|rma_msg");
+
+ /* Release the request returned by iStartMsg */
+ if (req != NULL) {
+ MPID_Request_release(req);
+ }
+ }
+ }
+
+ fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+/* ------------------------------------------------------------------------ */
+/*
+ * For debugging, we provide the following functions for printing the
+ * contents of an RMA packet
+ */
+/* ------------------------------------------------------------------------ */
+#ifdef MPICH_DBG_OUTPUT
+int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PUT\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype));
+ MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle));
+ /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype));
+ MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size));
+ MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle));
+ /*
+ * MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request));
+ * MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->get.win_ptr));
+ */
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_GetResp(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET_RESP\n"));
+ MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request_handle));
+ /*MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request)); */
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUMULATE\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype));
+ MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size));
+ MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle));
+ /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_Accum_Immed(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUM_IMMED\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum_immed.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum_immed.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum_immed.datatype));
+ MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum_immed.op));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum_immed.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum_immed.source_win_handle));
+ /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */
+ fflush(stdout);
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_Lock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK\n"));
+ MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock.lock_type));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock.source_win_handle));
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_LockPutUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_PUT_UNLOCK\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_put_unlock.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_put_unlock.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_put_unlock.datatype));
+ MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_put_unlock.lock_type));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_put_unlock.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_put_unlock.source_win_handle));
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_LockAccumUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_accum_unlock.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_accum_unlock.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_accum_unlock.datatype));
+ MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_accum_unlock.lock_type));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_accum_unlock.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle));
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GET_UNLOCK\n"));
+ MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_get_unlock.addr));
+ MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_get_unlock.count));
+ MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_get_unlock.datatype));
+ MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_get_unlock.lock_type));
+ MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_get_unlock.target_win_handle));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_get_unlock.source_win_handle));
+ MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->lock_get_unlock.request_handle));
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n"));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle));
+ return MPI_SUCCESS;
+}
+
+int MPIDI_CH3_PktPrint_LockGranted(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
+{
+ MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GRANTED\n"));
+ MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_granted.source_win_handle));
+ return MPI_SUCCESS;
+}
+#endif
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index 489309a..aeec089 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -15,16 +15,6 @@ categories:
description : cvars that control behavior of ch3
cvars:
- - name : MPIR_CVAR_CH3_RMA_ACC_IMMED
- category : CH3
- type : boolean
- default : true
- class : none
- verbosity : MPI_T_VERBOSITY_USER_BASIC
- scope : MPI_T_SCOPE_ALL_EQ
- description : >-
- Use the immediate accumulate optimization
-
- name : MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD
category : CH3
type : int
@@ -354,100 +344,16 @@ void MPIDI_CH3_RMA_Init_Pvars(void)
#define SYNC_POST_TAG 100
-static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr);
-static int send_unlock_msg(int dest, MPID_Win * win_ptr);
-/* static int send_flush_msg(int dest, MPID_Win *win_ptr); */
static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank);
-static int acquire_local_lock(MPID_Win * win_ptr, int lock_mode);
-static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info,
- void **dataloop, MPID_Request ** request);
-static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info,
- void **dataloop, MPID_Request ** request);
-static int send_contig_acc_msg(MPIDI_RMA_Op_t *, MPID_Win *,
- MPIDI_CH3_Pkt_flags_t flags, MPI_Win, MPI_Win, MPID_Request **);
-static int send_immed_rmw_msg(MPIDI_RMA_Op_t *, MPID_Win *,
- MPIDI_CH3_Pkt_flags_t flags, MPI_Win, MPI_Win, MPID_Request **);
static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
int *wait_for_rma_done_pkt, MPIDI_CH3_Pkt_flags_t sync_flags);
static int send_lock_put_or_acc(MPID_Win *, int);
static int send_lock_get(MPID_Win *, int);
-static inline int poke_progress_engine(void);
static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list);
static inline int rma_list_gc(MPID_Win * win_ptr,
MPIDI_RMA_Ops_list_t * ops_list,
MPIDI_RMA_Op_t * last_elm, int *nDone);
-static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info,
- const void *dataloop, MPI_Aint dataloop_sz,
- const void *o_addr, int o_count,
- MPI_Datatype o_datatype, MPID_Datatype ** combined_dtp);
-
-#define MPIDI_CH3I_ISSUE_RMA_OP(op_ptr_, win_ptr_, flags_, source_win_handle_, target_win_handle_,err_) \
- do { \
- switch ((op_ptr_)->type) \
- { \
- case (MPIDI_RMA_PUT): \
- case (MPIDI_RMA_ACCUMULATE): \
- (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \
- (target_win_handle_), &(op_ptr_)->dtype_info, \
- &(op_ptr_)->dataloop, &(op_ptr_)->request); \
- if (err_) { MPIU_ERR_POP(err_); } \
- break; \
- case (MPIDI_RMA_GET_ACCUMULATE): \
- if ((op_ptr_)->op == MPI_NO_OP) { \
- /* Note: Origin arguments are ignored for NO_OP, so we don't \
- * need to release a ref to the origin datatype. */ \
- \
- /* Convert the GAcc to a Get */ \
- (op_ptr_)->type = MPIDI_RMA_GET; \
- (op_ptr_)->origin_addr = (op_ptr_)->result_addr; \
- (op_ptr_)->origin_count = (op_ptr_)->result_count; \
- (op_ptr_)->origin_datatype = (op_ptr_)->result_datatype; \
- \
- (err_) = recv_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \
- (target_win_handle_), &(op_ptr_)->dtype_info, \
- &(op_ptr_)->dataloop, &(op_ptr_)->request); \
- } else { \
- (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \
- (target_win_handle_), &(op_ptr_)->dtype_info, \
- &(op_ptr_)->dataloop, &(op_ptr_)->request); \
- } \
- if (err_) { MPIU_ERR_POP(err_); } \
- break; \
- case MPIDI_RMA_ACC_CONTIG: \
- (err_) = send_contig_acc_msg((op_ptr_), (win_ptr_), (flags_), \
- (source_win_handle_), (target_win_handle_), \
- &(op_ptr_)->request); \
- if (err_) { MPIU_ERR_POP(err_); } \
- break; \
- case (MPIDI_RMA_GET): \
- (err_) = recv_rma_msg((op_ptr_), (win_ptr_), (flags_), \
- (source_win_handle_), (target_win_handle_), \
- &(op_ptr_)->dtype_info, \
- &(op_ptr_)->dataloop, &(op_ptr_)->request); \
- if (err_) { MPIU_ERR_POP(err_); } \
- break; \
- case (MPIDI_RMA_COMPARE_AND_SWAP): \
- case (MPIDI_RMA_FETCH_AND_OP): \
- (err_) = send_immed_rmw_msg((op_ptr_), (win_ptr_), (flags_), \
- (source_win_handle_), (target_win_handle_), \
- &(op_ptr_)->request); \
- if (err_) { MPIU_ERR_POP(err_); } \
- break; \
- \
- default: \
- MPIU_ERR_SETANDJUMP(err_,MPI_ERR_OTHER,"**winInvalidOp"); \
- } \
- } while (0)
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_fence
@@ -584,8 +490,10 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
source_win_handle = win_ptr->handle;
target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
- MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags,
- source_win_handle, target_win_handle, mpi_errno);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags,
+ source_win_handle, target_win_handle);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
i++;
curr_ops_cnt[curr_ptr->target_rank]++;
@@ -697,790 +605,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
/* --END ERROR HANDLING-- */
}
-/* create_datatype() creates a new struct datatype for the dtype_info
- and the dataloop of the target datatype together with the user data */
-#undef FUNCNAME
-#define FUNCNAME create_datatype
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info,
- const void *dataloop, MPI_Aint dataloop_sz,
- const void *o_addr, int o_count, MPI_Datatype o_datatype,
- MPID_Datatype ** combined_dtp)
-{
- int mpi_errno = MPI_SUCCESS;
- /* datatype_set_contents wants an array 'ints' which is the
- * blocklens array with count prepended to it. So blocklens
- * points to the 2nd element of ints to avoid having to copy
- * blocklens into ints later. */
- int ints[4];
- int *blocklens = &ints[1];
- MPI_Aint displaces[3];
- MPI_Datatype datatypes[3];
- const int count = 3;
- MPI_Datatype combined_datatype;
- MPIDI_STATE_DECL(MPID_STATE_CREATE_DATATYPE);
-
- MPIDI_FUNC_ENTER(MPID_STATE_CREATE_DATATYPE);
-
- /* create datatype */
- displaces[0] = MPIU_PtrToAint(dtype_info);
- blocklens[0] = sizeof(*dtype_info);
- datatypes[0] = MPI_BYTE;
-
- displaces[1] = MPIU_PtrToAint(dataloop);
- MPIU_Assign_trunc(blocklens[1], dataloop_sz, int);
- datatypes[1] = MPI_BYTE;
-
- displaces[2] = MPIU_PtrToAint(o_addr);
- blocklens[2] = o_count;
- datatypes[2] = o_datatype;
-
- mpi_errno = MPID_Type_struct(count, blocklens, displaces, datatypes, &combined_datatype);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
-
- ints[0] = count;
-
- MPID_Datatype_get_ptr(combined_datatype, *combined_dtp);
- mpi_errno = MPID_Datatype_set_contents(*combined_dtp, MPI_COMBINER_STRUCT, count + 1, /* ints (cnt,blklen) */
- count, /* aints (disps) */
- count, /* types */
- ints, displaces, datatypes);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
-
- /* Commit datatype */
-
- MPID_Dataloop_create(combined_datatype,
- &(*combined_dtp)->dataloop,
- &(*combined_dtp)->dataloop_size,
- &(*combined_dtp)->dataloop_depth, MPID_DATALOOP_HOMOGENEOUS);
-
- /* create heterogeneous dataloop */
- MPID_Dataloop_create(combined_datatype,
- &(*combined_dtp)->hetero_dloop,
- &(*combined_dtp)->hetero_dloop_size,
- &(*combined_dtp)->hetero_dloop_depth, MPID_DATALOOP_HETEROGENEOUS);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_CREATE_DATATYPE);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME send_rma_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
-{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put;
- MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
- MPID_IOV iov[MPID_IOV_LIMIT];
- int mpi_errno = MPI_SUCCESS;
- int origin_dt_derived, target_dt_derived, iovcnt;
- MPI_Aint origin_type_size;
- MPIDI_VC_t *vc;
- MPID_Comm *comm_ptr;
- MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL;
- MPID_Request *resp_req = NULL;
- MPIU_CHKPMEM_DECL(1);
- MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG);
- MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG);
-
- *request = NULL;
-
- if (rma_op->type == MPIDI_RMA_PUT) {
- MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
- put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- put_pkt->flags = flags;
- put_pkt->count = rma_op->target_count;
- put_pkt->datatype = rma_op->target_datatype;
- put_pkt->dataloop_size = 0;
- put_pkt->target_win_handle = target_win_handle;
- put_pkt->source_win_handle = source_win_handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*put_pkt);
- }
- else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) {
- /* Create a request for the GACC response. Store the response buf, count, and
- * datatype in it, and pass the request's handle in the GACC packet. When the
- * response comes from the target, it will contain the request handle. */
- resp_req = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
-
- MPIU_Object_set_ref(resp_req, 2);
-
- resp_req->dev.user_buf = rma_op->result_addr;
- resp_req->dev.user_count = rma_op->result_count;
- resp_req->dev.datatype = rma_op->result_datatype;
- resp_req->dev.target_win_handle = target_win_handle;
- resp_req->dev.source_win_handle = source_win_handle;
-
- if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) {
- MPID_Datatype *result_dtp = NULL;
- MPID_Datatype_get_ptr(resp_req->dev.datatype, result_dtp);
- resp_req->dev.datatype_ptr = result_dtp;
- /* this will cause the datatype to be freed when the
- * request is freed. */
- }
-
- /* Note: Get_accumulate uses the same packet type as accumulate */
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
- accum_pkt->request_handle = resp_req->handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
- }
- else {
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
- }
-
- /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
- * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
- * fflush(stdout);
- */
-
- comm_ptr = win_ptr->comm_ptr;
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
-
- if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) {
- origin_dt_derived = 1;
- MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp);
- }
- else {
- origin_dt_derived = 0;
- }
-
- if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
- target_dt_derived = 1;
- MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp);
- }
- else {
- target_dt_derived = 0;
- }
-
- if (target_dt_derived) {
- /* derived datatype on target. fill derived datatype info */
- dtype_info->is_contig = target_dtp->is_contig;
- dtype_info->max_contig_blocks = target_dtp->max_contig_blocks;
- dtype_info->size = target_dtp->size;
- dtype_info->extent = target_dtp->extent;
- dtype_info->dataloop_size = target_dtp->dataloop_size;
- dtype_info->dataloop_depth = target_dtp->dataloop_depth;
- dtype_info->eltype = target_dtp->eltype;
- dtype_info->dataloop = target_dtp->dataloop;
- dtype_info->ub = target_dtp->ub;
- dtype_info->lb = target_dtp->lb;
- dtype_info->true_ub = target_dtp->true_ub;
- dtype_info->true_lb = target_dtp->true_lb;
- dtype_info->has_sticky_ub = target_dtp->has_sticky_ub;
- dtype_info->has_sticky_lb = target_dtp->has_sticky_lb;
-
- MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop");
-
- MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
- MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size);
- MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
- /* the dataloop can have undefined padding sections, so we need to let
- * valgrind know that it is OK to pass this data to writev later on */
- MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size);
-
- if (rma_op->type == MPIDI_RMA_PUT) {
- put_pkt->dataloop_size = target_dtp->dataloop_size;
- }
- else {
- accum_pkt->dataloop_size = target_dtp->dataloop_size;
- }
- }
-
- MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
-
- if (!target_dt_derived) {
- /* basic datatype on target */
- if (!origin_dt_derived) {
- /* basic datatype on origin */
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
- iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
- iovcnt = 2;
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
- else {
- /* derived datatype on origin */
- *request = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
-
- MPIU_Object_set_ref(*request, 2);
- (*request)->kind = MPID_REQUEST_SEND;
-
- (*request)->dev.segment_ptr = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "MPID_Segment_alloc");
-
- (*request)->dev.datatype_ptr = origin_dtp;
- /* this will cause the datatype to be freed when the request
- * is freed. */
- MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count,
- rma_op->origin_datatype, (*request)->dev.segment_ptr, 0);
- (*request)->dev.segment_first = 0;
- (*request)->dev.segment_size = rma_op->origin_count * origin_type_size;
-
- (*request)->dev.OnFinal = 0;
- (*request)->dev.OnDataAvail = 0;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno =
- vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
- }
- else {
- /* derived datatype on target */
- MPID_Datatype *combined_dtp = NULL;
-
- *request = MPID_Request_create();
- if (*request == NULL) {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- }
-
- MPIU_Object_set_ref(*request, 2);
- (*request)->kind = MPID_REQUEST_SEND;
-
- (*request)->dev.segment_ptr = MPID_Segment_alloc();
- MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "MPID_Segment_alloc");
-
- /* create a new datatype containing the dtype_info, dataloop, and origin data */
-
- mpi_errno =
- create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr,
- rma_op->origin_count, rma_op->origin_datatype, &combined_dtp);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
-
- (*request)->dev.datatype_ptr = combined_dtp;
- /* combined_datatype will be freed when request is freed */
-
- MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0);
- (*request)->dev.segment_first = 0;
- (*request)->dev.segment_size = combined_dtp->size;
-
- (*request)->dev.OnFinal = 0;
- (*request)->dev.OnDataAvail = 0;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
-
- /* we're done with the datatypes */
- if (origin_dt_derived)
- MPID_Datatype_release(origin_dtp);
- MPID_Datatype_release(target_dtp);
- }
-
- /* This operation can generate two requests; one for inbound and one for
- * outbound data. */
- if (resp_req != NULL) {
- if (*request != NULL) {
- /* If we have both inbound and outbound requests (i.e. GACC
- * operation), we need to ensure that the source buffer is
- * available and that the response data has been received before
- * informing the origin that this operation is complete. Because
- * the update needs to be done atomically at the target, they will
- * not send back data until it has been received. Therefore,
- * completion of the response request implies that the send request
- * has completed.
- *
- * Therefore: refs on the response request are set to two: one is
- * held by the progress engine and the other by the RMA op
- * completion code. Refs on the outbound request are set to one;
- * it will be completed by the progress engine.
- */
-
- MPID_Request_release(*request);
- *request = resp_req;
-
- }
- else {
- *request = resp_req;
- }
-
- /* For error checking */
- resp_req = NULL;
- }
-
- fn_exit:
- MPIU_CHKPMEM_COMMIT();
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_RMA_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- if (resp_req) {
- MPID_Request_release(resp_req);
- }
- if (*request) {
- MPIU_CHKPMEM_REAP();
- if ((*request)->dev.datatype_ptr)
- MPID_Datatype_release((*request)->dev.datatype_ptr);
- MPID_Request_release(*request);
- }
- *request = NULL;
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-/*
- * Use this for contiguous accumulate operations
- */
-#undef FUNCNAME
-#define FUNCNAME send_contig_acc_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op,
- MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle, MPID_Request ** request)
-{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum;
- MPID_IOV iov[MPID_IOV_LIMIT];
- int mpi_errno = MPI_SUCCESS;
- int iovcnt;
- MPI_Aint origin_type_size;
- MPIDI_VC_t *vc;
- MPID_Comm *comm_ptr;
- size_t len;
- MPIDI_STATE_DECL(MPID_STATE_SEND_CONTIG_ACC_MSG);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG);
-
- *request = NULL;
-
- MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
- /* FIXME: Make this size check efficient and match the packet type */
- MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
- if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) {
- MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed;
- void *dest = accumi_pkt->data, *src = rma_op->origin_addr;
-
- MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED);
- accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- accumi_pkt->flags = flags;
- accumi_pkt->count = rma_op->target_count;
- accumi_pkt->datatype = rma_op->target_datatype;
- accumi_pkt->op = rma_op->op;
- accumi_pkt->target_win_handle = target_win_handle;
- accumi_pkt->source_win_handle = source_win_handle;
-
- switch (len) {
- case 1:
- *(uint8_t *) dest = *(uint8_t *) src;
- break;
- case 2:
- *(uint16_t *) dest = *(uint16_t *) src;
- break;
- case 4:
- *(uint32_t *) dest = *(uint32_t *) src;
- break;
- case 8:
- *(uint64_t *) dest = *(uint64_t *) src;
- break;
- default:
- MPIU_Memcpy(accumi_pkt->data, (void *) rma_op->origin_addr, len);
- }
- comm_ptr = win_ptr->comm_ptr;
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- goto fn_exit;
- }
-
- MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
- accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- accum_pkt->flags = flags;
- accum_pkt->count = rma_op->target_count;
- accum_pkt->datatype = rma_op->target_datatype;
- accum_pkt->dataloop_size = 0;
- accum_pkt->op = rma_op->op;
- accum_pkt->target_win_handle = target_win_handle;
- accum_pkt->source_win_handle = source_win_handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*accum_pkt);
-
- /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
- * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
- * fflush(stdout);
- */
-
- comm_ptr = win_ptr->comm_ptr;
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
-
-
- /* basic datatype on target */
- /* basic datatype on origin */
- /* FIXME: This is still very heavyweight for a small message operation,
- * such as a single word update */
- /* One possibility is to use iStartMsg with a buffer that is just large
- * enough, though note that nemesis has an optimization for this */
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
- iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size;
- iovcnt = 2;
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_CONTIG_ACC_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- if (*request) {
- MPID_Request_release(*request);
- }
- *request = NULL;
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-/*
- * Initiate an immediate RMW accumulate operation
- */
-#undef FUNCNAME
-#define FUNCNAME send_immed_rmw_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op,
- MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle, MPID_Request ** request)
-{
- int mpi_errno = MPI_SUCCESS;
- MPID_Request *rmw_req = NULL, *resp_req = NULL;
- MPIDI_VC_t *vc;
- MPID_Comm *comm_ptr;
- MPI_Aint len;
- MPIDI_STATE_DECL(MPID_STATE_SEND_IMMED_RMW_MSG);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG);
-
- *request = NULL;
-
- /* Create a request for the RMW response. Store the origin buf, count, and
- * datatype in it, and pass the request's handle RMW packet. When the
- * response comes from the target, it will contain the request handle. */
- resp_req = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- *request = resp_req;
-
- /* Set refs on the request to 2: one for the response message, and one for
- * the partial completion handler */
- MPIU_Object_set_ref(resp_req, 2);
-
- resp_req->dev.user_buf = rma_op->result_addr;
- resp_req->dev.user_count = rma_op->result_count;
- resp_req->dev.datatype = rma_op->result_datatype;
- resp_req->dev.target_win_handle = target_win_handle;
- resp_req->dev.source_win_handle = source_win_handle;
-
- /* REQUIRE: All datatype arguments must be of the same, builtin
- * type and counts must be 1. */
- MPID_Datatype_get_size_macro(rma_op->origin_datatype, len);
- comm_ptr = win_ptr->comm_ptr;
-
- if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) {
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas;
-
- MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u));
-
- MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS);
-
- cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- cas_pkt->flags = flags;
- cas_pkt->datatype = rma_op->target_datatype;
- cas_pkt->target_win_handle = target_win_handle;
- cas_pkt->request_handle = resp_req->handle;
-
- MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len);
- MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len);
-
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
-
- if (rmw_req != NULL) {
- MPID_Request_release(rmw_req);
- }
- }
-
- else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) {
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop;
-
- MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u));
-
- MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP);
-
- fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- fop_pkt->flags = flags;
- fop_pkt->datatype = rma_op->target_datatype;
- fop_pkt->target_win_handle = target_win_handle;
- fop_pkt->request_handle = resp_req->handle;
- fop_pkt->op = rma_op->op;
-
- if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) {
- /* Embed FOP data in the packet header */
- if (rma_op->op != MPI_NO_OP) {
- MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len);
- }
-
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &rmw_req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
-
- if (rmw_req != NULL) {
- MPID_Request_release(rmw_req);
- }
- }
- else {
- /* Data is too big to copy into the FOP header, use an IOV to send it */
- MPID_IOV iov[MPID_IOV_LIMIT];
-
- rmw_req = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(rmw_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- MPIU_Object_set_ref(rmw_req, 1);
-
- rmw_req->dev.OnFinal = 0;
- rmw_req->dev.OnDataAvail = 0;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*fop_pkt);
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr;
- iov[1].MPID_IOV_LEN = len; /* count == 1 */
-
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iSendv(vc, rmw_req, iov, 2);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
-
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
- }
- else {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_IMMED_RMW_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- if (*request) {
- MPID_Request_release(*request);
- }
- *request = NULL;
- if (rmw_req) {
- MPID_Request_release(rmw_req);
- }
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-
-#undef FUNCNAME
-#define FUNCNAME recv_rma_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
- MPIDI_CH3_Pkt_flags_t flags,
- MPI_Win source_win_handle,
- MPI_Win target_win_handle,
- MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request)
-{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_VC_t *vc;
- MPID_Comm *comm_ptr;
- MPID_Request *req = NULL;
- MPID_Datatype *dtp;
- MPID_IOV iov[MPID_IOV_LIMIT];
- MPIU_CHKPMEM_DECL(1);
- MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG);
- MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_RECV_RMA_MSG);
-
- /* create a request, store the origin buf, cnt, datatype in it,
- * and pass a handle to it in the get packet. When the get
- * response comes from the target, it will contain the request
- * handle. */
- req = MPID_Request_create();
- if (req == NULL) {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- }
-
- *request = req;
-
- MPIU_Object_set_ref(req, 2);
-
- req->dev.user_buf = rma_op->origin_addr;
- req->dev.user_count = rma_op->origin_count;
- req->dev.datatype = rma_op->origin_datatype;
- req->dev.target_win_handle = MPI_WIN_NULL;
- req->dev.source_win_handle = source_win_handle;
- if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) {
- MPID_Datatype_get_ptr(req->dev.datatype, dtp);
- req->dev.datatype_ptr = dtp;
- /* this will cause the datatype to be freed when the
- * request is freed. */
- }
-
- MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
- get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] +
- win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp;
- get_pkt->flags = flags;
- get_pkt->count = rma_op->target_count;
- get_pkt->datatype = rma_op->target_datatype;
- get_pkt->request_handle = req->handle;
- get_pkt->target_win_handle = target_win_handle;
- get_pkt->source_win_handle = source_win_handle;
-
-/* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type,
- rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]);
- fflush(stdout);
-*/
-
- comm_ptr = win_ptr->comm_ptr;
- MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
-
- if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) {
- /* basic datatype on target. simply send the get_pkt. */
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- }
- else {
- /* derived datatype on target. fill derived datatype info and
- * send it along with get_pkt. */
-
- MPID_Datatype_get_ptr(rma_op->target_datatype, dtp);
- dtype_info->is_contig = dtp->is_contig;
- dtype_info->max_contig_blocks = dtp->max_contig_blocks;
- dtype_info->size = dtp->size;
- dtype_info->extent = dtp->extent;
- dtype_info->dataloop_size = dtp->dataloop_size;
- dtype_info->dataloop_depth = dtp->dataloop_depth;
- dtype_info->eltype = dtp->eltype;
- dtype_info->dataloop = dtp->dataloop;
- dtype_info->ub = dtp->ub;
- dtype_info->lb = dtp->lb;
- dtype_info->true_ub = dtp->true_ub;
- dtype_info->true_lb = dtp->true_lb;
- dtype_info->has_sticky_ub = dtp->has_sticky_ub;
- dtype_info->has_sticky_lb = dtp->has_sticky_lb;
-
- MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop");
-
- MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
- MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size);
- MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
-
- /* the dataloop can have undefined padding sections, so we need to let
- * valgrind know that it is OK to pass this data to writev later on */
- MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size);
-
- get_pkt->dataloop_size = dtp->dataloop_size;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*get_pkt);
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info;
- iov[1].MPID_IOV_LEN = sizeof(*dtype_info);
- iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop;
- iov[2].MPID_IOV_LEN = dtp->dataloop_size;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, 3, &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
-
- /* release the target datatype */
- MPID_Datatype_release(dtp);
- }
-
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
-
- /* release the request returned by iStartMsg or iStartMsgv */
- if (req != NULL) {
- MPID_Request_release(req);
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_RECV_RMA_MSG);
- return mpi_errno;
-
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- MPIU_CHKPMEM_REAP();
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_post
@@ -1903,8 +1027,10 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
source_win_handle = win_ptr->handle;
target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
- MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags,
- source_win_handle, target_win_handle, mpi_errno);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags,
+ source_win_handle, target_win_handle);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
i++;
curr_ops_cnt[curr_ptr->target_rank]++;
@@ -1992,9 +1118,9 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
/* free the group stored in window */
MPIR_Group_release(win_ptr->start_group_ptr);
- win_ptr->start_group_ptr = NULL;
-
- fn_exit:
+ win_ptr->start_group_ptr = NULL;
+
+ fn_exit:
MPIU_CHKLMEM_FREEALL();
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_COMPLETE);
return mpi_errno;
@@ -2060,6 +1186,7 @@ int MPIDI_Win_wait(MPID_Win * win_ptr)
/* --END ERROR HANDLING-- */
}
+
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_test
#undef FCNAME
@@ -2106,8 +1233,6 @@ int MPIDI_Win_test(MPID_Win * win_ptr, int *flag)
}
-
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_lock
#undef FCNAME
@@ -2779,33 +1904,6 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr)
#undef FUNCNAME
-#define FUNCNAME MPIDI_Win_sync
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_Win_sync(MPID_Win * win_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SYNC);
-
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SYNC);
-
- MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
- win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL,
- mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
-
- OPA_read_write_barrier();
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SYNC);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
#define FUNCNAME do_passive_target_rma
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
@@ -2934,8 +2032,10 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
source_win_handle = win_ptr->handle;
}
- MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, source_win_handle,
- target_win_handle, mpi_errno);
+ mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, source_win_handle,
+ target_win_handle);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
@@ -2996,85 +2096,25 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
/* --END ERROR HANDLING-- */
}
-
#undef FUNCNAME
-#define FUNCNAME send_lock_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_lock_t *lock_pkt = &upkt.lock;
- MPID_Request *req = NULL;
- MPIDI_VC_t *vc;
- MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_MSG);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_LOCK_MSG);
-
- MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED);
-
- MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
-
- MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK);
- lock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
- lock_pkt->source_win_handle = win_ptr->handle;
- lock_pkt->lock_type = lock_type;
-
- win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED;
- win_ptr->targets[dest].remote_lock_mode = lock_type;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
-
- /* release the request returned by iStartMsg */
- if (req != NULL) {
- MPID_Request_release(req);
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME acquire_local_lock
+#define FUNCNAME MPIDI_Win_sync
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
+int MPIDI_Win_sync(MPID_Win * win_ptr)
{
int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_ACQUIRE_LOCAL_LOCK);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SYNC);
- /* poke the progress engine until the local lock is granted */
- if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
- MPID_Progress_state progress_state;
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SYNC);
- MPID_Progress_start(&progress_state);
- while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
- mpi_errno = MPID_Progress_wait(&progress_state);
- /* --BEGIN ERROR HANDLING-- */
- if (mpi_errno != MPI_SUCCESS) {
- MPID_Progress_end(&progress_state);
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress");
- }
- /* --END ERROR HANDLING-- */
- }
- MPID_Progress_end(&progress_state);
- }
+ MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
+ win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL,
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
- win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
- win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_mode = lock_type;
+ OPA_read_write_barrier();
fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_ACQUIRE_LOCAL_LOCK);
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SYNC);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
@@ -3129,99 +2169,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank)
#undef FUNCNAME
-#define FUNCNAME send_unlock_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_unlock_msg(int dest, MPID_Win * win_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &upkt.unlock;
- MPID_Request *req = NULL;
- MPIDI_VC_t *vc;
- MPIDI_STATE_DECL(MPID_STATE_SEND_UNLOCK_MSG);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_UNLOCK_MSG);
-
- MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED);
-
- MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
-
- /* Send a lock packet over to the target. wait for the lock_granted
- * reply. Then do all the RMA ops. */
-
- MPIDI_Pkt_init(unlock_pkt, MPIDI_CH3_PKT_UNLOCK);
- unlock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
-
- /* Reset the local state of the target to unlocked */
- win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, unlock_pkt, sizeof(*unlock_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
-
- /* Release the request returned by iStartMsg */
- if (req != NULL) {
- MPID_Request_release(req);
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_UNLOCK_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-/* Commented out function to squash a warning, but retaining the code
- * for later use. */
-#if 0
-#undef FUNCNAME
-#define FUNCNAME send_flush_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_flush_msg(int dest, MPID_Win * win_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush;
- MPID_Request *req = NULL;
- MPIDI_VC_t *vc;
- MPIDI_STATE_DECL(MPID_STATE_SEND_FLUSH_MSG);
- MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_FLUSH_MSG);
-
- MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_FLUSH);
-
- MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc);
-
- MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH);
- flush_pkt->target_win_handle = win_ptr->all_win_handles[dest];
- flush_pkt->source_win_handle = win_ptr->handle;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
-
- /* Release the request returned by iStartMsg */
- if (req != NULL) {
- MPID_Request_release(req);
- }
-
- fn_exit:
- MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_FLUSH_MSG);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-#endif
-
-
-#undef FUNCNAME
#define FUNCNAME send_lock_put_or_acc
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
@@ -3523,1549 +2470,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
}
/* ------------------------------------------------------------------------ */
-/*
- * Utility routines
- */
-/* ------------------------------------------------------------------------ */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle)
-{
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted;
- MPID_Request *req = NULL;
- int mpi_errno;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
-
- /* send lock granted packet */
- MPIDI_Pkt_init(lock_granted_pkt, MPIDI_CH3_PKT_LOCK_GRANTED);
- lock_granted_pkt->source_win_handle = source_win_handle;
- lock_granted_pkt->target_rank = win_ptr->comm_ptr->rank;
-
- MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE,
- (MPIU_DBG_FDEST, "sending lock granted pkt on vc=%p, source_win_handle=%#08x",
- vc, lock_granted_pkt->source_win_handle));
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_granted_pkt, sizeof(*lock_granted_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- if (mpi_errno) {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
-
- if (req != NULL) {
- MPID_Request_release(req);
- }
-
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT);
-
- return mpi_errno;
-}
-
-/* ------------------------------------------------------------------------ */
-/*
- * The following routines are the packet handlers for the packet types
- * used above in the implementation of the RMA operations in terms
- * of messages.
- */
-/* ------------------------------------------------------------------------ */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Put
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_put_t *put_pkt = &pkt->put;
- MPID_Request *req = NULL;
- MPI_Aint type_size;
- int complete = 0;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- MPID_Win *win_ptr;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received put pkt");
-
- if (put_pkt->count == 0) {
- /* it's a 0-byte message sent just to decrement the
- * completion counter. This happens only in
- * post/start/complete/wait sync model; therefore, no need
- * to check lock queue. */
- if (put_pkt->target_win_handle != MPI_WIN_NULL) {
- MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr);
- mpi_errno =
- MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, put_pkt->flags, MPI_WIN_NULL);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- }
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
- goto fn_exit;
- }
-
- MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, put_pkt->flags);
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- req = MPID_Request_create();
- MPIU_Object_set_ref(req, 1);
-
- req->dev.user_buf = put_pkt->addr;
- req->dev.user_count = put_pkt->count;
- req->dev.target_win_handle = put_pkt->target_win_handle;
- req->dev.source_win_handle = put_pkt->source_win_handle;
- req->dev.flags = put_pkt->flags;
-
- if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) {
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP);
- req->dev.datatype = put_pkt->datatype;
-
- MPID_Datatype_get_size_macro(put_pkt->datatype, type_size);
- req->dev.recv_data_sz = type_size * put_pkt->count;
-
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
- /* FIXME: Only change the handling of completion if
- * post_data_receive reset the handler. There should
- * be a cleaner way to do this */
- if (!req->dev.OnDataAvail) {
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
- }
-
- /* return the number of bytes processed in this function */
- *buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
-
- if (complete) {
- mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
- if (complete) {
- *rreqp = NULL;
- goto fn_exit;
- }
- }
- }
- else {
- /* derived datatype */
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT);
- req->dev.datatype = MPI_DATATYPE_NULL;
- req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
-
- req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
- MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
- if (!req->dev.dtype_info) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_RMA_dtype_info");
- }
-
- req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size);
- if (!req->dev.dataloop) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- put_pkt->dataloop_size);
- }
-
- /* if we received all of the dtype_info and dataloop, copy it
- * now and call the handler, otherwise set the iov and let the
- * channel copy it */
- if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) {
- /* copy all of dtype_info and dataloop */
- MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
- MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
- put_pkt->dataloop_size);
-
- *buflen =
- sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size;
-
- /* All dtype data has been received, call req handler */
- mpi_errno = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete(vc, req, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
- if (complete) {
- *rreqp = NULL;
- goto fn_exit;
- }
- }
- else {
- req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info);
- req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
- req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
- req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size;
- req->dev.iov_count = 2;
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
-
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete;
- }
-
- }
-
- *rreqp = req;
-
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT");
- }
-
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Get
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get;
- MPID_Request *req = NULL;
- MPID_IOV iov[MPID_IOV_LIMIT];
- int complete;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- MPID_Win *win_ptr;
- int mpi_errno = MPI_SUCCESS;
- MPI_Aint type_size;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get pkt");
-
- MPIU_Assert(get_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(get_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, get_pkt->flags);
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- req = MPID_Request_create();
- req->dev.target_win_handle = get_pkt->target_win_handle;
- req->dev.source_win_handle = get_pkt->source_win_handle;
- req->dev.flags = get_pkt->flags;
-
- /* here we increment the Active Target counter to guarantee the GET-like
- operation are completed when counter reaches zero. */
- win_ptr->at_completion_counter++;
-
- if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) {
- /* basic datatype. send the data. */
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp;
-
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
- req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete;
- req->kind = MPID_REQUEST_SEND;
-
- MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
- get_resp_pkt->request_handle = get_pkt->request_handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
-
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt->addr;
- MPID_Datatype_get_size_macro(get_pkt->datatype, type_size);
- iov[1].MPID_IOV_LEN = get_pkt->count * type_size;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- /* --BEGIN ERROR HANDLING-- */
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_Object_set_ref(req, 0);
- MPIDI_CH3_Request_destroy(req);
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
- /* --END ERROR HANDLING-- */
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
- }
- else {
- /* derived datatype. first get the dtype_info and dataloop. */
-
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP_DERIVED_DT);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete;
- req->dev.OnFinal = 0;
- req->dev.user_buf = get_pkt->addr;
- req->dev.user_count = get_pkt->count;
- req->dev.datatype = MPI_DATATYPE_NULL;
- req->dev.request_handle = get_pkt->request_handle;
-
- req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
- MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
- if (!req->dev.dtype_info) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_RMA_dtype_info");
- }
-
- req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size);
- if (!req->dev.dataloop) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- get_pkt->dataloop_size);
- }
-
- /* if we received all of the dtype_info and dataloop, copy it
- * now and call the handler, otherwise set the iov and let the
- * channel copy it */
- if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) {
- /* copy all of dtype_info and dataloop */
- MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
- MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
- get_pkt->dataloop_size);
-
- *buflen =
- sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size;
-
- /* All dtype data has been received, call req handler */
- mpi_errno = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete(vc, req, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET");
- if (complete)
- *rreqp = NULL;
- }
- else {
- req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info;
- req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
- req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
- req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size;
- req->dev.iov_count = 2;
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = req;
- }
-
- }
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
- return mpi_errno;
-
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_accum_t *accum_pkt = &pkt->accum;
- MPID_Request *req = NULL;
- MPI_Aint true_lb, true_extent, extent;
- void *tmp_buf = NULL;
- int complete = 0;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- MPID_Win *win_ptr;
- int mpi_errno = MPI_SUCCESS;
- MPI_Aint type_size;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt");
-
- MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- req = MPID_Request_create();
- MPIU_Object_set_ref(req, 1);
- *rreqp = req;
-
- req->dev.user_count = accum_pkt->count;
- req->dev.op = accum_pkt->op;
- req->dev.real_user_buf = accum_pkt->addr;
- req->dev.target_win_handle = accum_pkt->target_win_handle;
- req->dev.source_win_handle = accum_pkt->source_win_handle;
- req->dev.flags = accum_pkt->flags;
-
- if (accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM) {
- req->dev.resp_request_handle = accum_pkt->request_handle;
- }
- else {
- req->dev.resp_request_handle = MPI_REQUEST_NULL;
- }
-
- if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) {
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP);
- req->dev.datatype = accum_pkt->datatype;
-
- MPIR_Type_get_true_extent_impl(accum_pkt->datatype, &true_lb, &true_extent);
- MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent);
-
- /* Predefined types should always have zero lb */
- MPIU_Assert(true_lb == 0);
-
- tmp_buf = MPIU_Malloc(accum_pkt->count * (MPIR_MAX(extent, true_extent)));
- if (!tmp_buf) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- accum_pkt->count * MPIR_MAX(extent, true_extent));
- }
-
- req->dev.user_buf = tmp_buf;
-
- MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size);
- req->dev.recv_data_sz = type_size * accum_pkt->count;
-
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
- /* FIXME: Only change the handling of completion if
- * post_data_receive reset the handler. There should
- * be a cleaner way to do this */
- if (!req->dev.OnDataAvail) {
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
- }
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
-
- if (complete) {
- mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete);
- if (mpi_errno)
- MPIU_ERR_POP(mpi_errno);
- if (complete) {
- *rreqp = NULL;
- goto fn_exit;
- }
- }
- }
- else {
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete;
- req->dev.datatype = MPI_DATATYPE_NULL;
- req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
-
- req->dev.dtype_info = (MPIDI_RMA_dtype_info *)
- MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info));
- if (!req->dev.dtype_info) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_RMA_dtype_info");
- }
-
- req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size);
- if (!req->dev.dataloop) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- accum_pkt->dataloop_size);
- }
-
- if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) {
- /* copy all of dtype_info and dataloop */
- MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info));
- MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info),
- accum_pkt->dataloop_size);
-
- *buflen =
- sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size;
-
- /* All dtype data has been received, call req handler */
- mpi_errno = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete(vc, req, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_ACCUMULATE");
- if (complete) {
- *rreqp = NULL;
- goto fn_exit;
- }
- }
- else {
- req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info;
- req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info);
- req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop;
- req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size;
- req->dev.iov_count = 2;
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- }
-
- }
-
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-
-}
-
-/* Special accumulate for short data items entirely within the packet */
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate_Immed
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_accum_immed_t *accum_pkt = &pkt->accum_immed;
- MPID_Win *win_ptr;
- MPI_Aint extent;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt");
-
- MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
-
- /* return the number of bytes processed in this function */
- /* data_len == 0 (all within packet) */
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
-
- MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent);
-
- if (win_ptr->shm_allocated == TRUE)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
- /* Data is already present */
- if (accum_pkt->op == MPI_REPLACE) {
- /* no datatypes required */
- int len;
- MPIU_Assign_trunc(len, (accum_pkt->count * extent), int);
- /* FIXME: use immediate copy because this is short */
- MPIUI_Memcpy(accum_pkt->addr, accum_pkt->data, len);
- }
- else {
- if (HANDLE_GET_KIND(accum_pkt->op) == HANDLE_KIND_BUILTIN) {
- MPI_User_function *uop;
- /* get the function by indexing into the op table */
- uop = MPIR_OP_HDL_TO_FN(accum_pkt->op);
- (*uop) (accum_pkt->data, accum_pkt->addr, &(accum_pkt->count), &(accum_pkt->datatype));
- }
- else {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OP, "**opnotpredefined",
- "**opnotpredefined %d", accum_pkt->op);
- }
- }
- if (win_ptr->shm_allocated == TRUE)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
-
- /* There are additional steps to take if this is a passive
- * target RMA or the last operation from the source */
-
- /* Here is the code executed in PutAccumRespComplete after the
- * accumulation operation */
- MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
-
- mpi_errno = MPIDI_CH3_Finish_rma_op_target(vc, win_ptr, TRUE,
- accum_pkt->flags, accum_pkt->source_win_handle);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_CAS
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp;
- MPIDI_CH3_Pkt_cas_t *cas_pkt = &pkt->cas;
- MPID_Win *win_ptr;
- MPID_Request *req;
- MPI_Aint len;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt");
-
- MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags);
-
- /* return the number of bytes processed in this function */
- /* data_len == 0 (all within packet) */
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
-
- MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP);
- cas_resp_pkt->request_handle = cas_pkt->request_handle;
-
- /* Copy old value into the response packet */
- MPID_Datatype_get_size_macro(cas_pkt->datatype, len);
- MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u));
-
- if (win_ptr->shm_allocated == TRUE)
- MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
-
- MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len);
-
- /* Compare and replace if equal */
- if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) {
- MPIU_Memcpy(cas_pkt->addr, &cas_pkt->origin_data, len);
- }
-
- if (win_ptr->shm_allocated == TRUE)
- MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
-
- /* Send the response packet */
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
-
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
-
- if (req != NULL) {
- if (!MPID_Request_is_complete(req)) {
- /* sending process is not completed, set proper OnDataAvail
- (it is initialized to NULL by lower layer) */
- req->dev.target_win_handle = cas_pkt->target_win_handle;
- req->dev.flags = cas_pkt->flags;
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetAccumRespComplete;
-
- /* here we increment the Active Target counter to guarantee the GET-like
- operation are completed when counter reaches zero. */
- win_ptr->at_completion_counter++;
-
- MPID_Request_release(req);
- goto fn_exit;
- }
- else
- MPID_Request_release(req);
- }
-
-
- /* There are additional steps to take if this is a passive
- * target RMA or the last operation from the source */
-
- mpi_errno = MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, cas_pkt->flags, MPI_WIN_NULL);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_CASResp
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &pkt->cas_resp;
- MPID_Request *req;
- MPI_Aint len;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS response pkt");
-
- MPID_Request_get_ptr(cas_resp_pkt->request_handle, req);
- MPID_Datatype_get_size_macro(req->dev.datatype, len);
-
- MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len);
-
- MPIDI_CH3U_Request_complete(req);
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_FOP
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_fop_t *fop_pkt = &pkt->fop;
- MPID_Request *req;
- MPID_Win *win_ptr;
- int data_complete = 0;
- MPI_Aint len;
- MPIU_CHKPMEM_DECL(1);
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt");
-
- MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL);
- MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags);
-
- req = MPID_Request_create();
- MPIU_ERR_CHKANDJUMP(req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
- MPIU_Object_set_ref(req, 1); /* Ref is held by progress engine */
- *rreqp = NULL;
-
- req->dev.user_buf = NULL; /* will be set later */
- req->dev.user_count = 1;
- req->dev.datatype = fop_pkt->datatype;
- req->dev.op = fop_pkt->op;
- req->dev.real_user_buf = fop_pkt->addr;
- req->dev.target_win_handle = fop_pkt->target_win_handle;
- req->dev.request_handle = fop_pkt->request_handle;
- req->dev.flags = fop_pkt->flags;
-
- MPID_Datatype_get_size_macro(req->dev.datatype, len);
- MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u));
-
- /* Set up the user buffer and receive data if needed */
- if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) {
- req->dev.user_buf = fop_pkt->origin_data;
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- data_complete = 1;
- }
- else {
- /* Data won't fit in the header, allocate temp space and receive it */
- MPIDI_msg_sz_t data_len;
- void *data_buf;
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
- req->dev.recv_data_sz = len; /* count == 1 for FOP */
-
- MPIU_CHKPMEM_MALLOC(req->dev.user_buf, void *, len, mpi_errno, "**nomemreq");
-
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &data_complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE");
-
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPComplete;
-
- if (!data_complete) {
- *rreqp = req;
- }
-
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
- }
-
- if (data_complete) {
- int fop_complete = 0;
- mpi_errno = MPIDI_CH3_ReqHandler_FOPComplete(vc, req, &fop_complete);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- *rreqp = NULL;
- }
-
- fn_exit:
- MPIU_CHKPMEM_COMMIT();
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- MPIU_CHKPMEM_REAP();
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_FOPResp
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp;
- MPID_Request *req;
- int complete = 0;
- MPI_Aint len;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP response pkt");
-
- MPID_Request_get_ptr(fop_resp_pkt->request_handle, req);
- MPID_Datatype_get_size_macro(req->dev.datatype, len);
-
- if (len <= sizeof(fop_resp_pkt->data)) {
- MPIU_Memcpy(req->dev.user_buf, (void *) fop_resp_pkt->data, len);
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- complete = 1;
- }
- else {
- /* Data was too big to embed in the header */
- MPIDI_msg_sz_t data_len;
- void *data_buf;
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
- req->dev.recv_data_sz = len; /* count == 1 for FOP */
- *rreqp = req;
-
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER,
- "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP");
-
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
- }
-
- if (complete) {
- MPIDI_CH3U_Request_complete(req);
- *rreqp = NULL;
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Get_AccumResp
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp;
- MPID_Request *req;
- int complete;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- int mpi_errno = MPI_SUCCESS;
- MPI_Aint type_size;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt");
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req);
-
- MPID_Datatype_get_size_macro(req->dev.datatype, type_size);
- req->dev.recv_data_sz = type_size * req->dev.user_count;
-
- *rreqp = req;
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP");
- if (complete) {
- MPIDI_CH3U_Request_complete(req);
- *rreqp = NULL;
- }
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Lock
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock;
- MPID_Win *win_ptr = NULL;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr);
-
- if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) {
- /* send lock granted packet. */
- mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, lock_pkt->source_win_handle);
- }
-
- else {
- /* queue the lock information */
- MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
-
- /* Note: This code is reached by the fechandadd rma tests */
- /* FIXME: MT: This may need to be done atomically. */
-
- /* FIXME: Since we need to add to the tail of the list,
- * we should maintain a tail pointer rather than traversing the
- * list each time to find the tail. */
- curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
- prev_ptr = curr_ptr;
- while (curr_ptr != NULL) {
- prev_ptr = curr_ptr;
- curr_ptr = curr_ptr->next;
- }
-
- new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- if (!new_ptr) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_Win_lock_queue");
- }
- if (prev_ptr != NULL)
- prev_ptr->next = new_ptr;
- else
- win_ptr->lock_queue = new_ptr;
-
- new_ptr->next = NULL;
- new_ptr->lock_type = lock_pkt->lock_type;
- new_ptr->source_win_handle = lock_pkt->source_win_handle;
- new_ptr->vc = vc;
- new_ptr->pt_single_op = NULL;
- }
-
- *rreqp = NULL;
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_LockPutUnlock
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &pkt->lock_put_unlock;
- MPID_Win *win_ptr = NULL;
- MPID_Request *req = NULL;
- MPI_Aint type_size;
- int complete;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- int mpi_errno = MPI_SUCCESS;
- int (*fcn)( MPIDI_VC_t *, struct MPID_Request *, int * );
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_put_unlock pkt");
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- req = MPID_Request_create();
- MPIU_Object_set_ref(req, 1);
-
- req->dev.datatype = lock_put_unlock_pkt->datatype;
- MPID_Datatype_get_size_macro(lock_put_unlock_pkt->datatype, type_size);
- req->dev.recv_data_sz = type_size * lock_put_unlock_pkt->count;
- req->dev.user_count = lock_put_unlock_pkt->count;
- req->dev.target_win_handle = lock_put_unlock_pkt->target_win_handle;
-
- MPID_Win_get_ptr(lock_put_unlock_pkt->target_win_handle, win_ptr);
-
- if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_put_unlock_pkt->lock_type) == 1) {
- /* do the put. for this optimization, only basic datatypes supported. */
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
- req->dev.user_buf = lock_put_unlock_pkt->addr;
- req->dev.source_win_handle = lock_put_unlock_pkt->source_win_handle;
- req->dev.flags = lock_put_unlock_pkt->flags;
- }
-
- else {
- /* queue the information */
- MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
-
- new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- if (!new_ptr) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_Win_lock_queue");
- }
-
- new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
- if (new_ptr->pt_single_op == NULL) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_PT_single_op");
- }
-
- /* FIXME: MT: The queuing may need to be done atomically. */
-
- curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
- prev_ptr = curr_ptr;
- while (curr_ptr != NULL) {
- prev_ptr = curr_ptr;
- curr_ptr = curr_ptr->next;
- }
-
- if (prev_ptr != NULL)
- prev_ptr->next = new_ptr;
- else
- win_ptr->lock_queue = new_ptr;
-
- new_ptr->next = NULL;
- new_ptr->lock_type = lock_put_unlock_pkt->lock_type;
- new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
-
- new_ptr->pt_single_op->type = MPIDI_RMA_PUT;
- new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags;
- new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr;
- new_ptr->pt_single_op->count = lock_put_unlock_pkt->count;
- new_ptr->pt_single_op->datatype = lock_put_unlock_pkt->datatype;
- /* allocate memory to receive the data */
- new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz);
- if (new_ptr->pt_single_op->data == NULL) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- req->dev.recv_data_sz);
- }
-
- new_ptr->pt_single_op->data_recd = 0;
-
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_PUT);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete;
- req->dev.user_buf = new_ptr->pt_single_op->data;
- req->dev.lock_queue_entry = new_ptr;
- }
-
- fcn = req->dev.OnDataAvail;
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER,
- "**ch3|postrecv", "**ch3|postrecv %s",
- "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK");
- }
- req->dev.OnDataAvail = fcn;
- *rreqp = req;
-
- if (complete) {
- mpi_errno = fcn(vc, req, &complete);
- if (complete) {
- *rreqp = NULL;
- }
- }
-
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
-
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER,
- "**ch3|postrecv", "**ch3|postrecv %s",
- "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK");
- }
-
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK);
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_LockGetUnlock
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &pkt->lock_get_unlock;
- MPID_Win *win_ptr = NULL;
- MPI_Aint type_size;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_get_unlock pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Win_get_ptr(lock_get_unlock_pkt->target_win_handle, win_ptr);
-
- if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_get_unlock_pkt->lock_type) == 1) {
- /* do the get. for this optimization, only basic datatypes supported. */
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp;
- MPID_Request *req;
- MPID_IOV iov[MPID_IOV_LIMIT];
-
- req = MPID_Request_create();
- req->dev.target_win_handle = lock_get_unlock_pkt->target_win_handle;
- req->dev.source_win_handle = lock_get_unlock_pkt->source_win_handle;
- req->dev.flags = lock_get_unlock_pkt->flags;
-
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP);
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
- req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete;
- req->kind = MPID_REQUEST_SEND;
-
- /* here we increment the Active Target counter to guarantee the GET-like
- operation are completed when counter reaches zero. */
- win_ptr->at_completion_counter++;
-
- MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
- get_resp_pkt->request_handle = lock_get_unlock_pkt->request_handle;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
- iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
-
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_get_unlock_pkt->addr;
- MPID_Datatype_get_size_macro(lock_get_unlock_pkt->datatype, type_size);
- iov[1].MPID_IOV_LEN = lock_get_unlock_pkt->count * type_size;
-
- mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
- /* --BEGIN ERROR HANDLING-- */
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_Object_set_ref(req, 0);
- MPIDI_CH3_Request_destroy(req);
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
- }
- /* --END ERROR HANDLING-- */
- }
-
- else {
- /* queue the information */
- MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
-
- /* FIXME: MT: This may need to be done atomically. */
-
- curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
- prev_ptr = curr_ptr;
- while (curr_ptr != NULL) {
- prev_ptr = curr_ptr;
- curr_ptr = curr_ptr->next;
- }
-
- new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- if (!new_ptr) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_Win_lock_queue");
- }
- new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
- if (new_ptr->pt_single_op == NULL) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_PT_Single_op");
- }
-
- if (prev_ptr != NULL)
- prev_ptr->next = new_ptr;
- else
- win_ptr->lock_queue = new_ptr;
-
- new_ptr->next = NULL;
- new_ptr->lock_type = lock_get_unlock_pkt->lock_type;
- new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
-
- new_ptr->pt_single_op->type = MPIDI_RMA_GET;
- new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags;
- new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr;
- new_ptr->pt_single_op->count = lock_get_unlock_pkt->count;
- new_ptr->pt_single_op->datatype = lock_get_unlock_pkt->datatype;
- new_ptr->pt_single_op->data = NULL;
- new_ptr->pt_single_op->request_handle = lock_get_unlock_pkt->request_handle;
- new_ptr->pt_single_op->data_recd = 1;
- }
-
- *rreqp = NULL;
-
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK);
- return mpi_errno;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_LockAccumUnlock
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &pkt->lock_accum_unlock;
- MPID_Request *req = NULL;
- MPID_Win *win_ptr = NULL;
- MPIDI_Win_lock_queue *curr_ptr = NULL, *prev_ptr = NULL, *new_ptr = NULL;
- MPI_Aint type_size;
- int complete;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_accum_unlock pkt");
-
- /* no need to acquire the lock here because we need to receive the
- * data into a temporary buffer first */
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- req = MPID_Request_create();
- MPIU_Object_set_ref(req, 1);
-
- req->dev.datatype = lock_accum_unlock_pkt->datatype;
- MPID_Datatype_get_size_macro(lock_accum_unlock_pkt->datatype, type_size);
- req->dev.recv_data_sz = type_size * lock_accum_unlock_pkt->count;
- req->dev.user_count = lock_accum_unlock_pkt->count;
- req->dev.target_win_handle = lock_accum_unlock_pkt->target_win_handle;
- req->dev.flags = lock_accum_unlock_pkt->flags;
-
- /* queue the information */
-
- new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- if (!new_ptr) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_Win_lock_queue");
- }
-
- new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op));
- if (new_ptr->pt_single_op == NULL) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
- "MPIDI_PT_single_op");
- }
-
- MPID_Win_get_ptr(lock_accum_unlock_pkt->target_win_handle, win_ptr);
-
- /* FIXME: MT: The queuing may need to be done atomically. */
-
- curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
- prev_ptr = curr_ptr;
- while (curr_ptr != NULL) {
- prev_ptr = curr_ptr;
- curr_ptr = curr_ptr->next;
- }
-
- if (prev_ptr != NULL)
- prev_ptr->next = new_ptr;
- else
- win_ptr->lock_queue = new_ptr;
-
- new_ptr->next = NULL;
- new_ptr->lock_type = lock_accum_unlock_pkt->lock_type;
- new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle;
- new_ptr->vc = vc;
-
- new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE;
- new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags;
- new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr;
- new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count;
- new_ptr->pt_single_op->datatype = lock_accum_unlock_pkt->datatype;
- new_ptr->pt_single_op->op = lock_accum_unlock_pkt->op;
- /* allocate memory to receive the data */
- new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz);
- if (new_ptr->pt_single_op->data == NULL) {
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d",
- req->dev.recv_data_sz);
- }
-
- new_ptr->pt_single_op->data_recd = 0;
-
- MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_ACCUM);
- req->dev.user_buf = new_ptr->pt_single_op->data;
- req->dev.lock_queue_entry = new_ptr;
-
- *rreqp = req;
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- /* FIXME: Only change the handling of completion if
- * post_data_receive reset the handler. There should
- * be a cleaner way to do this */
- if (!req->dev.OnDataAvail) {
- req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete;
- }
- if (mpi_errno != MPI_SUCCESS) {
- MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv",
- "**ch3|postrecv %s", "MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK");
- }
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
-
- if (complete) {
- mpi_errno = MPIDI_CH3_ReqHandler_SinglePutAccumComplete(vc, req, &complete);
- if (complete) {
- *rreqp = NULL;
- }
- }
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK);
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_GetResp
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp;
- MPID_Request *req;
- int complete;
- char *data_buf = NULL;
- MPIDI_msg_sz_t data_len;
- int mpi_errno = MPI_SUCCESS;
- MPI_Aint type_size;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get response pkt");
-
- data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
- data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Request_get_ptr(get_resp_pkt->request_handle, req);
-
- MPID_Datatype_get_size_macro(req->dev.datatype, type_size);
- req->dev.recv_data_sz = type_size * req->dev.user_count;
-
- *rreqp = req;
- mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete);
- MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s",
- "MPIDI_CH3_PKT_GET_RESP");
- if (complete) {
- MPIDI_CH3U_Request_complete(req);
- *rreqp = NULL;
- }
- /* return the number of bytes processed in this function */
- *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_LockGranted
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &pkt->lock_granted;
- MPID_Win *win_ptr = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock granted pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Win_get_ptr(lock_granted_pkt->source_win_handle, win_ptr);
- /* set the remote_lock_state flag in the window */
- win_ptr->targets[lock_granted_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
-
- *rreqp = NULL;
- MPIDI_CH3_Progress_signal_completion();
-
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
- return MPI_SUCCESS;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done;
- MPID_Win *win_ptr = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
-
- MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr);
- MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state !=
- MPIDI_CH3_WIN_LOCK_NONE);
-
- if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state ==
- MPIDI_CH3_WIN_LOCK_FLUSH)
- win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state =
- MPIDI_CH3_WIN_LOCK_GRANTED;
- else
- win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
-
- *rreqp = NULL;
- MPIDI_CH3_Progress_signal_completion();
-
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE);
- return MPI_SUCCESS;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Unlock
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Unlock(MPIDI_VC_t * vc ATTRIBUTE((unused)),
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &pkt->unlock;
- MPID_Win *win_ptr = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received unlock pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
-
- MPID_Win_get_ptr(unlock_pkt->target_win_handle, win_ptr);
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
-
- MPIDI_CH3_Progress_signal_completion();
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_PktHandler_Flush
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_PktHandler_Flush(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_CH3_Pkt_flush_t *flush_pkt = &pkt->flush;
- MPID_Win *win_ptr = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received flush pkt");
-
- *buflen = sizeof(MPIDI_CH3_Pkt_t);
- *rreqp = NULL;
-
- /* This is a flush request packet */
- if (flush_pkt->target_win_handle != MPI_WIN_NULL) {
- MPID_Request *req = NULL;
-
- MPID_Win_get_ptr(flush_pkt->target_win_handle, win_ptr);
-
- flush_pkt->target_win_handle = MPI_WIN_NULL;
- flush_pkt->target_rank = win_ptr->comm_ptr->rank;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
-
- /* Release the request returned by iStartMsg */
- if (req != NULL) {
- MPID_Request_release(req);
- }
- }
-
- /* This is a flush response packet */
- else {
- MPID_Win_get_ptr(flush_pkt->source_win_handle, win_ptr);
- win_ptr->targets[flush_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
- MPIDI_CH3_Progress_signal_completion();
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-/* ------------------------------------------------------------------------ */
/* list_complete_timer/counter and list_block_timer defined above */
-static inline int poke_progress_engine(void)
-{
- int mpi_errno = MPI_SUCCESS;
- MPID_Progress_state progress_state;
-
- MPID_Progress_start(&progress_state);
- mpi_errno = MPID_Progress_poke();
- if (mpi_errno != MPI_SUCCESS)
- MPIU_ERR_POP(mpi_errno);
- MPID_Progress_end(&progress_state);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list)
{
int ntimes = 0, mpi_errno = 0;
@@ -5085,6 +2491,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
/* In some tests, this hung unless the test ensured that
* there was an incomplete request. */
curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list);
+
/* MT: avoid processing unissued operations enqueued by other
threads in MPID_Progress_wait() */
if (curr_ptr && !curr_ptr->request) {
@@ -5187,271 +2594,3 @@ static inline int rma_list_gc(MPID_Win * win_ptr,
fn_fail:
return mpi_errno;
}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_Start_rma_op_target
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_Start_rma_op_target(MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
-
- /* Lock with NOCHECK is piggybacked on this message. We should be able to
- * immediately grab the lock. Otherwise, there is a synchronization error. */
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK && flags & MPIDI_CH3_PKT_FLAG_RMA_NOCHECK) {
- int lock_acquired;
- int lock_mode;
-
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_SHARED) {
- lock_mode = MPI_LOCK_SHARED;
- }
- else if (flags & MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE) {
- lock_mode = MPI_LOCK_EXCLUSIVE;
- }
- else {
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|rma_flags");
- }
-
- lock_acquired = MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_mode);
- MPIU_ERR_CHKANDJUMP(!lock_acquired, mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|nocheck_invalid");
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPIDI_CH3_Finish_rma_op_target
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_rma_update,
- MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
-
- /* This function should be called by the target process after each RMA
- * operation is completed, to update synchronization state. */
-
- /* Last RMA operation from source. If active target RMA, decrement window
- * counter. */
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE) {
- MPIU_Assert(win_ptr->current_lock_type == MPID_LOCK_NONE);
-
- win_ptr->at_completion_counter -= 1;
- MPIU_Assert(win_ptr->at_completion_counter >= 0);
-
- /* Signal the local process when the op counter reaches 0. */
- if (win_ptr->at_completion_counter == 0)
- MPIDI_CH3_Progress_signal_completion();
- }
-
- /* If passive target RMA, release lock on window and grant next lock in the
- * lock queue if there is any. If requested by the origin, send an ack back
- * to indicate completion at the target. */
- else if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
- MPIU_Assert(win_ptr->current_lock_type != MPID_LOCK_NONE);
-
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) {
- MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL);
- mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
- }
-
- mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
- if (mpi_errno) {
- MPIU_ERR_POP(mpi_errno);
- }
-
- /* The local process may be waiting for the lock. Signal completion to
- * wake it up, so it can attempt to grab the lock. */
- MPIDI_CH3_Progress_signal_completion();
- }
- else if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) {
- /* Ensure store instructions have been performed before flush call is
- * finished on origin process. */
- OPA_read_write_barrier();
-
- if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) {
- MPIDI_CH3_Pkt_t upkt;
- MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush;
- MPID_Request *req = NULL;
-
- MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received piggybacked flush request");
- MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL);
-
- MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH);
- flush_pkt->source_win_handle = source_win_handle;
- flush_pkt->target_win_handle = MPI_WIN_NULL;
- flush_pkt->target_rank = win_ptr->comm_ptr->rank;
-
- MPIU_THREAD_CS_ENTER(CH3COMM, vc);
- mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req);
- MPIU_THREAD_CS_EXIT(CH3COMM, vc);
- MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER,
- "**ch3|rma_msg");
-
- /* Release the request returned by iStartMsg */
- if (req != NULL) {
- MPID_Request_release(req);
- }
- }
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET);
- return mpi_errno;
- /* --BEGIN ERROR HANDLING-- */
- fn_fail:
- goto fn_exit;
- /* --END ERROR HANDLING-- */
-}
-
-
-/* ------------------------------------------------------------------------ */
-/*
- * For debugging, we provide the following functions for printing the
- * contents of an RMA packet
- */
-/* ------------------------------------------------------------------------ */
-#ifdef MPICH_DBG_OUTPUT
-int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PUT\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype));
- MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle));
- /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype));
- MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size));
- MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle));
- /*
- * MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request));
- * MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->get.win_ptr));
- */
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_GetResp(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET_RESP\n"));
- MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request_handle));
- /*MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request)); */
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUMULATE\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype));
- MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size));
- MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle));
- /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_Accum_Immed(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUM_IMMED\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum_immed.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum_immed.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum_immed.datatype));
- MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum_immed.op));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum_immed.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum_immed.source_win_handle));
- /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */
- fflush(stdout);
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_Lock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK\n"));
- MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock.lock_type));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock.source_win_handle));
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_LockPutUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_PUT_UNLOCK\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_put_unlock.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_put_unlock.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_put_unlock.datatype));
- MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_put_unlock.lock_type));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_put_unlock.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_put_unlock.source_win_handle));
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_LockAccumUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_accum_unlock.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_accum_unlock.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_accum_unlock.datatype));
- MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_accum_unlock.lock_type));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_accum_unlock.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle));
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GET_UNLOCK\n"));
- MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_get_unlock.addr));
- MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_get_unlock.count));
- MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_get_unlock.datatype));
- MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_get_unlock.lock_type));
- MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_get_unlock.target_win_handle));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_get_unlock.source_win_handle));
- MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->lock_get_unlock.request_handle));
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n"));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle));
- return MPI_SUCCESS;
-}
-
-int MPIDI_CH3_PktPrint_LockGranted(FILE * fp, MPIDI_CH3_Pkt_t * pkt)
-{
- MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GRANTED\n"));
- MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_granted.source_win_handle));
- return MPI_SUCCESS;
-}
-#endif
diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c
new file mode 100644
index 0000000..5a7221c
--- /dev/null
+++ b/src/mpid/ch3/src/mpidi_rma.c
@@ -0,0 +1,128 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ * (C) 2001 by Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include "mpidimpl.h"
+#include "mpidrma.h"
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_Win_free
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_Win_free(MPID_Win ** win_ptr)
+{
+ int mpi_errno = MPI_SUCCESS;
+ int in_use;
+ MPID_Comm *comm_ptr;
+ int errflag = FALSE;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE);
+
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE);
+
+ MPIU_ERR_CHKANDJUMP((*win_ptr)->epoch_state != MPIDI_EPOCH_NONE,
+ mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
+
+ if (!(*win_ptr)->shm_allocated) {
+ /* when SHM is allocated, we already did a global barrier in
+ MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */
+ mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+ }
+
+ comm_ptr = (*win_ptr)->comm_ptr;
+ mpi_errno = MPIR_Comm_free_impl(comm_ptr);
+ if (mpi_errno)
+ MPIU_ERR_POP(mpi_errno);
+
+ MPIU_Free((*win_ptr)->targets);
+ MPIU_Free((*win_ptr)->base_addrs);
+ MPIU_Free((*win_ptr)->sizes);
+ MPIU_Free((*win_ptr)->disp_units);
+ MPIU_Free((*win_ptr)->all_win_handles);
+
+ /* Free the attached buffer for windows created with MPI_Win_allocate() */
+ if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE ||
+ (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) {
+ if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) {
+ MPIU_Free((*win_ptr)->base);
+ }
+ }
+
+ MPIU_Object_release_ref(*win_ptr, &in_use);
+ /* MPI windows don't have reference count semantics, so this should always be true */
+ MPIU_Assert(!in_use);
+ MPIU_Handle_obj_free(&MPID_Win_mem, *win_ptr);
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FREE);
+ return mpi_errno;
+
+ fn_fail:
+ goto fn_exit;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_Win_shared_query
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_Win_shared_query(MPID_Win * win_ptr, int target_rank, MPI_Aint * size,
+ int *disp_unit, void *baseptr)
+{
+ int mpi_errno = MPI_SUCCESS;
+
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
+ MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
+
+ *(void **) baseptr = win_ptr->base;
+ *size = win_ptr->size;
+ *disp_unit = win_ptr->disp_unit;
+
+ fn_exit:
+ MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SHARED_QUERY);
+ return mpi_errno;
+ /* --BEGIN ERROR HANDLING-- */
+ fn_fail:
+ goto fn_exit;
+ /* --END ERROR HANDLING-- */
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_Alloc_mem
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+void *MPIDI_Alloc_mem(size_t size, MPID_Info * info_ptr)
+{
+ void *ap;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_ALLOC_MEM);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_ALLOC_MEM);
+
+ ap = MPIU_Malloc(size);
+
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_ALLOC_MEM);
+ return ap;
+}
+
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_Free_mem
+#undef FCNAME
+#define FCNAME MPIDI_QUOTE(FUNCNAME)
+int MPIDI_Free_mem(void *ptr)
+{
+ int mpi_errno = MPI_SUCCESS;
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_FREE_MEM);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_FREE_MEM);
+
+ MPIU_Free(ptr);
+
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_FREE_MEM);
+ return mpi_errno;
+}
http://git.mpich.org/mpich.git/commitdiff/5c51303260595939cb15d658e5f369bc148c9a4a
commit 5c51303260595939cb15d658e5f369bc148c9a4a
Author: Xin Zhao <xinzhao3 at illinois.edu>
Date: Sun Nov 2 20:10:48 2014 -0600
Temporarily remove all RMA PVARs.
Because we are going to rewrite the RMA infrastructure
and many PVARs will no longer be used, here we temporarily
remove all PVARs and will add needed PVARs back after new
implementation is done.
Signed-off-by: Pavan Balaji <balaji at anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
index c786dcb..b28d50c 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
@@ -14,8 +14,6 @@
#define MPIDI_CH3_PAGESIZE_MASK (~(MPIDI_CH3_PAGESIZE-1))
#define MPIDI_CH3_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_CH3_PAGESIZE_MASK)) & MPIDI_CH3_PAGESIZE_MASK)
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather);
-
MPIDI_SHM_Wins_list_t shm_wins_list;
static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr,
@@ -317,7 +315,6 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *
node_size = node_comm_ptr->local_size;
node_rank = node_comm_ptr->rank;
- MPIR_T_PVAR_TIMER_START(RMA, rma_wincreate_allgather);
/* allocate memory for the base addresses, disp_units, and
completion counters of all processes */
MPIU_CHKPMEM_MALLOC((*win_ptr)->base_addrs, void **,
@@ -350,7 +347,6 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, 3 * sizeof(MPI_Aint), MPI_BYTE,
(*win_ptr)->comm_ptr, &errflag);
- MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h
index 652af02..3710086 100644
--- a/src/mpid/ch3/include/mpidrma.h
+++ b/src/mpid/ch3/include/mpidrma.h
@@ -9,10 +9,6 @@
#include "mpl_utlist.h"
#include "mpidi_ch3_impl.h"
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_rs);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_complete);
-
typedef enum MPIDI_RMA_Op_type {
MPIDI_RMA_PUT = 23,
MPIDI_RMA_GET = 24,
diff --git a/src/mpid/ch3/src/ch3u_rma_acc_ops.c b/src/mpid/ch3/src/ch3u_rma_acc_ops.c
index a7edf6a..5d72483 100644
--- a/src/mpid/ch3/src/ch3u_rma_acc_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_acc_ops.c
@@ -6,9 +6,6 @@
#include "mpidrma.h"
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Get_accumulate
#undef FCNAME
@@ -79,16 +76,13 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
MPIDI_RMA_Op_t *new_ptr = NULL;
/* Append the operation to the window's RMA ops queue */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
/* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
/* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
* contain both PUT and GET like ops */
@@ -103,7 +97,6 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
new_ptr->target_count = target_count;
new_ptr->target_datatype = target_datatype;
new_ptr->op = op;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -193,14 +186,11 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
MPIDI_RMA_Op_t *new_ptr = NULL;
/* Append this operation to the RMA ops queue */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
@@ -215,7 +205,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
new_ptr->compare_addr = (void *) compare_addr;
new_ptr->compare_count = 1;
new_ptr->compare_datatype = datatype;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
}
fn_exit:
@@ -288,14 +277,11 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
MPIDI_RMA_Op_t *new_ptr = NULL;
/* Append this operation to the RMA ops queue */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
@@ -308,7 +294,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
new_ptr->result_count = 1;
new_ptr->result_datatype = datatype;
new_ptr->op = op;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
}
fn_exit:
diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c
index 83b1e37..d91a626 100644
--- a/src/mpid/ch3/src/ch3u_rma_ops.c
+++ b/src/mpid/ch3/src/ch3u_rma_ops.c
@@ -8,9 +8,6 @@
static int enableShortACC = 1;
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);
-
#define MPIDI_PASSIVE_TARGET_DONE_TAG 348297
#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924
@@ -180,14 +177,11 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
MPIDI_RMA_Op_t *new_ptr = NULL;
/* queue it up */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->type = MPIDI_RMA_PUT;
/* Cast away const'ness for the origin address, as the
@@ -200,7 +194,6 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->target_disp = target_disp;
new_ptr->target_count = target_count;
new_ptr->target_datatype = target_datatype;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -291,14 +284,11 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
MPIDI_RMA_Op_t *new_ptr = NULL;
/* queue it up */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->type = MPIDI_RMA_GET;
new_ptr->origin_addr = origin_addr;
@@ -308,7 +298,6 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
new_ptr->target_disp = target_disp;
new_ptr->target_count = target_count;
new_ptr->target_datatype = target_datatype;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
@@ -400,9 +389,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
MPIDI_RMA_Op_t *new_ptr = NULL;
/* queue it up */
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -410,7 +397,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
/* If predefined and contiguous, use a simplified element */
if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) &&
MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && enableShortACC) {
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->type = MPIDI_RMA_ACC_CONTIG;
/* Only the information needed for the contig/predefined acc */
/* Cast away const'ness for origin_address as
@@ -423,11 +409,9 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->target_count = target_count;
new_ptr->target_datatype = target_datatype;
new_ptr->op = op;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
goto fn_exit;
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->type = MPIDI_RMA_ACCUMULATE;
/* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
* contain both PUT and GET like ops */
@@ -439,7 +423,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->target_count = target_count;
new_ptr->target_datatype = target_datatype;
new_ptr->op = op;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index 416f2f5..489309a 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -334,547 +334,8 @@ cvars:
PROC_SYNC with origin will see the latest data.
*/
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_lockqueue_alloc);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_clearlock);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_clearlock_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_rs);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_issue);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_issue_aux);
-MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_winfence_issue_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_complete);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_complete_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_wait);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_wait_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_block);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_block_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winpost_clearlock);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winpost_clearlock_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winpost_sendsync);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winpost_sendsync_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winstart_clearlock);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winstart_clearlock_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_issue);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_issue_aux);
-MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_wincomplete_issue_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_complete);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_complete_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_recvsync);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_recvsync_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_block);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_block_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winwait_wait);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winwait_wait_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winlock_getlocallock);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_getlock);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_issue);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_complete);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_complete_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_block);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_block_aux);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_predef);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_immed);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_immed_op);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_cas);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_fop);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get_accum);
-
-MPIR_T_PVAR_ULONG2_LEVEL_DECL(RMA, rma_winfence_reqs);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_reqs);
-MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_winfence_reqs);
-
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_reqs);
-MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_reqs);
-
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincreate_allgather);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfree_rs);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfree_complete);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmaqueue_alloc);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmaqueue_set);
-
void MPIDI_CH3_RMA_Init_Pvars(void)
{
- /* rma_lockqueue_alloc */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_lockqueue_alloc,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "Allocate Lock Queue element (in seconds)");
-
- /* rma_winfence_clearlock */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_clearlock,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Clear prior lock (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_clearlock_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Clear prior lock");
-
- /* rma_winfence_rs */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_rs,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:ReduceScatterBlock (in seconds)");
-
- /* rma_winfence_issue and auxiliaries */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_issue,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Issue RMA ops (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_issue_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Issue RMA ops");
-
- MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_issue_aux, 0, /* init value */
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Issue RMA ops");
-
- /* rma_winfence_complete */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_complete,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Complete RMA ops (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_complete_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Complete RMA ops");
-
- /* rma_winfence_wait */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_wait,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA",
- "WIN_FENCE:Wait for ops from other processes (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_wait_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Wait for ops from other processes");
-
- /* rma_winfence_block */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfence_block,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Wait for any progress (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_block_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Wait for any progress");
-
-
- /* rma_winpost_clearlock */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winpost_clearlock,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_POST:Clear prior lock (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winpost_clearlock_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_POST:Clear prior lock");
-
- /* rma_winpost_sendsync */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winpost_sendsync,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_POST:Senc sync messages (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winpost_sendsync_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_POST:Senc sync messages");
-
- /* rma_winstart_clearlock */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winstart_clearlock,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_START:Clear prior lock (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winstart_clearlock_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_START:Clear prior lock");
-
- /* rma_wincomplete_issue and auxiliaries */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_wincomplete_issue,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Issue RMA ops (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_wincomplete_issue_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Issue RMA ops");
-
- MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_wincomplete_issue_aux, 0, /* init value */
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Issue RMA ops");
-
- /* rma_wincomplete_complete */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_wincomplete_complete,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Complete RMA ops (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_wincomplete_complete_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Complete RMA ops");
-
- /* rma_wincomplete_recvsync */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_wincomplete_recvsync,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Recv sync messages (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_wincomplete_recvsync_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Recv sync messages");
-
- /* rma_wincomplete_block */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_wincomplete_block,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Wait for any progress (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_wincomplete_block_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Wait for any progress");
-
- /* rma_winwait_wait */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winwait_wait,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA",
- "WIN_WAIT:Wait for ops from other processes (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winwait_wait_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_WAIT:Wait for ops from other processes");
-
- /* rma_winlock_getlocallock */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winlock_getlocallock,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_LOCK:Get local lock (in seconds)");
-
- /* rma_winunlock_getlock */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winunlock_getlock,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Acquire lock (in seconds)");
-
- /* rma_winunlock_issue */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winunlock_issue,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Issue RMA ops (in seconds)");
-
- /* rma_winunlock_complete */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winunlock_complete,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Complete RMA ops (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winunlock_complete_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Complete RMA ops (in seconds)");
-
- /* rma_winunlock_block */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winunlock_block,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Wait for any progress (in seconds)");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winunlock_block_aux,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Wait for any progress");
-
- /* rma_rmapkt_acc */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_acc,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "RMA:PKTHANDLER for Accumulate (in seconds)");
-
- /* rma_rmapkt_acc_predef */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_acc_predef,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA",
- "RMA:PKTHANDLER for Accumulate: predef dtype (in seconds)");
-
- /* rma_rmapkt_acc_immed */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_acc_immed,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "RMA:PKTHANDLER for Accum immed (in seconds)");
-
- /* rma_rmapkt_acc_immed_op */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_acc_immed_op,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA",
- "RMA:PKTHANDLER for Accum immed operation (in seconds)");
-
- /* rma_rmapkt_cas */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_cas,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "RMA:PKTHANDLER for Compare-and-swap (in seconds)");
-
- /* rma_rmapkt_fop */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_fop,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "RMA:PKTHANDLER for Fetch-and-op (in seconds)");
-
- /* rma_rmapkt_get_accum */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmapkt_get_accum,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "RMA:PKTHANDLER for Get-Accumulate (in seconds)");
-
- /* Level, counter and highwatermark for rma_winfence_reqs */
- MPIR_T_PVAR_LEVEL_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_reqs, 0, /* init value */
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
- "RMA", "WIN_FENCE:Pending requests");
-
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winfence_reqs,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Pending requests");
-
- MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_reqs, 0, /* init value */
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FENCE:Pending requests");
-
- /* rma_winunlock_reqs */
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_winunlock_reqs,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_UNLOCK:Pending requests");
-
- /* rma_wincomplete_reqs */
- MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA,
- MPI_UNSIGNED_LONG_LONG,
- rma_wincomplete_reqs,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_COMPLETE:Pending requests");
-
- /* rma_wincreate_allgather */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_wincreate_allgather,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_CREATE:Allgather (in seconds)");
-
- /* rma_winfree_rs */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfree_rs,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FREE:ReduceScatterBlock (in seconds)");
-
- /* rma_winfree_complete */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_winfree_complete,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "WIN_FREE:Complete (in seconds)");
-
- /* rma_rmaqueue_alloc */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmaqueue_alloc,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "Allocate RMA Queue element (in seconds)");
-
- /* rma_rmaqueue_set */
- MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
- MPI_DOUBLE,
- rma_rmaqueue_set,
- MPI_T_VERBOSITY_MPIDEV_DETAIL,
- MPI_T_BIND_NO_OBJECT,
- MPIR_T_PVAR_FLAG_READONLY,
- "RMA", "Set fields in RMA Queue element (in seconds)");
}
/* These are used to use a common routine to complete lists of RMA
@@ -883,9 +344,6 @@ void MPIDI_CH3_RMA_Init_Pvars(void)
thread-safe; the best choice for thread-safety is to eliminate this
ability to discriminate between the different types of RMA synchronization.
*/
-static MPIR_T_pvar_timer_t *list_complete_timer; /* outer */
-static unsigned long long *list_complete_counter;
-static MPIR_T_pvar_timer_t *list_block_timer; /* Inner; while waiting */
/*
* These routines provide a default implementation of the MPI RMA operations
@@ -1046,7 +504,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
OPA_read_write_barrier();
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_rs);
/* This is the second or later fence. Do all the preceding RMA ops. */
comm_ptr = win_ptr->comm_ptr;
/* First inform every process whether it is a target of RMA
@@ -1096,7 +553,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
mpi_errno = MPIR_Reduce_scatter_block_impl(MPI_IN_PLACE, rma_target_proc, 1,
MPI_INT, MPI_SUM, comm_ptr, &errflag);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_rs);
/* result is stored in rma_target_proc[0] */
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
@@ -1114,15 +570,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
win_ptr->at_completion_counter -= comm_size;
win_ptr->at_completion_counter += rma_target_proc[0];
- MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_issue);
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_issue_aux, total_op_count);
- MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_winfence_issue_aux, total_op_count);
-
- MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_winfence_reqs,
- MPIR_T_PVAR_LEVEL_GET(rma_winfence_reqs));
- MPIR_T_PVAR_LEVEL_SET(RMA, rma_winfence_reqs, 0); /* reset the level */
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_reqs, 1);
-
i = 0;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list);
while (curr_ptr != NULL) {
@@ -1148,7 +595,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
}
else {
nRequest++;
- MPIR_T_PVAR_LEVEL_INC(RMA, rma_winfence_reqs, 1);
curr_ptr = curr_ptr->next;
/* The test on the difference is to reduce the number
* of times the partial complete routine is called. Without
@@ -1161,10 +607,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
mpi_errno = poke_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
- MPIR_T_PVAR_STMT(RMA, list_complete_timer =
- MPIR_T_PVAR_TIMER_ADDR(rma_winfence_complete));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_winfence_complete_aux));
mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone);
if (mpi_errno != MPI_SUCCESS)
@@ -1175,7 +617,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
}
}
}
- MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_issue);
/* We replaced a loop over an array of requests with a list of the
* incomplete requests. The reason to do
@@ -1189,11 +630,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
* needed while looping through the requests.
*/
if (total_op_count) {
- MPIR_T_PVAR_STMT(RMA, list_complete_timer =
- MPIR_T_PVAR_TIMER_ADDR(rma_winfence_complete));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_winfence_complete_aux));
- MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winfence_block));
mpi_errno = rma_list_complete(win_ptr, ops_list);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
@@ -1209,7 +645,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
finish_up:
/* wait for all operations from other processes to finish */
if (win_ptr->at_completion_counter) {
- MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_wait);
MPID_Progress_start(&progress_state);
while (win_ptr->at_completion_counter) {
mpi_errno = MPID_Progress_wait(&progress_state);
@@ -1219,10 +654,8 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr)
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**winnoprogress");
}
/* --END ERROR HANDLING-- */
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_wait_aux, 1);
}
MPID_Progress_end(&progress_state);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_wait);
}
if (assert & MPI_MODE_NOSUCCEED) {
@@ -2094,8 +1527,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr)
MPI_Request *req;
MPI_Status *status;
- MPIR_T_PVAR_TIMER_START(RMA, rma_winpost_sendsync);
-
/* NOCHECK not specified. We need to notify the source
* processes that Post has been called. */
@@ -2132,7 +1563,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr)
"status");
/* Send a 0-byte message to the source processes */
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winpost_sendsync_aux, post_grp_size);
for (i = 0; i < post_grp_size; i++) {
dst = ranks_in_win_grp[i];
@@ -2170,7 +1600,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr)
mpi_errno = MPIR_Group_free_impl(win_grp_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winpost_sendsync);
}
fn_exit:
@@ -2412,8 +1841,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
start_grp_size = win_ptr->start_group_ptr->size;
- MPIR_T_PVAR_TIMER_START(RMA, rma_wincomplete_recvsync);
-
MPIU_CHKLMEM_MALLOC(ranks_in_win_grp, int *, start_grp_size * sizeof(int),
mpi_errno, "ranks_in_win_grp");
@@ -2432,15 +1859,12 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_END(RMA, rma_wincomplete_recvsync);
/* keep track of no. of ops to each proc. Needed for knowing
* whether or not to decrement the completion counter. The
* completion counter is decremented only on the last
* operation. */
- MPIR_T_PVAR_TIMER_START(RMA, rma_wincomplete_issue);
-
/* Note, active target uses the following ops list, and passive
* target uses win_ptr->targets[..] */
ops_list = &win_ptr->at_rma_ops_list;
@@ -2457,9 +1881,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
curr_ptr = curr_ptr->next;
}
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_wincomplete_issue_aux, total_op_count);
- MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_wincomplete_issue_aux, total_op_count);
-
/* We allocate a few extra requests because if there are no RMA
* ops to a target process, we need to send a 0-byte message just
* to decrement the completion counter. */
@@ -2493,7 +1914,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
}
else {
nRequest++;
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_wincomplete_reqs, 1);
curr_ptr = curr_ptr->next;
if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD &&
nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
@@ -2501,10 +1921,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
mpi_errno = poke_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
- MPIR_T_PVAR_STMT(RMA, list_complete_timer =
- MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_complete));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_wincomplete_complete_aux));
mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
@@ -2513,7 +1929,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
}
}
}
- MPIR_T_PVAR_TIMER_END(RMA, rma_wincomplete_issue);
/* If the start_group included some processes that did not end up
* becoming targets of RMA operations from this process, we need
@@ -2555,16 +1970,12 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
if (request) {
MPIDI_RMA_Op_t *new_ptr = NULL;
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
new_ptr->request = request;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
}
j++;
new_total_op_count++;
@@ -2572,11 +1983,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr)
}
if (new_total_op_count) {
- MPIR_T_PVAR_STMT(RMA, list_complete_timer =
- MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_complete));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_wincomplete_complete_aux));
- MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_block));
mpi_errno = rma_list_complete(win_ptr, ops_list);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
@@ -2626,7 +2032,6 @@ int MPIDI_Win_wait(MPID_Win * win_ptr)
if (win_ptr->at_completion_counter) {
MPID_Progress_state progress_state;
- MPIR_T_PVAR_TIMER_START(RMA, rma_winwait_wait);
MPID_Progress_start(&progress_state);
while (win_ptr->at_completion_counter) {
mpi_errno = MPID_Progress_wait(&progress_state);
@@ -2637,10 +2042,8 @@ int MPIDI_Win_wait(MPID_Win * win_ptr)
return mpi_errno;
}
/* --END ERROR HANDLING-- */
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winwait_wait_aux, 1);
}
MPID_Progress_end(&progress_state);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winwait_wait);
}
/* Ensure ordering of load/store operations. */
@@ -3476,8 +2879,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
curr_ptr = curr_ptr->next;
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_winunlock_issue);
-
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list);
if (curr_ptr != NULL) {
@@ -3543,7 +2944,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
}
else {
nRequest++;
- MPIR_T_PVAR_COUNTER_INC(RMA, rma_winunlock_reqs, 1);
curr_ptr = curr_ptr->next;
if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD &&
nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
@@ -3551,10 +2951,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
mpi_errno = poke_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
- MPIR_T_PVAR_STMT(RMA, list_complete_timer =
- MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_complete));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_winunlock_complete_aux));
mpi_errno =
rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, curr_ptr,
&nDone);
@@ -3566,13 +2962,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank,
}
}
}
- MPIR_T_PVAR_TIMER_END(RMA, rma_winunlock_issue);
if (nops) {
- MPIR_T_PVAR_STMT(RMA, list_complete_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_complete));
- MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_block));
- MPIR_T_PVAR_STMT(RMA, list_complete_counter =
- MPIR_T_PVAR_COUNTER_ADDR(rma_winunlock_complete_aux));
mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
@@ -3666,7 +3057,6 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
MPID_Progress_state progress_state;
- MPIR_T_PVAR_TIMER_START(RMA, rma_winlock_getlocallock);
MPID_Progress_start(&progress_state);
while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) {
mpi_errno = MPID_Progress_wait(&progress_state);
@@ -3678,7 +3068,6 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
/* --END ERROR HANDLING-- */
}
MPID_Progress_end(&progress_state);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winlock_getlocallock);
}
win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED;
@@ -3716,7 +3105,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank)
if (win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_GRANTED) {
MPID_Progress_state progress_state;
- MPIR_T_PVAR_TIMER_START(RMA, rma_winunlock_getlock);
MPID_Progress_start(&progress_state);
while (win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_GRANTED) {
mpi_errno = MPID_Progress_wait(&progress_state);
@@ -3728,7 +3116,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank)
/* --END ERROR HANDLING-- */
}
MPID_Progress_end(&progress_state);
- MPIR_T_PVAR_TIMER_END(RMA, rma_winunlock_getlock);
}
fn_exit:
@@ -4497,7 +3884,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt");
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc);
MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
@@ -4524,7 +3910,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) {
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_predef);
MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP);
req->dev.datatype = accum_pkt->datatype;
@@ -4563,11 +3948,9 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_ERR_POP(mpi_errno);
if (complete) {
*rreqp = NULL;
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_predef);
goto fn_exit;
}
}
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_predef);
}
else {
MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT);
@@ -4623,7 +4006,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
return mpi_errno;
fn_fail:
@@ -4649,7 +4031,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt");
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_immed);
MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags);
@@ -4661,7 +4042,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt
MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent);
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_immed_op);
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
/* Data is already present */
@@ -4686,7 +4066,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt
}
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_immed_op);
/* There are additional steps to take if this is a passive
* target RMA or the last operation from the source */
@@ -4702,7 +4081,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt
}
fn_exit:
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_immed);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED);
return mpi_errno;
fn_fail:
@@ -4731,7 +4109,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt");
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas);
MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr);
mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags);
@@ -4797,7 +4174,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_cas);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
return mpi_errno;
fn_fail:
@@ -4861,7 +4237,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt");
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop);
MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr);
mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags);
@@ -4925,7 +4300,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
fn_exit:
MPIU_CHKPMEM_COMMIT();
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
@@ -5015,7 +4389,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt");
- MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum);
data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t);
data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t);
@@ -5037,7 +4410,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
*buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
fn_exit:
- MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
return mpi_errno;
fn_fail:
@@ -5087,9 +4459,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
curr_ptr = curr_ptr->next;
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc);
new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc);
if (!new_ptr) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIDI_Win_lock_queue");
@@ -5161,9 +4531,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
/* queue the information */
MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr;
- MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc);
new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc);
if (!new_ptr) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIDI_Win_lock_queue");
@@ -5320,9 +4688,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
curr_ptr = curr_ptr->next;
}
- MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc);
new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc);
if (!new_ptr) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIDI_Win_lock_queue");
@@ -5401,9 +4767,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
/* queue the information */
- MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc);
new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
- MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc);
if (!new_ptr) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIDI_Win_lock_queue");
@@ -5708,7 +5072,6 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
MPIDI_RMA_Op_t *curr_ptr;
MPID_Progress_state progress_state;
- MPIR_T_PVAR_TIMER_START_VAR(RMA, list_complete_timer);
MPID_Progress_start(&progress_state);
/* Process all operations until they are complete */
while (!MPIDI_CH3I_RMA_Ops_isempty(ops_list)) {
@@ -5729,7 +5092,6 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
break;
}
if (curr_ptr && !MPID_Request_is_complete(curr_ptr->request)) {
- MPIR_T_PVAR_TIMER_START_VAR(RMA, list_block_timer);
mpi_errno = MPID_Progress_wait(&progress_state);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
@@ -5737,12 +5099,9 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress");
}
/* --END ERROR HANDLING-- */
- MPIR_T_PVAR_TIMER_END_VAR(RMA, list_block_timer);
}
} /* While list of rma operation is non-empty */
MPID_Progress_end(&progress_state);
- MPIR_T_PVAR_COUNTER_INC_VAR(RMA, list_complete_counter, ntimes);
- MPIR_T_PVAR_TIMER_END_VAR(RMA, list_complete_timer);
fn_fail:
return mpi_errno;
@@ -5763,8 +5122,6 @@ static inline int rma_list_gc(MPID_Win * win_ptr,
int nComplete = 0;
int nVisit = 0;
- MPIR_T_PVAR_TIMER_START_VAR(RMA, list_complete_timer);
-
curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list);
do {
/* MT: avoid processing unissued operations enqueued by other threads
@@ -5824,8 +5181,6 @@ static inline int rma_list_gc(MPID_Win * win_ptr,
} while (curr_ptr && curr_ptr != last_elm);
/* if (nComplete) printf("Completed %d requests\n", nComplete); */
- MPIR_T_PVAR_COUNTER_INC_VAR(RMA, list_complete_counter, 1);
- MPIR_T_PVAR_TIMER_END_VAR(RMA, list_complete_timer);
*nDone = nComplete;
diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c
index 7beaafe..eb9398b 100644
--- a/src/mpid/ch3/src/ch3u_win_fns.c
+++ b/src/mpid/ch3/src/ch3u_win_fns.c
@@ -8,8 +8,6 @@
#include "mpiinfo.h"
#include "mpidrma.h"
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather);
-
#undef FUNCNAME
#define FUNCNAME MPIDI_Win_fns_init
#undef FCNAME
@@ -54,7 +52,6 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit,
/* RMA handlers should be set before calling this function */
mpi_errno = (*win_ptr)->RMAFns.Win_set_info(*win_ptr, info);
- MPIR_T_PVAR_TIMER_START(RMA, rma_wincreate_allgather);
/* allocate memory for the base addresses, disp_units, and
completion counters of all processes */
MPIU_CHKPMEM_MALLOC((*win_ptr)->base_addrs, void **,
@@ -87,7 +84,6 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit,
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, 4, MPI_AINT,
(*win_ptr)->comm_ptr, &errflag);
- MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c
index 428d7a4..139dd76 100644
--- a/src/mpid/ch3/src/mpid_rma.c
+++ b/src/mpid/ch3/src/mpid_rma.c
@@ -10,12 +10,6 @@
MPIU_THREADSAFE_INIT_DECL(initRMAoptions);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_rs);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_complete);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
-MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);
-
static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model,
MPID_Comm * comm_ptr, MPID_Win ** win_ptr);
-----------------------------------------------------------------------
Summary of changes:
src/mpi/errhan/errnames.txt | 3 +
src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 4 -
src/mpid/ch3/include/mpid_rma_oplist.h | 270 ++
src/mpid/ch3/include/{mpidrma.h => mpid_rma_shm.h} | 315 +--
src/mpid/ch3/include/mpid_rma_types.h | 102 +
src/mpid/ch3/include/mpidimpl.h | 4 +-
src/mpid/ch3/include/mpidpkt.h | 51 +-
src/mpid/ch3/include/mpidpre.h | 9 +
src/mpid/ch3/include/mpidrma.h | 1181 ++-----
src/mpid/ch3/src/Makefile.mk | 4 +-
src/mpid/ch3/src/ch3u_handle_recv_pkt.c | 4 +-
src/mpid/ch3/src/ch3u_handle_recv_req.c | 172 +-
src/mpid/ch3/src/ch3u_rma_acc_ops.c | 15 -
src/mpid/ch3/src/ch3u_rma_oplist.c | 822 +++++
src/mpid/ch3/src/ch3u_rma_ops.c | 543 ++-
src/mpid/ch3/src/ch3u_rma_pkthandler.c | 1757 ++++++++++
src/mpid/ch3/src/ch3u_rma_sync.c | 3699 +-------------------
src/mpid/ch3/src/ch3u_win_fns.c | 4 -
src/mpid/ch3/src/mpid_finalize.c | 2 +
src/mpid/ch3/src/mpid_init.c | 3 +
src/mpid/ch3/src/mpid_rma.c | 20 +-
src/mpid/ch3/src/mpidi_printf.c | 8 +-
src/mpid/ch3/src/mpidi_rma.c | 210 ++
23 files changed, 3986 insertions(+), 5216 deletions(-)
create mode 100644 src/mpid/ch3/include/mpid_rma_oplist.h
copy src/mpid/ch3/include/{mpidrma.h => mpid_rma_shm.h} (72%)
create mode 100644 src/mpid/ch3/include/mpid_rma_types.h
create mode 100644 src/mpid/ch3/src/ch3u_rma_oplist.c
create mode 100644 src/mpid/ch3/src/ch3u_rma_pkthandler.c
create mode 100644 src/mpid/ch3/src/mpidi_rma.c
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list