[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.2b2-15-g2069c15
Service Account
noreply at mpich.org
Mon Apr 27 13:31:06 CDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via 2069c15edcf2aba2abfeb2ceb10033e7f8c99e14 (commit)
via 34e57aa8061a911f0657997fdd522913ca8deac8 (commit)
via 4131f0fb817eb6cf579633fd889fdeb078e3b62e (commit)
via 2df00d453288a94ea759cbd381a70a8eb8d6c575 (commit)
via ec920e5f097727e72e00c37a805db45e053d2c79 (commit)
from eb69611ecdb0cd2f65dd4076a50c13d3fbc4531c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/2069c15edcf2aba2abfeb2ceb10033e7f8c99e14
commit 2069c15edcf2aba2abfeb2ceb10033e7f8c99e14
Author: Valentin Petrov <valentin.petrov at intel.com>
Date: Mon Apr 6 14:19:33 2015 +0400
OFI: Bug fix for RTS/CTS/DATA protocol.
MPID_nem_ofi_data_callback used to check sreq->cc in order to track progress of
the RTS/CTS/DATA protocol. The was an implicit assumption that fi_tsend with RTS
completes first. However this would cause a hang if fi_trecv completed earlier.
The fix is: don't rely on the cc but rather check the tag bits explicitly.
Note, the RTS/CTS/DATA bits are no longer accumulated (i.e., no more
"wc->tag | CTS/DATA").
Signed-off-by: Charles J Archer <charles.j.archer at intel.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
index b6a78f0..e3302ae 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
@@ -233,7 +233,7 @@ static inline int MPID_nem_ofi_preposted_callback(cq_tagged_entry_t * wc, MPID_R
REQ_OFI(new_rreq)->pack_buffer_size,
gl_data.mr,
VC_OFI(vc)->direct_addr,
- wc->tag | MPID_MSG_DATA, 0, &(REQ_OFI(new_rreq)->ofi_context)), trecv);
+ MPID_MSG_DATA, 0, &(REQ_OFI(new_rreq)->ofi_context)), trecv);
MPID_nem_ofi_create_req(&sreq, 1);
sreq->dev.OnDataAvail = NULL;
@@ -245,7 +245,7 @@ static inline int MPID_nem_ofi_preposted_callback(cq_tagged_entry_t * wc, MPID_R
0,
gl_data.mr,
VC_OFI(vc)->direct_addr,
- wc->tag | MPID_MSG_CTS, &(REQ_OFI(sreq)->ofi_context)), tsend);
+ MPID_MSG_CTS, &(REQ_OFI(sreq)->ofi_context)), tsend);
MPIU_Assert(gl_data.persistent_req == rreq);
rreq->dev.user_count = 0;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index 074b537..9352936 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -73,18 +73,18 @@
REQ_OFI(sreq)->vc = vc; \
REQ_OFI(sreq)->tag = match_bits; \
\
- MPID_nem_ofi_create_req(&cts_req, 1); \
- cts_req->dev.OnDataAvail = NULL; \
- cts_req->dev.next = NULL; \
- REQ_OFI(cts_req)->event_callback = MPID_nem_ofi_cts_recv_callback; \
- REQ_OFI(cts_req)->parent = sreq; \
+ MPID_nem_ofi_create_req(&cts_req, 1); \
+ cts_req->dev.OnDataAvail = NULL; \
+ cts_req->dev.next = NULL; \
+ REQ_OFI(cts_req)->event_callback = MPID_nem_ofi_cts_recv_callback; \
+ REQ_OFI(cts_req)->parent = sreq; \
\
- FI_RC(fi_trecv(gl_data.endpoint, \
+ FI_RC(fi_trecv(gl_data.endpoint, \
NULL, \
0, \
gl_data.mr, \
VC_OFI(vc)->direct_addr, \
- match_bits | MPID_MSG_CTS, \
+ MPID_MSG_CTS, \
0, /* Exact tag match, no ignore bits */ \
&(REQ_OFI(cts_req)->ofi_context)),trecv); \
if (gl_data.api_set == API_SET_1){ \
@@ -122,15 +122,16 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
req_fn reqFn;
uint64_t tag = 0;
BEGIN_FUNC(FCNAME);
- if (MPID_cc_get(sreq->cc) == 2) {
+ switch (wc->tag & MPID_PROTOCOL_MASK) {
+ case MPID_MSG_CTS:
vc = REQ_OFI(sreq)->vc;
- REQ_OFI(sreq)->tag = tag | MPID_MSG_DATA;
FI_RC(fi_tsend(gl_data.endpoint,
REQ_OFI(sreq)->pack_buffer,
REQ_OFI(sreq)->pack_buffer_size,
gl_data.mr,
VC_OFI(vc)->direct_addr,
MPID_MSG_DATA, (void *) &(REQ_OFI(sreq)->ofi_context)), tsend);
+ MPIDI_CH3U_Request_complete(sreq);
break;
case MPID_MSG_DATA:
if (REQ_OFI(sreq)->pack_buffer)
@@ -148,6 +149,7 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
break;
case MPID_MSG_RTS:
MPIDI_CH3U_Request_complete(sreq);
+ break;
}
END_FUNC_RC(FCNAME);
}
http://git.mpich.org/mpich.git/commitdiff/34e57aa8061a911f0657997fdd522913ca8deac8
commit 34e57aa8061a911f0657997fdd522913ca8deac8
Author: Valentin Petrov <valentin.petrov at intel.com>
Date: Mon Apr 6 14:24:31 2015 +0400
OFI: MPIR_Barrier_impl should not be called from MPID_nem_ofi_finalize.
It uses nemesis shared memory which is already cleaned up at this stage.
However, w/o any synchronization a hang in the close protocol is possible
since rts/cts/data messages may be on the fly. This change fixes the issue.
Signed-off-by: Charles J Archer <charles.j.archer at intel.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
index 619e6ff..5cab470 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
@@ -57,6 +57,7 @@ typedef struct {
MPID_Request *persistent_req; /* Unexpected request queue */
MPID_Request *conn_req; /* Connection request */
MPIDI_Comm_ops_t comm_ops;
+ int rts_cts_in_flight;
int api_set;
} MPID_nem_ofi_global_t;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
index 390d76a..40e6759 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
@@ -235,6 +235,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* Wait for all the ranks to publish */
/* their business card */
/* --------------------------------- */
+ gl_data.rts_cts_in_flight = 0;
PMI_Barrier();
/* --------------------------------- */
@@ -307,12 +308,9 @@ int MPID_nem_ofi_finalize(void)
mpir_errflag_t ret = MPIR_ERR_NONE;
BEGIN_FUNC(FCNAME);
- /* --------------------------------------------- */
- /* Syncronization */
- /* Barrier across all ranks in this world */
- /* --------------------------------------------- */
- MPIR_Barrier_impl(MPIR_Process.comm_world, &ret);
-
+ while(gl_data.rts_cts_in_flight) {
+ MPID_nem_ofi_poll(0);
+ }
/* --------------------------------------------- */
/* Finalize connection management routines */
/* Cancels any persistent/global requests and */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index c314946..074b537 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -60,17 +60,18 @@
/* v v */
/* ------------------------------------------------------------------------ */
#define START_COMM() \
- ({ \
- GET_PGID_AND_SET_MATCH(); \
- VC_READY_CHECK(vc); \
- c = 1; \
- MPID_cc_incr(sreq->cc_ptr, &c); \
- MPID_cc_incr(sreq->cc_ptr, &c); \
- REQ_OFI(sreq)->event_callback = MPID_nem_ofi_data_callback; \
- REQ_OFI(sreq)->pack_buffer = pack_buffer; \
- REQ_OFI(sreq)->pack_buffer_size = pkt_len; \
- REQ_OFI(sreq)->vc = vc; \
- REQ_OFI(sreq)->tag = match_bits; \
+ ({ \
+ gl_data.rts_cts_in_flight++; \
+ GET_PGID_AND_SET_MATCH(); \
+ VC_READY_CHECK(vc); \
+ c = 1; \
+ MPID_cc_incr(sreq->cc_ptr, &c); \
+ MPID_cc_incr(sreq->cc_ptr, &c); \
+ REQ_OFI(sreq)->event_callback = MPID_nem_ofi_data_callback; \
+ REQ_OFI(sreq)->pack_buffer = pack_buffer; \
+ REQ_OFI(sreq)->pack_buffer_size = pkt_len; \
+ REQ_OFI(sreq)->vc = vc; \
+ REQ_OFI(sreq)->tag = match_bits; \
\
MPID_nem_ofi_create_req(&cts_req, 1); \
cts_req->dev.OnDataAvail = NULL; \
@@ -143,8 +144,9 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
vc = REQ_OFI(sreq)->vc;
MPI_RC(reqFn(vc, sreq, &complete));
}
- }
- else {
+ gl_data.rts_cts_in_flight--;
+ break;
+ case MPID_MSG_RTS:
MPIDI_CH3U_Request_complete(sreq);
}
END_FUNC_RC(FCNAME);
http://git.mpich.org/mpich.git/commitdiff/4131f0fb817eb6cf579633fd889fdeb078e3b62e
commit 4131f0fb817eb6cf579633fd889fdeb078e3b62e
Author: Valentin Petrov <valentin.petrov at intel.com>
Date: Wed Apr 15 17:57:14 2015 +0400
OFI: Remove unnecessary DYNAMIC_MR capability
Signed-off-by: Charles J Archer <charles.j.archer at intel.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
index 4e0ffee..390d76a 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
@@ -81,7 +81,6 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
hints->mode = FI_CONTEXT;
hints->ep_attr->type = FI_EP_RDM; /* Reliable datagram */
hints->caps = FI_TAGGED; /* Tag matching interface */
- hints->caps |= FI_DYNAMIC_MR; /* Global dynamic mem region */
hints->ep_attr->mem_tag_format = MEM_TAG_FORMAT;
MPIU_Assert(pg_p->size < ((1 << MPID_RANK_BITS) - 1));
@@ -161,21 +160,11 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* The objects include: */
/* * completion queue for events */
/* * address vector of other endpoint addresses */
- /* * dynamic memory-spanning memory region */
/* Other objects could be created (for example), but are unused in netmod */
/* * counters for incoming writes */
/* * completion counters for put and get */
/* ------------------------------------------------------------------------ */
- FI_RC(fi_mr_reg(gl_data.domain, /* In: Domain Object */
- 0, /* In: Lower memory address */
- UINTPTR_MAX, /* In: Upper memory address */
- FI_SEND | FI_RECV, /* In: Expose MR for read/write */
- 0ULL, /* In: base MR offset */
- 0ULL, /* In: requested key */
- 0ULL, /* In: No flags */
- &gl_data.mr, /* Out: memregion object */
- NULL), mr_reg); /* Context: memregion events */
-
+ gl_data.mr = NULL;
memset(&cq_attr, 0, sizeof(cq_attr));
cq_attr.format = FI_CQ_FORMAT_TAGGED;
FI_RC(fi_cq_open(gl_data.domain, /* In: Domain Object */
@@ -193,7 +182,6 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* --------------------------------------------- */
/* Bind the MR, CQ and AV to the endpoint object */
/* --------------------------------------------- */
- FI_RC(fi_ep_bind(gl_data.endpoint, (fid_t) gl_data.mr, 0), bind);
FI_RC(fi_ep_bind(gl_data.endpoint, (fid_t) gl_data.cq, FI_SEND | FI_RECV), bind);
FI_RC(fi_ep_bind(gl_data.endpoint, (fid_t) gl_data.av, 0), bind);
@@ -332,7 +320,6 @@ int MPID_nem_ofi_finalize(void)
/* --------------------------------------------- */
MPI_RC(MPID_nem_ofi_cm_finalize());
- FI_RC(fi_close((fid_t) gl_data.mr), mrclose);
FI_RC(fi_close((fid_t) gl_data.endpoint), epclose);
FI_RC(fi_close((fid_t) gl_data.av), avclose);
FI_RC(fi_close((fid_t) gl_data.cq), cqclose);
http://git.mpich.org/mpich.git/commitdiff/2df00d453288a94ea759cbd381a70a8eb8d6c575
commit 2df00d453288a94ea759cbd381a70a8eb8d6c575
Author: Valentin Petrov <valentin.petrov at intel.com>
Date: Fri Mar 27 11:48:28 2015 +0400
OFI: Fix memory leak in anysource cancel path
Signed-off-by: Charles J Archer <charles.j.archer at intel.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
index b0ab745..cab6c11 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
@@ -102,7 +102,7 @@ int MPID_nem_ofi_cancel_recv(struct MPIDI_VC *vc ATTRIBUTE((unused)), struct MPI
#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_matched)
int MPID_nem_ofi_anysource_matched(MPID_Request * rreq)
{
- int mpi_errno = TRUE;
+ int matched = FALSE;
int ret;
BEGIN_FUNC(FCNAME);
/* ----------------------------------------------------- */
@@ -112,11 +112,23 @@ int MPID_nem_ofi_anysource_matched(MPID_Request * rreq)
/* ----------------------------------------------------- */
ret = fi_cancel((fid_t) gl_data.endpoint, &(REQ_OFI(rreq)->ofi_context));
if (ret == 0) {
- /* --------------------------------------------------- */
- /* Request cancelled: cancel and complete the request */
- /* --------------------------------------------------- */
- mpi_errno = FALSE;
+ /* Cancel succeded. This means that the actual message has been
+ * received via nemesis shared memory. We need to return
+ * matched=False. The request will be completed at the nemesis level.
+ *
+ * If anysource was posted for non-contig dtype then don't forget
+ * to clean up tmp space.
+ */
+ if (REQ_OFI(rreq)->pack_buffer) {
+ MPIU_Free(REQ_OFI(rreq)->pack_buffer);
+ }
+ matched = FALSE;
+ }else{
+ /* Cancel failed. We can only fail in the case of the message
+ * being already actually received via ofi fabric. return TRUE.
+ */
+ matched = TRUE;
}
END_FUNC(FCNAME);
- return mpi_errno;
+ return matched;
}
http://git.mpich.org/mpich.git/commitdiff/ec920e5f097727e72e00c37a805db45e053d2c79
commit ec920e5f097727e72e00c37a805db45e053d2c79
Author: Valentin Petrov <valentin.petrov at intel.com>
Date: Fri Apr 3 18:17:35 2015 +0400
OFI: Add support for large tags using immediate data and OFI tag layouts
This patch modifies the OFI netmod to support large tag layouts, while preserving the old
tag layout. OFI defines a 64 bit tag, but also provides for a 64 bit tag and immediate data.
In some OFI providers, we may want to select different tag layouts. This patch currently
does not query for the proper tag layout or attempt to make a choice of the optimal layout,
it provides macro/templatized support for different tag formats. Additional selection
criteria will be added in subsequent patches.
* Tag layout is moved to a separate file.
Added init_sendtag_M2, init_recvtag_M2 (M2 stands for MODE #2, i.e. the mode
that uses fi_tsenddata and does not pack source into tag).
* Created a template file for ofi_tagged.c
Moved do_isend into template file which is included twice into ofi_tagged.c thus providing for the two
versions of do_isend and do_isend_2 corresponding to the two API sets.
* All send functions are available in two versions.
Added macro that declares a function for the two API sets. The first set has the namings inherited from
the previous netmod version. The functions of the second API set have the "_2" suffix.
* Recv_posted, anysource_posted, recv_callback, ofi_probe are templatized.
* ofi_tag_to_vc renamed ofi_wc_to_vc
Note, for the API_SET_2 the pgid is stored in the imm data while
psource and port will be packed the same way as in API_SET_1.
* Adds api_set member in gl_data struct. Initialize routines based on api_set
* Added RCD (RtsCtsData) protocol identifiers
* Added support for OFI MEM_TAG_FORMAT
* PGID placement modified
Signed-off-by: Charles J Archer <charles.j.archer at intel.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
index 5f12566..b6a78f0 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c
@@ -22,14 +22,21 @@
/* ------------------------------------------------------------------------ */
#undef FCNAME
#define FCNAME DECL_FUNC(ofi_tag_to_vc)
-static inline MPIDI_VC_t *ofi_tag_to_vc(uint64_t match_bits)
+static inline MPIDI_VC_t *ofi_wc_to_vc(cq_tagged_entry_t * wc)
{
int pgid = 0, port = 0;
MPIDI_VC_t *vc = NULL;
MPIDI_PG_t *pg = NULL;
-
+ uint64_t match_bits = wc->tag;
+ int wc_pgid;
BEGIN_FUNC(FCNAME);
- if (NO_PGID == get_pgid(match_bits)) {
+ if (gl_data.api_set == API_SET_1) {
+ wc_pgid = get_pgid(match_bits);
+ } else {
+ wc_pgid = wc->data;
+ }
+
+ if (NO_PGID == wc_pgid) {
/* -------------------------------------------------------------------- */
/* Dynamic path -- This uses a linear search, but number of cm vc's is */
/* a small number, and they should be ephemeral. This lookup should */
@@ -53,7 +60,7 @@ static inline MPIDI_VC_t *ofi_tag_to_vc(uint64_t match_bits)
pg = gl_data.pg_p;
while (pg) {
MPIDI_PG_IdToNum(pg, &pgid);
- if (get_pgid(match_bits) == pgid) {
+ if (wc_pgid == pgid) {
break;
}
pg = pg->next;
@@ -102,13 +109,13 @@ static inline int MPID_nem_ofi_conn_req_callback(cq_tagged_entry_t * wc, MPID_Re
bc[wc->len] = '\0';
MPIU_Assert(gl_data.conn_req == rreq);
FI_RC(fi_trecv(gl_data.endpoint,
- gl_data.conn_req->dev.user_buf,
- OFI_KVSAPPSTRLEN,
- gl_data.mr,
- FI_ADDR_UNSPEC,
- MPID_CONN_REQ,
- ~MPID_PROTOCOL_MASK,
- (void *) &(REQ_OFI(gl_data.conn_req)->ofi_context)), trecv);
+ gl_data.conn_req->dev.user_buf,
+ OFI_KVSAPPSTRLEN,
+ gl_data.mr,
+ FI_ADDR_UNSPEC,
+ MPID_CONN_REQ,
+ GET_RCD_IGNORE_MASK(),
+ (void *) &(REQ_OFI(gl_data.conn_req)->ofi_context)), trecv);
addr = MPIU_Malloc(gl_data.bound_addrlen);
MPIU_Assertp(addr);
@@ -204,7 +211,7 @@ static inline int MPID_nem_ofi_preposted_callback(cq_tagged_entry_t * wc, MPID_R
MPID_Request *new_rreq, *sreq;
BEGIN_FUNC(FCNAME);
- vc = ofi_tag_to_vc(wc->tag);
+ vc = ofi_wc_to_vc(wc);
MPIU_Assert(vc);
VC_READY_CHECK(vc);
@@ -243,12 +250,13 @@ static inline int MPID_nem_ofi_preposted_callback(cq_tagged_entry_t * wc, MPID_R
rreq->dev.user_count = 0;
FI_RC(fi_trecv(gl_data.endpoint,
- &rreq->dev.user_count,
- sizeof rreq->dev.user_count,
- gl_data.mr,
- FI_ADDR_UNSPEC,
- MPID_MSG_RTS,
- ~MPID_PROTOCOL_MASK, &(REQ_OFI(rreq)->ofi_context)), trecv);
+ &rreq->dev.user_count,
+ sizeof rreq->dev.user_count,
+ gl_data.mr,
+ FI_ADDR_UNSPEC,
+ MPID_MSG_RTS,
+ GET_RCD_IGNORE_MASK(),
+ &(REQ_OFI(rreq)->ofi_context)), trecv);
END_FUNC_RC(FCNAME);
}
@@ -289,10 +297,17 @@ int MPID_nem_ofi_cm_init(MPIDI_PG_t * pg_p, int pg_rank ATTRIBUTE((unused)))
/* ------------------------------------- */
/* Set up CH3 and netmod data structures */
/* ------------------------------------- */
- MPI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted,
- MPID_nem_ofi_anysource_matched));
- MPIDI_Anysource_iprobe_fn = MPID_nem_ofi_anysource_iprobe;
- MPIDI_Anysource_improbe_fn = MPID_nem_ofi_anysource_improbe;
+ if (gl_data.api_set == API_SET_1) {
+ MPI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted,
+ MPID_nem_ofi_anysource_matched));
+ MPIDI_Anysource_iprobe_fn = MPID_nem_ofi_anysource_iprobe;
+ MPIDI_Anysource_improbe_fn = MPID_nem_ofi_anysource_improbe;
+ } else {
+ MPI_RC(MPIDI_CH3I_Register_anysource_notification(MPID_nem_ofi_anysource_posted_2,
+ MPID_nem_ofi_anysource_matched));
+ MPIDI_Anysource_iprobe_fn = MPID_nem_ofi_anysource_iprobe_2;
+ MPIDI_Anysource_improbe_fn = MPID_nem_ofi_anysource_improbe_2;
+ }
gl_data.pg_p = pg_p;
/* ----------------------------------- */
@@ -304,13 +319,13 @@ int MPID_nem_ofi_cm_init(MPIDI_PG_t * pg_p, int pg_rank ATTRIBUTE((unused)))
REQ_OFI(persistent_req)->vc = NULL;
REQ_OFI(persistent_req)->event_callback = MPID_nem_ofi_preposted_callback;
FI_RC(fi_trecv(gl_data.endpoint,
- &persistent_req->dev.user_count,
- sizeof persistent_req->dev.user_count,
- gl_data.mr,
- FI_ADDR_UNSPEC,
- MPID_MSG_RTS,
- ~MPID_PROTOCOL_MASK,
- (void *) &(REQ_OFI(persistent_req)->ofi_context)), trecv);
+ &persistent_req->dev.user_count,
+ sizeof persistent_req->dev.user_count,
+ gl_data.mr,
+ FI_ADDR_UNSPEC,
+ MPID_MSG_RTS,
+ GET_RCD_IGNORE_MASK(),
+ (void *) &(REQ_OFI(persistent_req)->ofi_context)), trecv);
gl_data.persistent_req = persistent_req;
/* --------------------------------- */
@@ -323,12 +338,13 @@ int MPID_nem_ofi_cm_init(MPIDI_PG_t * pg_p, int pg_rank ATTRIBUTE((unused)))
REQ_OFI(conn_req)->vc = NULL; /* We don't know the source yet */
REQ_OFI(conn_req)->event_callback = MPID_nem_ofi_conn_req_callback;
FI_RC(fi_trecv(gl_data.endpoint,
- conn_req->dev.user_buf,
- OFI_KVSAPPSTRLEN,
- gl_data.mr,
- FI_ADDR_UNSPEC,
- MPID_CONN_REQ,
- ~MPID_PROTOCOL_MASK, (void *) &(REQ_OFI(conn_req)->ofi_context)), trecv);
+ conn_req->dev.user_buf,
+ OFI_KVSAPPSTRLEN,
+ gl_data.mr,
+ FI_ADDR_UNSPEC,
+ MPID_CONN_REQ,
+ GET_RCD_IGNORE_MASK(),
+ (void *) &(REQ_OFI(conn_req)->ofi_context)), trecv);
gl_data.conn_req = conn_req;
@@ -536,13 +552,24 @@ int MPID_nem_ofi_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc)
sreq->dev.next = NULL;
REQ_OFI(sreq)->event_callback = MPID_nem_ofi_connect_to_root_callback;
REQ_OFI(sreq)->pack_buffer = my_bc;
- conn_req_send_bits = init_sendtag(0, MPIR_Process.comm_world->rank, 0, MPID_CONN_REQ);
- FI_RC(fi_tsend(gl_data.endpoint,
- REQ_OFI(sreq)->pack_buffer,
- my_bc_len,
- gl_data.mr,
- VC_OFI(new_vc)->direct_addr,
- conn_req_send_bits, &(REQ_OFI(sreq)->ofi_context)), tsend);
+ if (gl_data.api_set == API_SET_1) {
+ conn_req_send_bits = init_sendtag(0, MPIR_Process.comm_world->rank, 0, MPID_CONN_REQ);
+ FI_RC(fi_tsend(gl_data.endpoint,
+ REQ_OFI(sreq)->pack_buffer,
+ my_bc_len,
+ gl_data.mr,
+ VC_OFI(new_vc)->direct_addr,
+ conn_req_send_bits, &(REQ_OFI(sreq)->ofi_context)), tsend);
+ } else {
+ conn_req_send_bits = init_sendtag_2(0, 0, MPID_CONN_REQ);
+ FI_RC(fi_tsenddata(gl_data.endpoint,
+ REQ_OFI(sreq)->pack_buffer,
+ my_bc_len,
+ gl_data.mr,
+ MPIR_Process.comm_world->rank,
+ VC_OFI(new_vc)->direct_addr,
+ conn_req_send_bits, &(REQ_OFI(sreq)->ofi_context)), tsend);
+ }
MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL);
VC_OFI(new_vc)->is_cmvc = 1;
VC_OFI(new_vc)->next = gl_data.cm_vcs;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
index 065920a..619e6ff 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_impl.h
@@ -21,6 +21,7 @@
#include <rdma/fi_cm.h>
#include <netdb.h>
+#include "ofi_tag_layout.h"
/* ************************************************************************** */
/* Type Definitions */
/* ************************************************************************** */
@@ -56,6 +57,7 @@ typedef struct {
MPID_Request *persistent_req; /* Unexpected request queue */
MPID_Request *conn_req; /* Connection request */
MPIDI_Comm_ops_t comm_ops;
+ int api_set;
} MPID_nem_ofi_global_t;
/* ******************************** */
@@ -173,37 +175,11 @@ fn_fail: \
#define NO_PGID 0
-/* **************************************************************************
- * match/ignore bit manipulation
- * **************************************************************************
- * 0123 4567 01234567 0123 4567 01234567 0123 4567 01234567 01234567 01234567
- * | | |
- * ^ | context id | source | message tag
- * | | | |
- * +---- protocol
- * ************************************************************************** */
-#define MPID_PROTOCOL_MASK (0xF000000000000000ULL)
-#define MPID_CONTEXT_MASK (0x0FFFF00000000000ULL)
-#define MPID_SOURCE_MASK (0x00000FFFF0000000ULL)
-#define MPID_TAG_MASK (0x000000000FFFFFFFULL)
-#define MPID_PGID_MASK (0x00000000FFFFFFFFULL)
-#define MPID_PSOURCE_MASK (0x0000FFFF00000000ULL)
-#define MPID_PORT_NAME_MASK (0x0FFF000000000000ULL)
-#define MPID_SYNC_SEND (0x1000000000000000ULL)
-#define MPID_SYNC_SEND_ACK (0x2000000000000000ULL)
-#define MPID_MSG_RTS (0x3000000000000000ULL)
-#define MPID_MSG_CTS (0x4000000000000000ULL)
-#define MPID_MSG_DATA (0x5000000000000000ULL)
-#define MPID_CONN_REQ (0x6000000000000000ULL)
-#define MPID_SOURCE_SHIFT (16)
-#define MPID_TAG_SHIFT (28)
-#define MPID_PSOURCE_SHIFT (16)
-#define MPID_PORT_SHIFT (32)
-#define OFI_KVSAPPSTRLEN 1024
#define PEEK_INIT 0
#define PEEK_FOUND 1
+#define MEM_TAG_FORMAT (0xFFFF00000000LLU)
/* ******************************** */
/* Request manipulation inlines */
/* ******************************** */
@@ -224,97 +200,42 @@ static inline int MPID_nem_ofi_create_req(MPID_Request ** request, int refcnt)
return mpi_errno;
}
-/* ******************************** */
-/* Tag Manipulation inlines */
-/* ******************************** */
-static inline uint64_t init_sendtag(MPIR_Context_id_t contextid, int source, int tag, uint64_t type)
-{
- uint64_t match_bits;
- match_bits = contextid;
- match_bits = (match_bits << MPID_SOURCE_SHIFT);
- match_bits |= source;
- match_bits = (match_bits << MPID_TAG_SHIFT);
- match_bits |= (MPID_TAG_MASK & tag) | type;
- return match_bits;
-}
-
-/* receive posting */
-static inline uint64_t init_recvtag(uint64_t * mask_bits,
- MPIR_Context_id_t contextid, int source, int tag)
-{
- uint64_t match_bits = 0;
- *mask_bits = MPID_SYNC_SEND;
- match_bits = contextid;
- match_bits = (match_bits << MPID_SOURCE_SHIFT);
- if (MPI_ANY_SOURCE == source) {
- match_bits = (match_bits << MPID_TAG_SHIFT);
- *mask_bits |= MPID_SOURCE_MASK;
- }
- else {
- match_bits |= source;
- match_bits = (match_bits << MPID_TAG_SHIFT);
- }
- if (MPI_ANY_TAG == tag)
- *mask_bits |= MPID_TAG_MASK;
- else
- match_bits |= (MPID_TAG_MASK & tag);
-
- return match_bits;
-}
-
-static inline int get_tag(uint64_t match_bits)
-{
- return ((int) (match_bits & MPID_TAG_MASK));
-}
-
-static inline int get_source(uint64_t match_bits)
-{
- return ((int) ((match_bits & MPID_SOURCE_MASK) >> (MPID_TAG_SHIFT)));
-}
-
-static inline int get_psource(uint64_t match_bits)
-{
- return ((int) ((match_bits & MPID_PSOURCE_MASK) >> (MPID_PORT_SHIFT)));
-}
-
-static inline int get_pgid(uint64_t match_bits)
-{
- return ((int) (match_bits & MPID_PGID_MASK));
-}
-
-static inline int get_port(uint64_t match_bits)
-{
- return ((int) ((match_bits & MPID_PORT_NAME_MASK) >> MPID_TAG_SHIFT));
-}
/* ************************************************************************** */
/* MPICH Comm Override and Netmod functions */
/* ************************************************************************** */
-int MPID_nem_ofi_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req);
-int MPID_nem_ofi_send(struct MPIDI_VC *vc, const void *buf, int count,
- MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
- int context_offset, struct MPID_Request **request);
-int MPID_nem_ofi_isend(struct MPIDI_VC *vc, const void *buf, int count,
- MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
- int context_offset, struct MPID_Request **request);
-int MPID_nem_ofi_ssend(struct MPIDI_VC *vc, const void *buf, int count,
- MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
- int context_offset, struct MPID_Request **request);
-int MPID_nem_ofi_issend(struct MPIDI_VC *vc, const void *buf, int count,
- MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
- int context_offset, struct MPID_Request **request);
+#define DECLARE_TWO_API_SETS(_ret, _fc_name, ...) \
+ _ret _fc_name(__VA_ARGS__); \
+ _ret _fc_name##_2(__VA_ARGS__);
+
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_recv_posted, struct MPIDI_VC *vc, struct MPID_Request *req);
+
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_send, struct MPIDI_VC *vc, const void *buf, int count,\
+ MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,\
+ int context_offset, struct MPID_Request **request);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_isend, struct MPIDI_VC *vc, const void *buf, int count,\
+ MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,\
+ int context_offset, struct MPID_Request **request);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_ssend, struct MPIDI_VC *vc, const void *buf, int count,\
+ MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
+ int context_offset, struct MPID_Request **request);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_issend, struct MPIDI_VC *vc, const void *buf, int count,\
+ MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,\
+ int context_offset, struct MPID_Request **request);
int MPID_nem_ofi_cancel_send(struct MPIDI_VC *vc, struct MPID_Request *sreq);
int MPID_nem_ofi_cancel_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq);
-int MPID_nem_ofi_iprobe(struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
- int context_offset, int *flag, MPI_Status * status);
-int MPID_nem_ofi_improbe(struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
- int context_offset, int *flag, MPID_Request ** message,
- MPI_Status * status);
-int MPID_nem_ofi_anysource_iprobe(int tag, MPID_Comm * comm, int context_offset,
- int *flag, MPI_Status * status);
-int MPID_nem_ofi_anysource_improbe(int tag, MPID_Comm * comm, int context_offset,
- int *flag, MPID_Request ** message, MPI_Status * status);
-void MPID_nem_ofi_anysource_posted(MPID_Request * rreq);
+
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_iprobe, struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
+ int context_offset, int *flag, MPI_Status * status);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_improbe,struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
+ int context_offset, int *flag, MPID_Request ** message,
+ MPI_Status * status);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_anysource_iprobe,int tag, MPID_Comm * comm, int context_offset,
+ int *flag, MPI_Status * status);
+DECLARE_TWO_API_SETS(int, MPID_nem_ofi_anysource_improbe,int tag, MPID_Comm * comm, int context_offset,
+ int *flag, MPID_Request ** message, MPI_Status * status);
+DECLARE_TWO_API_SETS(void, MPID_nem_ofi_anysource_posted, MPID_Request * rreq);
+
int MPID_nem_ofi_anysource_matched(MPID_Request * rreq);
int MPID_nem_ofi_send_data(cq_tagged_entry_t * wc, MPID_Request * sreq);
int MPID_nem_ofi_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq,
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
index 91576f6..4e0ffee 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c
@@ -83,6 +83,9 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
hints->caps = FI_TAGGED; /* Tag matching interface */
hints->caps |= FI_DYNAMIC_MR; /* Global dynamic mem region */
+ hints->ep_attr->mem_tag_format = MEM_TAG_FORMAT;
+ MPIU_Assert(pg_p->size < ((1 << MPID_RANK_BITS) - 1));
+
/* ------------------------------------------------------------------------ */
/* FI_VERSION provides binary backward and forward compatibility support */
/* Specify the version of OFI is coded to, the provider will select struct */
@@ -124,6 +127,8 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
&gl_data.fabric, /* Out: Fabric descriptor */
NULL), openfabric); /* Context: fabric events */
+ gl_data.api_set = API_SET_1;
+
/* ------------------------------------------------------------------------ */
/* Create the access domain, which is the physical or virtual network or */
/* hardware port/collection of ports. Returns a domain object that can be */
@@ -236,7 +241,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_
/* -------------------------------- */
/* Set the MPI maximum tag value */
/* -------------------------------- */
- MPIR_Process.attrs.tag_ub = (1 << MPID_TAG_SHIFT) - 1;
+ MPIR_Process.attrs.tag_ub = (1 << MPID_TAG_BITS) - 1;
/* --------------------------------- */
/* Wait for all the ranks to publish */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
index fece219..c314946 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c
@@ -24,13 +24,18 @@
} else { \
pgid = NO_PGID; \
} \
- match_bits = (uint64_t)MPIR_Process.comm_world->rank << \
- (MPID_PORT_SHIFT); \
- if (0 == pgid) { \
+ if (gl_data.api_set == API_SET_1){ \
+ match_bits = ((uint64_t)pgid << MPID_PGID_SHIFT); \
+ }else{ \
+ match_bits = 0; \
+ } \
+ if (NO_PGID == pgid) { \
match_bits |= (uint64_t)vc->port_name_tag<< \
- (MPID_PORT_SHIFT+MPID_PSOURCE_SHIFT); \
+ (MPID_PORT_SHIFT); \
+ }else{ \
+ match_bits |= (uint64_t)MPIR_Process.comm_world->rank << \
+ (MPID_PSOURCE_SHIFT); \
} \
- match_bits |= pgid; \
match_bits |= MPID_MSG_RTS; \
})
@@ -80,15 +85,26 @@
VC_OFI(vc)->direct_addr, \
match_bits | MPID_MSG_CTS, \
0, /* Exact tag match, no ignore bits */ \
- &(REQ_OFI(cts_req)->ofi_context)),trecv); \
- FI_RC(fi_tsend(gl_data.endpoint, \
- &REQ_OFI(sreq)->pack_buffer_size, \
- sizeof(REQ_OFI(sreq)->pack_buffer_size), \
- gl_data.mr, \
- VC_OFI(vc)->direct_addr, \
- match_bits, \
- &(REQ_OFI(sreq)->ofi_context)),tsend); \
- })
+ &(REQ_OFI(cts_req)->ofi_context)),trecv); \
+ if (gl_data.api_set == API_SET_1){ \
+ FI_RC(fi_tsend(gl_data.endpoint, \
+ &REQ_OFI(sreq)->pack_buffer_size, \
+ sizeof(REQ_OFI(sreq)->pack_buffer_size), \
+ gl_data.mr, \
+ VC_OFI(vc)->direct_addr, \
+ match_bits, \
+ &(REQ_OFI(sreq)->ofi_context)),tsend); \
+ }else{ \
+ FI_RC(fi_tsenddata(gl_data.endpoint, \
+ &REQ_OFI(sreq)->pack_buffer_size, \
+ sizeof(REQ_OFI(sreq)->pack_buffer_size), \
+ gl_data.mr, \
+ pgid, \
+ VC_OFI(vc)->direct_addr, \
+ match_bits, \
+ &(REQ_OFI(sreq)->ofi_context)),tsend); \
+ } \
+ })
/* ------------------------------------------------------------------------ */
@@ -109,13 +125,13 @@ static int MPID_nem_ofi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sre
vc = REQ_OFI(sreq)->vc;
REQ_OFI(sreq)->tag = tag | MPID_MSG_DATA;
FI_RC(fi_tsend(gl_data.endpoint,
- REQ_OFI(sreq)->pack_buffer,
- REQ_OFI(sreq)->pack_buffer_size,
- gl_data.mr,
- VC_OFI(vc)->direct_addr,
- wc->tag | MPID_MSG_DATA, (void *) &(REQ_OFI(sreq)->ofi_context)), tsend);
- }
- if (MPID_cc_get(sreq->cc) == 1) {
+ REQ_OFI(sreq)->pack_buffer,
+ REQ_OFI(sreq)->pack_buffer_size,
+ gl_data.mr,
+ VC_OFI(vc)->direct_addr,
+ MPID_MSG_DATA, (void *) &(REQ_OFI(sreq)->ofi_context)), tsend);
+ break;
+ case MPID_MSG_DATA:
if (REQ_OFI(sreq)->pack_buffer)
MPIU_Free(REQ_OFI(sreq)->pack_buffer);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
new file mode 100644
index 0000000..58a619a
--- /dev/null
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
@@ -0,0 +1,176 @@
+#if (API_SET != API_SET_1) && (API_SET != API_SET_2)
+#error Undefined API SET
+#endif
+
+/* ------------------------------------------------------------------------ */
+/* peek_callback called when a successful peek is completed */
+/* ------------------------------------------------------------------------ */
+#undef FCNAME
+#define FCNAME DECL_FUNC(peek_callback)
+static int
+ADD_SUFFIX(peek_callback)(cq_tagged_entry_t * wc, MPID_Request * rreq)
+{
+ int mpi_errno = MPI_SUCCESS;
+ BEGIN_FUNC(FCNAME);
+ REQ_OFI(rreq)->match_state = PEEK_FOUND;
+#if API_SET == API_SET_1
+ rreq->status.MPI_SOURCE = get_source(wc->tag);
+#elif API_SET == API_SET_2
+ rreq->status.MPI_SOURCE = wc->data;
+#endif
+ rreq->status.MPI_TAG = get_tag(wc->tag);
+ MPIR_STATUS_SET_COUNT(rreq->status, wc->len);
+ rreq->status.MPI_ERROR = MPI_SUCCESS;
+ END_FUNC(FCNAME);
+ return mpi_errno;
+}
+
+#undef FCNAME
+#define FCNAME DECL_FUNC(MPID_nem_ofi_iprobe_impl)
+int ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(struct MPIDI_VC *vc,
+ int source,
+ int tag,
+ MPID_Comm * comm,
+ int context_offset,
+ int *flag, MPI_Status * status, MPID_Request ** rreq_ptr)
+{
+ int ret, mpi_errno = MPI_SUCCESS;
+ fi_addr_t remote_proc = 0;
+ uint64_t match_bits, mask_bits;
+ size_t len;
+ MPID_Request rreq_s, *rreq;
+
+ BEGIN_FUNC(FCNAME);
+ if (rreq_ptr) {
+ MPIDI_Request_create_rreq(rreq, mpi_errno, goto fn_exit);
+ *rreq_ptr = rreq;
+ rreq->comm = comm;
+ rreq->dev.match.parts.rank = source;
+ rreq->dev.match.parts.tag = tag;
+ rreq->dev.match.parts.context_id = comm->context_id;
+ MPIR_Comm_add_ref(comm);
+ }
+ else {
+ rreq = &rreq_s;
+ rreq->dev.OnDataAvail = NULL;
+ }
+ REQ_OFI(rreq)->event_callback = ADD_SUFFIX(peek_callback);
+ REQ_OFI(rreq)->match_state = PEEK_INIT;
+ OFI_ADDR_INIT(source, vc, remote_proc);
+#if API_SET == API_SET_1
+ match_bits = init_recvtag(&mask_bits, comm->context_id + context_offset, source, tag);
+#elif API_SET == API_SET_2
+ match_bits = init_recvtag_2(&mask_bits, comm->context_id + context_offset, tag);
+#endif
+
+ /* ------------------------------------------------------------------------- */
+ /* fi_recvmsg with FI_PEEK: */
+ /* Initiate a search for a match in the hardware or software queue. */
+ /* The search can complete immediately with -ENOMSG. */
+ /* I successful, libfabric will enqueue a context entry into the completion */
+ /* queue to make the search nonblocking. This code will poll until the */
+ /* entry is enqueued. */
+ /* ------------------------------------------------------------------------- */
+ msg_tagged_t msg;
+ uint64_t msgflags = FI_PEEK;
+ msg.msg_iov = NULL;
+ msg.desc = NULL;
+ msg.iov_count = 0;
+ msg.addr = remote_proc;
+ msg.tag = match_bits;
+ msg.ignore = mask_bits;
+ msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
+ msg.data = 0;
+ if(*flag == CLAIM_PEEK)
+ msgflags|=FI_CLAIM;
+ ret = fi_trecvmsg(gl_data.endpoint,&msg,msgflags);
+ if(ret == -ENOMSG) {
+ if (rreq_ptr) {
+ MPIDI_CH3_Request_destroy(rreq);
+ *rreq_ptr = NULL;
+ *flag = 0;
+ }
+ MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL);
+ goto fn_exit;
+ }
+ MPIU_ERR_CHKANDJUMP4((ret < 0), mpi_errno, MPI_ERR_OTHER,
+ "**ofi_peek", "**ofi_peek %s %d %s %s",
+ __SHORT_FILE__, __LINE__, FCNAME, fi_strerror(-ret));
+
+ while (PEEK_INIT == REQ_OFI(rreq)->match_state)
+ MPID_nem_ofi_poll(MPID_BLOCKING_POLL);
+ *status = rreq->status;
+ *flag = 1;
+ END_FUNC_RC(FCNAME);
+}
+
+
+#undef FCNAME
+#define FCNAME DECL_FUNC(MPID_nem_ofi_iprobe)
+int ADD_SUFFIX(MPID_nem_ofi_iprobe)(struct MPIDI_VC *vc,
+ int source,
+ int tag,
+ MPID_Comm * comm, int context_offset, int *flag, MPI_Status * status)
+{
+ int rc;
+ BEGIN_FUNC(FCNAME);
+ *flag = 0;
+ rc = ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(vc, source,
+ tag, comm, context_offset, flag, status, NULL);
+ END_FUNC(FCNAME);
+ return rc;
+}
+
+#undef FCNAME
+#define FCNAME DECL_FUNC(MPID_nem_ofi_improbe)
+int ADD_SUFFIX(MPID_nem_ofi_improbe)(struct MPIDI_VC *vc,
+ int source,
+ int tag,
+ MPID_Comm * comm,
+ int context_offset,
+ int *flag, MPID_Request ** message, MPI_Status * status)
+{
+ int old_error = status->MPI_ERROR;
+ int s;
+ BEGIN_FUNC(FCNAME);
+ *flag = NORMAL_PEEK;
+ s = ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(vc, source,
+ tag, comm, context_offset, flag, status, message);
+ if (*flag) {
+ status->MPI_ERROR = old_error;
+ (*message)->kind = MPID_REQUEST_MPROBE;
+ }
+ END_FUNC(FCNAME);
+ return s;
+}
+
+#undef FCNAME
+#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_iprobe)
+int ADD_SUFFIX(MPID_nem_ofi_anysource_iprobe)(int tag,
+ MPID_Comm * comm,
+ int context_offset, int *flag, MPI_Status * status)
+{
+ int rc;
+ BEGIN_FUNC(FCNAME);
+ *flag = NORMAL_PEEK;
+ rc = ADD_SUFFIX(MPID_nem_ofi_iprobe)(NULL, MPI_ANY_SOURCE,
+ tag, comm, context_offset, flag, status);
+ END_FUNC(FCNAME);
+ return rc;
+}
+
+#undef FCNAME
+#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_improbe)
+int ADD_SUFFIX(MPID_nem_ofi_anysource_improbe)(int tag,
+ MPID_Comm * comm,
+ int context_offset,
+ int *flag, MPID_Request ** message, MPI_Status * status)
+{
+ int rc;
+ BEGIN_FUNC(FCNAME);
+ *flag = CLAIM_PEEK;
+ rc = ADD_SUFFIX(MPID_nem_ofi_improbe)(NULL, MPI_ANY_SOURCE, tag, comm,
+ context_offset, flag, message, status);
+ END_FUNC(FCNAME);
+ return rc;
+}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
index cdf5535..34d85ec 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_progress.c
@@ -20,165 +20,16 @@ static inline MPID_Request *context_to_req(void *ofi_context)
return (MPID_Request *) container_of(ofi_context, MPID_Request, ch.netmod_area.padding);
}
-/* ------------------------------------------------------------------------ */
-/* peek_callback called when a successful peek is completed */
-/* ------------------------------------------------------------------------ */
-#undef FCNAME
-#define FCNAME DECL_FUNC(peek_callback)
-static int peek_callback(cq_tagged_entry_t * wc, MPID_Request * rreq)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- REQ_OFI(rreq)->match_state = PEEK_FOUND;
- rreq->status.MPI_SOURCE = get_source(wc->tag);
- rreq->status.MPI_TAG = get_tag(wc->tag);
- MPIR_STATUS_SET_COUNT(rreq->status, wc->len);
- rreq->status.MPI_ERROR = MPI_SUCCESS;
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_iprobe_impl)
-int MPID_nem_ofi_iprobe_impl(struct MPIDI_VC *vc,
- int source,
- int tag,
- MPID_Comm * comm,
- int context_offset,
- int *flag, MPI_Status * status, MPID_Request ** rreq_ptr)
-{
- int ret, mpi_errno = MPI_SUCCESS;
- fi_addr_t remote_proc = 0;
- uint64_t match_bits, mask_bits;
- size_t len;
- MPID_Request rreq_s, *rreq;
-
- BEGIN_FUNC(FCNAME);
- if (rreq_ptr) {
- MPIDI_Request_create_rreq(rreq, mpi_errno, goto fn_exit);
- *rreq_ptr = rreq;
- rreq->comm = comm;
- rreq->dev.match.parts.rank = source;
- rreq->dev.match.parts.tag = tag;
- rreq->dev.match.parts.context_id = comm->context_id;
- MPIR_Comm_add_ref(comm);
- }
- else {
- rreq = &rreq_s;
- rreq->dev.OnDataAvail = NULL;
- }
- REQ_OFI(rreq)->event_callback = peek_callback;
- REQ_OFI(rreq)->match_state = PEEK_INIT;
- OFI_ADDR_INIT(source, vc, remote_proc);
- match_bits = init_recvtag(&mask_bits, comm->context_id + context_offset, source, tag);
-
- /* ------------------------------------------------------------------------- */
- /* fi_recvmsg with FI_PEEK: */
- /* Initiate a search for a match in the hardware or software queue. */
- /* The search can complete immediately with -ENOMSG. */
- /* I successful, libfabric will enqueue a context entry into the completion */
- /* queue to make the search nonblocking. This code will poll until the */
- /* entry is enqueued. */
- /* ------------------------------------------------------------------------- */
- msg_tagged_t msg;
- uint64_t msgflags = FI_PEEK;
- msg.msg_iov = NULL;
- msg.desc = NULL;
- msg.iov_count = 0;
- msg.addr = remote_proc;
- msg.tag = match_bits;
- msg.ignore = mask_bits;
- msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
- msg.data = 0;
- if(*flag == CLAIM_PEEK)
- msgflags|=FI_CLAIM;
- ret = fi_trecvmsg(gl_data.endpoint,&msg,msgflags);
- if(ret == -ENOMSG) {
- if (rreq_ptr) {
- MPIDI_CH3_Request_destroy(rreq);
- *rreq_ptr = NULL;
- }
- MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL);
- *flag = 0;
- goto fn_exit;
- }
- MPIU_ERR_CHKANDJUMP4((ret < 0), mpi_errno, MPI_ERR_OTHER,
- "**ofi_peek", "**ofi_peek %s %d %s %s",
- __SHORT_FILE__, __LINE__, FCNAME, fi_strerror(-ret));
-
- while (PEEK_INIT == REQ_OFI(rreq)->match_state)
- MPID_nem_ofi_poll(MPID_BLOCKING_POLL);
- *status = rreq->status;
- *flag = 1;
- END_FUNC_RC(FCNAME);
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_iprobe)
-int MPID_nem_ofi_iprobe(struct MPIDI_VC *vc,
- int source,
- int tag,
- MPID_Comm * comm, int context_offset, int *flag, MPI_Status * status)
-{
- int rc;
- BEGIN_FUNC(FCNAME);
- *flag = 0;
- rc = MPID_nem_ofi_iprobe_impl(vc, source, tag, comm, context_offset, flag, status, NULL);
- END_FUNC(FCNAME);
- return rc;
-}
+#define ADD_SUFFIX(name) name
+#undef API_SET
+#define API_SET API_SET_1
+#include "ofi_probe_template.c"
+#undef ADD_SUFFIX
+#define ADD_SUFFIX(name) name##_2
+#undef API_SET
+#define API_SET API_SET_2
+#include "ofi_probe_template.c"
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_improbe)
-int MPID_nem_ofi_improbe(struct MPIDI_VC *vc,
- int source,
- int tag,
- MPID_Comm * comm,
- int context_offset,
- int *flag, MPID_Request ** message, MPI_Status * status)
-{
- int old_error = status->MPI_ERROR;
- int s;
- BEGIN_FUNC(FCNAME);
- *flag = NORMAL_PEEK;
- s = MPID_nem_ofi_iprobe_impl(vc, source, tag, comm, context_offset, flag, status, message);
- if (*flag) {
- status->MPI_ERROR = old_error;
- (*message)->kind = MPID_REQUEST_MPROBE;
- }
- END_FUNC(FCNAME);
- return s;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_iprobe)
-int MPID_nem_ofi_anysource_iprobe(int tag,
- MPID_Comm * comm,
- int context_offset, int *flag, MPI_Status * status)
-{
- int rc;
- BEGIN_FUNC(FCNAME);
- *flag = NORMAL_PEEK;
- rc = MPID_nem_ofi_iprobe(NULL, MPI_ANY_SOURCE, tag, comm, context_offset, flag, status);
- END_FUNC(FCNAME);
- return rc;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_improbe)
-int MPID_nem_ofi_anysource_improbe(int tag,
- MPID_Comm * comm,
- int context_offset,
- int *flag, MPID_Request ** message, MPI_Status * status)
-{
- int rc;
- BEGIN_FUNC(FCNAME);
- *flag = CLAIM_PEEK;
- rc = MPID_nem_ofi_improbe(NULL, MPI_ANY_SOURCE, tag, comm,
- context_offset, flag, message, status);
- END_FUNC(FCNAME);
- return rc;
-}
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_poll)
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tag_layout.h b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tag_layout.h
new file mode 100644
index 0000000..e6492d5
--- /dev/null
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tag_layout.h
@@ -0,0 +1,144 @@
+#ifndef OFI_TAG_LAYOUT_H
+#define OFI_TAG_LAYOUT_H
+
+
+
+
+/* *******************************************************************************
+ * match/ignore bit manipulation
+ * *******************************************************************************
+ * 01234567 01234567 | 01234567 01234567 | 0123 | 4567 01234567 01234567 01234567
+ * | | proto| |
+ * source | ctxid | col | message tag |
+ *
+ *********************************************************************************/
+#define MPID_PROTOCOL_MASK (0x00000000F0000000ULL)
+#define MPID_CONTEXT_MASK (0x0000FFFF00000000ULL)
+#define MPID_SOURCE_MASK (0xFFFF000000000000ULL)
+#define MPID_TAG_MASK (0x000000000FFFFFFFULL)
+#define MPID_PGID_MASK (0xFFFFFFFF00000000ULL)
+#define MPID_PSOURCE_MASK (0x000000000000FFFFULL)
+#define MPID_PORT_NAME_MASK (0x000000000000FFFFULL)
+#define MPID_SYNC_SEND (0x0000000010000000ULL)
+#define MPID_SYNC_SEND_ACK (0x0000000020000000ULL)
+#define MPID_MSG_RTS (0x0000000030000000ULL)
+#define MPID_MSG_CTS (0x0000000040000000ULL)
+#define MPID_MSG_DATA (0x0000000050000000ULL)
+#define MPID_CONN_REQ (0x0000000060000000ULL)
+#define MPID_SOURCE_SHIFT (48)
+#define MPID_CTXID_SHIFT (32)
+#define MPID_PGID_SHIFT (32)
+#define MPID_PSOURCE_SHIFT (0)
+#define MPID_PORT_SHIFT (0)
+#define MPID_TAG_BITS (28)
+#define MPID_RANK_BITS (16)
+#define MPID_RCD_IGNORE_MASK (0xFFFFFFFF0FFFFFFFULL)
+#define OFI_KVSAPPSTRLEN 1024
+
+
+/* ******************************** */
+/* Tag Manipulation inlines */
+/* ******************************** */
+static inline uint64_t init_sendtag(MPIR_Context_id_t contextid, int source, int tag, uint64_t type)
+{
+ uint64_t match_bits = 0;
+ match_bits |= ((uint64_t)source) << MPID_SOURCE_SHIFT;
+ match_bits |= ((uint64_t)contextid) << MPID_CTXID_SHIFT;
+ match_bits |= (MPID_TAG_MASK & tag) | type;
+ return match_bits;
+}
+
+/* receive posting */
+static inline uint64_t init_recvtag(uint64_t * mask_bits,
+ MPIR_Context_id_t contextid, int source, int tag)
+{
+ uint64_t match_bits = 0;
+ *mask_bits = MPID_SYNC_SEND;
+ match_bits |= ((uint64_t)contextid) << MPID_CTXID_SHIFT;
+
+ if (MPI_ANY_SOURCE == source) {
+ *mask_bits |= MPID_SOURCE_MASK;
+ }
+ else {
+ match_bits |= ((uint64_t)source) << MPID_SOURCE_SHIFT;
+ }
+ if (MPI_ANY_TAG == tag)
+ *mask_bits |= MPID_TAG_MASK;
+ else
+ match_bits |= (MPID_TAG_MASK & tag);
+
+ return match_bits;
+}
+
+static inline int get_tag(uint64_t match_bits)
+{
+ return ((int) (match_bits & MPID_TAG_MASK));
+}
+
+static inline int get_source(uint64_t match_bits)
+{
+ return ((int) ((match_bits & MPID_SOURCE_MASK) >> (MPID_SOURCE_SHIFT)));
+}
+
+static inline int get_psource(uint64_t match_bits)
+{
+ return ((int) ((match_bits & MPID_PSOURCE_MASK) >> (MPID_PSOURCE_SHIFT)));
+}
+
+static inline int get_pgid(uint64_t match_bits)
+{
+ return ((int) ((match_bits & MPID_PGID_MASK) >> MPID_PGID_SHIFT));
+}
+
+static inline int get_port(uint64_t match_bits)
+{
+ return ((int) ((match_bits & MPID_PORT_NAME_MASK) >> MPID_PORT_SHIFT));
+}
+
+
+/*********************************************************************************
+ * SECOND MODE TAG LAUOUT MACROS *
+ *********************************************************************************/
+
+/*********************************************************************************
+ * 01234567 01234567 | 01234567 01234567 | 0123 | 4567 01234567 01234567 01234567
+ * | | proto| |
+ * Not used | ctxid | col | message tag |
+ *
+ *********************************************************************************/
+#define MPID_RCD_IGNORE_MASK_M2 (0x000000000FFFFFFFULL)
+
+
+/* ******************************** */
+/* Tag Manipulation inlines */
+/* ******************************** */
+static inline uint64_t init_sendtag_2(MPIR_Context_id_t contextid, int tag, uint64_t type)
+{
+ uint64_t match_bits = 0;
+ match_bits |= ((uint64_t)contextid) << MPID_CTXID_SHIFT;
+ match_bits |= (MPID_TAG_MASK & tag) | type;
+ return match_bits;
+}
+
+/* receive posting */
+static inline uint64_t init_recvtag_2(uint64_t * mask_bits,
+ MPIR_Context_id_t contextid, int tag)
+{
+ uint64_t match_bits = 0;
+ *mask_bits = MPID_SYNC_SEND;
+ match_bits |= ((uint64_t)contextid) << MPID_CTXID_SHIFT;
+
+ if (MPI_ANY_TAG == tag)
+ *mask_bits |= MPID_TAG_MASK;
+ else
+ match_bits |= (MPID_TAG_MASK & tag);
+
+ return match_bits;
+}
+
+#define GET_RCD_IGNORE_MASK() (gl_data.api_set == API_SET_1 ? \
+ MPID_RCD_IGNORE_MASK : MPID_RCD_IGNORE_MASK_M2)
+#define API_SET_1 1
+#define API_SET_2 2
+
+#endif /*OFI_TAG_LAYOUT_H*/
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
index 66b9130..b0ab745 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
@@ -10,6 +10,23 @@
#include "ofi_impl.h"
#define MPID_NORMAL_SEND 0
+static inline int
+MPID_nem_ofi_sync_recv_callback(cq_tagged_entry_t * wc ATTRIBUTE((unused)),
+ MPID_Request * rreq);
+
+static inline int
+MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((unused)),
+ MPID_Request * sreq);
+
+#define ADD_SUFFIX(name) name
+#undef API_SET
+#define API_SET API_SET_1
+#include "ofi_tagged_template.c"
+#undef ADD_SUFFIX
+#define ADD_SUFFIX(name) name##_2
+#undef API_SET
+#define API_SET API_SET_2
+#include "ofi_tagged_template.c"
/* ------------------------------------------------------------------------ */
/* Receive callback called after sending a syncronous send acknowledgement. */
@@ -49,298 +66,6 @@ static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((u
return mpi_errno;
}
-/* ------------------------------------------------------------------------ */
-/* Receive done callback */
-/* Handle an incoming receive completion event */
-/* ------------------------------------------------------------------------ */
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_recv_callback)
-static inline int MPID_nem_ofi_recv_callback(cq_tagged_entry_t * wc, MPID_Request * rreq)
-{
- int err0, err1, src, mpi_errno = MPI_SUCCESS;
- uint64_t ssend_bits;
- MPIDI_msg_sz_t sz;
- MPIDI_VC_t *vc;
- MPID_Request *sync_req;
- BEGIN_FUNC(FCNAME);
- /* ---------------------------------------------------- */
- /* Populate the MPI Status and unpack noncontig buffer */
- /* ---------------------------------------------------- */
- rreq->status.MPI_ERROR = MPI_SUCCESS;
- rreq->status.MPI_SOURCE = get_source(wc->tag);
- rreq->status.MPI_TAG = get_tag(wc->tag);
- REQ_OFI(rreq)->req_started = 1;
- MPIR_STATUS_SET_COUNT(rreq->status, wc->len);
- if (REQ_OFI(rreq)->pack_buffer) {
- MPIDI_CH3U_Buffer_copy(REQ_OFI(rreq)->pack_buffer,
- MPIR_STATUS_GET_COUNT(rreq->status),
- MPI_BYTE, &err0, rreq->dev.user_buf,
- rreq->dev.user_count, rreq->dev.datatype, &sz, &err1);
- MPIR_STATUS_SET_COUNT(rreq->status, sz);
- MPIU_Free(REQ_OFI(rreq)->pack_buffer);
- if (err0 || err1) {
- rreq->status.MPI_ERROR = MPI_ERR_TYPE;
- }
- }
-
- if ((wc->tag & MPID_PROTOCOL_MASK) == MPID_SYNC_SEND) {
- /* ---------------------------------------------------- */
- /* Ack the sync send and wait for the send request */
- /* completion(when callback executed. A protocol bit */
- /* MPID_SYNC_SEND_ACK is set in the tag bits to provide */
- /* separation of MPI messages and protocol messages */
- /* ---------------------------------------------------- */
- vc = REQ_OFI(rreq)->vc;
- if (!vc) { /* MPI_ANY_SOURCE -- Post message from status, complete the VC */
- src = get_source(wc->tag);
- vc = rreq->comm->vcr[src];
- MPIU_Assert(vc);
- }
- ssend_bits = init_sendtag(rreq->dev.match.parts.context_id,
- rreq->comm->rank, rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK);
- MPID_nem_ofi_create_req(&sync_req, 1);
- sync_req->dev.OnDataAvail = NULL;
- sync_req->dev.next = NULL;
- REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback;
- REQ_OFI(sync_req)->parent = rreq;
- FI_RC(fi_tsend(gl_data.endpoint,
- NULL,
- 0,
- gl_data.mr,
- VC_OFI(vc)->direct_addr,
- ssend_bits, &(REQ_OFI(sync_req)->ofi_context)), tsend);
- }
- else {
- /* ---------------------------------------------------- */
- /* Non-syncronous send, complete normally */
- /* by removing from the CH3 queue and completing the */
- /* request object */
- /* ---------------------------------------------------- */
- MPIDI_CH3U_Recvq_DP(rreq);
- MPIDI_CH3U_Request_complete(rreq);
- }
- END_FUNC_RC(FCNAME);
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(do_isend)
-static inline int do_isend(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag,
- MPID_Comm * comm,
- int context_offset, struct MPID_Request **request, uint64_t type)
-{
- int err0, err1, dt_contig, mpi_errno = MPI_SUCCESS;
- char *send_buffer;
- uint64_t match_bits, ssend_match, ssend_mask;
- MPI_Aint dt_true_lb;
- MPID_Request *sreq = NULL, *sync_req = NULL;
- MPIDI_msg_sz_t data_sz;
- MPID_Datatype *dt_ptr;
- BEGIN_FUNC(FCNAME);
- VC_READY_CHECK(vc);
-
- /* ---------------------------------------------------- */
- /* Create the MPI request */
- /* ---------------------------------------------------- */
- MPID_nem_ofi_create_req(&sreq, 2);
- sreq->kind = MPID_REQUEST_SEND;
- sreq->dev.OnDataAvail = NULL;
- REQ_OFI(sreq)->event_callback = MPID_nem_ofi_send_callback;
- REQ_OFI(sreq)->vc = vc;
-
- /* ---------------------------------------------------- */
- /* Create the pack buffer (if required), and allocate */
- /* a send request */
- /* ---------------------------------------------------- */
- match_bits = init_sendtag(comm->context_id + context_offset, comm->rank, tag, type);
- sreq->dev.match.parts.tag = match_bits;
- MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
- send_buffer = (char *) buf + dt_true_lb;
- if (!dt_contig) {
- send_buffer = (char *) MPIU_Malloc(data_sz);
- MPIU_ERR_CHKANDJUMP1(send_buffer == NULL, mpi_errno,
- MPI_ERR_OTHER, "**nomem", "**nomem %s", "Send buffer alloc");
- MPIDI_CH3U_Buffer_copy(buf, count, datatype, &err0,
- send_buffer, data_sz, MPI_BYTE, &data_sz, &err1);
- REQ_OFI(sreq)->pack_buffer = send_buffer;
- }
-
- if (type == MPID_SYNC_SEND) {
- /* ---------------------------------------------------- */
- /* For syncronous send, we post a receive to catch the */
- /* match ack, but use the tag protocol bits to avoid */
- /* matching with MPI level messages. */
- /* ---------------------------------------------------- */
- int c = 1;
- MPID_cc_incr(sreq->cc_ptr, &c);
- MPID_nem_ofi_create_req(&sync_req, 1);
- sync_req->dev.OnDataAvail = NULL;
- sync_req->dev.next = NULL;
- REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback;
- REQ_OFI(sync_req)->parent = sreq;
- ssend_match = init_recvtag(&ssend_mask, comm->context_id + context_offset, dest, tag);
- ssend_match |= MPID_SYNC_SEND_ACK;
- FI_RC(fi_trecv(gl_data.endpoint, /* endpoint */
- NULL, /* recvbuf */
- 0, /* data sz */
- gl_data.mr, /* dynamic mr */
- VC_OFI(vc)->direct_addr, /* remote proc */
- ssend_match, /* match bits */
- 0ULL, /* mask bits */
- &(REQ_OFI(sync_req)->ofi_context)), trecv);
- }
- FI_RC(fi_tsend(gl_data.endpoint, /* Endpoint */
- send_buffer, /* Send buffer(packed or user) */
- data_sz, /* Size of the send */
- gl_data.mr, /* Dynamic memory region */
- VC_OFI(vc)->direct_addr, /* Use the address of this VC */
- match_bits, /* Match bits */
- &(REQ_OFI(sreq)->ofi_context)), tsend);
- *request = sreq;
- END_FUNC_RC(FCNAME);
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_recv_posted)
-int MPID_nem_ofi_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *rreq)
-{
- int mpi_errno = MPI_SUCCESS, dt_contig, src, tag;
- uint64_t match_bits = 0, mask_bits = 0;
- fi_addr_t remote_proc = 0;
- MPIDI_msg_sz_t data_sz;
- MPI_Aint dt_true_lb;
- MPID_Datatype *dt_ptr;
- MPIR_Context_id_t context_id;
- char *recv_buffer;
- BEGIN_FUNC(FCNAME);
-
- /* ------------------------ */
- /* Initialize the request */
- /* ------------------------ */
- MPID_nem_ofi_init_req(rreq);
- REQ_OFI(rreq)->event_callback = MPID_nem_ofi_recv_callback;
- REQ_OFI(rreq)->vc = vc;
-
- /* ---------------------------------------------------- */
- /* Fill out the match info, and allocate the pack buffer */
- /* a send request */
- /* ---------------------------------------------------- */
- src = rreq->dev.match.parts.rank;
- tag = rreq->dev.match.parts.tag;
- context_id = rreq->dev.match.parts.context_id;
- match_bits = init_recvtag(&mask_bits, context_id, src, tag);
- OFI_ADDR_INIT(src, vc, remote_proc);
- MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
- dt_contig, data_sz, dt_ptr, dt_true_lb);
- if (dt_contig) {
- recv_buffer = (char *) rreq->dev.user_buf + dt_true_lb;
- }
- else {
- recv_buffer = (char *) MPIU_Malloc(data_sz);
- MPIU_ERR_CHKANDJUMP1(recv_buffer == NULL, mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "Recv Pack Buffer alloc");
- REQ_OFI(rreq)->pack_buffer = recv_buffer;
- }
-
- /* ---------------- */
- /* Post the receive */
- /* ---------------- */
- uint64_t msgflags;
- iovec_t iov;
- msg_tagged_t msg;
- iov.iov_base = recv_buffer;
- iov.iov_len = data_sz;
- if(REQ_OFI(rreq)->match_state == PEEK_FOUND)
- msgflags = FI_CLAIM;
- else
- msgflags = 0ULL;
-
- msg.msg_iov = &iov;
- msg.desc = NULL;
- msg.iov_count = 1;
- msg.addr = remote_proc;
- msg.tag = match_bits;
- msg.ignore = mask_bits;
- msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
- msg.data = 0;
- FI_RC(fi_trecvmsg(gl_data.endpoint,&msg,msgflags), trecv);
- END_FUNC_RC(FCNAME);
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_send)
-int MPID_nem_ofi_send(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag, MPID_Comm * comm, int context_offset, struct MPID_Request **request)
-{
- int mpi_errno = MPI_SUCCESS;
-
- BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest, tag,
- comm, context_offset, request, MPID_NORMAL_SEND);
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_isend)
-int MPID_nem_ofi_isend(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag, MPID_Comm * comm, int context_offset, struct MPID_Request **request)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest,
- tag, comm, context_offset, request, MPID_NORMAL_SEND);
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_ssend)
-int MPID_nem_ofi_ssend(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag, MPID_Comm * comm, int context_offset, struct MPID_Request **request)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest,
- tag, comm, context_offset, request, MPID_SYNC_SEND);
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_issend)
-int MPID_nem_ofi_issend(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag,
- MPID_Comm * comm, int context_offset, struct MPID_Request **request)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest,
- tag, comm, context_offset, request, MPID_SYNC_SEND);
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
#define DO_CANCEL(req) \
({ \
int mpi_errno = MPI_SUCCESS; \
@@ -372,16 +97,6 @@ int MPID_nem_ofi_cancel_recv(struct MPIDI_VC *vc ATTRIBUTE((unused)), struct MPI
DO_CANCEL(rreq);
}
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_posted)
-void MPID_nem_ofi_anysource_posted(MPID_Request * rreq)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- mpi_errno = MPID_nem_ofi_recv_posted(NULL, rreq);
- MPIU_Assert(mpi_errno == MPI_SUCCESS);
- END_FUNC(FCNAME);
-}
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_matched)
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
similarity index 65%
copy from src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
copy to src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
index 66b9130..3c6140b 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tagged_template.c
@@ -1,53 +1,7 @@
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Portions of this code were written by Intel Corporation.
- * Copyright (C) 2011-2012 Intel Corporation. Intel provides this material
- * to Argonne National Laboratory subject to Software Grant and Corporate
- * Contributor License Agreement dated February 8, 2012.
- */
-#include "ofi_impl.h"
+#if (API_SET != API_SET_1) && (API_SET != API_SET_2)
+#error Undefined API SET
+#endif
-#define MPID_NORMAL_SEND 0
-
-/* ------------------------------------------------------------------------ */
-/* Receive callback called after sending a syncronous send acknowledgement. */
-/* ------------------------------------------------------------------------ */
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_sync_recv_callback)
-static inline int MPID_nem_ofi_sync_recv_callback(cq_tagged_entry_t * wc ATTRIBUTE((unused)),
- MPID_Request * rreq)
-{
- int mpi_errno = MPI_SUCCESS;
-
- BEGIN_FUNC(FCNAME);
-
- MPIDI_CH3U_Recvq_DP(REQ_OFI(rreq)->parent);
- MPIDI_CH3U_Request_complete(REQ_OFI(rreq)->parent);
- MPIDI_CH3U_Request_complete(rreq);
-
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-
-/* ------------------------------------------------------------------------ */
-/* Send done callback */
-/* Free any temporary/pack buffers and complete the send request */
-/* ------------------------------------------------------------------------ */
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_send_callback)
-static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((unused)),
- MPID_Request * sreq)
-{
- int mpi_errno = MPI_SUCCESS;
- BEGIN_FUNC(FCNAME);
- if (REQ_OFI(sreq)->pack_buffer)
- MPIU_Free(REQ_OFI(sreq)->pack_buffer);
- MPIDI_CH3U_Request_complete(sreq);
- END_FUNC(FCNAME);
- return mpi_errno;
-}
/* ------------------------------------------------------------------------ */
/* Receive done callback */
@@ -55,7 +9,8 @@ static inline int MPID_nem_ofi_send_callback(cq_tagged_entry_t * wc ATTRIBUTE((u
/* ------------------------------------------------------------------------ */
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_recv_callback)
-static inline int MPID_nem_ofi_recv_callback(cq_tagged_entry_t * wc, MPID_Request * rreq)
+static inline
+int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request * rreq)
{
int err0, err1, src, mpi_errno = MPI_SUCCESS;
uint64_t ssend_bits;
@@ -67,10 +22,17 @@ static inline int MPID_nem_ofi_recv_callback(cq_tagged_entry_t * wc, MPID_Reques
/* Populate the MPI Status and unpack noncontig buffer */
/* ---------------------------------------------------- */
rreq->status.MPI_ERROR = MPI_SUCCESS;
+#if API_SET == API_SET_1
rreq->status.MPI_SOURCE = get_source(wc->tag);
- rreq->status.MPI_TAG = get_tag(wc->tag);
+#elif API_SET == API_SET_2
+ rreq->status.MPI_SOURCE = wc->data;
+#endif
+ src = rreq->status.MPI_SOURCE;
+ rreq->status.MPI_TAG = get_tag(wc->tag);
+
REQ_OFI(rreq)->req_started = 1;
MPIR_STATUS_SET_COUNT(rreq->status, wc->len);
+
if (REQ_OFI(rreq)->pack_buffer) {
MPIDI_CH3U_Buffer_copy(REQ_OFI(rreq)->pack_buffer,
MPIR_STATUS_GET_COUNT(rreq->status),
@@ -92,21 +54,32 @@ static inline int MPID_nem_ofi_recv_callback(cq_tagged_entry_t * wc, MPID_Reques
/* ---------------------------------------------------- */
vc = REQ_OFI(rreq)->vc;
if (!vc) { /* MPI_ANY_SOURCE -- Post message from status, complete the VC */
- src = get_source(wc->tag);
vc = rreq->comm->vcr[src];
MPIU_Assert(vc);
}
+#if API_SET == API_SET_1
ssend_bits = init_sendtag(rreq->dev.match.parts.context_id,
rreq->comm->rank, rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK);
+#elif API_SET == API_SET_2
+ ssend_bits = init_sendtag_2(rreq->dev.match.parts.context_id,
+ rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK);
+#endif
MPID_nem_ofi_create_req(&sync_req, 1);
sync_req->dev.OnDataAvail = NULL;
sync_req->dev.next = NULL;
REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback;
REQ_OFI(sync_req)->parent = rreq;
+#if API_SET == API_SET_1
FI_RC(fi_tsend(gl_data.endpoint,
+#elif API_SET == API_SET_2
+ FI_RC(fi_tsenddata(gl_data.endpoint,
+#endif
NULL,
0,
gl_data.mr,
+#if API_SET == API_SET_2
+ rreq->comm->rank,
+#endif
VC_OFI(vc)->direct_addr,
ssend_bits, &(REQ_OFI(sync_req)->ofi_context)), tsend);
}
@@ -122,16 +95,18 @@ static inline int MPID_nem_ofi_recv_callback(cq_tagged_entry_t * wc, MPID_Reques
END_FUNC_RC(FCNAME);
}
+
#undef FCNAME
#define FCNAME DECL_FUNC(do_isend)
-static inline int do_isend(struct MPIDI_VC *vc,
- const void *buf,
- int count,
- MPI_Datatype datatype,
- int dest,
- int tag,
- MPID_Comm * comm,
- int context_offset, struct MPID_Request **request, uint64_t type)
+static inline int
+ADD_SUFFIX(do_isend)(struct MPIDI_VC *vc,
+ const void *buf,
+ int count,
+ MPI_Datatype datatype,
+ int dest,
+ int tag,
+ MPID_Comm * comm,
+ int context_offset, struct MPID_Request **request, uint64_t type)
{
int err0, err1, dt_contig, mpi_errno = MPI_SUCCESS;
char *send_buffer;
@@ -156,7 +131,12 @@ static inline int do_isend(struct MPIDI_VC *vc,
/* Create the pack buffer (if required), and allocate */
/* a send request */
/* ---------------------------------------------------- */
+#if API_SET == API_SET_1
match_bits = init_sendtag(comm->context_id + context_offset, comm->rank, tag, type);
+#elif API_SET == API_SET_2
+ match_bits = init_sendtag_2(comm->context_id + context_offset, tag, type);
+#endif
+
sreq->dev.match.parts.tag = match_bits;
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
send_buffer = (char *) buf + dt_true_lb;
@@ -182,7 +162,11 @@ static inline int do_isend(struct MPIDI_VC *vc,
sync_req->dev.next = NULL;
REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback;
REQ_OFI(sync_req)->parent = sreq;
+#if API_SET == API_SET_1
ssend_match = init_recvtag(&ssend_mask, comm->context_id + context_offset, dest, tag);
+#elif API_SET == API_SET_2
+ ssend_match = init_recvtag_2(&ssend_mask, comm->context_id + context_offset, tag);
+#endif
ssend_match |= MPID_SYNC_SEND_ACK;
FI_RC(fi_trecv(gl_data.endpoint, /* endpoint */
NULL, /* recvbuf */
@@ -193,87 +177,27 @@ static inline int do_isend(struct MPIDI_VC *vc,
0ULL, /* mask bits */
&(REQ_OFI(sync_req)->ofi_context)), trecv);
}
+#if API_SET == API_SET_1
FI_RC(fi_tsend(gl_data.endpoint, /* Endpoint */
- send_buffer, /* Send buffer(packed or user) */
- data_sz, /* Size of the send */
- gl_data.mr, /* Dynamic memory region */
- VC_OFI(vc)->direct_addr, /* Use the address of this VC */
- match_bits, /* Match bits */
- &(REQ_OFI(sreq)->ofi_context)), tsend);
+#elif API_SET == API_SET_2
+ FI_RC(fi_tsenddata(gl_data.endpoint, /* Endpoint */
+#endif
+ send_buffer, /* Send buffer(packed or user) */
+ data_sz, /* Size of the send */
+ gl_data.mr, /* Dynamic memory region */
+#if API_SET == API_SET_2
+ comm->rank,
+#endif
+ VC_OFI(vc)->direct_addr, /* Use the address of this VC */
+ match_bits, /* Match bits */
+ &(REQ_OFI(sreq)->ofi_context)), tsend);
*request = sreq;
END_FUNC_RC(FCNAME);
}
#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_recv_posted)
-int MPID_nem_ofi_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *rreq)
-{
- int mpi_errno = MPI_SUCCESS, dt_contig, src, tag;
- uint64_t match_bits = 0, mask_bits = 0;
- fi_addr_t remote_proc = 0;
- MPIDI_msg_sz_t data_sz;
- MPI_Aint dt_true_lb;
- MPID_Datatype *dt_ptr;
- MPIR_Context_id_t context_id;
- char *recv_buffer;
- BEGIN_FUNC(FCNAME);
-
- /* ------------------------ */
- /* Initialize the request */
- /* ------------------------ */
- MPID_nem_ofi_init_req(rreq);
- REQ_OFI(rreq)->event_callback = MPID_nem_ofi_recv_callback;
- REQ_OFI(rreq)->vc = vc;
-
- /* ---------------------------------------------------- */
- /* Fill out the match info, and allocate the pack buffer */
- /* a send request */
- /* ---------------------------------------------------- */
- src = rreq->dev.match.parts.rank;
- tag = rreq->dev.match.parts.tag;
- context_id = rreq->dev.match.parts.context_id;
- match_bits = init_recvtag(&mask_bits, context_id, src, tag);
- OFI_ADDR_INIT(src, vc, remote_proc);
- MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
- dt_contig, data_sz, dt_ptr, dt_true_lb);
- if (dt_contig) {
- recv_buffer = (char *) rreq->dev.user_buf + dt_true_lb;
- }
- else {
- recv_buffer = (char *) MPIU_Malloc(data_sz);
- MPIU_ERR_CHKANDJUMP1(recv_buffer == NULL, mpi_errno, MPI_ERR_OTHER,
- "**nomem", "**nomem %s", "Recv Pack Buffer alloc");
- REQ_OFI(rreq)->pack_buffer = recv_buffer;
- }
-
- /* ---------------- */
- /* Post the receive */
- /* ---------------- */
- uint64_t msgflags;
- iovec_t iov;
- msg_tagged_t msg;
- iov.iov_base = recv_buffer;
- iov.iov_len = data_sz;
- if(REQ_OFI(rreq)->match_state == PEEK_FOUND)
- msgflags = FI_CLAIM;
- else
- msgflags = 0ULL;
-
- msg.msg_iov = &iov;
- msg.desc = NULL;
- msg.iov_count = 1;
- msg.addr = remote_proc;
- msg.tag = match_bits;
- msg.ignore = mask_bits;
- msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
- msg.data = 0;
- FI_RC(fi_trecvmsg(gl_data.endpoint,&msg,msgflags), trecv);
- END_FUNC_RC(FCNAME);
-}
-
-#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_send)
-int MPID_nem_ofi_send(struct MPIDI_VC *vc,
+int ADD_SUFFIX(MPID_nem_ofi_send)(struct MPIDI_VC *vc,
const void *buf,
int count,
MPI_Datatype datatype,
@@ -283,7 +207,7 @@ int MPID_nem_ofi_send(struct MPIDI_VC *vc,
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest, tag,
+ mpi_errno = ADD_SUFFIX(do_isend)(vc, buf, count, datatype, dest, tag,
comm, context_offset, request, MPID_NORMAL_SEND);
END_FUNC(FCNAME);
return mpi_errno;
@@ -291,7 +215,7 @@ int MPID_nem_ofi_send(struct MPIDI_VC *vc,
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_isend)
-int MPID_nem_ofi_isend(struct MPIDI_VC *vc,
+int ADD_SUFFIX(MPID_nem_ofi_isend)(struct MPIDI_VC *vc,
const void *buf,
int count,
MPI_Datatype datatype,
@@ -300,7 +224,7 @@ int MPID_nem_ofi_isend(struct MPIDI_VC *vc,
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest,
+ mpi_errno = ADD_SUFFIX(do_isend)(vc, buf, count, datatype, dest,
tag, comm, context_offset, request, MPID_NORMAL_SEND);
END_FUNC(FCNAME);
return mpi_errno;
@@ -308,7 +232,7 @@ int MPID_nem_ofi_isend(struct MPIDI_VC *vc,
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_ssend)
-int MPID_nem_ofi_ssend(struct MPIDI_VC *vc,
+int ADD_SUFFIX(MPID_nem_ofi_ssend)(struct MPIDI_VC *vc,
const void *buf,
int count,
MPI_Datatype datatype,
@@ -317,7 +241,7 @@ int MPID_nem_ofi_ssend(struct MPIDI_VC *vc,
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- mpi_errno = do_isend(vc, buf, count, datatype, dest,
+ mpi_errno = ADD_SUFFIX(do_isend)(vc, buf, count, datatype, dest,
tag, comm, context_offset, request, MPID_SYNC_SEND);
END_FUNC(FCNAME);
return mpi_errno;
@@ -325,7 +249,7 @@ int MPID_nem_ofi_ssend(struct MPIDI_VC *vc,
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_issend)
-int MPID_nem_ofi_issend(struct MPIDI_VC *vc,
+int ADD_SUFFIX(MPID_nem_ofi_issend)(struct MPIDI_VC *vc,
const void *buf,
int count,
MPI_Datatype datatype,
@@ -341,67 +265,86 @@ int MPID_nem_ofi_issend(struct MPIDI_VC *vc,
return mpi_errno;
}
-#define DO_CANCEL(req) \
-({ \
- int mpi_errno = MPI_SUCCESS; \
- int ret; \
- BEGIN_FUNC(FCNAME); \
- MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL); \
- ret = fi_cancel((fid_t)gl_data.endpoint, \
- &(REQ_OFI(req)->ofi_context)); \
- if (ret == 0) { \
- MPIR_STATUS_SET_CANCEL_BIT(req->status, TRUE); \
- } else { \
- MPIR_STATUS_SET_CANCEL_BIT(req->status, FALSE); \
- } \
- END_FUNC(FCNAME); \
- return mpi_errno; \
-})
#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_cancel_send)
-int MPID_nem_ofi_cancel_send(struct MPIDI_VC *vc ATTRIBUTE((unused)), struct MPID_Request *sreq)
+#define FCNAME DECL_FUNC(MPID_nem_ofi_recv_posted)
+int ADD_SUFFIX(MPID_nem_ofi_recv_posted)(struct MPIDI_VC *vc, struct MPID_Request *rreq)
{
- DO_CANCEL(sreq);
-}
+ int mpi_errno = MPI_SUCCESS, dt_contig, src, tag;
+ uint64_t match_bits = 0, mask_bits = 0;
+ fi_addr_t remote_proc = 0;
+ MPIDI_msg_sz_t data_sz;
+ MPI_Aint dt_true_lb;
+ MPID_Datatype *dt_ptr;
+ MPIR_Context_id_t context_id;
+ char *recv_buffer;
+ BEGIN_FUNC(FCNAME);
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_cancel_recv)
-int MPID_nem_ofi_cancel_recv(struct MPIDI_VC *vc ATTRIBUTE((unused)), struct MPID_Request *rreq)
-{
- DO_CANCEL(rreq);
+ /* ------------------------ */
+ /* Initialize the request */
+ /* ------------------------ */
+ MPID_nem_ofi_init_req(rreq);
+ REQ_OFI(rreq)->event_callback = ADD_SUFFIX(MPID_nem_ofi_recv_callback);
+ REQ_OFI(rreq)->vc = vc;
+
+ /* ---------------------------------------------------- */
+ /* Fill out the match info, and allocate the pack buffer */
+ /* a send request */
+ /* ---------------------------------------------------- */
+ src = rreq->dev.match.parts.rank;
+ tag = rreq->dev.match.parts.tag;
+ context_id = rreq->dev.match.parts.context_id;
+#if API_SET == API_SET_1
+ match_bits = init_recvtag(&mask_bits, context_id, src, tag);
+#elif API_SET == API_SET_2
+ match_bits = init_recvtag_2(&mask_bits, context_id, tag);
+#endif
+ OFI_ADDR_INIT(src, vc, remote_proc);
+ MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
+ dt_contig, data_sz, dt_ptr, dt_true_lb);
+ if (dt_contig) {
+ recv_buffer = (char *) rreq->dev.user_buf + dt_true_lb;
+ }
+ else {
+ recv_buffer = (char *) MPIU_Malloc(data_sz);
+ MPIU_ERR_CHKANDJUMP1(recv_buffer == NULL, mpi_errno, MPI_ERR_OTHER,
+ "**nomem", "**nomem %s", "Recv Pack Buffer alloc");
+ REQ_OFI(rreq)->pack_buffer = recv_buffer;
+ }
+
+ /* ---------------- */
+ /* Post the receive */
+ /* ---------------- */
+ uint64_t msgflags;
+ iovec_t iov;
+ msg_tagged_t msg;
+ iov.iov_base = recv_buffer;
+ iov.iov_len = data_sz;
+ if(REQ_OFI(rreq)->match_state == PEEK_FOUND)
+ msgflags = FI_CLAIM;
+ else
+ msgflags = 0ULL;
+
+ msg.msg_iov = &iov;
+ msg.desc = NULL;
+ msg.iov_count = 1;
+ msg.addr = remote_proc;
+ msg.tag = match_bits;
+ msg.ignore = mask_bits;
+ msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
+ msg.data = 0;
+ FI_RC(fi_trecvmsg(gl_data.endpoint,&msg,msgflags), trecv);
+ END_FUNC_RC(FCNAME);
}
+
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_posted)
-void MPID_nem_ofi_anysource_posted(MPID_Request * rreq)
+void ADD_SUFFIX(MPID_nem_ofi_anysource_posted)(MPID_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
BEGIN_FUNC(FCNAME);
- mpi_errno = MPID_nem_ofi_recv_posted(NULL, rreq);
+ mpi_errno = ADD_SUFFIX(MPID_nem_ofi_recv_posted)(NULL, rreq);
MPIU_Assert(mpi_errno == MPI_SUCCESS);
END_FUNC(FCNAME);
}
-
-#undef FCNAME
-#define FCNAME DECL_FUNC(MPID_nem_ofi_anysource_matched)
-int MPID_nem_ofi_anysource_matched(MPID_Request * rreq)
-{
- int mpi_errno = TRUE;
- int ret;
- BEGIN_FUNC(FCNAME);
- /* ----------------------------------------------------- */
- /* Netmod has notified us that it has matched an any */
- /* source request on another device. We have the chance */
- /* to cancel this shared request if it has been posted */
- /* ----------------------------------------------------- */
- ret = fi_cancel((fid_t) gl_data.endpoint, &(REQ_OFI(rreq)->ofi_context));
- if (ret == 0) {
- /* --------------------------------------------------- */
- /* Request cancelled: cancel and complete the request */
- /* --------------------------------------------------- */
- mpi_errno = FALSE;
- }
- END_FUNC(FCNAME);
- return mpi_errno;
-}
-----------------------------------------------------------------------
Summary of changes:
src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_cm.c | 115 ++++---
.../ch3/channels/nemesis/netmod/ofi/ofi_impl.h | 146 ++-------
.../ch3/channels/nemesis/netmod/ofi/ofi_init.c | 32 +--
src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_msg.c | 106 ++++---
.../nemesis/netmod/ofi/ofi_probe_template.c | 176 ++++++++++
.../ch3/channels/nemesis/netmod/ofi/ofi_progress.c | 167 +---------
.../channels/nemesis/netmod/ofi/ofi_tag_layout.h | 144 ++++++++
.../ch3/channels/nemesis/netmod/ofi/ofi_tagged.c | 343 ++------------------
.../ofi/{ofi_tagged.c => ofi_tagged_template.c} | 323 ++++++++-----------
9 files changed, 676 insertions(+), 876 deletions(-)
create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_probe_template.c
create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_tag_layout.h
copy src/mpid/ch3/channels/nemesis/netmod/ofi/{ofi_tagged.c => ofi_tagged_template.c} (65%)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list