[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1.3-116-g5c1bd3d
Service Account
noreply at mpich.org
Tue Nov 4 11:21:11 CST 2014
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via 5c1bd3d411c7e40c9d8f9f868f00fd48a525c198 (commit)
via c443c8524733d9a60fda03b9ace634ce563df2ba (commit)
from 9ea630d09e1f4e4b278579c76d878a48ea975dc5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/5c1bd3d411c7e40c9d8f9f868f00fd48a525c198
commit 5c1bd3d411c7e40c9d8f9f868f00fd48a525c198
Author: Ken Raffenetti <raffenet at mcs.anl.gov>
Date: Fri Oct 31 11:19:39 2014 -0500
portals4: create separate EQ for origin events
An EQ for origin events is useful for rate-limiting operations so that
a process does not locally trigger a flow control event on its portal.
We will implement the rate-limiting logic in the rportals layer.
Signed-off-by: Antonio J. Pena <apenya at mcs.anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
index 497a51d..46d33f7 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h
@@ -19,7 +19,8 @@ extern ptl_pt_index_t MPIDI_nem_ptl_pt;
extern ptl_pt_index_t MPIDI_nem_ptl_get_pt; /* portal for gets by receiver */
extern ptl_pt_index_t MPIDI_nem_ptl_control_pt; /* portal for MPICH control messages */
extern ptl_pt_index_t MPIDI_nem_ptl_rpt_pt; /* portal for MPICH control messages */
-extern ptl_handle_eq_t MPIDI_nem_ptl_eq;
+extern ptl_handle_eq_t MPIDI_nem_ptl_target_eq;
+extern ptl_handle_eq_t MPIDI_nem_ptl_origin_eq;
extern ptl_handle_md_t MPIDI_nem_ptl_global_md;
extern ptl_ni_limits_t MPIDI_nem_ptl_ni_limits;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
index 06a8be6..e69f4e1 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
@@ -26,7 +26,8 @@ ptl_pt_index_t MPIDI_nem_ptl_pt;
ptl_pt_index_t MPIDI_nem_ptl_get_pt; /* portal for gets by receiver */
ptl_pt_index_t MPIDI_nem_ptl_control_pt; /* portal for MPICH control messages */
ptl_pt_index_t MPIDI_nem_ptl_rpt_pt; /* portal for rportals control messages */
-ptl_handle_eq_t MPIDI_nem_ptl_eq;
+ptl_handle_eq_t MPIDI_nem_ptl_target_eq;
+ptl_handle_eq_t MPIDI_nem_ptl_origin_eq;
ptl_handle_md_t MPIDI_nem_ptl_global_md;
ptl_ni_limits_t MPIDI_nem_ptl_ni_limits;
@@ -178,26 +179,31 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
PTL_PID_ANY, &desired, &MPIDI_nem_ptl_ni_limits, &MPIDI_nem_ptl_ni);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret));
- ret = PtlEQAlloc(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_ni_limits.max_eqs, &MPIDI_nem_ptl_eq);
+ ret = PtlEQAlloc(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_ni_limits.max_eqs, &MPIDI_nem_ptl_target_eq);
+ MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret));
+
+ /* allocate a separate EQ for origin events. with this, we can implement rate-limit operations
+ to prevent a locally triggered flow control even */
+ ret = PtlEQAlloc(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_ni_limits.max_eqs, &MPIDI_nem_ptl_origin_eq);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret));
/* allocate portal for matching messages */
- ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_eq,
+ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_target_eq,
PTL_PT_ANY, &MPIDI_nem_ptl_pt);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
/* allocate portal for large messages where receiver does a get */
- ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_eq,
+ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_target_eq,
PTL_PT_ANY, &MPIDI_nem_ptl_get_pt);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
/* allocate portal for MPICH control messages */
- ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_eq,
+ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_target_eq,
PTL_PT_ANY, &MPIDI_nem_ptl_control_pt);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
/* allocate portal for MPICH control messages */
- ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_eq,
+ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_target_eq,
PTL_PT_ANY, &MPIDI_nem_ptl_rpt_pt);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
@@ -205,7 +211,7 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
md.start = 0;
md.length = (ptl_size_t)-1;
md.options = 0x0;
- md.eq_handle = MPIDI_nem_ptl_eq;
+ md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &MPIDI_nem_ptl_global_md);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
@@ -215,7 +221,7 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret));
/* allow rportal to manage the primary portal and retransmit if needed */
- ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_eq, MPIDI_nem_ptl_pt, MPIDI_nem_ptl_rpt_pt);
+ ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_pt, MPIDI_nem_ptl_rpt_pt);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
/* allow rportal to manage the get and control portals, but we
@@ -223,10 +229,10 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
* we pass PTL_PT_ANY as the dummy portal. unfortunately, portals
* does not have an "invalid" PT constant, which would have been
* more appropriate to pass over here. */
- ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_eq, MPIDI_nem_ptl_get_pt, PTL_PT_ANY);
+ ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_get_pt, PTL_PT_ANY);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
- ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_eq, MPIDI_nem_ptl_control_pt, PTL_PT_ANY);
+ ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_control_pt, PTL_PT_ANY);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret));
/* create business card */
@@ -255,6 +261,7 @@ static int ptl_finalize(void)
{
int mpi_errno = MPI_SUCCESS;
int ret;
+ ptl_handle_eq_t eqs[2];
MPIDI_STATE_DECL(MPID_STATE_PTL_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_PTL_FINALIZE);
@@ -266,7 +273,9 @@ static int ptl_finalize(void)
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* shut down portals */
- ret = MPID_nem_ptl_rptl_drain_eq(1, &MPIDI_nem_ptl_eq);
+ eqs[0] = MPIDI_nem_ptl_target_eq;
+ eqs[1] = MPIDI_nem_ptl_origin_eq;
+ ret = MPID_nem_ptl_rptl_drain_eq(2, eqs);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptfree", "**ptlptfree %s", MPID_nem_ptl_strerror(ret));
ret = MPID_nem_ptl_rptl_ptfini(MPIDI_nem_ptl_pt);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c
index 60e07a0..26a1eb2 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c
@@ -131,10 +131,17 @@ int MPID_nem_ptl_poll(int is_blocking_poll)
/* MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_POLL); */
while (1) {
- ret = MPID_nem_ptl_rptl_eqget(MPIDI_nem_ptl_eq, &event);
- if (ret == PTL_EQ_EMPTY)
- break;
+ /* check both origin and target EQs for events */
+ ret = MPID_nem_ptl_rptl_eqget(MPIDI_nem_ptl_target_eq, &event);
MPIU_ERR_CHKANDJUMP(ret == PTL_EQ_DROPPED, mpi_errno, MPI_ERR_OTHER, "**eqdropped");
+ if (ret == PTL_EQ_EMPTY) {
+ ret = MPID_nem_ptl_rptl_eqget(MPIDI_nem_ptl_origin_eq, &event);
+ MPIU_ERR_CHKANDJUMP(ret == PTL_EQ_DROPPED, mpi_errno, MPI_ERR_OTHER, "**eqdropped");
+
+ /* if both queues are empty, exit the loop */
+ if (ret == PTL_EQ_EMPTY)
+ break;
+ }
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqget", "**ptleqget %s", MPID_nem_ptl_strerror(ret));
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "Received event %s ni_fail=%s list=%s user_ptr=%p hdr_data=%#lx mlength=%lu",
MPID_nem_ptl_strevent(&event), MPID_nem_ptl_strnifail(event.ni_fail_type),
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
index fd694f5..cca6d4c 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c
@@ -255,7 +255,7 @@ static int handler_recv_dequeue_large(const ptl_event_t *e)
md.start = rreq->dev.iov;
md.length = rreq->dev.iov_count;
md.options = PTL_IOVEC;
- md.eq_handle = MPIDI_nem_ptl_eq;
+ md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(rreq)->md);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
@@ -680,7 +680,7 @@ int MPID_nem_ptl_lmt_start_recv(MPIDI_VC_t *vc, MPID_Request *rreq, MPID_IOV s_
md.start = rreq->dev.iov;
md.length = rreq->dev.iov_count;
md.options = PTL_IOVEC;
- md.eq_handle = MPIDI_nem_ptl_eq;
+ md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(rreq)->md);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s",
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
index a6fab84..796f559 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c
@@ -240,7 +240,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
md.start = sreq->dev.iov;
md.length = sreq->dev.iov_count;
md.options = PTL_IOVEC;
- md.eq_handle = MPIDI_nem_ptl_eq;
+ md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(sreq)->md);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
@@ -352,7 +352,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
md.start = sreq->dev.iov;
md.length = initial_iov_count;
md.options = PTL_IOVEC;
- md.eq_handle = MPIDI_nem_ptl_eq;
+ md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(sreq)->md);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
http://git.mpich.org/mpich.git/commitdiff/c443c8524733d9a60fda03b9ace634ce563df2ba
commit c443c8524733d9a60fda03b9ace634ce563df2ba
Author: Ken Raffenetti <raffenet at mcs.anl.gov>
Date: Fri Oct 31 11:13:01 2014 -0500
portals4: create EQ using limit from NI init
Rather than use an EQ limit that may be lower than the system default,
just create our EQ using the returned maximum from PtlNIInit.
Signed-off-by: Antonio J. Pena <apenya at mcs.anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
index 4a53a3d..06a8be6 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
@@ -178,7 +178,7 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
PTL_PID_ANY, &desired, &MPIDI_nem_ptl_ni_limits, &MPIDI_nem_ptl_ni);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret));
- ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EQ_COUNT, &MPIDI_nem_ptl_eq);
+ ret = PtlEQAlloc(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_ni_limits.max_eqs, &MPIDI_nem_ptl_eq);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret));
/* allocate portal for matching messages */
-----------------------------------------------------------------------
Summary of changes:
.../channels/nemesis/netmod/portals4/ptl_impl.h | 3 +-
.../channels/nemesis/netmod/portals4/ptl_init.c | 31 +++++++++++++-------
.../channels/nemesis/netmod/portals4/ptl_poll.c | 13 ++++++--
.../channels/nemesis/netmod/portals4/ptl_recv.c | 4 +-
.../channels/nemesis/netmod/portals4/ptl_send.c | 4 +-
5 files changed, 36 insertions(+), 19 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list