[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.0.4-178-ge9579e1
mysql vizuser
noreply at mpich.org
Thu May 9 13:58:03 CDT 2013
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via e9579e1c6f2ed201225b51901129870e90be734e (commit)
via 87e84a5e3892a68c503c2059918b31506d9ddc73 (commit)
via 5ef1a628b2feb1ce786d95691213171493fecaf9 (commit)
via c60642ccede4a913d2b025fd6f65c340fe5a8f5f (commit)
via f29783c21fd7c5ea74be0ca3b0a920f48b92c730 (commit)
via ef90f1b9309ab8ffc67658b5a4cbf81f3e2a9468 (commit)
via 1daeae3155739627d291277c89669f0c633c1e5a (commit)
via a45954b594b5a5ba4a54acdcb8fda7c2e819a40f (commit)
via dcf62e917496501a74fe6cac918c8967316056dc (commit)
via 77999d6f565e65e7f1c5ee9f00dda99b28186f0d (commit)
from c478a4fe8d4cad97a2412c13b02d445181f78a56 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/e9579e1c6f2ed201225b51901129870e90be734e
commit e9579e1c6f2ed201225b51901129870e90be734e
Author: Charles Archer <archerc at us.ibm.com>
Date: Fri Apr 5 07:24:24 2013 -0400
use void* for sizeof detection instead of int
(ibm) D189618
(ibm) f30c0544fcc37fc18cb0f9b27f414373d5d5308b
Signed-off-by: Bob Cernohous <bobc at us.ibm.com>
diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c
index 9a4dfc4..44e0137 100644
--- a/src/mpid/pamid/src/mpid_init.c
+++ b/src/mpid/pamid/src/mpid_init.c
@@ -1344,7 +1344,7 @@ int MPIDI_Banner(char * bufPtr) {
}
}
- if(sizeof(int) == 8)
+ if(sizeof(void*) == 8)
strcpy(type, "64bit (MPI over PAMI)");
else if(sizeof(int) == 4)
strcpy(type, "32bit (MPI over PAMI)");
diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c
index bf9c423..a41ad77 100644
--- a/src/mpid/pamid/src/mpidi_util.c
+++ b/src/mpid/pamid/src/mpidi_util.c
@@ -562,7 +562,7 @@ int MPIDI_Print_mpenv(int rank,int size)
else strcpy(sender.libpath, "NOT SET");
getcwd(sender.directory, FILENAME_MAX+1);
- if(sizeof(int) == 4)
+ if(sizeof(void*) == 4)
sender.mode_64 = 0;
else
sender.mode_64 = 1;
http://git.mpich.org/mpich.git/commitdiff/87e84a5e3892a68c503c2059918b31506d9ddc73
commit 87e84a5e3892a68c503c2059918b31506d9ddc73
Author: Charles Archer <archerc at us.ibm.com>
Date: Thu Apr 4 11:58:52 2013 -0400
detect 32 bit vs 64 bit via sizeof
(ibm) D189618
(ibm) 73870926a4a9598deacbc5c0b91fa6150ef8f7ed
Signed-off-by: Bob Cernohous <bobc at us.ibm.com>
diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c
index 7e7d00c..9a4dfc4 100644
--- a/src/mpid/pamid/src/mpid_init.c
+++ b/src/mpid/pamid/src/mpid_init.c
@@ -1344,11 +1344,12 @@ int MPIDI_Banner(char * bufPtr) {
}
}
-#ifndef __32BIT__
- strcpy(type, "64bit (MPI over PAMI)");
-#else
- strcpy(type, "32bit (MPI over PAMI)");
-#endif
+ if(sizeof(int) == 8)
+ strcpy(type, "64bit (MPI over PAMI)");
+ else if(sizeof(int) == 4)
+ strcpy(type, "32bit (MPI over PAMI)");
+ else
+ strcpy(type, "UNKNOWN-bit (MPI over PAMI)");
sprintf(msgBuf,"MPICH library was compiled on");
diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c
index 5f9f2cf..bf9c423 100644
--- a/src/mpid/pamid/src/mpidi_util.c
+++ b/src/mpid/pamid/src/mpidi_util.c
@@ -561,11 +561,12 @@ int MPIDI_Print_mpenv(int rank,int size)
if (cp) strncpy(sender.libpath, cp, MAXPATHLEN-1);
else strcpy(sender.libpath, "NOT SET");
getcwd(sender.directory, FILENAME_MAX+1);
-#ifdef __32BIT__
- sender.mode_64 = 0;
-#else
- sender.mode_64 = 1;
-#endif
+
+ if(sizeof(int) == 4)
+ sender.mode_64 = 0;
+ else
+ sender.mode_64 = 1;
+
sender.threaded = 1; /* Always 1 */
cp = getenv("AIXTHREAD_SCOPE");
if (cp) strncpy(sender.thread_scope, cp, 8);
http://git.mpich.org/mpich.git/commitdiff/5ef1a628b2feb1ce786d95691213171493fecaf9
commit 5ef1a628b2feb1ce786d95691213171493fecaf9
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Tue Mar 26 09:15:47 2013 -0500
Fix alltoall handling of non-PAMI datatypes and non-continuous datatypes.
(ibm) 36f5ac777f738068690184193b557fab53b3472f
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c
index 646b89f..d970069 100644
--- a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c
+++ b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c
@@ -45,7 +45,10 @@ int MPIDO_Alltoallv(const void *sendbuf,
{
TRACE_ERR("Entering MPIDO_Alltoallv\n");
volatile unsigned active = 1;
- int sndtypelen, rcvtypelen, snd_contig, rcv_contig;
+ void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL;
+ void *sbuf = NULL, *rbuf = NULL;
+ int recvok=PAMI_SUCCESS, sendok=PAMI_SUCCESS;
+ int sndtypelen, rcvtypelen, snd_contig=0, rcv_contig=0;
MPID_Datatype *sdt, *rdt;
pami_type_t stype, rtype;
MPI_Aint sdt_true_lb, rdt_true_lb;
@@ -61,17 +64,20 @@ int MPIDO_Alltoallv(const void *sendbuf,
#endif
const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid);
const int selected_type = mpid->user_selected_type[PAMI_XFER_ALLTOALLV_INT];
-
- if((sendbuf != MPI_IN_PLACE) && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS))
- pamidt = 0;
- if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS)
- pamidt = 0;
-
- MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb);
- if(!rcv_contig) pamidt = 0;
-
- if((selected_type == MPID_COLL_USE_MPICH) ||
- pamidt == 0)
+ const int size = comm_ptr->local_size;
+ int sendcontinuous , recvcontinuous=0;
+ size_t recv_size=0, send_size=0;
+ size_t totalrecvcount=0;
+ int *lrecvdispls = NULL; /* possible local displs calculated for noncontinous */
+ int *lsenddispls = NULL;/* possible local displs calculated for noncontinous */
+ int *lrecvcounts = NULL;/* possible local counts calculated for noncontinous */
+ int *lsendcounts = NULL;/* possible local counts calculated for noncontinous */
+ const int *precvdispls = recvdispls; /* pointer to displs to use as pami parmi */
+ const int *psenddispls = senddispls; /* pointer to displs to use as pami parmi */
+ const int *precvcounts = recvcounts; /* pointer to counts to use as pami parmi */
+ const int *psendcounts = sendcounts; /* pointer to counts to use as pami parmi */
+ int inplace = sendbuf == MPI_IN_PLACE? 1 : 0;
+ if(selected_type == MPID_COLL_USE_MPICH)
{
if(unlikely(verbose))
fprintf(stderr,"Using MPICH alltoallv algorithm\n");
@@ -80,6 +86,148 @@ int MPIDO_Alltoallv(const void *sendbuf,
comm_ptr, mpierrno);
}
+ if(!inplace)
+ {
+ sendok = MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp);
+ MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndtypelen, sdt, sdt_true_lb);
+ sbuf = (char *)sendbuf + sdt_true_lb;
+ if(!snd_contig || (sendok != PAMI_SUCCESS))
+ {
+ stype = PAMI_TYPE_UNSIGNED_CHAR;
+ size_t totalsendcount = sendcounts[0];
+ sendcontinuous = senddispls[0] == 0? 1 : 0 ;
+ int i;
+ psenddispls = lsenddispls = MPIU_Malloc(size*sizeof(int));
+ psendcounts = lsendcounts = MPIU_Malloc(size*sizeof(int));
+ lsenddispls[0]= 0;
+ lsendcounts[0]= sndtypelen * sendcounts[0];
+ for(i=1; i<size; ++i)
+ {
+ lsenddispls[i]= sndtypelen * totalsendcount;
+ totalsendcount += sendcounts[i];
+ if(senddispls[i] != (senddispls[i-1] + (sendcounts[i-1]*sndtypelen)))
+ sendcontinuous = 0;
+ lsendcounts[i]= sndtypelen * sendcounts[i];
+ }
+ send_size = sndtypelen * totalsendcount;
+ TRACE_ERR("Pack receive sndv_contig %zu, sendok %zd, totalsendcount %zu, sendcontinuous %zu, sndtypelen %zu, send_size %zu\n",
+ (size_t)snd_contig, (size_t)sendok, (size_t)totalsendcount, (size_t)sendcontinuous, (size_t) sndtypelen, (size_t)send_size);
+ snd_noncontig_buff = MPIU_Malloc(send_size);
+ sbuf = snd_noncontig_buff;
+ if(snd_noncontig_buff == NULL)
+ {
+ MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
+ "Fatal: Cannot allocate pack buffer");
+ }
+ if(sendcontinuous)
+ {
+ MPIR_Localcopy(sendbuf, totalsendcount, sendtype,
+ snd_noncontig_buff, send_size,MPI_CHAR);
+ }
+ else
+ {
+ size_t extent;
+ MPID_Datatype_get_extent_macro(sendtype,extent);
+ for(i=0; i<size; ++i)
+ {
+ char* scbuf = (char*)sendbuf + senddispls[i]*extent;
+ char* rcbuf = (char*)snd_noncontig_buff + psenddispls[i];
+ MPIR_Localcopy(scbuf, sendcounts[i], sendtype,
+ rcbuf, psendcounts[i], MPI_CHAR);
+ TRACE_ERR("Pack send src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)extent, (size_t)i,(size_t)senddispls[i],(size_t)i,(size_t)sendcounts[i],(size_t)senddispls[i], *(int*)scbuf);
+ TRACE_ERR("Pack send dest displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)i,(size_t)psenddispls[i],(size_t)i,(size_t)psendcounts[i],(size_t)psenddispls[i], *(int*)rcbuf);
+ }
+ }
+ }
+ }
+ MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb);
+ recvok = MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp);
+ rbuf = (char *)recvbuf + rdt_true_lb;
+ if(!rcv_contig || (recvok != PAMI_SUCCESS))
+ {
+ rtype = PAMI_TYPE_UNSIGNED_CHAR;
+ totalrecvcount = recvcounts[0];
+ recvcontinuous = recvdispls[0] == 0? 1 : 0 ;
+ int i;
+ precvdispls = lrecvdispls = MPIU_Malloc(size*sizeof(int));
+ precvcounts = lrecvcounts = MPIU_Malloc(size*sizeof(int));
+ lrecvdispls[0]= 0;
+ lrecvcounts[0]= rcvtypelen * recvcounts[0];
+ for(i=1; i<size; ++i)
+ {
+ lrecvdispls[i]= rcvtypelen * totalrecvcount;
+ totalrecvcount += recvcounts[i];
+ if(recvdispls[i] != (recvdispls[i-1] + (recvcounts[i-1]*rcvtypelen)))
+ recvcontinuous = 0;
+ lrecvcounts[i]= rcvtypelen * recvcounts[i];
+ }
+ recv_size = rcvtypelen * totalrecvcount;
+ TRACE_ERR("Pack receive rcv_contig %zu, recvok %zd, totalrecvcount %zu, recvcontinuous %zu, rcvtypelen %zu, recv_size %zu\n",
+ (size_t)rcv_contig, (size_t)recvok, (size_t)totalrecvcount, (size_t)recvcontinuous,(size_t)rcvtypelen, (size_t)recv_size);
+ rcv_noncontig_buff = MPIU_Malloc(recv_size);
+ rbuf = rcv_noncontig_buff;
+ if(rcv_noncontig_buff == NULL)
+ {
+ MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
+ "Fatal: Cannot allocate pack buffer");
+ }
+ /* need to copy it now if it's used for the send buffer and then do not do in place */
+ if(inplace)
+ {
+ inplace = 0;
+ stype = PAMI_TYPE_UNSIGNED_CHAR;
+ size_t totalsendcount = recvcounts[0];
+ sendcontinuous = recvdispls[0] == 0? 1 : 0 ;
+ int i;
+ psenddispls = lsenddispls = MPIU_Malloc(size*sizeof(int));
+ psendcounts = lsendcounts = MPIU_Malloc(size*sizeof(int));
+ lsenddispls[0]= 0;
+ lsendcounts[0]= rcvtypelen * recvcounts[0];
+ for(i=1; i<size; ++i)
+ {
+ lsenddispls[i]= rcvtypelen * totalsendcount;
+ totalsendcount += recvcounts[i];
+ if(recvdispls[i] != (recvdispls[i-1] + (recvcounts[i-1]*rcvtypelen)))
+ sendcontinuous = 0;
+ lsendcounts[i]= rcvtypelen * recvcounts[i];
+ }
+ send_size = rcvtypelen * totalsendcount;
+ TRACE_ERR("Pack MPI_IN_PLACE receive sndv_contig %zu, sendok %zd, totalsendcount %zu, sendcontinuous %zu, rcvtypelen %zu, send_size %zu\n",
+ (size_t)snd_contig, (size_t)sendok, (size_t)totalsendcount, (size_t)sendcontinuous, (size_t) rcvtypelen, (size_t)send_size);
+ snd_noncontig_buff = MPIU_Malloc(send_size);
+ sbuf = snd_noncontig_buff;
+ if(snd_noncontig_buff == NULL)
+ {
+ MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1,
+ "Fatal: Cannot allocate pack buffer");
+ }
+ if(sendcontinuous)
+ {
+ MPIR_Localcopy(recvbuf, totalsendcount, recvtype,
+ snd_noncontig_buff, send_size,MPI_CHAR);
+ }
+ else
+ {
+ size_t extent;
+ MPID_Datatype_get_extent_macro(recvtype,extent);
+ for(i=0; i<size; ++i)
+ {
+ char* scbuf = (char*)recvbuf + recvdispls[i]*extent;
+ char* rcbuf = (char*)snd_noncontig_buff + psenddispls[i];
+ MPIR_Localcopy(scbuf, recvcounts[i], recvtype,
+ rcbuf, psendcounts[i], MPI_CHAR);
+ TRACE_ERR("Pack send src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)extent, (size_t)i,(size_t)recvdispls[i],(size_t)i,(size_t)recvcounts[i],(size_t)recvdispls[i], *(int*)scbuf);
+ TRACE_ERR("Pack send dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)extent, (size_t)i,(size_t)psenddispls[i],(size_t)i,(size_t)psendcounts[i],(size_t)psenddispls[i], *(int*)rcbuf);
+ }
+ }
+ }
+ }
+
+
pami_xfer_t alltoallv;
pami_algorithm_t my_alltoallv;
const pami_metadata_t *my_md = (pami_metadata_t *)NULL;
@@ -106,27 +254,26 @@ int MPIDO_Alltoallv(const void *sendbuf,
alltoallv.cb_done = cb_alltoallv;
alltoallv.cookie = (void *)&active;
/* We won't bother with alltoallv since MPI is always going to be ints. */
- if(sendbuf == MPI_IN_PLACE)
+ if(inplace)
{
if(unlikely(verbose))
fprintf(stderr,"alltoallv MPI_IN_PLACE buffering\n");
alltoallv.cmd.xfer_alltoallv_int.stype = rtype;
- alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) recvdispls;
- alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) recvcounts;
+ alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) precvdispls;
+ alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) precvcounts;
alltoallv.cmd.xfer_alltoallv_int.sndbuf = PAMI_IN_PLACE;
}
else
{
- MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndtypelen, sdt, sdt_true_lb);
alltoallv.cmd.xfer_alltoallv_int.stype = stype;
- alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) senddispls;
- alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) sendcounts;
- alltoallv.cmd.xfer_alltoallv_int.sndbuf = (char *)sendbuf+sdt_true_lb;
+ alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) psenddispls;
+ alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) psendcounts;
+ alltoallv.cmd.xfer_alltoallv_int.sndbuf = sbuf;
}
- alltoallv.cmd.xfer_alltoallv_int.rcvbuf = (char *)recvbuf+rdt_true_lb;
+ alltoallv.cmd.xfer_alltoallv_int.rcvbuf = rbuf;
- alltoallv.cmd.xfer_alltoallv_int.rdispls = (int *) recvdispls;
- alltoallv.cmd.xfer_alltoallv_int.rtypecounts = (int *) recvcounts;
+ alltoallv.cmd.xfer_alltoallv_int.rdispls = (int *) precvdispls;
+ alltoallv.cmd.xfer_alltoallv_int.rtypecounts = (int *) precvcounts;
alltoallv.cmd.xfer_alltoallv_int.rtype = rtype;
if(unlikely(queryreq == MPID_COLL_ALWAYS_QUERY ||
@@ -205,6 +352,37 @@ int MPIDO_Alltoallv(const void *sendbuf,
TRACE_ERR("%d waiting on active %d\n", rank, active);
MPID_PROGRESS_WAIT_WHILE(active);
+ if(!rcv_contig || (recvok != PAMI_SUCCESS))
+ {
+ if(recvcontinuous)
+ {
+ MPIR_Localcopy(rcv_noncontig_buff, recv_size,MPI_CHAR,
+ recvbuf, totalrecvcount, recvtype);
+ }
+ else
+ {
+ size_t extent;
+ int i;
+ MPID_Datatype_get_extent_macro(recvtype,extent);
+ for(i=0; i<size; ++i)
+ {
+ char* scbuf = (char*)rcv_noncontig_buff+ precvdispls[i];
+ char* rcbuf = (char*)recvbuf + recvdispls[i]*extent;
+ MPIR_Localcopy(scbuf, precvcounts[i], MPI_CHAR,
+ rcbuf, recvcounts[i], recvtype);
+ TRACE_ERR("Pack recv src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)extent, (size_t)i,(size_t)precvdispls[i],(size_t)i,(size_t)precvcounts[i],(size_t)precvdispls[i], *(int*)scbuf);
+ TRACE_ERR("Pack recv dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n",
+ (size_t)extent, (size_t)i,(size_t)recvdispls[i],(size_t)i,(size_t)recvcounts[i],(size_t)recvdispls[i], *(int*)rcbuf);
+ }
+ }
+ MPIU_Free(rcv_noncontig_buff);
+ }
+ if(!snd_contig || (sendok != PAMI_SUCCESS)) MPIU_Free(snd_noncontig_buff);
+ if(lrecvdispls) MPIU_Free(lrecvdispls);
+ if(lsenddispls) MPIU_Free(lsenddispls);
+ if(lrecvcounts) MPIU_Free(lrecvcounts);
+ if(lsendcounts) MPIU_Free(lsendcounts);
TRACE_ERR("Leaving alltoallv\n");
http://git.mpich.org/mpich.git/commitdiff/c60642ccede4a913d2b025fd6f65c340fe5a8f5f
commit c60642ccede4a913d2b025fd6f65c340fe5a8f5f
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Wed Mar 6 15:41:26 2013 -0600
BGQ must create geometries with (existing) tasklists
(ibm) Issue 7967
(ibm) 6b7ba8eb718d8323c8dd638c27cca6a05baa202b
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/include/mpidi_macros.h b/src/mpid/pamid/include/mpidi_macros.h
index 14d0195..144b0b6 100644
--- a/src/mpid/pamid/include/mpidi_macros.h
+++ b/src/mpid/pamid/include/mpidi_macros.h
@@ -114,7 +114,14 @@ _dt_contig_out, _data_sz_out, _dt_ptr, _dt_true_lb) \
vcr[index]->taskid; \
})
-
+#ifdef __BGQ__
+/* BGQ just shares the MPICH vcr/tasklist.
+ This relies on the VCR being a simple task list which is asserted
+ in static_assertions() in mpid_init.c */
+#define MPID_VCR_GET_LPIDS(comm, taskids) taskids = &((*comm->vcr)->taskid);
+#define MPID_VCR_FREE_LPIDS(taskids)
+#else
+/* non-BGQ mallocs and copies the MPICH vcr/tasklist */
#define MPID_VCR_GET_LPIDS(comm, taskids) \
({ \
int i; \
@@ -123,7 +130,9 @@ _dt_contig_out, _data_sz_out, _dt_ptr, _dt_true_lb) \
for(i=0; i<comm->local_size; i++) \
taskids[i] = comm->vcr[i]->taskid; \
})
+#define MPID_VCR_FREE_LPIDS(taskids) MPIU_Free(taskids)
+#endif
#define MPID_GPID_Get(comm_ptr, rank, gpid) \
({ \
diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c
index 8ee9532..a8be80c 100644
--- a/src/mpid/pamid/src/comm/mpid_comm.c
+++ b/src/mpid/pamid/src/comm/mpid_comm.c
@@ -245,13 +245,13 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm)
comm->mpid.tasks = NULL;
for(i=1;i<comm->local_size;i++)
{
- /* only if sequential tasks should we use a (single) range.
- Multi or reordered ranges are inefficient */
- if(MPID_VCR_GET_LPID(comm->vcr, i) != (MPID_VCR_GET_LPID(comm->vcr, i-1) + 1)) {
- /* not sequential, use tasklist */
- MPID_VCR_GET_LPIDS(comm, comm->mpid.tasks);
- break;
- }
+ /* only if sequential tasks should we use a (single) range.
+ Multi or reordered ranges are inefficient */
+ if(MPID_VCR_GET_LPID(comm->vcr, i) != (MPID_VCR_GET_LPID(comm->vcr, i-1) + 1)) {
+ /* not sequential, use tasklist */
+ MPID_VCR_GET_LPIDS(comm, comm->mpid.tasks);
+ break;
+ }
}
/* Should we use a range? (no task list set) */
if(comm->mpid.tasks == NULL)
@@ -421,7 +421,7 @@ void MPIDI_Coll_comm_destroy(MPID_Comm *comm)
TRACE_ERR("Waiting for geom destroy to finish\n");
MPID_PROGRESS_WAIT_WHILE(geom_destroy);
- MPIU_Free(comm->mpid.tasks);
+ MPID_VCR_FREE_LPIDS(comm->mpid.tasks);
/* TRACE_ERR("Freeing geometry ranges\n");
MPIU_TestFree(&comm->mpid.tasks_descriptor.ranges);
*/
diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c
index 12983fc..7e7d00c 100644
--- a/src/mpid/pamid/src/mpid_init.c
+++ b/src/mpid/pamid/src/mpid_init.c
@@ -1374,6 +1374,7 @@ int MPIDI_Banner(char * bufPtr) {
sprintf(bufPtr, "%s %s %s %s ", type, ver_buf, msgBuf, ASC_time);
} else {
sprintf(bufPtr, "%s %s %s ", type, msgBuf, ASC_time);
+
}
MPIU_Free(tmx);
@@ -1388,6 +1389,11 @@ static_assertions()
MPID_assert_static(sizeof(void*) == sizeof(size_t));
MPID_assert_static(sizeof(uintptr_t) == sizeof(size_t));
#ifdef __BGQ__
+ /* MPID_VCR_GET_LPIDS relies on the VCR being a simple task list */
+ MPID_VCR vcr=NULL;
+ MPID_assert_static(sizeof(*vcr) == sizeof(pami_task_t));/* VCR is a simple task list */
+ MPID_assert_static(sizeof(vcr->taskid) == sizeof(*vcr));/* VCR is a simple task list */
+
MPID_assert_static(sizeof(MPIDI_MsgInfo) == 16);
MPID_assert_static(sizeof(uint64_t) == sizeof(size_t));
#endif
http://git.mpich.org/mpich.git/commitdiff/f29783c21fd7c5ea74be0ca3b0a920f48b92c730
commit f29783c21fd7c5ea74be0ca3b0a920f48b92c730
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Thu Mar 21 11:58:55 2013 -0500
Remove ticket #595 circumvention
(ibm) Trac #659
(ibm) 9d36aed8f988859eba1a284292f0a1a6fc75d01d
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/comm/mpid_selectcolls.c b/src/mpid/pamid/src/comm/mpid_selectcolls.c
index cd30053..571eb68 100644
--- a/src/mpid/pamid/src/comm/mpid_selectcolls.c
+++ b/src/mpid/pamid/src/comm/mpid_selectcolls.c
@@ -370,15 +370,6 @@ void MPIDI_Comm_coll_envvars(MPID_Comm *comm)
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_SCATTERV", NULL};
MPIDI_Check_protocols(names, comm, "scatterv", PAMI_XFER_SCATTERV_INT);
-
- /* Use MPICH on large communicators (Issue 7516 and ticket 595)*/
- if((comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] ==
- MPID_COLL_NOSELECTION) /* no env var selected */
- && (comm->local_size > (16*1024))) /* and > 16k ranks */
- {
- comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_USE_MPICH;
- comm->mpid.user_selected[PAMI_XFER_SCATTERV_INT] = 0;
- }
}
TRACE_ERR("Checking scatter\n");
http://git.mpich.org/mpich.git/commitdiff/ef90f1b9309ab8ffc67658b5a4cbf81f3e2a9468
commit ef90f1b9309ab8ffc67658b5a4cbf81f3e2a9468
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Mon Mar 11 18:17:03 2013 -0500
Update scatter[v] protocol selection
Do not tie scatterv to scatter selection.
(ibm) Issue 9416
(ibm) 4901e679071e489e38ee1a6925da36eab9c3ffac
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c
index 70a25ee..8ee9532 100644
--- a/src/mpid/pamid/src/comm/mpid_comm.c
+++ b/src/mpid/pamid/src/comm/mpid_comm.c
@@ -237,7 +237,7 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm)
if(comm->mpid.geometry != MPIDI_Process.world_geometry)
{
- if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL)
+ if(unlikely(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0))
fprintf(stderr,"world geom: %p parent geom: %p\n", MPIDI_Process.world_geometry, comm->mpid.parent);
TRACE_ERR("Creating subgeom\n");
/* Change to this at some point */
diff --git a/src/mpid/pamid/src/comm/mpid_optcolls.c b/src/mpid/pamid/src/comm/mpid_optcolls.c
index 23f63d7..03192f3 100644
--- a/src/mpid/pamid/src/comm/mpid_optcolls.c
+++ b/src/mpid/pamid/src/comm/mpid_optcolls.c
@@ -273,6 +273,128 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr)
opt_proto = -1;
mustquery = 0;
+ /* Scatter */
+ /* If the user has forced a selection, don't bother setting it here */
+ if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_NOSELECTION)
+ {
+ TRACE_ERR("No scatter env var, so setting optimized scatter\n");
+ /* The best scatter seems to be I0:Binomial (at least better than I0:Flat) */
+ /* Loop over the protocols until we find the one we want */
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTER][0]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][0][i].name, "I0:Binomial:-:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][0][i].name, "I0:Binomial:SHMEM:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ }
+ if(opt_proto == -1) /* check other list */
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTER][1]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][1][i].name, "I0:Binomial:-:MU") == 0)
+ {
+ opt_proto = i;
+ mustquery = 1;
+ break;
+ }
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][1][i].name, "I0:Binomial:SHMEM:MU") == 0)
+ {
+ opt_proto = i;
+ mustquery = 1;
+ break;
+ }
+ }
+ /* Actually, MPICH looks better right now, so use MPICH
+ if(opt_proto != -1)
+ {
+ TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n",
+ PAMI_XFER_SCATTER, opt_proto,
+ comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][mustquery][opt_proto].name);
+ comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTER][0] =
+ comm_ptr->mpid.coll_algorithm[PAMI_XFER_SCATTER][mustquery][opt_proto];
+ memcpy(&comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTER][0],
+ &comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][mustquery][opt_proto],
+ sizeof(pami_metadata_t));
+ comm_ptr->mpid.must_query[PAMI_XFER_SCATTER][0] = mustquery?MPID_COLL_ALWAYS_QUERY:MPID_COLL_NOQUERY;
+ comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] = MPID_COLL_OPTIMIZED;
+ }
+ else
+ */
+ {
+ TRACE_ERR("Scatter reverting to MPICH\n");
+ comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] = MPID_COLL_USE_MPICH;
+ comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTER][0] = 0;
+ }
+ TRACE_ERR("Done setting optimized scatter\n");
+ }
+ opt_proto = -1;
+ mustquery = 0;
+ /* Scatterv */
+ /* If the user has forced a selection, don't bother setting it here */
+ if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_NOSELECTION)
+ {
+ TRACE_ERR("No scatterv env var, so setting optimized scatterv\n");
+ /* The best scatterv seems to be I0:SoftwareTree: */
+ /* Loop over the protocols until we find the one we want */
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTERV_INT][0]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][0][i].name, "I0:SoftwareTree:-:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][0][i].name, "I0:SoftwareTree:SHMEM:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ }
+ if(opt_proto == -1) /* check other list */
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTERV_INT][1]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][1][i].name, "I0:SoftwareTree:-:MU") == 0)
+ {
+ opt_proto = i;
+ mustquery = 1;
+ break;
+ }
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][1][i].name, "I0:SoftwareTree:SHMEM:MU") == 0)
+ {
+ opt_proto = i;
+ mustquery = 1;
+ break;
+ }
+ }
+ if(opt_proto != -1)
+ {
+ TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n",
+ PAMI_XFER_SCATTERV_INT, opt_proto,
+ comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto].name);
+ comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTERV_INT][0] =
+ comm_ptr->mpid.coll_algorithm[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto];
+ memcpy(&comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTERV_INT][0],
+ &comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto],
+ sizeof(pami_metadata_t));
+ comm_ptr->mpid.must_query[PAMI_XFER_SCATTERV_INT][0] = mustquery?MPID_COLL_ALWAYS_QUERY:MPID_COLL_NOQUERY;
+ comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_OPTIMIZED;
+ }
+ else
+ {
+ TRACE_ERR("Scatterv reverting to MPICH\n");
+ comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_USE_MPICH;
+ comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTERV_INT][0] = 0;
+ }
+ TRACE_ERR("Done setting optimized scatterv\n");
+ }
+
+
+ opt_proto = -1;
+ mustquery = 0;
/* Alltoall */
/* If the user has forced a selection, don't bother setting it here */
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION)
@@ -859,6 +981,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr)
fprintf(stderr,"Selecting MPICH for allgatherv below %d size comm %p\n", comm_ptr->mpid.cutoff_size[PAMI_XFER_ALLGATHERV_INT][0], comm_ptr);
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_GATHER] == MPID_COLL_USE_MPICH)
fprintf(stderr,"Selecting MPICH for gather comm %p\n", comm_ptr);
+ if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_USE_MPICH)
+ fprintf(stderr,"Selecting MPICH for scatter comm %p\n", comm_ptr);
+ else if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_OPTIMIZED)
+ fprintf(stderr,"Selecting %s for scatter comm %p\n",
+ comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTER][0].name,
+ comm_ptr);
+ if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_USE_MPICH)
+ fprintf(stderr,"Selecting MPICH for scatterv comm %p\n", comm_ptr);
+ else if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_OPTIMIZED)
+ fprintf(stderr,"Selecting %s for scatterv comm %p\n",
+ comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTERV_INT][0].name,
+ comm_ptr);
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_BROADCAST] == MPID_COLL_OPTIMIZED)
fprintf(stderr,"Selecting %s for opt bcast up to size %d comm %p\n", comm_ptr->mpid.opt_protocol_md[PAMI_XFER_BROADCAST][0].name,
comm_ptr->mpid.cutoff_size[PAMI_XFER_BROADCAST][0], comm_ptr);
http://git.mpich.org/mpich.git/commitdiff/1daeae3155739627d291277c89669f0c633c1e5a
commit 1daeae3155739627d291277c89669f0c633c1e5a
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Mon Mar 11 17:24:14 2013 -0500
Fix M2M protocol selection
(ibm) Issue 9416
(ibm) 723fb4ce39b5e06ef63cc37231acdaeb6d1c08cf
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/comm/mpid_optcolls.c b/src/mpid/pamid/src/comm/mpid_optcolls.c
index 85bfd45..23f63d7 100644
--- a/src/mpid/pamid/src/comm/mpid_optcolls.c
+++ b/src/mpid/pamid/src/comm/mpid_optcolls.c
@@ -278,10 +278,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr)
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION)
{
TRACE_ERR("No alltoall env var, so setting optimized alltoall\n");
- /* the best alltoall is always I0:M2MComposite:MU:MU, though there are
- * displacement array memory issues today.... */
+ /* The best alltoall is always I0:M2MComposite:MU:MU */
/* Loop over the protocols until we find the one we want */
if(use_threaded_collectives)
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALL][0]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALL][0][i].name, "I0:M2MComposite:MU:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ }
+ if(use_threaded_collectives && (opt_proto == -1)) /* check other list */
for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALL][1]; i++)
{
if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALL][1][i].name, "I0:M2MComposite:MU:MU") == 0)
@@ -290,7 +298,7 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr)
mustquery = 1;
break;
}
- }
+ }
if(opt_proto != -1)
{
TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n",
@@ -320,10 +328,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr)
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALLV_INT] == MPID_COLL_NOSELECTION)
{
TRACE_ERR("No alltoallv env var, so setting optimized alltoallv\n");
- /* the best alltoallv is always I0:M2MComposite:MU:MU, though there are
- * displacement array memory issues today.... */
+ /* The best alltoall is always I0:M2MComposite:MU:MU */
/* Loop over the protocols until we find the one we want */
if(use_threaded_collectives)
+ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALLV_INT][0]; i++)
+ {
+ if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALLV_INT][0][i].name, "I0:M2MComposite:MU:MU") == 0)
+ {
+ opt_proto = i;
+ break;
+ }
+ }
+ if(use_threaded_collectives && (opt_proto == -1)) /* check other list */
for(i = 0; i <comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALLV_INT][1]; i++)
{
if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALLV_INT][1][i].name, "I0:M2MComposite:MU:MU") == 0)
http://git.mpich.org/mpich.git/commitdiff/a45954b594b5a5ba4a54acdcb8fda7c2e819a40f
commit a45954b594b5a5ba4a54acdcb8fda7c2e819a40f
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Mon Mar 11 12:16:38 2013 -0500
Need to read some environment variables before creating the client
(ibm) 26edfa903df4881e1ab0190bc1859f5dc30e868e
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c
index 2f5394f..12983fc 100644
--- a/src/mpid/pamid/src/mpid_init.c
+++ b/src/mpid/pamid/src/mpid_init.c
@@ -294,6 +294,14 @@ MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char **
pami_result_t rc = PAMI_ERROR;
pami_configuration_t config[2];
+
+ /* Set the status for memory optimized collectives */
+ {
+ char* env = getenv("PAMID_COLLECTIVES_MEMORY_OPTIMIZED");
+ if (env != NULL)
+ MPIDI_atoi(env,&MPIDI_Process.optimized.memory);
+ }
+
config[0].name = PAMI_CLIENT_NONCONTIG;
if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG)
config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives so save memory
http://git.mpich.org/mpich.git/commitdiff/dcf62e917496501a74fe6cac918c8967316056dc
commit dcf62e917496501a74fe6cac918c8967316056dc
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Thu Mar 7 14:21:02 2013 -0600
Use MPIDO and not MPIR internally in PAMID
(ibm) 88338876fc003a9fe936298097b42df1e54ead32
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/src/coll/gather/mpido_gather.c b/src/mpid/pamid/src/coll/gather/mpido_gather.c
index 1047a44..8829c2b 100644
--- a/src/mpid/pamid/src/coll/gather/mpido_gather.c
+++ b/src/mpid/pamid/src/coll/gather/mpido_gather.c
@@ -102,8 +102,8 @@ int MPIDO_Gather_reduce(void * sendbuf,
memset(tempbuf, 0, sbytes * size * sizeof(char));
memcpy(tempbuf+(rank*sbytes), sendbuf, sbytes);
}
- /* #warning TODO need an optimal reduce */
- rc = MPIR_Reduce(MPI_IN_PLACE,
+ /* Switch to comm->coll_fns->fn() */
+ rc = MPIDO_Reduce(MPI_IN_PLACE,
tempbuf,
(sbytes * size)/4,
MPI_INT,
diff --git a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c
index 00946a3..bc47e41 100644
--- a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c
+++ b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c
@@ -175,7 +175,8 @@ int MPIDO_Scatterv_alltoallv(void * sendbuf,
rcounts[root] = rbytes;
- rc = MPIR_Alltoallv(sbuf,
+ /* Switch to comm->coll_fns->fn() */
+ rc = MPIDO_Alltoallv(sbuf,
scounts,
sdispls,
sendtype,
diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c
index d3b03c9..70a25ee 100644
--- a/src/mpid/pamid/src/comm/mpid_comm.c
+++ b/src/mpid/pamid/src/comm/mpid_comm.c
@@ -353,7 +353,8 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm)
MPIDI_Comm_coll_select(comm);
TRACE_ERR("mpir barrier\n");
int mpierrno;
- MPIR_Barrier(comm, &mpierrno);
+ /* Switch to comm->coll_fns->fn() */
+ MPIDO_Barrier(comm, &mpierrno);
TRACE_ERR("MPIDI_Coll_comm_create exit\n");
http://git.mpich.org/mpich.git/commitdiff/77999d6f565e65e7f1c5ee9f00dda99b28186f0d
commit 77999d6f565e65e7f1c5ee9f00dda99b28186f0d
Author: Bob Cernohous <bobc at us.ibm.com>
Date: Mon Mar 18 14:51:05 2013 -0500
Specify PAMID_COLLECTIVES_MEMORY_OPTIMIZED levels
(ibm) CPS 95PHUZ
(ibm) 1b4e2da6d5fb0990d530be6f896f539dc7571620
Signed-off-by: sssharka <sssharka at us.ibm.com>
diff --git a/src/mpid/pamid/include/mpidi_constants.h b/src/mpid/pamid/include/mpidi_constants.h
index a16292e..42d4bd8 100644
--- a/src/mpid/pamid/include/mpidi_constants.h
+++ b/src/mpid/pamid/include/mpidi_constants.h
@@ -117,4 +117,10 @@ enum
MPID_AUTO_SELECT_COLLS_ALL = 0xFFFFFFFF,
};
+enum /* PAMID_COLLECTIVES_MEMORY_OPTIMIZED levels */
+
+{
+ MPID_OPT_LVL_IRREG = 1, /**< Do not optimize irregular communicators */
+ MPID_OPT_LVL_NONCONTIG = 2, /**< Disable some non-contig collectives */
+};
#endif
diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h
index 42f0da0..7e33c76 100644
--- a/src/mpid/pamid/include/mpidi_datatypes.h
+++ b/src/mpid/pamid/include/mpidi_datatypes.h
@@ -113,7 +113,7 @@ typedef struct
unsigned subcomms; /**< Enable hardware optimized subcomm's */
unsigned select_colls; /**< Enable collective selection */
unsigned auto_select_colls; /**< Enable automatic collective selection */
- unsigned memory; /**< Enable memory optimized subcomm's */
+ unsigned memory; /**< Enable memory optimized subcomm's - See MPID_OPT_LVL_xxxx */
unsigned num_requests; /**< Number of requests between flow control barriers */
}
optimized;
diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c
index 3201595..d3b03c9 100644
--- a/src/mpid/pamid/src/comm/mpid_comm.c
+++ b/src/mpid/pamid/src/comm/mpid_comm.c
@@ -266,7 +266,11 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm)
pami_configuration_t config[3];
config[0].name = PAMI_GEOMETRY_NONCONTIG;
- config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives
+ if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG)
+ config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives
+ else
+ config[0].value.intval = 1; // Enable non-contig even though pamid doesn't use pami for non-contig data collectives,
+ // we still possibly want those collectives for other reasons.
size_t numconfigs = 1;
if(MPIDI_Process.optimized.subcomms)
{
@@ -281,7 +285,7 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm)
++numconfigs;
}
- if(MPIDI_Process.optimized.memory && (comm->local_size & (comm->local_size-1)))
+ if((MPIDI_Process.optimized.memory & MPID_OPT_LVL_IRREG) && (comm->local_size & (comm->local_size-1)))
{
/* Don't create irregular geometries. Fallback to MPICH only collectives */
geom_init = 0;
diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c
index 4f85bd6..2f5394f 100644
--- a/src/mpid/pamid/src/mpid_init.c
+++ b/src/mpid/pamid/src/mpid_init.c
@@ -295,7 +295,11 @@ MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char **
pami_configuration_t config[2];
config[0].name = PAMI_CLIENT_NONCONTIG;
- config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data
+ if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG)
+ config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives so save memory
+ else
+ config[0].value.intval = 1; // Enable non-contig even though pamid doesn't use pami for non-contig data collectives,
+ // we still possibly want those collectives for other reasons.
size_t numconfigs = 1;
if(MPIDI_Process.optimized.memory)
{
diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c
index f08acb7..f32fccd 100644
--- a/src/mpid/pamid/src/mpidi_env.c
+++ b/src/mpid/pamid/src/mpidi_env.c
@@ -110,8 +110,9 @@
* optimized to reduce memory usage. This may disable some PAMI collectives.
* Possible values:
* - 0 - Collectives are not memory optimized.
- * - n - Collectives are memory optimized. 'n' may represent different
- * levels of optimization.
+ * - n - Collectives are memory optimized. Levels are bitwise values :
+ * MPID_OPT_LVL_IRREG = 1, Do not optimize irregular communicators
+ * MPID_OPT_LVL_NONCONTIG = 2, Disable some non-contig collectives
*
* PAMID_OPTIMIZED_SUBCOMMS - Use PAMI 'optimized' collectives. Defaullt is 1.
* - 0 - Some optimized protocols may be disabled.
-----------------------------------------------------------------------
Summary of changes:
src/mpid/pamid/include/mpidi_constants.h | 6 +
src/mpid/pamid/include/mpidi_datatypes.h | 2 +-
src/mpid/pamid/include/mpidi_macros.h | 11 +-
.../pamid/src/coll/alltoallv/mpido_alltoallv.c | 222 ++++++++++++++++++--
src/mpid/pamid/src/coll/gather/mpido_gather.c | 4 +-
src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c | 3 +-
src/mpid/pamid/src/comm/mpid_comm.c | 29 ++-
src/mpid/pamid/src/comm/mpid_optcolls.c | 160 ++++++++++++++-
src/mpid/pamid/src/comm/mpid_selectcolls.c | 9 -
src/mpid/pamid/src/mpid_init.c | 31 +++-
src/mpid/pamid/src/mpidi_env.c | 5 +-
src/mpid/pamid/src/mpidi_util.c | 11 +-
12 files changed, 427 insertions(+), 66 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list