[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1rc2-41-gb0c0f93
mysql vizuser
noreply at mpich.org
Thu Dec 5 13:33:51 CST 2013
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via b0c0f93fb68821d3dd4b24582094739740836dd5 (commit)
via 3a780c43ef6f7a0af1565c6cb395b4026311c2ce (commit)
from b8c870e7239098c8cc05a80505d2140adbbeff86 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/b0c0f93fb68821d3dd4b24582094739740836dd5
commit b0c0f93fb68821d3dd4b24582094739740836dd5
Author: Masamichi Takagi <masamichi.takagi at gmail.com>
Date: Wed Dec 4 15:43:01 2013 +0900
Enable --with-dcfa* configure options
Add PAC_SET_HEADER_LIB_PATH(dcfa) in
src/mpid/ch3/channels/nemesis/subconfigure.m4
so that --with-dcfa, --with-dcfa-lib and --with-dcfa-include
is enabled.
Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/subconfigure.m4
index 87b7e5d..2ebaa97 100644
--- a/src/mpid/ch3/channels/nemesis/subconfigure.m4
+++ b/src/mpid/ch3/channels/nemesis/subconfigure.m4
@@ -94,6 +94,7 @@ AC_CHECK_FUNCS(signal)
# Check for netmod relevant headers and libraries
PAC_SET_HEADER_LIB_PATH(mx)
PAC_SET_HEADER_LIB_PATH(scif)
+PAC_SET_HEADER_LIB_PATH(dcfa)
nemesis_nets_dirs=""
nemesis_nets_strings=""
http://git.mpich.org/mpich.git/commitdiff/3a780c43ef6f7a0af1565c6cb395b4026311c2ce
commit 3a780c43ef6f7a0af1565c6cb395b4026311c2ce
Author: Masamichi Takagi <masamichi.takagi at gmail.com>
Date: Wed Dec 4 15:01:59 2013 +0900
Modify macro name for DCFA specific lines
Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c
index d9e624a..18df872 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c
@@ -114,7 +114,7 @@ static int modify_qp_to_rtr(struct ibv_qp *qp, uint32_t remote_qpn, uint16_t dli
/* In dcfa gid is not set and for testing here it is also not set */
#if 1
-#ifdef DCFA /* DCFA doesn't use gid */
+#ifdef HAVE_LIBDCFA /* DCFA doesn't use gid */
#else
if (gid_idx >= 0) {
attr.ah_attr.is_global = 1;
@@ -187,7 +187,7 @@ static int ibcomDeviceinit()
return -1;
}
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
for (i = 0; i < dev_num; i++) {
if (ib_devlist[i]) {
goto dev_found;
@@ -216,7 +216,7 @@ static int ibcomDeviceinit()
goto err_exit;
}
ib_ctx_export = ib_ctx;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
dev_name = strdup(ibv_get_device_name(ib_devlist[i]));
dprintf("ibcomDeviceinit,dev_name=%s\n", dev_name);
@@ -368,7 +368,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->rdmabuf_occupancy_notify_lstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW;
//dprintf("ibcomOpen,ptr=%p,rsr_seq_num_poll=%d\n", conp, conp->rsr_seq_num_poll);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
if (ibv_query_port(ib_ctx, ib_port, &conp->icom_pattr)) {
dprintf("ibv_query_port on port %u failed\n", ib_port);
@@ -380,7 +380,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
switch (ibcom_open_flag) {
case IBCOM_OPEN_RC:
if (!rc_shared_scq) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -390,7 +390,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->icom_scq = rc_shared_scq;
if (!rc_shared_rcq) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -401,7 +401,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
break;
case IBCOM_OPEN_SCRATCH_PAD:
if (!rc_shared_scq_scratch_pad) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -411,7 +411,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->icom_scq = rc_shared_scq_scratch_pad;
if (!rc_shared_rcq_scratch_pad) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -422,7 +422,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
break;
case IBCOM_OPEN_RC_LMT_PUT:
if (!rc_shared_scq_lmt_put) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -432,7 +432,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->icom_scq = rc_shared_scq_lmt_put;
if (!rc_shared_rcq_lmt_put) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -443,7 +443,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
break;
case IBCOM_OPEN_UD:
if (!ud_shared_scq) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ud_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
ud_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -453,7 +453,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->icom_scq = ud_shared_scq;
if (!ud_shared_rcq) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ud_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY);
#else
ud_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0);
@@ -499,13 +499,13 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
qp_init_attr.cap.max_send_wr, qp_init_attr.cap.max_recv_wr,
qp_init_attr.cap.max_inline_data);
dprintf("ibcomOpen,fd=%d,qpn=%08x\n", *condesc, conp->icom_qp->qp_num);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
dprintf("ibcomOpen,fd=%d,lid=%04x\n", *condesc, ib_ctx->lid);
#else
dprintf("ibcomOpen,fd=%d,lid=%04x\n", *condesc, conp->icom_pattr.lid);
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
/* DCFA doesn't use gid */
for (i = 0; i < 16; i++) {
conp->icom_gid.raw[i] = 0;
@@ -584,7 +584,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags);
goto err_exit;
}
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
dprintf("ibcomOpen,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[IBCOM_RDMAWR_TO]->buf);
#else
dprintf("ibcomOpen,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[IBCOM_RDMAWR_TO]->addr);
@@ -697,7 +697,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
int i;
for (i = 0; i < IBCOM_SMT_INLINE_NCHAIN; i++) {
/* SGE (RDMA-send-from memory) template */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[0]), 0,
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -709,7 +709,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
(i ==
IBCOM_SMT_INLINE_NCHAIN - 1) ? NULL : &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 +
i + 1];
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list = sge;
#endif
@@ -719,7 +719,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
}
{
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[0]), 0,
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -729,7 +729,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SMT_NOINLINE_INITIATOR_NSGE);
#endif
conp->icom_sr[IBCOM_SMT_NOINLINE].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list = sge;
#endif
@@ -738,7 +738,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
}
{
/* SR (send request) template for IBCOM_LMT_INITIATOR */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[0]), 0,
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -747,7 +747,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_INITIATOR_NSGE);
#endif
conp->icom_sr[IBCOM_LMT_INITIATOR].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list = sge;
#endif
@@ -756,14 +756,14 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
}
/* SR (send request) template for IBCOM_LMT_PUT *//* for lmt-put-done */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE);
#endif
conp->icom_sr[IBCOM_LMT_PUT].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_LMT_PUT].sg_list = sge;
#endif
@@ -783,7 +783,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
/* create one dummy RR to ibv_post_recv */
conp->icom_rr[IBCOM_RDMAWR_RESPONDER].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_rr[IBCOM_RDMAWR_RESPONDER].sg_list = NULL;
#endif
@@ -798,7 +798,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_SCRATCH_PAD_SR_NTEMPLATE);
/* SR (send request) template for IBCOM_SCRATCH_PAD_INITIATOR */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0]), 0,
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -808,7 +808,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SCRATCH_PAD_INITIATOR_NSGE);
#endif
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list = sge;
#endif
@@ -825,14 +825,14 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
IBCOM_RC_SR_LMT_PUT_NTEMPLATE);
memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_RC_SR_LMT_PUT_NTEMPLATE);
/* SR (send request) template for IBCOM_LMT_PUT */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
memset(&(conp->icom_sr[IBCOM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE);
#endif
conp->icom_sr[IBCOM_LMT_PUT].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_LMT_PUT].sg_list = sge;
#endif
@@ -842,7 +842,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
case IBCOM_OPEN_UD:
/* SGE (RDMA-send-from memory) template for IBCOM_UD_INITIATOR */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
sge = &(conp->icom_sr[IBCOM_UD_INITIATOR].sg_list[0]);
memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -876,7 +876,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
(struct ibv_send_wr *) calloc(IBCOM_UD_SR_NTEMPLATE, sizeof(struct ibv_send_wr));
conp->icom_sr[IBCOM_UD_INITIATOR].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_UD_INITIATOR].sg_list = sge;
#endif
@@ -887,7 +887,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
conp->icom_sr[IBCOM_UD_INITIATOR].wr.ud.remote_qkey = IBCOM_QKEY;
/* SGE (scatter gather element) template for recv */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
sge = &(conp->icom_rr[IBCOM_UD_RESPONDER].sg_list[0]);
memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM);
#else
@@ -903,7 +903,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc)
/* create one dummy RR to ibv_post_recv */
conp->icom_rr[IBCOM_UD_RESPONDER].next = NULL;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_rr[IBCOM_UD_RESPONDER].sg_list = sge;
#endif
@@ -955,7 +955,7 @@ int ibcom_alloc(int condesc, int sz)
IBCOM_ERR_CHKANDJUMP(!conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO], -1,
dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
dprintf("ibcom_alloc,fd=%d,rmem=%p\n", condesc,
conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO]->buf);
#else
@@ -1112,7 +1112,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *
void *hdr_copy = buf_from + sizeof(sz_hdrmagic_t);
memcpy(hdr_copy, prefix, sz_prefix);
memcpy(hdr_copy + sz_prefix, hdr, sz_hdr);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic;
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr =
conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) sz_hdrmagic -
@@ -1131,7 +1131,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *
//dprintf("ibcom_isend,data=%p,sz_data=%d\n", data, sz_data);
struct ibv_mr *mr_data = ibcom_reg_mr_fetch(data, sz_data);
IBCOM_ERR_CHKANDJUMP(!mr_data, -1, printf("ibcom_isend,ibv_reg_mr_fetch failed\n"));
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) data;
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr =
mr_data->host_addr + ((uint64_t) data - (uint64_t) data);
@@ -1147,7 +1147,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *
tailmagic_t *tailmagic =
(tailmagic_t *) (buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad);
tailmagic->magic = IBCOM_MAGIC;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr =
(uint64_t) buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr;
conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr =
@@ -1192,7 +1192,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *
*copied = 0;
}
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_NOINLINE]);
IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_isend, ibv_post_send, rc=%d\n", ib_errno));
#else
@@ -1267,7 +1267,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
sz_hdrmagic->sz = sumsz;
sz_hdrmagic->magic = IBCOM_MAGIC;
memcpy(buf_from + sizeof(sz_hdrmagic_t), hdr, sz_hdr);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) buf_from;
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
@@ -1291,7 +1291,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
//tscs = MPID_nem_dcfa_rdtsc();
if (sz_data_rem > 0) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
(uint64_t) data + sz_data - sz_data_rem;
@@ -1310,7 +1310,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
mr_data = ibcom_reg_mr_fetch(data, sz_data);
IBCOM_ERR_CHKANDJUMP(!mr_data, -1, printf("ibcom_isend,ibv_reg_mr_fetch failed\n"));
}
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) data + sz_data - sz_data_rem;
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
@@ -1330,7 +1330,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
int sz_pad = sz_data_pow2 - sz_data;
tailmagic_t *tailmagic = (tailmagic_t *) (buf_from + sz_pad);
tailmagic->magic = IBCOM_MAGIC;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) buf_from;
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
@@ -1358,7 +1358,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
printf
("ibcom_isend_chain,tail-magic gets over packet-boundary\n"));
int sz_pad = IBCOM_INLINE_DATA - sz_used;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) buf_from;
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
@@ -1397,7 +1397,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s].send_flags |= IBV_SEND_FENCE;
}
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s]);
#else
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s], &bad_wr);
@@ -1408,7 +1408,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *
conp->after_rdma_rd = 0;
}
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_isend, ibv_post_send, rc=%d\n", ib_errno));
#else
IBCOM_ERR_CHKANDJUMP(ib_errno, -1,
@@ -1437,13 +1437,13 @@ int ibcom_irecv(int condesc, uint64_t wr_id)
//dprintf("ibcom_irecv,condesc=%d,wr_id=%016lx\n", condesc, wr_id);
conp->icom_rr[IBCOM_RDMAWR_RESPONDER].wr_id = wr_id;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_RDMAWR_RESPONDER]);
#else
ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_RDMAWR_RESPONDER], &bad_wr);
#endif
if (ib_errno) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
fprintf(stderr, "ibcom_irecv: failed to post receive, ib_errno=%d\n", ib_errno);
#else
fprintf(stderr, "ibcom_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", ib_errno,
@@ -1467,7 +1467,7 @@ int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, ui
RANGE_CHECK_WITH_ERROR(condesc, conp);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
IBCOM_ERR_CHKANDJUMP(1, -1,
dprintf
("ibcom_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n"));
@@ -1502,7 +1502,7 @@ int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, ui
}
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_UD_INITIATOR]);
#else
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_UD_INITIATOR], &bad_wr);
@@ -1530,7 +1530,7 @@ int ibcom_udrecv(int condesc)
conp->icom_rr[IBCOM_UD_RESPONDER].wr_id = 0;
/* Post RR to RQ */
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_UD_RESPONDER]);
#else
ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_UD_RESPONDER], &bad_wr);
@@ -1565,7 +1565,7 @@ int ibcom_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t
/* Erase magic, super bug!! */
//((tailmagic_t*)(laddr + sz_data - sizeof(tailmagic_t)))->magic = 0;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].mic_addr = (uint64_t) laddr;
conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].addr =
mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr);
@@ -1581,7 +1581,7 @@ int ibcom_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t
conp->icom_sr[IBCOM_LMT_INITIATOR].wr.rdma.remote_addr = (uint64_t) raddr;
conp->icom_sr[IBCOM_LMT_INITIATOR].wr.rdma.rkey = rkey;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_INITIATOR]);
IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_lrecv, ibv_post_send, rc=%d\n", ib_errno));
#else
@@ -1625,7 +1625,7 @@ int ibcom_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_
struct ibv_mr *mr_data = ibcom_reg_mr_fetch(laddr, sz_data);
IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_lmt,ibv_reg_mr_fetch failed\n"));
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].mic_addr = (uint64_t) laddr;
conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].addr =
mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr);
@@ -1641,7 +1641,7 @@ int ibcom_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_
conp->icom_sr[IBCOM_LMT_PUT].wr.rdma.remote_addr = (uint64_t) raddr;
conp->icom_sr[IBCOM_LMT_PUT].wr.rdma.rkey = rkey;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_PUT]);
IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_put_lmt, ibv_post_send, rc=%d\n", ib_errno));
#else
@@ -1683,7 +1683,7 @@ int ibcom_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz,
IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
dprintf("ibcom_put_scratch_pad,");
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr;
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr =
mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr);
@@ -1702,7 +1702,7 @@ int ibcom_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz,
dprintf("ibcom_put_scratch_pad,wr.rdma.remote_addr=%llx\n",
(unsigned long long) conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR]);
IBCOM_ERR_CHKANDJUMP(ib_errno, -1,
dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno));
@@ -1746,7 +1746,7 @@ int ibcom_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t
IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
dprintf("ibcom_put_scratch_pad,");
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr;
conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr =
mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr);
@@ -1767,7 +1767,7 @@ int ibcom_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t
dprintf("ibcom_put_scratch_pad,wr.rdma.remote_addr=%llx\n",
(unsigned long long) conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR]);
IBCOM_ERR_CHKANDJUMP(ib_errno, -1,
dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno));
@@ -1869,7 +1869,7 @@ int ibcom_get_info_conn(int condesc, int key, void *out, uint32_t out_len)
memcpy(out, &conp->icom_qp->qp_num, out_len);
break;
case IBCOM_INFOKEY_PORT_LID:
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
dprintf("ibcom_get_info_conn,lid=%04x\n", ib_ctx->lid);
memcpy(out, &ib_ctx->lid, out_len);
#else
@@ -1881,7 +1881,7 @@ int ibcom_get_info_conn(int condesc, int key, void *out, uint32_t out_len)
memcpy(out, &conp->icom_gid, out_len);
break;
case IBCOM_INFOKEY_PATTR_MAX_MSG_SZ:{
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
uint32_t max_msg_sz = 1073741824; /* ConnectX-3 */
memcpy(out, &max_msg_sz, out_len);
#else
@@ -1912,7 +1912,7 @@ int ibcom_get_info_mr(int condesc, int memid, int key, void *out, int out_len)
switch (key) {
case IBCOM_INFOKEY_MR_ADDR:
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
/* host_addr is created by ibv_reg_mr in ibcomOpen, */
/* dcfa_init read this host-addr, put it into KVS, the counter-party read it through KVS */
memcpy(out, &mr->host_addr, out_len);
@@ -1921,7 +1921,7 @@ int ibcom_get_info_mr(int condesc, int memid, int key, void *out, int out_len)
#endif
break;
case IBCOM_INFOKEY_MR_LENGTH:{
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
assert(out_len == sizeof(size_t));
size_t length = mr->size; /* type of mr->size is int */
memcpy(out, &length, out_len);
@@ -2134,7 +2134,7 @@ void ibcomShow(int condesc)
RANGE_CHECK(condesc, conp);
fprintf(stdout, "qp_num = %d\n", conp->icom_qp->qp_num);
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
fprintf(stdout, "lid = %d\n", ib_ctx->lid);
#else
fprintf(stdout, "lid = %d\n", conp->icom_pattr.lid);
@@ -2203,7 +2203,7 @@ int ibcom_dereg_mr(struct ibv_mr *mr)
fprintf(stderr, "cannot deregister memory\n");
goto fn_fail;
}
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
dprintf("ibcom_dereg_mr, addr=%p\n", mr->buf);
#else
dprintf("ibcom_dereg_mr, addr=%p\n", mr->addr);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h
index 3a3f5b4..b5970a8 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h
@@ -13,6 +13,7 @@
#include <stdint.h>
#include <sys/types.h>
#include <sys/mman.h>
+#include "mpid_nem_impl.h"
//#define DEBUG_ON 1
#ifdef DEBUG_ON
@@ -21,7 +22,7 @@
#define DEBUG if (0)
#endif
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#include "dcfa.h"
/*
@@ -271,7 +272,7 @@ typedef struct IbCom {
short icom_used;
short icom_connected;
int icom_port;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
struct ibv_port_attr icom_pattr; /* IB port attributes */
#endif
diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c
index 5c9b2f4..c145343 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c
@@ -82,7 +82,7 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts
("lmt_initate_lmt,dt_contig=%d,write_from_buf=%p,req->dev.user_buf=%p,REQ_FIELD(req, lmt_pack_buf)=%p\n",
dt_contig, write_from_buf, req->dev.user_buf, REQ_FIELD(req, lmt_pack_buf));
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
#else
s_cookie_buf->addr = write_from_buf;
#endif
@@ -111,7 +111,7 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts
/* put IB rkey */
struct ibv_mr *mr = ibcom_reg_mr_fetch(write_from_buf, data_sz);
MPIU_ERR_CHKANDJUMP(!mr, mpi_errno, MPI_ERR_OTHER, "**ibcom_reg_mr_fetch");
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
s_cookie_buf->addr = (void *) mr->host_addr;
dprintf("lmt_initiate_lmt,s_cookie_buf->addr=%p\n", s_cookie_buf->addr);
#endif
diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c
index 43a43b3..4ed0e39 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c
@@ -63,7 +63,7 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress)
}
entered_drain_scq = 1;
-#if 0 /*def DCFA */
+#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(rc_shared_scq, 1, &cqe[0]);
#else
result = ibv_poll_cq(rc_shared_scq, /*3 */ IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
@@ -95,7 +95,7 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress)
printf("%d\n", *(int *) 0);
}
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
if (cqe[i].status != IBV_WC_SUCCESS) {
dprintf("drain_scq,kind=%d,req_type=%d,msg_type=%d,cqe.status=%08x\n", kind, req_type,
msg_type, cqe[i].status);
@@ -411,7 +411,7 @@ int MPID_nem_dcfa_drain_scq_lmt_put()
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_LMT_PUT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_LMT_PUT);
-#if 0 /*def DCFA */
+#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(rc_shared_scq_lmt_put, 1, &cqe[0]);
#else
result = ibv_poll_cq(rc_shared_scq_lmt_put, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
@@ -427,7 +427,7 @@ int MPID_nem_dcfa_drain_scq_lmt_put()
MPID_Request_kind_t kind;
int req_type, msg_type;
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
if (cqe[i].status != IBV_WC_SUCCESS) {
dprintf("drain_scq_lmt_put,status=%08x\n", cqe[i].status);
}
@@ -521,7 +521,7 @@ int MPID_nem_dcfa_drain_scq_scratch_pad()
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_SCRATCH_PAD);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_SCRATCH_PAD);
-#if 0 /*def DCFA */
+#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(rc_shared_scq_scratch_pad, 1, &cqe[0]);
#else
result = ibv_poll_cq(rc_shared_scq_scratch_pad, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
@@ -533,7 +533,7 @@ int MPID_nem_dcfa_drain_scq_scratch_pad()
}
for (i = 0; i < result; i++) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
if (cqe[i].status != IBV_WC_SUCCESS) {
dprintf("drain_scq_scratch_pad,status=%08x\n", cqe[i].status);
}
@@ -1872,7 +1872,7 @@ int MPID_nem_dcfa_cm_drain_scq()
}
for (i = 0; i < result; i++) {
-#ifdef DCFA
+#ifdef HAVE_LIBDCFA
if (cqe[i].status != IBV_WC_SUCCESS) {
dprintf("cm_drain_scq,status=%08x\n", cqe[i].status);
}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c
index a8c020a..3b08b0b 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c
@@ -175,7 +175,7 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len)
int key;
struct ibcom_reg_mr_cache_entry_t *e;
-#if 1 /*def DCFA */
+#if 1 /*def HAVE_LIBDCFA */
/* we can't change addr because ibv_post_send assumes mr->host_addr (output of this function)
* must have an exact mirror value of addr (input of this function) */
void *addr_aligned = addr;
-----------------------------------------------------------------------
Summary of changes:
.../ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c | 126 ++++++++++----------
.../ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h | 5 +-
.../ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c | 4 +-
.../ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c | 14 +-
.../ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c | 2 +-
src/mpid/ch3/channels/nemesis/subconfigure.m4 | 1 +
6 files changed, 77 insertions(+), 75 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list