[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1rc2-90-gb2b9e1f
mysql vizuser
noreply at mpich.org
Thu Dec 19 02:08:14 CST 2013
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via b2b9e1f9a5bbaf79126867531a7c685835319a04 (commit)
from 37012b672626a5113b2d5707d8c8af6a850de71e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/b2b9e1f9a5bbaf79126867531a7c685835319a04
commit b2b9e1f9a5bbaf79126867531a7c685835319a04
Author: Pavan Balaji <balaji at mcs.anl.gov>
Date: Thu Dec 19 13:44:23 2013 +0800
White space cleanup.
Signed-off-by: Masamichi Takagi <masamichi.takagi at gmail.com>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c
index 3fae51f..0948cda 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c
@@ -36,9 +36,9 @@ static int ib_initialized = 0;
static int maxcon;
static struct ibv_device **ib_devlist;
static struct ibv_context *ib_ctx;
-struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */
+struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */
static struct ibv_pd *ib_pd;
-struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */
+struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */
struct ibv_cq *MPID_nem_ib_rc_shared_scq;
struct ibv_cq *MPID_nem_ib_rc_shared_scq_lmt_put;
struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad;
@@ -210,7 +210,8 @@ static int MPID_nem_ib_com_device_init()
else {
ib_ctx = ibv_open_device(ib_devlist[i]);
}
- dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n", MPID_nem_ib_ctx_export, ib_ctx);
+ dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n",
+ MPID_nem_ib_ctx_export, ib_ctx);
if (!ib_ctx) {
fprintf(stderr, "failed to open IB device\n");
goto err_exit;
@@ -228,7 +229,8 @@ static int MPID_nem_ib_com_device_init()
else {
ib_pd = ibv_alloc_pd(ib_ctx);
}
- dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_pd_export=%p,ib_pd=%p\n", MPID_nem_ib_pd_export, ib_pd);
+ dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_pd_export=%p,ib_pd=%p\n",
+ MPID_nem_ib_pd_export, ib_pd);
if (!ib_pd) {
fprintf(stderr, "ibv_alloc_pd failed\n");
goto err_exit;
@@ -323,7 +325,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
int open_flag_conn = open_flag;
if (open_flag_conn != MPID_NEM_IB_COM_OPEN_RC &&
open_flag_conn != MPID_NEM_IB_COM_OPEN_RC_LMT_PUT &&
- open_flag_conn != MPID_NEM_IB_COM_OPEN_UD && open_flag_conn != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD) {
+ open_flag_conn != MPID_NEM_IB_COM_OPEN_UD &&
+ open_flag_conn != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD) {
dprintf("MPID_nem_ib_com_open,bad flag\n");
ibcom_errno = -1;
goto fn_fail;
@@ -383,9 +386,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
#ifdef HAVE_LIBDCFA
MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_scq =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq, -1, dprintf("MPID_nem_ib_rc_shared_scq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq, -1,
+ dprintf("MPID_nem_ib_rc_shared_scq"));
}
conp->icom_scq = MPID_nem_ib_rc_shared_scq;
@@ -393,51 +398,65 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
#ifdef HAVE_LIBDCFA
MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_rcq =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq, -1, dprintf("MPID_nem_ib_rc_shared_rcq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq, -1,
+ dprintf("MPID_nem_ib_rc_shared_rcq"));
}
conp->icom_rcq = MPID_nem_ib_rc_shared_rcq;
break;
case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD:
if (!MPID_nem_ib_rc_shared_scq_scratch_pad) {
#ifdef HAVE_LIBDCFA
- MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
+ MPID_nem_ib_rc_shared_scq_scratch_pad =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_scq_scratch_pad =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_scq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_scratch_pad, -1,
+ dprintf("MPID_nem_ib_rc_shared_scq"));
}
conp->icom_scq = MPID_nem_ib_rc_shared_scq_scratch_pad;
if (!MPID_nem_ib_rc_shared_rcq_scratch_pad) {
#ifdef HAVE_LIBDCFA
- MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
+ MPID_nem_ib_rc_shared_rcq_scratch_pad =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_rcq_scratch_pad =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_rcq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_scratch_pad, -1,
+ dprintf("MPID_nem_ib_rc_shared_rcq"));
}
conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_scratch_pad;
break;
case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT:
if (!MPID_nem_ib_rc_shared_scq_lmt_put) {
#ifdef HAVE_LIBDCFA
- MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
+ MPID_nem_ib_rc_shared_scq_lmt_put =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_scq_lmt_put =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_scq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_lmt_put, -1,
+ dprintf("MPID_nem_ib_rc_shared_scq"));
}
conp->icom_scq = MPID_nem_ib_rc_shared_scq_lmt_put;
if (!MPID_nem_ib_rc_shared_rcq_lmt_put) {
#ifdef HAVE_LIBDCFA
- MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
+ MPID_nem_ib_rc_shared_rcq_lmt_put =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_rc_shared_rcq_lmt_put =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_rcq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_lmt_put, -1,
+ dprintf("MPID_nem_ib_rc_shared_rcq"));
}
conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_lmt_put;
break;
@@ -446,9 +465,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
#ifdef HAVE_LIBDCFA
MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_ud_shared_scq =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_scq, -1, dprintf("MPID_nem_ib_ud_shared_scq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_scq, -1,
+ dprintf("MPID_nem_ib_ud_shared_scq"));
}
conp->icom_scq = MPID_nem_ib_ud_shared_scq;
@@ -456,9 +477,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
#ifdef HAVE_LIBDCFA
MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY);
#else
- MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
+ MPID_nem_ib_ud_shared_rcq =
+ ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0);
#endif
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_rcq, -1, dprintf("MPID_nem_ib_ud_shared_rcq"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_rcq, -1,
+ dprintf("MPID_nem_ib_ud_shared_rcq"));
}
conp->icom_rcq = MPID_nem_ib_ud_shared_rcq;
break;
@@ -538,17 +561,20 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
/* RDMA-write-from local memory area */
conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM] = MPID_NEM_IB_COM_RDMABUF_SZ;
conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] =
- mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ);
+ mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n",
+ conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ);
if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] == (void *) -1) {
fprintf(stderr, "failed to allocate buffer\n");
goto err_exit;
}
- memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], 0,
+ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]);
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM] =
MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM],
- conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]);
if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]) {
fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags);
goto err_exit;
@@ -568,28 +594,34 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
}
#else
conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] =
- mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ);
+ mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO],
+ MPID_NEM_IB_COM_RDMABUF_SZ);
#endif
if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] == (void *) -1) {
fprintf(stderr, "failed to allocate buffer\n");
goto err_exit;
}
- memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO]);
+ memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], 0,
+ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO]);
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO] =
- ibv_reg_mr(ib_pd, conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO],
- mr_flags);
+ ibv_reg_mr(ib_pd, conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO],
+ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO], mr_flags);
if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]) {
fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags);
goto err_exit;
}
#ifdef HAVE_LIBDCFA
- dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->buf);
+ dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc,
+ conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->buf);
#else
- dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->addr);
+ dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc,
+ conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->addr);
#endif
- dprintf("MPID_nem_ib_com_open,fd=%d,rkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->rkey);
+ dprintf("MPID_nem_ib_com_open,fd=%d,rkey=%08x\n", *condesc,
+ conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->rkey);
/* RDMA-write-to remote memory area */
conp->icom_rmem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA);
@@ -644,40 +676,51 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE;
/* UD-write-from local memory area */
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1,
+ dprintf("buf_size too short\n"));
conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM] = MPID_NEM_IB_COM_UDBUF_SZ;
conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] =
- mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], MPID_NEM_IB_COM_UDBUF_SZ);
+ mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM],
+ MPID_NEM_IB_COM_UDBUF_SZ);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] == (void *) -1, -1,
- dprintf("failed to allocate buffer\n"));
- memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]);
+ dprintf("failed to allocate buffer\n"));
+ memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], 0,
+ conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]);
conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM] =
- MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]);
+ MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM],
+ conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM], -1,
- dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
+ dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
/* UD-write-to local memory area */
/* addr to addr+39 are not filled, addr+40 to addr+length-1 are filled with payload */
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1,
+ dprintf("buf_size too short\n"));
conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO] = MPID_NEM_IB_COM_UDBUF_SZ;
conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] =
- mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], MPID_NEM_IB_COM_UDBUF_SZ);
+ mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO],
+ MPID_NEM_IB_COM_UDBUF_SZ);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] == (void *) -1, -1,
- dprintf("failed to allocate buffer\n"));
- memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]);
+ dprintf("failed to allocate buffer\n"));
+ memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], 0,
+ conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]);
conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO] =
- MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]);
+ MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO],
+ conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO], -1,
- dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
+ dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
/* initialize arena allocator for MPID_NEM_IB_COM_UDWR_TO */
//MPID_nem_ib_com_udbuf_init(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO]);
- dprintf("MPID_nem_ib_com_open,ud,fd=%d,lkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey);
+ dprintf("MPID_nem_ib_com_open,ud,fd=%d,lkey=%08x\n", *condesc,
+ conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey);
break;
default:
MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, dprintf("invalid open_flag\n"));
@@ -691,7 +734,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
/* SR (send request) template */
conp->icom_sr =
- (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE);
+ (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) *
+ MPID_NEM_IB_COM_RC_SR_NTEMPLATE);
memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE);
for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) {
@@ -701,13 +745,14 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge =
- (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE);
+ (struct ibv_sge *) malloc(sizeof(struct ibv_sge) *
+ MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE);
#endif
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].next =
(i ==
- MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) ? NULL : &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 +
- i + 1];
+ MPID_NEM_IB_COM_SMT_INLINE_NCHAIN -
+ 1) ? NULL : &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i + 1];
#ifdef HAVE_LIBDCFA
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list = sge;
@@ -742,7 +787,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge =
- (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE);
+ (struct ibv_sge *) malloc(sizeof(struct ibv_sge) *
+ MPID_NEM_IB_COM_LMT_INITIATOR_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE);
#endif
conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].next = NULL;
@@ -756,7 +802,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
/* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT *//* for lmt-put-done */
#ifdef HAVE_LIBDCFA
- memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM);
+ memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0,
+ sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE);
@@ -777,7 +824,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
/* RR (receive request) template for MPID_NEM_IB_COM_RDMAWR_RESPONDER */
conp->icom_rr =
- (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE);
+ (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) *
+ MPID_NEM_IB_COM_RC_RR_NTEMPLATE);
memset(conp->icom_rr, 0, sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE);
/* create one dummy RR to ibv_post_recv */
@@ -794,7 +842,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
conp->icom_sr =
(struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) *
MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE);
- memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE);
+ memset(conp->icom_sr, 0,
+ sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE);
/* SR (send request) template for MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR */
#ifdef HAVE_LIBDCFA
@@ -822,10 +871,12 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
conp->icom_sr =
(struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) *
MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE);
- memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE);
+ memset(conp->icom_sr, 0,
+ sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE);
/* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT */
#ifdef HAVE_LIBDCFA
- memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM);
+ memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0,
+ sizeof(struct ibv_sge) * WR_SG_NUM);
#else
sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE);
memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE);
@@ -854,25 +905,27 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
conp->icom_ah_attr =
- (struct ibv_ah_attr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_ah_attr));
+ (struct ibv_ah_attr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE,
+ sizeof(struct ibv_ah_attr));
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].sl = 0;
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].src_path_bits = 0;
- conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */
+ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 0;
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].port_num = conp->icom_port;
#if 0
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 1;
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.flow_label = 0;
- conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */
+ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.hop_limit = 1;
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.traffic_class = 0;
#endif
/* SR (send request) template for MPID_NEM_IB_COM_UD_INITIATOR */
conp->icom_sr =
- (struct ibv_send_wr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_send_wr));
+ (struct ibv_send_wr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE,
+ sizeof(struct ibv_send_wr));
conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].next = NULL;
#ifdef HAVE_LIBDCFA
@@ -898,7 +951,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc)
/* RR (receive request) template for MPID_NEM_IB_COM_UD_RESPONDER */
conp->icom_rr =
- (struct ibv_recv_wr *) calloc(MPID_NEM_IB_COM_UD_RR_NTEMPLATE, sizeof(struct ibv_recv_wr));
+ (struct ibv_recv_wr *) calloc(MPID_NEM_IB_COM_UD_RR_NTEMPLATE,
+ sizeof(struct ibv_recv_wr));
/* create one dummy RR to ibv_post_recv */
conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].next = NULL;
@@ -941,7 +995,7 @@ int MPID_nem_ib_com_alloc(int condesc, int sz)
scratch_pad = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
dprintf("MPID_nem_ib_com_alloc,mmap=%p,len=%d\n", scratch_pad, sz);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(scratch_pad == (void *) -1, -1,
- dprintf("failed to allocate buffer\n"));
+ dprintf("failed to allocate buffer\n"));
dprintf("MPID_nem_ib_com_alloc,scratch_pad=%p\n", scratch_pad);
memset(scratch_pad, 0, sz);
}
@@ -950,9 +1004,9 @@ int MPID_nem_ib_com_alloc(int condesc, int sz)
conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO] =
MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO],
- conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO]);
+ conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO]);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO], -1,
- dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
+ dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags));
#ifdef HAVE_LIBDCFA
dprintf("MPID_nem_ib_com_alloc,fd=%d,rmem=%p\n", condesc,
@@ -966,7 +1020,8 @@ int MPID_nem_ib_com_alloc(int condesc, int sz)
break;
default:
MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1,
- dprintf("MPID_nem_ib_com_alloc, invalid open_flag=%d\n", conp->open_flag));
+ dprintf("MPID_nem_ib_com_alloc, invalid open_flag=%d\n",
+ conp->open_flag));
break;
}
@@ -993,7 +1048,8 @@ int MPID_nem_ib_com_close(int condesc)
goto fn_exit;
}
-int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid)
+int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid,
+ union ibv_gid *remote_gid)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1073,8 +1129,8 @@ int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, unio
}
#define MPID_NEM_IB_ENABLE_INLINE
-int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, int sz_hdr,
- void *data, int sz_data, int *copied)
+int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr,
+ int sz_hdr, void *data, int sz_data, int *copied)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1082,8 +1138,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
int ib_errno;
int num_sge;
- dprintf("MPID_nem_ib_com_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n", prefix,
- sz_prefix, hdr, sz_hdr, data, sz_data);
+ dprintf("MPID_nem_ib_com_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n",
+ prefix, sz_prefix, hdr, sz_hdr, data, sz_data);
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
if (conp->icom_connected == 0) {
@@ -1103,7 +1159,9 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG);
MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf_from;
- sz_hdrmagic->sz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t);
+ sz_hdrmagic->sz =
+ sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data +
+ sizeof(MPID_nem_ib_tailmagic_t);
sz_hdrmagic->magic = MPID_NEM_IB_COM_MAGIC;
/* memcpy hdr is needed because hdr resides in stack when sending close-VC command */
@@ -1115,8 +1173,9 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic;
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) sz_hdrmagic -
- (uint64_t) conp->
- icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ (uint64_t)
+ conp->icom_mem
+ [MPID_NEM_IB_COM_RDMAWR_FROM]);
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) sz_hdrmagic;
#endif
@@ -1129,7 +1188,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
if (sz_data) {
//dprintf("MPID_nem_ib_com_isend,data=%p,sz_data=%d\n", data, sz_data);
struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n"));
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) data;
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr =
@@ -1144,27 +1204,31 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
int sz_pad = sz_data_pow2 - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data);
MPID_nem_ib_tailmagic_t *tailmagic =
- (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad);
+ (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) +
+ sz_prefix + sz_hdr + sz_pad);
tailmagic->magic = MPID_NEM_IB_COM_MAGIC;
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr =
(uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr;
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from +
- sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix +
- sz_hdr -
- (uint64_t) conp->
- icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ sizeof
+ (MPID_nem_ib_sz_hdrmagic_t) +
+ sz_prefix + sz_hdr -
+ (uint64_t)
+ conp->icom_mem
+ [MPID_NEM_IB_COM_RDMAWR_FROM]);
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr =
(uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr;
#endif
- conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = sz_pad + sizeof(MPID_nem_ib_tailmagic_t);
+ conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length =
+ sz_pad + sizeof(MPID_nem_ib_tailmagic_t);
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].lkey =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey;
num_sge += 1;
- dprintf("MPID_nem_ib_com_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data, sz_data_pow2, sz_pad,
- num_sge);
+ dprintf("MPID_nem_ib_com_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data,
+ sz_data_pow2, sz_pad, num_sge);
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge = num_sge;
conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id = wr_id;
@@ -1193,12 +1257,15 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
#endif
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE]);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
+ dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE], &bad_wr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno,
- bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
#ifdef MPID_NEM_IB_ENABLE_INLINE
if (sumsz <= conp->max_inline_data) {
@@ -1221,7 +1288,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref
goto fn_exit;
}
-int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, int sz_data)
+int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data,
+ int sz_data)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1230,13 +1298,14 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
int sz_data_rem = sz_data;
int i;
struct ibv_mr *mr_data;
- uint32_t sumsz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t);
+ uint32_t sumsz =
+ sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t);
unsigned long tscs, tsce;
dprintf("MPID_nem_ib_com_isend_chain,enter\n");
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_connected == 0, -1,
- printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n"));
+ printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n"));
void *buf_from =
(uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] +
@@ -1248,12 +1317,17 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
/* let the last command icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAIN-1] which has IBV_WR_RDMA_WRITE_WITH_IMM */
int s =
- MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2 +
- sizeof(MPID_nem_ib_tailmagic_t) + MPID_NEM_IB_COM_INLINE_DATA - 1) / MPID_NEM_IB_COM_INLINE_DATA;
- MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2) % 4 != 0, -1,
- printf("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n"));
+ MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr +
+ sz_data_pow2 + sizeof(MPID_nem_ib_tailmagic_t) +
+ MPID_NEM_IB_COM_INLINE_DATA -
+ 1) / MPID_NEM_IB_COM_INLINE_DATA;
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr +
+ sz_data_pow2) % 4 != 0, -1,
+ printf
+ ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n"));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(s < 0 ||
- s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1, printf("MPID_nem_ib_com_isend_chain,s\n"));
+ s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1,
+ printf("MPID_nem_ib_com_isend_chain,s\n"));
dprintf("MPID_nem_ib_com_isend_chain,sz_hdr=%d,sz_data=%d,s=%d\n", sz_hdr, sz_data, s);
for (i = s; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) {
@@ -1271,8 +1345,9 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
(uint64_t) buf_from;
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from -
- (uint64_t) conp->
- icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ (uint64_t)
+ conp->icom_mem
+ [MPID_NEM_IB_COM_RDMAWR_FROM]);
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
(uint64_t) buf_from;
@@ -1298,16 +1373,22 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
int sz_data_red =
sz_used + sz_data_rem + sizeof(MPID_nem_ib_tailmagic_t) <=
MPID_NEM_IB_COM_INLINE_DATA ? sz_data_rem : sz_data_rem <=
- MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA - sz_used;
- conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_data_red;
+ MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA -
+ sz_used;
+ conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length =
+ sz_data_red;
sz_used += sz_data_red;
sz_data_rem -= sz_data_red;
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1,
+ printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n"));
if (i == s) {
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, printf("MPID_nem_ib_com_isend_chain,sz_data==0\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1,
+ printf("MPID_nem_ib_com_isend_chain,sz_data==0\n"));
mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ printf
+ ("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n"));
}
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
@@ -1315,9 +1396,11 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
mr_data->host_addr + ((uint64_t) data + sz_data - sz_data_rem - (uint64_t) data);
#endif
- conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = mr_data->lkey;
+ conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey =
+ mr_data->lkey;
num_sge += 1;
- dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, sz_data_rem);
+ dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used,
+ sz_data_rem);
}
else { /* tailmagic only packet is being generated */
@@ -1325,17 +1408,19 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
//tsce = MPID_nem_ib_rdtsc(); printf("1,%ld\n", tsce-tscs);
//tscs = MPID_nem_ib_rdtsc();
- if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */
+ if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */
int sz_pad = sz_data_pow2 - sz_data;
- MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sz_pad);
+ MPID_nem_ib_tailmagic_t *tailmagic =
+ (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sz_pad);
tailmagic->magic = MPID_NEM_IB_COM_MAGIC;
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) buf_from;
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from -
- (uint64_t) conp->
- icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ (uint64_t)
+ conp->icom_mem
+ [MPID_NEM_IB_COM_RDMAWR_FROM]);
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
(uint64_t) buf_from;
@@ -1343,27 +1428,29 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length =
sz_pad + sizeof(MPID_nem_ib_tailmagic_t);
sz_used += sz_pad + sizeof(MPID_nem_ib_tailmagic_t);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1,
+ printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n"));
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey;
num_sge += 1;
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].imm_data = conp->sseq_num;
- dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, sz_used,
- num_sge);
+ dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad,
+ sz_used, num_sge);
}
- else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */
+ else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */
MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1,
- printf
- ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n"));
+ printf
+ ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n"));
int sz_pad = MPID_NEM_IB_COM_INLINE_DATA - sz_used;
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr =
(uint64_t) buf_from;
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from -
- (uint64_t) conp->
- icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]);
+ (uint64_t)
+ conp->icom_mem
+ [MPID_NEM_IB_COM_RDMAWR_FROM]);
#else
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr =
(uint64_t) buf_from;
@@ -1372,7 +1459,7 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad;
sz_used += sz_pad;
MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1,
- printf("MPID_nem_ib_com_isend_chain, sz_used\n"));
+ printf("MPID_nem_ib_com_isend_chain, sz_used\n"));
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey =
conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey;
num_sge += 1;
@@ -1380,7 +1467,7 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
}
else { /* packet is full with data */
MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1,
- printf("MPID_nem_ib_com_isend_chain, sz_used\n"));
+ printf("MPID_nem_ib_com_isend_chain, sz_used\n"));
}
//tsce = MPID_nem_ib_rdtsc(); printf("2,%ld\n", tsce-tscs);
@@ -1388,8 +1475,8 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr_id = wr_id;
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.remote_addr =
(uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] +
- MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG) + MPID_NEM_IB_COM_INLINE_DATA * (i -
- s);
+ MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG) +
+ MPID_NEM_IB_COM_INLINE_DATA * (i - s);
}
#if 0
if (conp->after_rdma_rd) {
@@ -1397,9 +1484,12 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
}
#endif
#ifdef HAVE_LIBDCFA
- ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]);
+ ib_errno =
+ ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]);
#else
- ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s], &bad_wr);
+ ib_errno =
+ ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s],
+ &bad_wr);
#endif
#if 0
if (i == 0 && conp->after_rdma_rd) {
@@ -1408,11 +1498,14 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h
}
#endif
#ifdef HAVE_LIBDCFA
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
+ dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno,
- bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
conp->ncom += (MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - s);
conp->sseq_num += 1;
@@ -1439,14 +1532,15 @@ int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id)
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER]);
#else
- ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER], &bad_wr);
+ ib_errno =
+ ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER], &bad_wr);
#endif
if (ib_errno) {
#ifdef HAVE_LIBDCFA
fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d\n", ib_errno);
#else
- fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", ib_errno,
- bad_wr);
+ fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n",
+ ib_errno, bad_wr);
#endif
ibcom_errno = ib_errno;
goto fn_fail;
@@ -1457,8 +1551,8 @@ int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id)
goto fn_exit;
}
-int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, uint32_t remote_qpn,
- uint32_t imm_data, uint64_t wr_id)
+int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid,
+ uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id)
{
MPID_nem_ib_com_t *conp;
struct ibv_send_wr *bad_wr;
@@ -1468,8 +1562,8 @@ int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remo
#ifdef HAVE_LIBDCFA
MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1,
- dprintf
- ("MPID_nem_ib_com_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n"));
+ dprintf
+ ("MPID_nem_ib_com_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n"));
#else
/* prepare ibv_ah_attr */
conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].dlid = remote_lid;
@@ -1542,7 +1636,8 @@ int MPID_nem_ib_com_udrecv(int condesc)
goto fn_exit;
}
-int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr)
+int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey,
+ void *laddr)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1553,14 +1648,16 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data,
dprintf("MPID_nem_ib_com_lrecv,enter,raddr=%p,sz_data=%d,laddr=%p\n", raddr, sz_data, laddr);
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_lrecv,not connected\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1,
+ dprintf("MPID_nem_ib_com_lrecv,not connected\n"));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_lrecv,sz_data==0\n"));
num_sge = 0;
/* register memory area containing data */
struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_lrecv,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ dprintf("MPID_nem_ib_com_lrecv,ibv_reg_mr_fetch failed\n"));
/* Erase magic, super bug!! */
//((MPID_nem_ib_tailmagic_t*)(laddr + sz_data - sizeof(MPID_nem_ib_tailmagic_t)))->magic = 0;
@@ -1582,12 +1679,15 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data,
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR]);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d\n", ib_errno));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
+ dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR], &bad_wr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno,
- bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
/* other commands can be executed before RDMA-rd command */
@@ -1604,7 +1704,8 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data,
}
/* use the same QP as isend */
-int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr)
+int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey,
+ void *laddr)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1615,14 +1716,16 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat
dprintf("MPID_nem_ib_com_put_lmt,enter,sz_data=%d,laddr=%p\n", sz_data, laddr);
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_put_lmt,not connected\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1,
+ dprintf("MPID_nem_ib_com_put_lmt,not connected\n"));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_put_lmt,sz_data==0\n"));
num_sge = 0;
/* register memory area containing data */
struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_lmt,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ dprintf("MPID_nem_ib_com_put_lmt,ibv_reg_mr_fetch failed\n"));
#ifdef HAVE_LIBDCFA
conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].mic_addr = (uint64_t) laddr;
@@ -1642,12 +1745,15 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT]);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d\n", ib_errno));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
+ dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT], &bad_wr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno,
- bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
conp->ncom += 1;
@@ -1659,7 +1765,8 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat
goto fn_exit;
}
-int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr)
+int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz,
+ void *laddr)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1672,14 +1779,17 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag));
+ dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n",
+ conp->open_flag));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n"));
+ dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n"));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n"));
/* register memory area containing data */
struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
dprintf("MPID_nem_ib_com_put_scratch_pad,");
#ifdef HAVE_LIBDCFA
@@ -1699,17 +1809,23 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
/* rkey is defined in MPID_nem_ib_com_reg_mr_connect */
dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n",
- (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr);
+ (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.
+ remote_addr);
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno));
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
- ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr);
+ ib_errno =
+ ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR],
+ &bad_wr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n",
- ib_errno, bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
conp->ncom_scratch_pad += 1;
@@ -1722,7 +1838,7 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
#ifdef MPID_NEM_IB_ONDEMAND
int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t compare,
- uint64_t swap)
+ uint64_t swap)
{
MPID_nem_ib_com_t *conp;
int ibcom_errno = 0;
@@ -1735,14 +1851,17 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag));
+ dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n",
+ conp->open_flag));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n"));
+ dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n"));
MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n"));
/* register memory area containing data */
struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1,
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n"));
dprintf("MPID_nem_ib_com_put_scratch_pad,");
#ifdef HAVE_LIBDCFA
@@ -1764,17 +1883,23 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.atomic.swap = swap;
dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n",
- (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr);
+ (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.
+ remote_addr);
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno));
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n",
+ ib_errno));
#else
- ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr);
+ ib_errno =
+ ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR],
+ &bad_wr);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1,
- dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n",
- ib_errno, bad_wr));
+ dprintf
+ ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n",
+ ib_errno, bad_wr));
#endif
conp->ncom_scratch_pad += 1;
@@ -1807,8 +1932,9 @@ int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result)
}
if (*result < 0) {
- dprintf("MPID_nem_ib_com_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n",
- wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id);
+ dprintf
+ ("MPID_nem_ib_com_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n",
+ wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id);
ibcom_errno = *result;
goto fn_fail;
}
@@ -1831,7 +1957,8 @@ int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey)
case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT:
conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] = rmem;
conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO] = rkey;
- conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey = conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO];
+ conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey =
+ conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO];
for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) {
conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.rkey =
conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO];
@@ -1906,7 +2033,7 @@ int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
MPID_NEM_IB_COM_ERR_CHKANDJUMP(memid >= conp->icom_mrlen, -1,
- dprintf("MPID_nem_ib_com_get_info_mr,wrong mem_id=%d\n", memid));
+ dprintf("MPID_nem_ib_com_get_info_mr,wrong mem_id=%d\n", memid));
mr = conp->icom_mrlist[memid];
switch (key) {
@@ -1965,7 +2092,9 @@ int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out)
int ibcom_errno = 0;
MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp);
- *out = (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG);
+ *out =
+ (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] +
+ MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG);
fn_exit:
return ibcom_errno;
@@ -2179,7 +2308,8 @@ int MPID_nem_ib_com_reg_mr(void *addr, int len, struct ibv_mr **mr)
ibv_reg_mr(ib_pd, addr, len,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
- MPID_NEM_IB_COM_ERR_CHKANDJUMP(*mr == 0, -1, dprintf("MPID_nem_ib_com_reg_mr,cannot register memory\n"));
+ MPID_NEM_IB_COM_ERR_CHKANDJUMP(*mr == 0, -1,
+ dprintf("MPID_nem_ib_com_reg_mr,cannot register memory\n"));
fn_exit:
return ibcom_errno;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h
index e77091e..930fbf1 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h
@@ -175,15 +175,15 @@ extern struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad;
extern struct ibv_cq *MPID_nem_ib_ud_shared_rcq;
#define MPID_NEM_IB_COM_SIZE 2048 /* one process uses 2-4 fds */
-#define MPID_NEM_IB_COM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */
+#define MPID_NEM_IB_COM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */
#define MPID_NEM_IB_COM_MAX_SQ_CAPACITY (256/1)
-#define MPID_NEM_IB_COM_MAX_RQ_CAPACITY ((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv MPID_NEM_IB_COM_MAX_SQ_CAPACITY of commands */
-#define MPID_NEM_IB_COM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */
+#define MPID_NEM_IB_COM_MAX_RQ_CAPACITY ((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv MPID_NEM_IB_COM_MAX_SQ_CAPACITY of commands */
+#define MPID_NEM_IB_COM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */
#define MPID_NEM_IB_COM_MAX_CQ_CAPACITY MPID_NEM_IB_COM_MAX_RQ_CAPACITY
-#define MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */
-#define MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */
-#define MPID_NEM_IB_COM_AMT_CQ_DRAIN ((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2) /* drain this amount */
+#define MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */
+#define MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */
+#define MPID_NEM_IB_COM_AMT_CQ_DRAIN ((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2) /* drain this amount */
#define MPID_NEM_IB_COM_MAX_RD_ATOMIC 4
#define MPID_NEM_IB_COM_MAX_TRIES 1
@@ -200,65 +200,65 @@ extern struct ibv_cq *MPID_nem_ib_ud_shared_rcq;
/* buffers */
-#define MPID_NEM_IB_COM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */
-#define MPID_NEM_IB_COM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */
-#define MPID_NEM_IB_COM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */
+#define MPID_NEM_IB_COM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */
+#define MPID_NEM_IB_COM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */
+#define MPID_NEM_IB_COM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */
/* assuming that the unit (32768) is equals to eager-RDMA-write threashold
assuming that the multiplier (256) is
equals to max number of outstanding eager-RDMA-write transactions */
-#define MPID_NEM_IB_COM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see ib_impl.h, ib_com.c, src/mpid/ch3/src/mpid_isend.c */
-#define MPID_NEM_IB_COM_RDMABUF_SZ ((MPID_NEM_IB_COM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */
+#define MPID_NEM_IB_COM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see ib_impl.h, ib_com.c, src/mpid/ch3/src/mpid_isend.c */
+#define MPID_NEM_IB_COM_RDMABUF_SZ ((MPID_NEM_IB_COM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */
#define MPID_NEM_IB_COM_RDMABUF_NSEG ((MPID_NEM_IB_COM_RDMABUF_SZ) / (MPID_NEM_IB_COM_RDMABUF_SZSEG))
-#define MPID_NEM_IB_COM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */
+#define MPID_NEM_IB_COM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */
#define MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>1)+((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2))
#define MPID_NEM_IB_COM_RDMABUF_LOW_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2))
#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW 1
#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW 2
#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW /*1*/(((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4) == 0 ? 1 : ((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4))
-#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */
-#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */
+#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */
+#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */
-#define MPID_NEM_IB_COM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */
-#define MPID_NEM_IB_COM_UDWR_FROM 0 /* index to UD-write-from buffer */
-#define MPID_NEM_IB_COM_UDWR_TO 1 /* index to UD-write-to buffer */
-#define MPID_NEM_IB_COM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */
+#define MPID_NEM_IB_COM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */
+#define MPID_NEM_IB_COM_UDWR_FROM 0 /* index to UD-write-from buffer */
+#define MPID_NEM_IB_COM_UDWR_TO 1 /* index to UD-write-to buffer */
+#define MPID_NEM_IB_COM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */
#define MPID_NEM_IB_COM_UDBUF_SZSEG (128)
#define MPID_NEM_IB_COM_UDBUF_NSEG (MPID_NEM_IB_COM_UDBUF_SZ / MPID_NEM_IB_COM_UDBUF_SZSEG)
-#define MPID_NEM_IB_COM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */
-#define MPID_NEM_IB_COM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */
+#define MPID_NEM_IB_COM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */
+#define MPID_NEM_IB_COM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */
/* send command templates */
-#define MPID_NEM_IB_COM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */
-#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 0 /* index to it */
+#define MPID_NEM_IB_COM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */
+#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 0 /* index to it */
#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED7 7
#define MPID_NEM_IB_COM_SMT_NOINLINE 8
-#define MPID_NEM_IB_COM_LMT_INITIATOR 9 /* FIXME: bad naming */
+#define MPID_NEM_IB_COM_LMT_INITIATOR 9 /* FIXME: bad naming */
-#define MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE MPID_NEM_IB_COM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named MPID_NEM_IB_COM_RC_SR shares MPID_NEM_IB_COM_LMT_PUT */
+#define MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE MPID_NEM_IB_COM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named MPID_NEM_IB_COM_RC_SR shares MPID_NEM_IB_COM_LMT_PUT */
#define MPID_NEM_IB_COM_LMT_PUT 10
/* recv command templates */
-#define MPID_NEM_IB_COM_RC_RR_NTEMPLATE 1 /* 1 for smt, */
-#define MPID_NEM_IB_COM_RDMAWR_RESPONDER 0 /* index to recv request template */
+#define MPID_NEM_IB_COM_RC_RR_NTEMPLATE 1 /* 1 for smt, */
+#define MPID_NEM_IB_COM_RDMAWR_RESPONDER 0 /* index to recv request template */
/* sge template */
-#define MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */
-#define MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */
-#define MPID_NEM_IB_COM_LMT_INITIATOR_NSGE 1 /* data x1 */
-#define MPID_NEM_IB_COM_LMT_PUT_NSGE 1 /* data x1 */
-#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */
+#define MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */
+#define MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */
+#define MPID_NEM_IB_COM_LMT_INITIATOR_NSGE 1 /* data x1 */
+#define MPID_NEM_IB_COM_LMT_PUT_NSGE 1 /* data x1 */
+#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */
#define MPID_NEM_IB_COM_UD_SR_NTEMPLATE 1
#define MPID_NEM_IB_COM_UD_RR_NTEMPLATE 1
-#define MPID_NEM_IB_COM_UD_INITIATOR 0 /* index to send request template */
-#define MPID_NEM_IB_COM_UD_RESPONDER 0 /* index to recv request template */
+#define MPID_NEM_IB_COM_UD_INITIATOR 0 /* index to send request template */
+#define MPID_NEM_IB_COM_UD_RESPONDER 0 /* index to recv request template */
#define MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE 2
#define MPID_NEM_IB_COM_SCRATCH_PAD_RR_NTEMPLATE 1
-#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */
+#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */
#define MPID_NEM_IB_COM_SCRATCH_PAD_CAS 1
-#define MPID_NEM_IB_COM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */
+#define MPID_NEM_IB_COM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */
typedef struct MPID_nem_ib_com {
@@ -310,23 +310,25 @@ typedef struct MPID_nem_ib_com {
extern int MPID_nem_ib_com_open(int ib_port, int MPID_nem_ib_com_open_flag, int *condesc);
extern int MPID_nem_ib_com_alloc(int condesc, int sz);
extern int MPID_nem_ib_com_close(int);
-extern int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid);
+extern int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid,
+ union ibv_gid *remote_gid);
extern int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey);
-extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr,
- int sz_hdr, void *data, int sz_data, int *copied);
-extern int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data,
- int sz_data);
-extern int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr);
+extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix,
+ void *hdr, int sz_hdr, void *data, int sz_data, int *copied);
+extern int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr,
+ void *data, int sz_data);
+extern int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz,
+ void *laddr);
//extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void* hdr, int sz_hdr, void* data, int sz_data);
extern int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id);
extern int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid,
- uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id);
+ uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id);
extern int MPID_nem_ib_com_udrecv(int condesc);
-extern int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey,
- void *laddr);
-extern int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey,
- void *laddr);
+extern int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data,
+ uint32_t rkey, void *laddr);
+extern int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data,
+ uint32_t rkey, void *laddr);
extern int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result);
extern int MPID_nem_ib_com_obtain_pointer(int condesc, MPID_nem_ib_com_t ** MPID_nem_ib_com);
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h
index aaddb5c..e8d230d 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h
@@ -15,7 +15,7 @@
#include <linux/mman.h> /* make it define MAP_ANONYMOUS */
#include <sys/mman.h>
-#define MPID_NEM_IB_LMT_GET_CQE /* detect RDMA completion by CQE */
+#define MPID_NEM_IB_LMT_GET_CQE /* detect RDMA completion by CQE */
#define MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* lmt-put:
(1) receiver sends cts to sender (2) sender RDMA-write to receiver
@@ -44,7 +44,7 @@ typedef struct {
MPID_nem_ib_conn_t *sc;
int pending_sends; /* number of send in flight */
MPID_nem_ib_com_t *ibcom, *ibcom_lmt_put;
- MPID_nem_ib_sendq_t sendq; /* overflow queue for IB commands */
+ MPID_nem_ib_sendq_t sendq; /* overflow queue for IB commands */
MPID_nem_ib_sendq_t sendq_lmt_put;
int is_connected; /* dynamic connection, checked in iSendContig, protocol processed there and in progress engine */
} MPID_nem_ib_vc_area;
@@ -112,7 +112,7 @@ typedef union {
typedef struct MPID_nem_ib_cm_sendq_entry {
MPID_nem_ib_cm_pkt_t pending_pkt;
- struct MPID_nem_ib_cm_sendq_entry *sendq_next; /* for software command queue */
+ struct MPID_nem_ib_cm_sendq_entry *sendq_next; /* for software command queue */
} MPID_nem_ib_cm_sendq_entry_t;
#ifdef MPID_NEM_IB_ONDEMAND
@@ -157,10 +157,10 @@ typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_ib_cm_sendq_t;
/* counting bloom filter to detect multiple lmt-sends in one send-wait period to
avoid overwriting the last byte in the receive buffer */
-#define MPID_nem_ib_cbf_nslot 16 /* slots */
-#define MPID_nem_ib_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */
+#define MPID_nem_ib_cbf_nslot 16 /* slots */
+#define MPID_nem_ib_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */
#define MPID_nem_ib_cbf_lognslot 4
-#define MPID_nem_ib_cbf_nhash 3 /* number of hash functions */
+#define MPID_nem_ib_cbf_nhash 3 /* number of hash functions */
#define MPID_nem_ib_getpos \
int pos_8b = pos / (8 / MPID_nem_ib_cbf_bitsperslot);\
assert(0 <= pos_8b && pos_8b < MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot / 8);\
@@ -266,13 +266,13 @@ static inline int MPID_nem_ib_cbf_would_overflow(uint64_t addr, uint8_t * array)
//dprintf("cbf_would_overflow,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr)));
return
MPID_nem_ib_cbf_get(array,
- MPID_nem_ib_cbf_hash1(addr)) ==
+ MPID_nem_ib_cbf_hash1(addr)) ==
(1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 ||
MPID_nem_ib_cbf_get(array,
- MPID_nem_ib_cbf_hash2(addr)) ==
+ MPID_nem_ib_cbf_hash2(addr)) ==
(1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 ||
MPID_nem_ib_cbf_get(array,
- MPID_nem_ib_cbf_hash3(addr)) ==
+ MPID_nem_ib_cbf_hash3(addr)) ==
(1ULL << MPID_nem_ib_cbf_bitsperslot) - 1;
}
@@ -295,22 +295,22 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc);
int MPID_nem_ib_pkthandler_init(MPIDI_CH3_PktHandler_Fcn * pktArray[], int arraySize);
int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header,
- MPIDI_msg_sz_t hdr_sz);
+ MPIDI_msg_sz_t hdr_sz);
/* CH3 send/recv functions */
int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
- MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz);
+ MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz);
int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data,
- MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr);
+ MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr);
/* used by ib_poll.c */
int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib);
/* CH3--lmt send/recv functions */
int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt,
- struct MPID_Request *req);
+ struct MPID_Request *req);
int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey,
- void *write_to_buf);
+ void *write_to_buf);
int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie);
int MPID_nem_ib_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie);
int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req);
@@ -345,24 +345,24 @@ extern void *MPID_nem_ib_fl[18];
extern int MPID_nem_ib_nranks;
//extern char *MPID_nem_ib_recv_buf;
extern int MPID_nem_ib_myrank;
-extern uint64_t MPID_nem_ib_tsc_poll; /* to throttle ib_poll in recv_posted (in ib_poll.c) */
-extern int MPID_nem_ib_ncqe; /* for lazy poll scq */
-extern int MPID_nem_ib_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */
+extern uint64_t MPID_nem_ib_tsc_poll; /* to throttle ib_poll in recv_posted (in ib_poll.c) */
+extern int MPID_nem_ib_ncqe; /* for lazy poll scq */
+extern int MPID_nem_ib_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */
#ifdef MPID_NEM_IB_ONDEMAND
extern MPID_nem_ib_cm_map_t MPID_nem_ib_cm_state;
-extern int MPID_nem_ib_ncqe_connect; /* couting outstanding connection requests */
+extern int MPID_nem_ib_ncqe_connect; /* couting outstanding connection requests */
#endif
extern int MPID_nem_ib_ncqe_scratch_pad;
-extern int MPID_nem_ib_ncqe_to_drain; /* count put in lmt-put-done protocol */
-extern int MPID_nem_ib_ncqe_nces; /* counting non-copied eager-send */
-extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq; /* poll queue for lmt */
-extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */
+extern int MPID_nem_ib_ncqe_to_drain; /* count put in lmt-put-done protocol */
+extern int MPID_nem_ib_ncqe_nces; /* counting non-copied eager-send */
+extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq; /* poll queue for lmt */
+extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */
extern MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib;
/* to detect multiple lmt-sends in one send-wait period to
avoid overwriting the last byte in the receive buffer */
extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot *
- MPID_nem_ib_cbf_bitsperslot / 8];
+ MPID_nem_ib_cbf_bitsperslot / 8];
#define MPID_NEM_IB_MAX_POLLINGSET 65536
@@ -374,8 +374,8 @@ extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot *
#define MPID_NEM_IB_SYNC_SYNACK 1
#define MPID_NEM_IB_SYNC_NACK 2
-#define MPID_NEM_IB_EAGER_MAX_MSG_SZ (MPID_NEM_IB_COM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(MPID_nem_ib_sz_hdrmagic_t)-sizeof(MPID_nem_ib_pkt_prefix_t)-sizeof(MPID_nem_ib_tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */
-#define MPID_NEM_IB_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous ib_poll to ib_poll in recv_posted */
+#define MPID_NEM_IB_EAGER_MAX_MSG_SZ (MPID_NEM_IB_COM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(MPID_nem_ib_sz_hdrmagic_t)-sizeof(MPID_nem_ib_pkt_prefix_t)-sizeof(MPID_nem_ib_tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */
+#define MPID_NEM_IB_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous ib_poll to ib_poll in recv_posted */
#define MPID_NEM_IB_POLL_PERIOD_SEND_POSTED 2000
typedef struct {
@@ -440,32 +440,32 @@ typedef struct MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t {
} MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t;
int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */);
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */);
int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */);
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */);
int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */);
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */);
int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */);
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */);
int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */);
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */);
int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp);
int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc,
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen,
- MPID_Request ** rreqp);
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen,
+ MPID_Request ** rreqp);
/* MPID_nem_ib_PktHandler_lmt_done is a wrapper of pkt_DONE_handler and calls it */
/* pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) is not exported */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c
index 25cd464..1b969d8 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c
@@ -40,8 +40,8 @@ MPID_nem_netmod_funcs_t MPIDI_nem_ib_funcs = {
MPIDI_CH3_PktHandler_Fcn *MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS];
static MPIDI_Comm_ops_t comm_ops = {
- /*NULL, */ MPID_nem_ib_recv_posted,
- /* recv_posted */
+ /*NULL, */ MPID_nem_ib_recv_posted,
+ /* recv_posted */
NULL, /* send */
NULL, /* rsend */
@@ -88,7 +88,7 @@ int MPID_nem_ib_ncqe_nces;
MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq = { NULL, NULL };
MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq = { NULL, NULL };
uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot /
- 8] = { 0 };
+ 8] = { 0 };
static uint32_t MPID_nem_ib_rand_next = 1;
MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib;
static int listen_fd;
@@ -112,7 +112,8 @@ uint64_t MPID_nem_ib_rdtsc()
#define FUNCNAME MPID_nem_ib_kvs_put_binary
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int MPID_nem_ib_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length)
+static int MPID_nem_ib_kvs_put_binary(int from, const char *postfix, const uint8_t * buf,
+ int length)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
@@ -223,7 +224,9 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
/* prepare UD QPN for dynamic connection */
ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_UD, &MPID_nem_ib_conn_ud_fd);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open");
- ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conn_ud_fd, &MPID_nem_ib_conn_ud_MPID_nem_ib_com);
+ ibcom_errno =
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conn_ud_fd,
+ &MPID_nem_ib_conn_ud_MPID_nem_ib_com);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
ibcom_errno = MPID_nem_ib_com_rts(MPID_nem_ib_conn_ud_fd, 0, 0, 0);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts");
@@ -244,11 +247,11 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
uint16_t my_lid;
union ibv_gid my_gid;
MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid,
- sizeof(uint16_t));
+ sizeof(uint16_t));
MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid,
- sizeof(union ibv_gid));
+ sizeof(union ibv_gid));
char *kvs_name;
mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name);
@@ -295,15 +298,13 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
MPID_nem_ib_conn_ud[i].gid.raw[j] = strtol(str, NULL, 16);
strp += 2;
}
- sscanf(strp, ":%04x:%08x", &MPID_nem_ib_conn_ud[i].lid,
- &MPID_nem_ib_conn_ud[i].qpn);
+ sscanf(strp, ":%04x:%08x", &MPID_nem_ib_conn_ud[i].lid, &MPID_nem_ib_conn_ud[i].qpn);
dprintf("remote rank=%d,gid=", i);
for (j = 0; j < sizeof(union ibv_gid); j++) {
dprintf("%02x", MPID_nem_ib_conn_ud[i].gid.raw[j]);
}
- dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_ib_conn_ud[i].lid,
- MPID_nem_ib_conn_ud[i].qpn);
+ dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_ib_conn_ud[i].lid, MPID_nem_ib_conn_ud[i].qpn);
}
}
#endif
@@ -315,12 +316,14 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
/* prepare scrath-pad QP and malloc scratch-pad */
for (i = 0; i < MPID_nem_ib_nranks; i++) {
- ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, &MPID_nem_ib_scratch_pad_fds[i]);
+ ibcom_errno =
+ MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_SCRATCH_PAD,
+ &MPID_nem_ib_scratch_pad_fds[i]);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open");
ibcom_errno =
MPID_nem_ib_com_alloc(MPID_nem_ib_scratch_pad_fds[i],
- MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t));
+ MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_alloc");
}
@@ -355,14 +358,16 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
if (i == 0) {
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_LID,
- &my_lid, sizeof(uint16_t));
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i],
+ MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid,
+ sizeof(uint16_t));
dprintf("ib_init,scratch pad,lid=%04x\n", my_lid);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/lid", (uint8_t *) & my_lid,
- sizeof(uint16_t));
+ sizeof(uint16_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_kvs_put_binary");
@@ -372,13 +377,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
}
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_GID,
- &my_gid, sizeof(union ibv_gid));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i],
+ MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid,
+ sizeof(union ibv_gid));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/gid", (uint8_t *) & my_gid,
- sizeof(union ibv_gid));
+ sizeof(union ibv_gid));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_kvs_put_binary");
@@ -394,14 +401,16 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
sprintf(remote_rank_str, "/%x", i);
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
- sizeof(uint32_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i],
+ MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
+ sizeof(uint32_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
dprintf("ib_init,scratch pad,qpn=%08x\n", my_qpnum);
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
dprintf("ib_init,scratch pad,kvs put done\n");
@@ -410,14 +419,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i],
+ MPID_NEM_IB_COM_SCRATCH_PAD_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
dprintf("ib_init,scratch_pad,rmem=%p\n", my_rmem);
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem,
- sizeof(void *));
+ sizeof(void *));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
strcpy(key_str, "sp/rkey");
@@ -425,14 +435,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i],
+ MPID_NEM_IB_COM_SCRATCH_PAD_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
dprintf("ib_init,scratch_pad,rkey=%08x\n", my_rkey);
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
}
@@ -447,7 +458,7 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
mpi_errno =
MPID_nem_ib_kvs_get_binary(i, "sp/gid", (char *) &remote_gid,
- sizeof(union ibv_gid));
+ sizeof(union ibv_gid));
dprintf("ib_init,after kvs get\n");
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
@@ -472,7 +483,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
dprintf("ib_init,get KVS,remote_qpnum=%08x\n", remote_qpnum);
ibcom_errno =
- MPID_nem_ib_com_rts(MPID_nem_ib_scratch_pad_fds[i], remote_qpnum, remote_lid, &remote_gid);
+ MPID_nem_ib_com_rts(MPID_nem_ib_scratch_pad_fds[i], remote_qpnum, remote_lid,
+ &remote_gid);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts");
strcpy(key_str, "sp/rmem");
@@ -497,8 +509,10 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
dprintf("ib_init,get KVS,remote_rkey=%08x\n", remote_rkey);
ibcom_errno =
- MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_scratch_pad_fds[i], remote_rmem, remote_rkey);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect");
+ MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_scratch_pad_fds[i], remote_rmem,
+ remote_rkey);
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_reg_mr_connect");
}
}
@@ -507,7 +521,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier");
MPIU_CHKPMEM_MALLOC(MPID_nem_ib_conns, MPID_nem_ib_conn_t *,
- MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t), mpi_errno, "connection table");
+ MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t), mpi_errno,
+ "connection table");
memset(MPID_nem_ib_conns, 0, MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t));
MPIU_CHKPMEM_MALLOC(MPID_nem_ib_pollingset, MPIDI_VC_t **,
@@ -517,14 +532,17 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
/* prepare eager-send QP */
for (i = 0; i < MPID_nem_ib_nranks; i++) {
- ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[i].fd);
+ ibcom_errno =
+ MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[i].fd);
dprintf("init,fd=%d\n", MPID_nem_ib_conns[i].fd);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open");
}
#if 0
for (i = 0; i < MPID_nem_ib_nranks; i++) {
- ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC_LMT_PUT, &MPID_nem_ib_conns[i].fd_lmt_put);
+ ibcom_errno =
+ MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC_LMT_PUT,
+ &MPID_nem_ib_conns[i].fd_lmt_put);
dprintf("init,fd_lmt_put=%d\n", MPID_nem_ib_conns[i].fd_lmt_put);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open");
}
@@ -546,13 +564,13 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
mpi_errno =
MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_LID_KEY, (char *) &remote_lid,
- sizeof(uint16_t));
+ sizeof(uint16_t));
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
mpi_errno =
MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_GID_KEY, (char *) &remote_gid,
- sizeof(union ibv_gid));
+ sizeof(union ibv_gid));
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -593,20 +611,27 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
/* report me-to-you eager-send QP becomes RTR */
MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad;
ibcom_errno =
- MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[i], &MPID_nem_ib_com_scratch_pad);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[i],
+ &MPID_nem_ib_com_scratch_pad);
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_obtain_pointer");
MPID_nem_ib_com_qp_state_t state = {.state = MPID_NEM_IB_COM_QP_STATE_RTR };
ibcom_errno =
MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[i],
- (uint64_t) MPID_nem_ib_com_scratch_pad,
- sizeof(MPID_nem_ib_com_qp_state_t) * MPID_nem_ib_myrank,
- sizeof(MPID_nem_ib_com_qp_state_t), (void *) &state);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad");
+ (uint64_t) MPID_nem_ib_com_scratch_pad,
+ sizeof(MPID_nem_ib_com_qp_state_t) *
+ MPID_nem_ib_myrank,
+ sizeof(MPID_nem_ib_com_qp_state_t),
+ (void *) &state);
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_put_scratch_pad");
MPID_nem_ib_ncqe_scratch_pad += 1;
- ibcom_errno = MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[i].fd, remote_rmem, remote_rkey);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect");
+ ibcom_errno =
+ MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[i].fd, remote_rmem, remote_rkey);
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_reg_mr_connect");
dprintf("ib_init,after mr_connect for me-to-you eager-send QP\n");
#if 0
@@ -622,7 +647,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
}
ibcom_errno =
- MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd_lmt_put, remote_qpnum, remote_lid, &remote_gid);
+ MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd_lmt_put, remote_qpnum, remote_lid,
+ &remote_gid);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts");
#endif
}
@@ -685,7 +711,7 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
for (i = 0, nranks = MPID_nem_ib_nranks; nranks > 0; nranks /= 10, i++) {
}
MPIU_CHKLMEM_MALLOC(remote_rank_str, char *, i + 1, mpi_errno, "key_str"); /* alloca */
- MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPID_NEM_IB_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */
+ MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPID_NEM_IB_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */
/* We have one local qp and remote qp for each rank-pair,
* so a rank should perform
@@ -700,20 +726,23 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
/* lid and gid are common for all remote-ranks */
if (i == 0) {
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid,
- sizeof(uint16_t));
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd,
+ MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid,
+ sizeof(uint16_t));
dprintf("get_business_card,lid=%04x\n", my_lid);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_LID_KEY,
- (uint8_t *) & my_lid, sizeof(uint16_t));
+ (uint8_t *) & my_lid, sizeof(uint16_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_kvs_put_binary");
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid,
- sizeof(union ibv_gid));
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd,
+ MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid,
+ sizeof(union ibv_gid));
dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p);
dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid));
@@ -722,11 +751,12 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
}
dprintf("\n");
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_GID_KEY,
- (uint8_t *) & my_gid, sizeof(union ibv_gid));
+ (uint8_t *) & my_gid, sizeof(union ibv_gid));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_kvs_put_binary");
dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p);
@@ -743,13 +773,14 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
sprintf(remote_rank_str, "/%x", i);
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
- sizeof(uint32_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN,
+ &my_qpnum, sizeof(uint32_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
#if 0
@@ -759,13 +790,15 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
sprintf(remote_rank_str, "/%x", i);
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd_lmt_put, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
- sizeof(uint32_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd_lmt_put,
+ MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum,
+ sizeof(uint32_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
#endif
@@ -774,14 +807,14 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_ADDR,
- &my_rmem, sizeof(void *));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
dprintf("rmem=%p\n", my_rmem);
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem,
- sizeof(void *));
+ sizeof(void *));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
strcpy(key_str, MPID_NEM_IB_RKEY_KEY);
@@ -789,13 +822,13 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *
strcat(key_str, remote_rank_str);
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_RKEY,
- &my_rkey, sizeof(int));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
mpi_errno =
MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey,
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary");
}
@@ -857,7 +890,8 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
ibcom_errno =
- MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd_lmt_put, &vc_ib->ibcom_lmt_put);
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd_lmt_put,
+ &vc_ib->ibcom_lmt_put);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
//dprintf("vc_init,open,fd=%d,ptr=%p,rsr_seq_num_poll=%d\n", MPID_nem_ib_conns[vc->pg_rank].fd, vc_ib->MPID_nem_ib_com, vc_ib->ibcom->rsr_seq_num_poll);
@@ -878,13 +912,15 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
/* wait until you-to-me eager-send QP becomes RTR */
MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad;
ibcom_errno =
- MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[vc->pg_rank], &MPID_nem_ib_com_scratch_pad);
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[vc->pg_rank],
+ &MPID_nem_ib_com_scratch_pad);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
int ntrial = 0;
volatile MPID_nem_ib_com_qp_state_t *rstate =
- (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] +
- vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t));
+ (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) MPID_nem_ib_com_scratch_pad->
+ icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] +
+ vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t));
dprintf("ib_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate));
while (rstate->state != MPID_NEM_IB_COM_QP_STATE_RTR) {
__asm__ __volatile__("pause;":::"memory");
@@ -901,7 +937,8 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
int i;
for (i = 0; i < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; i++) {
//dprintf("irecv,%d->%d\n", MPID_nem_ib_myrank, vc->pg_rank);
- ibcom_errno = MPID_nem_ib_com_irecv(MPID_nem_ib_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank);
+ ibcom_errno =
+ MPID_nem_ib_com_irecv(MPID_nem_ib_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_irecv");
}
@@ -909,8 +946,9 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
uint32_t max_msg_sz;
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
- &max_msg_sz, sizeof(max_msg_sz));
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd,
+ MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz,
+ sizeof(max_msg_sz));
VC_FIELD(vc, pending_sends) = 0;
#ifdef MPID_NEM_IB_ONDEMAND
VC_FIELD(vc, is_connected) = 0;
@@ -946,7 +984,7 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
dprintf("ib_vc_init,vc->eager_max_msg_sz=%d\n", vc->eager_max_msg_sz);
/* vc->rndvSend_fn is set in MPID_nem_vc_init (in src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c) */
- ;
+ ;
vc->sendNoncontig_fn = MPID_nem_ib_SendNoncontig;
vc->comm_ops = &comm_ops;
@@ -962,8 +1000,7 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_ACCUMULATE] = MPID_nem_ib_PktHandler_Accumulate;
MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_GET_DONE] = MPID_nem_ib_pkt_GET_DONE_handler;
MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REQ_SEQ_NUM] = MPID_nem_ib_PktHandler_req_seq_num;
- MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM] =
- MPID_nem_ib_PktHandler_reply_seq_num;
+ MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM] = MPID_nem_ib_PktHandler_reply_seq_num;
MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_CHG_RDMABUF_OCC_NOTIFY_STATE] =
MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state;
@@ -1025,10 +1062,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc)
dprintf("init,before,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n",
MPID_nem_ib_myrank, vc->pg_rank,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
- MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc,
- pending_sends));
+ MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends));
/* update remote RDMA-write-to buffer occupancy */
#if 0 /* we can't send it when the other party has closed QP */
@@ -1053,10 +1089,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc)
dprintf("init,middle,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n",
MPID_nem_ib_myrank, vc->pg_rank,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
- MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc,
- pending_sends));
+ MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends));
if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) {
usleep(1000);
@@ -1065,10 +1100,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc)
dprintf("init,middle2,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n",
MPID_nem_ib_myrank, vc->pg_rank,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
- MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc,
- pending_sends));
+ MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends));
if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) {
usleep(1000);
@@ -1087,10 +1121,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc)
dprintf("init,after ,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n",
MPID_nem_ib_myrank, vc->pg_rank,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
- MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc,
- pending_sends));
+ MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends));
/* drain scratch-pad scq */
ibcom_errno = MPID_nem_ib_drain_scq_scratch_pad();
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c
index f0c6ea4..e5da2f9 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c
@@ -23,7 +23,7 @@
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt,
- struct MPID_Request *req)
+ struct MPID_Request *req)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
@@ -137,7 +137,7 @@ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_p
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey,
- void *write_to_buf)
+ void *write_to_buf)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -149,7 +149,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3
ibcom_errno =
MPID_nem_ib_com_lrecv(vc_ib->sc->fd, (uint64_t) req, raddr, req->ch.lmt_data_sz, rkey,
- write_to_buf);
+ write_to_buf);
MPID_nem_ib_ncqe += 1;
//dprintf("start_recv,ncqe=%d\n", MPID_nem_ib_ncqe);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lrecv");
@@ -161,7 +161,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3
*((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))));
#ifdef MPID_NEM_IB_LMT_GET_CQE
- MPID_nem_ib_ncqe_to_drain += 1; /* use CQE instead of polling */
+ MPID_nem_ib_ncqe_to_drain += 1; /* use CQE instead of polling */
#else
/* drain_scq and ib_poll is not ordered, so both can decrement ref_count */
MPIR_Request_add_ref(req);
@@ -250,7 +250,7 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP
if (MPID_nem_ib_sendq_empty(vc_ib->sendq) &&
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack &&
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) {
- mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */
+ mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -280,8 +280,7 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
//dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
- MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
//dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
//dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail));
@@ -292,10 +291,9 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG);
+ vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG);
}
- if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) &&
- MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) {
+ if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) {
dprintf("lmt_start_recv,send_progress\n");
fflush(stdout);
MPID_nem_ib_send_progress(vc_ib);
@@ -360,7 +358,8 @@ int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req)
}
//assert(dt_true_lb == 0);
- uint8_t *tailp = (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t));
+ uint8_t *tailp =
+ (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t));
#if 0
*is_end_flag_same = (r_cookie_buf->tail == *tailp) ? 1 : 0;
#else
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c
index 021e354..77d4c48 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c
@@ -65,7 +65,8 @@ int MPID_nem_ib_drain_scq(int dont_call_progress)
#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, 1, &cqe[0]);
#else
- result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
+ result =
+ ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
#endif
MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq");
@@ -190,7 +191,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress)
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) <
+ vc_ib->ibcom->lsr_seq_num_tail) <
MPID_NEM_IB_COM_RDMABUF_NSEG);
MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq);
@@ -356,7 +357,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress)
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) <
+ vc_ib->ibcom->lsr_seq_num_tail) <
MPID_NEM_IB_COM_RDMABUF_NSEG);
MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq);
int msg_type_sreq = MPIDI_Request_get_msg_type(sreq);
@@ -413,7 +414,9 @@ int MPID_nem_ib_drain_scq_lmt_put()
#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, 1, &cqe[0]);
#else
- result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
+ result =
+ ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN,
+ &cqe[0]);
#endif
MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq");
@@ -463,7 +466,7 @@ int MPID_nem_ib_drain_scq_lmt_put()
MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc);
vc_ib->ibcom->ncom_lmt_put -= 1;
MPID_nem_ib_ncqe_lmt_put -= 1;
- dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_ib_ncqe_lmt_put); /*suspicious */
+ dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_ib_ncqe_lmt_put); /*suspicious */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
(VC_FIELD(req->ch.vc, pending_sends)) -= 1;
@@ -523,7 +526,9 @@ int MPID_nem_ib_drain_scq_scratch_pad()
#if 0 /*def HAVE_LIBDCFA */
result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, 1, &cqe[0]);
#else
- result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
+ result =
+ ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN,
+ &cqe[0]);
#endif
MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq");
@@ -580,7 +585,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
//dprintf("ib_poll,ld,rsr_seq_num_poll=%d\n", vc_ib->ibcom->rsr_seq_num_poll);
volatile void *buf =
(uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] +
- MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint32_t) vc_ib->ibcom->rsr_seq_num_poll % MPID_NEM_IB_COM_RDMABUF_NSEG);
+ MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint32_t) vc_ib->ibcom->rsr_seq_num_poll %
+ MPID_NEM_IB_COM_RDMABUF_NSEG);
volatile MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf;
if (sz_hdrmagic->magic != MPID_NEM_IB_COM_MAGIC) {
goto fn_exit;
@@ -599,7 +605,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
int sz_data_pow2;
MPID_NEM_IB_SZ_DATA_POW2(sz_hdrmagic->sz);
- volatile MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + sz_data_pow2);
+ volatile MPID_nem_ib_tailmagic_t *tailmagic =
+ (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + sz_data_pow2);
dprintf("poll,sz_data_pow2=%d,tailmagic=%p,sz=%d\n", sz_data_pow2, tailmagic, sz_hdrmagic->sz);
int k = 0;
//tsce = MPID_nem_ib_rdtsc(); printf("9,%ld\n", tsce - tscs); // 55 for 512-byte
@@ -638,7 +645,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
#if 1
void *rsi;
- for (rsi = (void *) buf; rsi < (uint8_t *) buf + sz_hdrmagic->sz; rsi = (uint8_t *) rsi + 64 * 4) {
+ for (rsi = (void *) buf; rsi < (uint8_t *) buf + sz_hdrmagic->sz;
+ rsi = (uint8_t *) rsi + 64 * 4) {
#ifdef __MIC__
__asm__ __volatile__
("movq %0, %%rsi;"
@@ -656,9 +664,11 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
}
#endif
- MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t));
+ MPIDI_CH3_Pkt_eager_send_t *pkt =
+ (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t));
MPIU_Assert(sz_hdrmagic->sz >=
- sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t));
+ sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) +
+ sizeof(MPID_nem_ib_tailmagic_t));
MPIDI_CH3_Pkt_eager_send_t *pkt2 =
(MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) +
sizeof(MPID_nem_ib_pkt_prefix_t));
@@ -691,16 +701,17 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
int notify_rate;
ibcom_errno =
- MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, ¬ify_rate);
+ MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd,
+ ¬ify_rate);
dprintf("poll_eager,sendq=%d,ncom=%d,ncqe=%d,ldiff=%d(%d-%d),rdiff=%d(%d-%d),rate=%d\n",
- MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
+ MPID_nem_ib_sendq_empty(vc_ib->sendq),
+ vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
- vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent,
- notify_rate);
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, notify_rate);
//dprintf("ib_poll,current pcc=%d\n", MPIDI_CH3I_progress_completion_count.v);
@@ -715,12 +726,14 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc)
dprintf("poll_eager,released,type=%d,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM=%d\n", pkt->type,
MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM);
MPID_nem_ib_recv_buf_released(vc,
- (void *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) +
- sizeof(MPIDI_CH3_Pkt_t)));
+ (void *) ((uint8_t *) buf +
+ sizeof(MPID_nem_ib_sz_hdrmagic_t) +
+ sizeof(MPIDI_CH3_Pkt_t)));
}
else {
if (sz_hdrmagic->sz ==
- sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t)) {
+ sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) +
+ sizeof(MPID_nem_ib_tailmagic_t)) {
if (pkt->type == MPIDI_CH3_PKT_EAGERSHORT_SEND
//|| pkt->type == MPIDI_CH3_PKT_GET
) {
@@ -1096,8 +1109,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED);
- dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank,
- user_data);
+ dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank, user_data);
#if 1 /* moving from ib_poll */
/* unmark magic */
/* magic is located at MPID_NEM_IB_COM_INLINE_DATA boundary and variable length entails multiple prospective locations for the future use */
@@ -1112,10 +1124,12 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
}
MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= user_data &&
- user_data < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ);
+ user_data <
+ vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ);
unsigned long mod =
(unsigned long) ((uint8_t *) user_data -
- (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) & (MPID_NEM_IB_COM_RDMABUF_SZSEG - 1);
+ (uint8_t *) vc_ib->ibcom->
+ icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) & (MPID_NEM_IB_COM_RDMABUF_SZSEG - 1);
void *buf = (void *) ((uint8_t *) user_data - mod);
//dprintf("recv_buf_released,clearing,buf=%p\n", buf);
@@ -1130,10 +1144,12 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
offset =
offset ? ((((offset + 1) << 1) - 1) >
MPID_NEM_IB_MAX_DATA_POW2 ? MPID_NEM_IB_MAX_DATA_POW2 : (((offset + 1) << 1) -
- 1)) : 15) {
- volatile MPID_nem_ib_tailmagic_t *ptr = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + offset);
+ 1)) : 15) {
+ volatile MPID_nem_ib_tailmagic_t *ptr =
+ (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + offset);
MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= ptr &&
- ptr < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ);
+ ptr <
+ vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ);
ptr->magic = 0 /*0xde */ ;
if (offset == sz_data_pow2) {
break;
@@ -1146,7 +1162,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
/* mark that one eager-send RDMA-write-to buffer has been released */
int index_slot =
(unsigned long) ((uint8_t *) user_data -
- (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) / MPID_NEM_IB_COM_RDMABUF_SZSEG;
+ (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) /
+ MPID_NEM_IB_COM_RDMABUF_SZSEG;
MPIU_Assert(0 <= index_slot && index_slot < MPID_NEM_IB_COM_RDMABUF_NSEG);
//dprintf("user_data=%p,mem=%p,sub=%08lx,index_slot=%d\n", user_data, vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], (unsigned long)user_data - (unsigned long)vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], index_slot);
//dprintf("index_slot=%d,released=%016lx\n", index_slot, vc_ib->ibcom->rsr_seq_num_released[index_slot / 64]);
@@ -1158,11 +1175,10 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
//dprintf("released=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]);
if (1 || (index_tail & 7) || MPID_nem_ib_diff32(index_slot, index_tail) >= MPID_NEM_IB_COM_RDMABUF_NSEG - 8) { /* avoid wrap-around */
while (1) {
- if (((vc_ib->ibcom->
- rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) {
+ if (((vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 1) ==
+ 1) {
vc_ib->ibcom->rsr_seq_num_tail += 1;
- vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &=
- ~(1ULL << (index_tail & 63));
+ vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(1ULL << (index_tail & 63));
dprintf("rsr_seq_num_tail,incremented to %d\n", vc_ib->ibcom->rsr_seq_num_tail);
}
else {
@@ -1174,8 +1190,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
if (((vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 0xff) ==
0xff) {
vc_ib->ibcom->rsr_seq_num_tail += 8;
- vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &=
- ~(0xffULL << (index_tail & 63));
+ vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(0xffULL << (index_tail & 63));
//dprintf("released[index_tail/64]=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]);
}
}
@@ -1184,7 +1199,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
int notify_rate;
ibcom_errno =
- MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, ¬ify_rate);
+ MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd,
+ ¬ify_rate);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get");
@@ -1201,7 +1217,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
int msg_type = MPIDI_Request_get_msg_type(sreq);
if (msg_type == MPIDI_REQUEST_EAGER_MSG && /* guard for the following pointer dereference */
((MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF)->type == MPIDI_NEM_PKT_NETMOD &&
- ((MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF)->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) {
+ ((MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF)->subtype ==
+ MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) {
goto skip;
}
}
@@ -1226,7 +1243,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -1273,8 +1290,8 @@ int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */)
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */)
{
MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt;
MPIDI_CH3_Pkt_eager_send_t *ch3_pkt =
@@ -1303,20 +1320,18 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
int *lsr_seq_num_tail;
MPID_nem_ib_vc_area *vc_ib = VC_IB(vc);
ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_lsr_seq_num_tail_get");
dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail=%d,netmod_pkt->seq_num_tail=%d\n",
*lsr_seq_num_tail, netmod_pkt->seq_num_tail);
*lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, netmod_pkt->seq_num_tail);
- dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n",
- *lsr_seq_num_tail);
+ dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n", *lsr_seq_num_tail);
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
- dprintf("pkthandler,eagersend,old rstate=%d\n",
- vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
+ dprintf("pkthandler,eagersend,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, lsr_seq_num_tail);
- dprintf("pkthandler,eagersend,new rstate=%d\n",
- vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
+ dprintf("pkthandler,eagersend,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
dprintf("pkthandler,eagersend,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n",
@@ -1367,8 +1382,8 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */)
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */)
{
MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt;
MPIDI_CH3_Pkt_put_t *ch3_pkt =
@@ -1396,8 +1411,7 @@ int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
- MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n",
@@ -1435,8 +1449,8 @@ int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */)
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */)
{
MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt;
MPIDI_CH3_Pkt_accum_t *ch3_pkt =
@@ -1464,8 +1478,7 @@ int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
- MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n",
@@ -1502,8 +1515,8 @@ int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */)
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */)
{
MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt;
MPIDI_CH3_Pkt_get_t *ch3_pkt =
@@ -1531,8 +1544,7 @@ int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
- MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n",
@@ -1568,8 +1580,8 @@ int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen /* out */ ,
- MPID_Request ** rreqp /* out */)
+ MPIDI_msg_sz_t * buflen /* out */ ,
+ MPID_Request ** rreqp /* out */)
{
MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt;
MPIDI_CH3_Pkt_get_t *ch3_pkt =
@@ -1597,8 +1609,7 @@ int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
/* change remote notification policy of RDMA-write-to buf */
dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
- MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail);
dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n",
@@ -1633,7 +1644,7 @@ int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -1671,7 +1682,7 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
/* change remote notification policy of RDMA-write-to buf */
//dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib,
- &vc_ib->ibcom->lsr_seq_num_tail);
+ &vc_ib->ibcom->lsr_seq_num_tail);
//dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
//dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail));
@@ -1682,8 +1693,9 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG,
- vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail);
+ vc_ib->ibcom->lsr_seq_num_tail) <
+ MPID_NEM_IB_COM_RDMABUF_NSEG, vc_ib->ibcom->sseq_num,
+ vc_ib->ibcom->lsr_seq_num_tail);
}
dprintf("get_done_handler,send_progress\n");
fflush(stdout);
@@ -1711,7 +1723,7 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -1737,7 +1749,7 @@ int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG,
+ vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG,
vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail);
/* send reply */
@@ -1756,7 +1768,7 @@ int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
+ MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -1778,7 +1790,8 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
dprintf("pkthandler,reply_seq_num,old lsr_seq_num=%d,reply_pkt->seq_num_tail=%d\n",
vc_ib->ibcom->lsr_seq_num_tail, reply_pkt->seq_num_tail);
ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_lsr_seq_num_tail_get");
*lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, reply_pkt->seq_num_tail);
//dprintf("pkthandler,reply_seq_num,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail);
@@ -1793,8 +1806,8 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
/* try to send from sendq because at least one RDMA-write-to buffer has been released */
//dprintf("pkthandler,reply_seq_num,send_progress\n");
dprintf("pkthandler,reply_seq_num,send_progress\n");
- MPID_NEM_IB_CHECK_AND_SEND_PROGRESS fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM);
+ MPID_NEM_IB_CHECK_AND_SEND_PROGRESS fn_exit:
+ MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM);
return mpi_errno;
fn_fail:
goto fn_exit;
@@ -1805,9 +1818,9 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc,
- MPIDI_CH3_Pkt_t * pkt,
- MPIDI_msg_sz_t * buflen,
- MPID_Request ** rreqp)
+ MPIDI_CH3_Pkt_t * pkt,
+ MPIDI_msg_sz_t * buflen,
+ MPID_Request ** rreqp)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -1831,7 +1844,7 @@ int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc
int *rdmabuf_occupancy_notify_lstate;
ibcom_errno =
MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(vc_ib->sc->fd,
- &rdmabuf_occupancy_notify_lstate);
+ &rdmabuf_occupancy_notify_lstate);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get");
*rdmabuf_occupancy_notify_lstate = reply_pkt->state;
@@ -1861,7 +1874,9 @@ int MPID_nem_ib_cm_drain_scq()
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ);
- result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]);
+ result =
+ ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN,
+ &cqe[0]);
MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq");
if (result > 0) {
@@ -1922,23 +1937,26 @@ int MPID_nem_ib_cm_poll()
dprintf("cm_poll,enter\n");
- volatile uint32_t *owner = (uint32_t *) (MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]);
+ volatile uint32_t *owner =
+ (uint32_t *) (MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]);
if (*owner == (uint32_t) - 1) {
goto fn_exit;
} /* not acquired */
MPID_nem_ib_com *ibcom_scratch_pad;
- ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[*owner], &ibcom_scratch_pad);
+ ibcom_errno =
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[*owner], &ibcom_scratch_pad);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
MPID_nem_ib_cm_cmd_t *received =
(MPID_nem_ib_cm_cmd_t *) (ibcom_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] +
- sizeof(uint32_t));
+ sizeof(uint32_t));
MPID_nem_ib_cm_cmd_t cmd;
MPID_nem_ib_vc_area *vc_ib;
switch (received->type) {
case MPID_NEM_IB_CM_SYN:
- ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[*owner].fd);
+ ibcom_errno =
+ MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[*owner].fd);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open");
cmd.type = MPID_NEM_IB_CM_SYNACK;
goto common_tail;
@@ -1946,38 +1964,49 @@ int MPID_nem_ib_cm_poll()
case MPID_NEM_IB_CM_BUSINESSCARD:
ibcom_errno =
MPID_nem_ib_com_rts(MPID_nem_ib_conns[*owner].fd, received->qpnum, received->lid,
- &(received->gid));
+ &(received->gid));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts");
ibcom_errno =
- MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[*owner].fd, received->rmem, received->rkey);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect");
+ MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[*owner].fd, received->rmem,
+ received->rkey);
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_reg_mr_connect");
VC_FIELD(MPID_nem_ib_conns[owner].vc, is_connected) = 1;
cmd.type = MPID_NEM_IB_CM_ACK;
common_tail:
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &(cmd.lid),
- sizeof(uint16_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd,
+ MPID_NEM_IB_COM_INFOKEY_PORT_LID, &(cmd.lid),
+ sizeof(uint16_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &(cmd.gid),
- sizeof(union ibv_gid));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd,
+ MPID_NEM_IB_COM_INFOKEY_PORT_GID, &(cmd.gid),
+ sizeof(union ibv_gid));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &(cmd.qpnum),
- sizeof(uint32_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd,
+ MPID_NEM_IB_COM_INFOKEY_QP_QPN, &(cmd.qpnum),
+ sizeof(uint32_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &(cmd.rmem), sizeof(void *));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd,
+ MPID_NEM_IB_COM_SCRATCH_PAD_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &(cmd.rmem),
+ sizeof(void *));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int));
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd,
+ MPID_NEM_IB_COM_SCRATCH_PAD_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int));
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
*owner = (uint32_t) - 1; /* release */
@@ -2030,30 +2059,38 @@ int MPID_nem_ib_cm_accept()
MPID_nem_ib_conn_pkt_t *rpkt = (MPID_nem_ib_conn_pkt_t *) (rbuf + 40);
if (rpkt->type == MPID_NEM_IB_SYN) {
- dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_ib_myrank, rpkt->remote_rank,
- rpkt->type);
+ dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_ib_myrank, rpkt->remote_rank, rpkt->type);
void *sbuf;
ibcom_errno = MPID_nem_ib_com_mem_udwr_from(MPID_nem_ib_conn_ud_fd, &sbuf);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_mem_udwr_from");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_mem_udwr_from");
MPID_nem_ib_conn_pkt_t *spkt = (MPID_nem_ib_conn_pkt_t *) (sbuf + 40);
spkt->remote_rank = MPID_nem_ib_myrank;
spkt->type = MPID_NEM_IB_SYNACK;
ibcom_errno =
- MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[rpkt->remote_rank].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN,
- &spkt->qpn, sizeof(uint32_t));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn");
+ MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[rpkt->remote_rank].fd,
+ MPID_NEM_IB_COM_INFOKEY_QP_QPN, &spkt->qpn,
+ sizeof(uint32_t));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_conn");
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &spkt->rmem, sizeof(void *));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd,
+ MPID_NEM_IB_COM_RDMAWR_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &spkt->rmem,
+ sizeof(void *));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_mr");
ibcom_errno =
- MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO,
- MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &spkt->rkey, sizeof(int));
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr");
+ MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd,
+ MPID_NEM_IB_COM_RDMAWR_TO,
+ MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &spkt->rkey,
+ sizeof(int));
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_get_info_mr");
/* kokomade. add udsend_core(synack) */
if (MPID_nem_ib_conn_ibcom->ncom < &&MPID_nem_ib_ncqe_connect <) {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c
index 145ae53..ab9448e 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c
@@ -57,11 +57,13 @@ static inline void *aalloc(size_t sz, int id)
arena_flist[id] = ((free_list_t *) arena_flist[id])->next;
}
else {
- q = mmap(NULL, MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096), PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ q = mmap(NULL,
+ MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096),
+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#if MPID_NEM_IB_NCLUST_SLAB > 1
arena_flist[id] = q + MPID_NEM_IB_SZARENA;
- for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA; p += MPID_NEM_IB_SZARENA) {
+ for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA;
+ p += MPID_NEM_IB_SZARENA) {
((free_list_t *) p)->next = p + MPID_NEM_IB_SZARENA;
}
((free_list_t *) p)->next = 0;
@@ -105,7 +107,8 @@ struct MPID_nem_ib_com_reg_mr_cache_entry_t {
int refc;
};
-static struct MPID_nem_ib_com_reg_mr_listnode_t MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE];
+static struct MPID_nem_ib_com_reg_mr_listnode_t
+ MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE];
__inline__ int MPID_nem_ib_com_hash_func(char *addr)
{
@@ -115,7 +118,8 @@ __inline__ int MPID_nem_ib_com_hash_func(char *addr)
return (int) v;
}
-static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c, struct MPID_nem_ib_com_reg_mr_listnode_t *e)
+static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c,
+ struct MPID_nem_ib_com_reg_mr_listnode_t *e)
{
struct MPID_nem_ib_com_reg_mr_listnode_t *next;
struct MPID_nem_ib_com_reg_mr_listnode_t *prev;
@@ -142,7 +146,9 @@ static inline void __lru_queue_display()
int i = 0;
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
dprintf("---- hash %d\n", i);
- for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next;
+ for (p =
+ (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].
+ lru_next;
p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];
p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) {
if (p && p->addr) {
@@ -161,8 +167,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
#if 0 /* debug */
struct ibv_mr *mr;
int ibcom_errno = MPID_nem_ib_com_reg_mr(addr, len, &mr);
- printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, mr->lkey,
- mr->rkey);
+ printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len,
+ mr->lkey, mr->rkey);
if (ibcom_errno != 0) {
goto fn_fail;
}
@@ -183,8 +189,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
#else
void *addr_aligned = (void *) ((unsigned long) addr & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1));
int len_aligned =
- ((((unsigned long) addr + len) - (unsigned long) addr_aligned + MPID_NEM_IB_COM_REG_MR_SZPAGE -
- 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1));
+ ((((unsigned long) addr + len) - (unsigned long) addr_aligned +
+ MPID_NEM_IB_COM_REG_MR_SZPAGE - 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1));
#endif
key = MPID_nem_ib_com_hash_func(addr);
@@ -193,12 +199,14 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
//__lru_queue_display();
int way = 0;
- for (e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next;
+ for (e =
+ (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next;
e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[key];
e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_next, way++) {
//dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next);
- if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) {
+ if (e->addr <= addr_aligned &&
+ (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) {
//dprintf
//("MPID_nem_ib_com_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n",
//e->addr, e->len, e->mr->addr, e->mr->length, addr, len);
@@ -224,7 +232,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
}
- e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t), MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
+ e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t),
+ MPID_NEM_IB_COM_AALLOC_ID_MRCACHE);
/* reference counter is used when evicting entry */
e->refc = 1;
@@ -242,7 +251,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
//key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey);
/* register to cache */
- MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
+ MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key],
+ (struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
//__lru_queue_display();
@@ -254,9 +264,12 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len)
e->refc++;
#if 0 /* disable for debug */
/* move to head of the list */
- if (e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next) {
+ if (e !=
+ (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].
+ lru_next) {
MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
- MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
+ MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key],
+ (struct MPID_nem_ib_com_reg_mr_listnode_t *) e);
}
#endif
//dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e,
@@ -291,8 +304,10 @@ void MPID_nem_ib_com_register_cache_init()
/* Using the address to the start node to express the end of the list
* instead of using NULL */
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
- MPID_nem_ib_com_reg_mr_cache[i].lru_next = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
- MPID_nem_ib_com_reg_mr_cache[i].lru_prev = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
+ MPID_nem_ib_com_reg_mr_cache[i].lru_next =
+ (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
+ MPID_nem_ib_com_reg_mr_cache[i].lru_prev =
+ (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i];
}
dprintf("[MrCache] cache initializes %d entries\n", MPID_NEM_IB_COM_REG_MR_NLINE);
@@ -304,7 +319,9 @@ void MPID_nem_ib_com_register_cache_destroy()
int i = 0, cnt = 0;
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
- for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next;
+ for (p =
+ (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].
+ lru_next;
p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];
p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) {
if (p && p->addr > 0) {
diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c
index c0ca7e0..48b9b83 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c
@@ -54,7 +54,8 @@ void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_lengt
}
int s1_minus_s2;
while (1) {
- int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree);
+ int lmin =
+ key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree);
int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree);
s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin);
@@ -66,24 +67,24 @@ void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_lengt
else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) {
// psArg is "smaller" OR same substring, psArg is shorter
if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) {
- MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write
+ MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write
/* left child */
MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val);
dprintf("stored as left child\n");
return;
}
- pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child
+ pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child
}
else {
// psArg is "larger" OR same substring, psArg is longer
if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) {
- MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write
+ MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write
/* right child */
MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val);
dprintf("stored as right child\n");
return;
}
- pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child
+ pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child
}
}
}
@@ -102,7 +103,8 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length
}
int s1_minus_s2;
while (1) {
- int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree);
+ int lmin =
+ key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree);
int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree);
s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin);
@@ -118,7 +120,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length
dprintf("left is null\n");
goto fn_fail;
}
- pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child
+ pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child
}
else {
// psArg is "larger" OR same substring, psArg is longer
@@ -127,7 +129,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length
dprintf("right is null\n");
goto fn_fail;
}
- pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child
+ pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child
}
}
fn_exit:
@@ -142,7 +144,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
- MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
+ MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -159,7 +161,8 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
/* remote SR sequence number which is last sent */
int *rsr_seq_num_tail;
ibcom_errno = MPID_nem_ib_com_rsr_seq_num_tail_get(vc_ib->sc->fd, &rsr_seq_num_tail);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rsr_seq_num_tail_get");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_rsr_seq_num_tail_get");
/* remote SR sequence number which is last sent */
int *rsr_seq_num_tail_last_sent;
@@ -256,14 +259,14 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
int copied;
ibcom_errno =
- MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr, hdr_sz, data,
- (int) data_sz, &copied);
+ MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr,
+ hdr_sz, data, (int) data_sz, &copied);
MPIU_ERR_CHKFATALANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend");
MPID_nem_ib_ncqe += 1;
//dprintf("isendcontig_core,ncqe=%d\n", MPID_nem_ib_ncqe);
dprintf("isendcontig_core,isend,kind=%d,msg_type=%d,copied=%d\n", sreq->kind, msg_type, copied); /*suspicious lines,(if1,on,on,off,if0) works */
#if 0
-#define MPID_NEM_IB_TLBPREF_SEND 20 //20
+#define MPID_NEM_IB_TLBPREF_SEND 20 //20
int tlb_pref_ahd = 4096 * MPID_NEM_IB_TLBPREF_SEND;
__asm__ __volatile__
("movq %0, %%rsi;"
@@ -336,16 +339,17 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
}
}
else {
- MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */
+ MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */
}
#else
- MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */
+ MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */
#endif
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
//dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
int *notify_rstate;
- ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, ¬ify_rstate);
+ ibcom_errno =
+ MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, ¬ify_rstate);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get");
@@ -354,13 +358,14 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
/* if the number of slots in RMDA-write-to buffer have hit the high water-mark */
if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW &&
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) {
+ vc_ib->ibcom->lsr_seq_num_tail) >
+ MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) {
dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num);
/* remember remote notifying policy so that local can know when to change remote policy back to LW */
*notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW;
/* change remote notifying policy of RDMA-write-to buf occupancy info */
MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc,
- MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW);
+ MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW);
}
//dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
@@ -377,7 +382,7 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
- MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
+ MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -399,10 +404,12 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
#if 0
/* aggressively perform drain_scq */
/* try to clear the road blocks, i.e. ncom, ncqe */
- if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
- MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
+ if (vc_ib->ibcom->ncom >=
+ /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
+ MPID_nem_ib_ncqe >=
+ /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
//printf("isendcontig,kick drain_scq\n");
- ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
+ ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq");
}
#endif
@@ -425,7 +432,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
int *lsr_seq_num_tail;
/* sequence number of (largest) completed send command */
ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail);
- MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_lsr_seq_num_tail_get");
int lsr_seq_num_head;
/* sequence number of (largest) in-flight send command */
@@ -438,7 +446,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent);
dprintf("isendcontig,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n",
MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe,
@@ -459,7 +467,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
/* make control packet bringing sequence number go ahead of
* queued packets to avoid dead-lock */
int goahead =
- (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REQ_SEQ_NUM)
+ (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD &&
+ netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REQ_SEQ_NUM)
|| (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD &&
netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) ||
(ch3_hdr->type == MPIDI_NEM_PKT_NETMOD &&
@@ -474,7 +483,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
(goahead || MPID_nem_ib_sendq_empty(vc_ib->sendq)) &&
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack &&
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack &&
- MPID_nem_ib_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) {
+ MPID_nem_ib_diff32(lsr_seq_num_head,
+ *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) {
mpi_errno = MPID_nem_ib_iSendContig_core(vc, sreq, hdr, hdr_sz, data, data_sz);
if (mpi_errno) {
@@ -526,7 +536,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
dprintf("isendcontig,enqueuing,type=%d,\n", ((MPIDI_CH3_Pkt_t *) hdr)->type);
#if 0
if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD) {
- switch(((MPID_nem_pkt_netmod_t *) hdr)->subtype) {
+ switch (((MPID_nem_pkt_netmod_t *) hdr)->subtype) {
case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:
dprintf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank,
MPID_nem_ib_ncqe);
@@ -534,8 +544,9 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
default:
break;
}
- } else {
- switch(((MPIDI_CH3_Pkt_t *) hdr)->type) {
+ }
+ else {
+ switch (((MPIDI_CH3_Pkt_t *) hdr)->type) {
case MPIDI_CH3_PKT_ACCUMULATE:
dprintf("enqueuing ACCUMULATE\n");
break;
@@ -581,7 +592,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz,
- void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr)
+ void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr)
{
MPID_Request *sreq = NULL;
int mpi_errno = MPI_SUCCESS;
@@ -649,7 +660,7 @@ int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_s
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
- MPIDI_msg_sz_t hdr_sz)
+ MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -677,14 +688,15 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
}
ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &sseq_num);
- MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get");
+ MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER,
+ "**MPID_nem_ib_com_sseq_num_get");
int copied;
dprintf("sendnoncontig_core,isend,%d->%d,seq_num=%d\n", MPID_nem_ib_myrank, vc->pg_rank,
sseq_num);
ibcom_errno =
MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, NULL, 0, hdr, sizeof(MPIDI_CH3_Pkt_t),
- (void *) REQ_FIELD(sreq, lmt_pack_buf), (int) last, &copied);
+ (void *) REQ_FIELD(sreq, lmt_pack_buf), (int) last, &copied);
MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend");
MPID_nem_ib_ncqe += 1;
dprintf("sendnoncontig_core,ncqe=%d\n", MPID_nem_ib_ncqe);
@@ -727,17 +739,18 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
}
}
else {
- MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */
+ MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */
}
#else
- MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */
+ MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */
#endif
#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE
#if 1
//dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
int *notify_rstate;
- ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, ¬ify_rstate);
+ ibcom_errno =
+ MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, ¬ify_rstate);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get");
@@ -746,13 +759,14 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
/* if the number of slots in RMDA-write-to buffer have hit the high water-mark */
if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW &&
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) {
+ vc_ib->ibcom->lsr_seq_num_tail) >
+ MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) {
dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num);
/* remember remote notifying policy so that local can know when to change remote policy back to LW */
*notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW;
/* change remote notifying policy of RDMA-write-to buf occupancy info */
MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc,
- MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW);
+ MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW);
}
//dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate);
#endif
@@ -770,7 +784,7 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
- MPIDI_msg_sz_t hdr_sz)
+ MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int ibcom_errno;
@@ -789,10 +803,12 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
#if 0
/* aggressively perform drain_scq */
/* try to clear the road blocks, i.e. ncom, ncqe */
- if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
- MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
+ if (vc_ib->ibcom->ncom >=
+ /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
+ MPID_nem_ib_ncqe >=
+ /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
//printf("isendcontig,kick drain_scq\n");
- ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
+ ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq");
}
#endif
@@ -801,12 +817,12 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
/* if IB command overflow-queue is empty AND local IB command queue isn't full AND remote RDMA-write-to buf isn't getting overrun */
/* set it for drain_scq */
- int slack = MPID_NEM_IB_COM_AMT_SLACK; /* slack for control packet bringing sequence number */
+ int slack = MPID_NEM_IB_COM_AMT_SLACK; /* slack for control packet bringing sequence number */
if (MPID_nem_ib_sendq_empty(vc_ib->sendq) &&
vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack &&
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack &&
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) {
+ vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) {
mpi_errno = MPID_nem_ib_SendNoncontig_core(vc, sreq, hdr, hdr_sz);
if (mpi_errno) {
@@ -871,10 +887,12 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
#if 0
/* aggressively perform drain_scq */
/* try to clear the road blocks, i.e. ncom, ncqe */
- if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
- MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
+ if (vc_ib->ibcom->ncom >=
+ /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN ||
+ MPID_nem_ib_ncqe >=
+ /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) {
dprintf("send_progress,kick drain_scq\n");
- ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
+ ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_drain_scq");
}
@@ -892,11 +910,13 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
(ch3_hdr->type != MPIDI_NEM_PKT_NETMOD ||
netmod_hdr->subtype != MPIDI_NEM_IB_PKT_LMT_GET_DONE) &&
ch3_hdr->type != MPIDI_NEM_PKT_LMT_RTS &&
- ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0) : MPID_NEM_IB_COM_AMT_SLACK;
+ ch3_hdr->type !=
+ MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0) :
+ MPID_NEM_IB_COM_AMT_SLACK;
if (vc_ib->ibcom->ncom >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack ||
MPID_nem_ib_ncqe >= MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack ||
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail) >=
+ vc_ib->ibcom->lsr_seq_num_tail) >=
MPID_NEM_IB_COM_RDMABUF_NSEG - slack) {
break;
}
@@ -912,10 +932,10 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
dprintf("send_progress,%d->%d,rdiff=%d(%d-%d),ldiff=%d(%d-%d),slack=%d\n",
MPID_nem_ib_myrank, sreq->ch.vc->pg_rank,
MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail,
- vc_ib->ibcom->rsr_seq_num_tail_last_sent),
+ vc_ib->ibcom->rsr_seq_num_tail_last_sent),
vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent,
MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num,
- vc_ib->ibcom->lsr_seq_num_tail),
+ vc_ib->ibcom->lsr_seq_num_tail),
vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, slack);
if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_EAGER_MSG) {
if (!sreq->ch.noncontig) {
@@ -969,8 +989,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
* might have happened while being queued */
case MPIDI_NEM_IB_PKT_LMT_GET_DONE:{
MPID_nem_ib_pkt_lmt_get_done_t *_done_pkt =
- (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev.iov[0].
- MPID_IOV_BUF;
+ (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev.
+ iov[0].MPID_IOV_BUF;
dprintf
("send_progress,MPIDI_NEM_IB_PKT_LMT_GET_DONE,rsr_seq_num_tail=%d\n",
vc_ib->ibcom->rsr_seq_num_tail);
@@ -983,8 +1003,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
}
case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{
MPID_nem_ib_pkt_reply_seq_num_t *_pkt =
- (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.iov[0].
- MPID_IOV_BUF;
+ (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.
+ iov[0].MPID_IOV_BUF;
dprintf
("send_progress,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n",
vc_ib->ibcom->rsr_seq_num_tail);
@@ -1003,10 +1023,10 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
mpi_errno =
MPID_nem_ib_iSendContig_core(sreq->ch.vc, sreq,
- sreq->dev.iov[0].MPID_IOV_BUF,
- sreq->dev.iov[0].MPID_IOV_LEN,
- sreq->dev.iov[1].MPID_IOV_BUF,
- sreq->dev.iov[1].MPID_IOV_LEN);
+ sreq->dev.iov[0].MPID_IOV_BUF,
+ sreq->dev.iov[0].MPID_IOV_LEN,
+ sreq->dev.iov[1].MPID_IOV_BUF,
+ sreq->dev.iov[1].MPID_IOV_LEN);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1015,8 +1035,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
dprintf("send_progress,send,noncontig\n");
mpi_errno =
MPID_nem_ib_SendNoncontig_core(sreq->ch.vc, sreq,
- sreq->dev.iov[0].MPID_IOV_BUF,
- sreq->dev.iov[0].MPID_IOV_LEN);
+ sreq->dev.iov[0].MPID_IOV_BUF,
+ sreq->dev.iov[0].MPID_IOV_LEN);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1028,8 +1048,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
MPID_nem_ib_sendq_next(sreq));
mpi_errno =
MPID_nem_ib_lmt_start_recv_core(sreq, REQ_FIELD(sreq, lmt_raddr),
- REQ_FIELD(sreq, lmt_rkey), REQ_FIELD(sreq,
- lmt_write_to_buf));
+ REQ_FIELD(sreq, lmt_rkey), REQ_FIELD(sreq,
+ lmt_write_to_buf));
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
@@ -1079,7 +1099,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib)
int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd)
{
MPID_nem_ib_com *ibcom_scratch_pad;
- ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[rank], &ibcom_scratch_pad);
+ ibcom_errno =
+ MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[rank], &ibcom_scratch_pad);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
if (MPID_nem_ib_ncqe_scratch_pad >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY ||
@@ -1090,8 +1111,9 @@ int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd)
}
ibcom_errno =
- MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) ibcom_scratch_pad,
- sizeof(uint32_t), sizeof(MPID_nem_ib_cm_cmd_t), (void *) cmd);
+ MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank],
+ (uint64_t) ibcom_scratch_pad, sizeof(uint32_t),
+ sizeof(MPID_nem_ib_cm_cmd_t), (void *) cmd);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad");
MPID_nem_ib_ncqe_scratch_pad += 1;
-----------------------------------------------------------------------
Summary of changes:
src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c | 474 +++++++++++++-------
src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h | 94 ++--
src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h | 84 ++--
src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c | 223 ++++++----
src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c | 21 +-
src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c | 253 ++++++-----
.../ch3/channels/nemesis/netmod/ib/ib_reg_mr.c | 55 ++-
src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c | 154 ++++---
8 files changed, 799 insertions(+), 559 deletions(-)
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list