[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1.3-32-gaad52bc
Service Account
noreply at mpich.org
Mon Oct 20 11:54:21 CDT 2014
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".
The branch, master has been updated
via aad52bcc7414ecf80a8242b5177d0b33f3067b5e (commit)
via 2df2063305764000b8cdeaba5735a43de4511d46 (commit)
from c2ce2188e32780d846b8366f372258df3b7025c0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/aad52bcc7414ecf80a8242b5177d0b33f3067b5e
commit aad52bcc7414ecf80a8242b5177d0b33f3067b5e
Author: Pavan Balaji <balaji at anl.gov>
Date: Thu Oct 16 18:20:30 2014 -0500
Delete the unsupported mx and wintcp netmods.
Signed-off-by: Wesley Bland <wbland at anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk
index c85abf9..256ae99 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk
+++ b/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk
@@ -6,7 +6,6 @@
## See COPYRIGHT in top-level directory.
##
-include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/mx/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/tcp/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/none/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/newmad/Makefile.mk
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/mx/Makefile.mk
deleted file mode 100644
index edcc31a..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/Makefile.mk
+++ /dev/null
@@ -1,24 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2012 by Argonne National Laboratory.
-## See COPYRIGHT in top-level directory.
-##
-
-if BUILD_NEMESIS_NETMOD_MX
-
-mpi_core_sources += \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_finalize.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c
-
-noinst_HEADERS += \
- src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h \
- src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h
-
-endif BUILD_NEMESIS_NETMOD_MX
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/TODO.txt b/src/mpid/ch3/channels/nemesis/netmod/mx/TODO.txt
deleted file mode 100644
index acc39e1..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/TODO.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-* Cancellation of requests
-Currently, cancellation of requests is implemented in the module
-itself but needs to be propagated at the ADI level.
-The code snipets are included in mx_cancel.c.
-However, since MX does not support the cancellation of send requests,
-it's unlikely that the MX module will.
-Note that for recv request cancellation, the VC is need only to find
-the module function. It's therefore MPI_ANY_SOURCE ready.
-
-* On demand connections
-Currently, all connections are established on start-up. Some framework
-for on-demand connection is implemented but not yet functionnal.
-
-* Maximum tag length.
-Currently, the MX module only supports tags that are at most 31-bits long.
-Support for full, 32-bits tags is implemented but some work need to be
-done at higher level. The current fix is the following :
-1- Uncomment the /* #define USE_CTXT_AS_MARK */ line in the mx_impl.h file
-2- Replace line 439 in the src/mpi/comm/commutil.c file :
- context_mask[0] = 0xFFFFFFF8;
- with the following code:
- context_mask[0] = 0xFFFFFFF0;
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c
deleted file mode 100644
index a5d6e31..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-#include "mx_impl.h"
-
-#define NUM_REQ (1024)
-
-static MPID_nem_mx_internal_req_t *MPID_nem_mx_internal_req_queue_head;
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_internal_req_queue_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_internal_req_queue_init(void)
-{
- MPID_nem_mx_internal_req_t *curr_req;
- MPID_nem_mx_internal_req_t *next_req;
- int mpi_errno = MPI_SUCCESS;
- int index;
-
- MPID_nem_mx_internal_req_queue_head = (MPID_nem_mx_internal_req_t *)MPIU_Malloc(sizeof(MPID_nem_mx_internal_req_t));
- curr_req = MPID_nem_mx_internal_req_queue_head;
-
- for(index = 0 ; index < (NUM_REQ - 1) ; index++)
- {
- next_req = (MPID_nem_mx_internal_req_t *)MPIU_Malloc(sizeof(MPID_nem_mx_internal_req_t));
- curr_req->next = next_req;
- curr_req = next_req;
- }
- curr_req->next = NULL;
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_internal_req_queue_destroy
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_internal_req_queue_destroy(void)
-{
- MPID_nem_mx_internal_req_t *curr_req;
- MPID_nem_mx_internal_req_t *req_to_del;
- int mpi_errno = MPI_SUCCESS;
-
- curr_req = MPID_nem_mx_internal_req_queue_head;
-
- if(curr_req->next == NULL)
- {
- MPIU_Free(curr_req);
- }
- else
- {
- do
- {
- req_to_del = curr_req;
- curr_req = curr_req->next;
- MPIU_Free(req_to_del);
- }
- while(curr_req != NULL);
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_internal_req_dequeue
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_internal_req_dequeue(MPID_nem_mx_internal_req_t **param_req)
-{
- int mpi_errno = MPI_SUCCESS;
-
- *param_req = MPID_nem_mx_internal_req_queue_head;
- MPID_nem_mx_internal_req_queue_head = (*param_req)->next;
- (*param_req)->next = NULL;
-
- if(MPID_nem_mx_internal_req_queue_head == NULL)
- {
- MPID_nem_mx_internal_req_t *curr_req;
- MPID_nem_mx_internal_req_t *next_req;
- int index;
-
- MPID_nem_mx_internal_req_queue_head = (MPID_nem_mx_internal_req_t *)MPIU_Malloc(sizeof(MPID_nem_mx_internal_req_t));
- curr_req = MPID_nem_mx_internal_req_queue_head;
- for(index = 0 ; index < (NUM_REQ - 1) ; index++)
- {
- next_req = (MPID_nem_mx_internal_req_t *)MPIU_Malloc(sizeof(MPID_nem_mx_internal_req_t));
- curr_req->next = next_req;
- curr_req = next_req;
- }
- curr_req->next = NULL;
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_internal_req_enqueue
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_internal_req_enqueue(MPID_nem_mx_internal_req_t *req)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_Assert(MPID_nem_mx_internal_req_queue_head != NULL);
- req->next = MPID_nem_mx_internal_req_queue_head;
- MPID_nem_mx_internal_req_queue_head = req;
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
deleted file mode 100644
index 755c327..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mx_impl.h"
-#include "my_papi_defs.h"
-
-/* code in mpid_cancel_send */
-/*
- #ifdef ENABLE_COMM_OVERRIDES
- if (vc->comm_ops && vc->comm_ops->cancel_send)
- {
- int handled;
- handled = vc->comm_ops->cancel_send(vc, sreq);
- if (handled)
- goto fn_exit;
- }
- #endif
-*/
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_cancel_send
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_cancel_send(MPIDI_VC_t *vc, MPID_Request *sreq)
-{
- mx_request_t *mx_request = NULL;
- mx_return_t ret;
- uint32_t result;
- int mpi_errno = MPI_SUCCESS;
- int handled = FALSE;
-
- if (!vc->ch.is_local)
- {
- mx_request = &(REQ_FIELD(sreq,mx_request));
- ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
- MPIU_ERR_CHKANDJUMP1(ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_cancel", "**mx_cancel %s", mx_strerror(ret));
-
- if (result)
- {
- MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
- sreq->cc = 0;
- MPIU_Object_set_ref(sreq, 1);
- (VC_FIELD(vc,pending_sends)) -= 1;
- }
- else
- {
- MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
- }
- handled = TRUE;
- }
-
- fn_exit:
- return handled;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_cancel_recv
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_cancel_recv(MPIDI_VC_t *vc, MPID_Request *rreq)
-{
- mx_request_t *mx_request = NULL;
- mx_return_t ret;
- uint32_t result;
- int mpi_errno = MPI_SUCCESS;
- int handled = FALSE;
-
- mx_request = &(REQ_FIELD(rreq,mx_request));
- /* FIXME this test is probably not correct with multiple netmods */
- /* We need to know to which netmod a recv request actually "belongs" to */
- if(mx_request != NULL)
- {
- ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
- MPIU_ERR_CHKANDJUMP1(ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_cancel", "**mx_cancel %s", mx_strerror(ret));
-
- if (result)
- {
- int found;
- MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
- found = MPIDI_CH3U_Recvq_DP(rreq);
- MPIU_Assert(found);
- MPIR_STATUS_SET_COUNT(rreq->status, 0);
- MPID_REQUEST_SET_COMPLETED(rreq);
- MPID_Request_release(rreq);
- }
- else
- {
- MPIR_STATUS_SET_CANCEL_BIT(rreq->status, FALSE);
- MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
- "request 0x%08x already matched, unable to cancel", rreq->handle);
- }
- handled = TRUE;
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_finalize.c
deleted file mode 100644
index e8743fe..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_finalize.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mx_impl.h"
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_finalize
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_finalize()
-{
- int mpi_errno = MPI_SUCCESS;
- int ret ;
-
- ret = mx_close_endpoint(MPID_nem_mx_local_endpoint);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_close_endpoint", "**mx_close_endpoint %s", mx_strerror (ret));
-
- ret = mx_finalize();
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_finalize", "**mx_finalize %s", mx_strerror (ret));
-
- MPID_nem_mx_internal_req_queue_destroy();
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h
deleted file mode 100644
index f634bf8..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#ifndef MX_MODULE_IMPL_H
-#define MX_MODULE_IMPL_H
-#ifdef USE_PMI2_API
-#include "pmi2.h"
-#else
-#include "pmi.h"
-#endif
-#include <myriexpress.h>
-#include "mx_extensions.h"
-#include "mpid_nem_impl.h"
-
-/* #define USE_CTXT_AS_MARK */
-/* #define DEBUG_IOV */
-/* #define ONDEMAND */
-
-int MPID_nem_mx_init (MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p);
-int MPID_nem_mx_finalize (void);
-int MPID_nem_mx_poll(int in_blocking_progress);
-int MPID_nem_mx_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p);
-int MPID_nem_mx_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc);
-int MPID_nem_mx_vc_init (MPIDI_VC_t *vc);
-int MPID_nem_mx_vc_destroy(MPIDI_VC_t *vc);
-int MPID_nem_mx_vc_terminate (MPIDI_VC_t *vc);
-int MPID_nem_mx_get_from_bc(const char *business_card, uint32_t *remote_endpoint_id, uint64_t *remote_nic_id);
-
-/* alternate interface */
-int MPID_nem_mx_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
- void *data, MPIDI_msg_sz_t data_sz);
-int MPID_nem_mx_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data,
- MPIDI_msg_sz_t data_sz, MPID_Request **sreq_ptr);
-int MPID_nem_mx_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz);
-
-/* Direct Routines */
-int MPID_nem_mx_directSend(MPIDI_VC_t *vc, const void * buf, int count, MPI_Datatype datatype, int dest, int tag,
- MPID_Comm * comm, int context_offset, MPID_Request **sreq_p);
-int MPID_nem_mx_directSsend(MPIDI_VC_t *vc, const void * buf, int count, MPI_Datatype datatype, int dest, int tag,
- MPID_Comm * comm, int context_offset,MPID_Request **sreq_p);
-int MPID_nem_mx_directRecv(MPIDI_VC_t *vc, MPID_Request *rreq);
-int MPID_nem_mx_cancel_send(MPIDI_VC_t *vc, MPID_Request *sreq);
-int MPID_nem_mx_cancel_recv(MPIDI_VC_t *vc, MPID_Request *rreq);
-int MPID_nem_mx_probe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, MPI_Status *status);
-int MPID_nem_mx_iprobe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, int *flag, MPI_Status *status);
-int MPID_nem_mx_improbe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, int *flag,
- MPID_Request **message, MPI_Status *status);
-
-int MPID_nem_mx_anysource_iprobe(int tag, MPID_Comm *comm, int context_offset, int *flag, MPI_Status *status);
-int MPID_nem_mx_anysource_improbe(int tag, MPID_Comm *comm, int context_offset, int *flag,
- MPID_Request **message,MPI_Status *status);
-/* Callback routine for unex msgs in MX */
-mx_unexp_handler_action_t MPID_nem_mx_get_adi_msg(void *context,mx_endpoint_addr_t source,
- uint64_t match_info,uint32_t length,void *data);
-/* Any source management */
-void MPID_nem_mx_anysource_posted(MPID_Request *rreq);
-int MPID_nem_mx_anysource_matched(MPID_Request *rreq);
-
-/* Dtype management */
-int MPID_nem_mx_process_sdtype(MPID_Request **sreq_p, MPI_Datatype datatype, MPID_Datatype * dt_ptr, const void *buf,
- int count, MPIDI_msg_sz_t data_sz, mx_segment_t *mx_iov, uint32_t *num_seg,int first_free_slot);
-int MPID_nem_mx_process_rdtype(MPID_Request **rreq_p, MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, mx_segment_t *mx_iov,
- uint32_t *num_seg);
-
-/* Connection management*/
-int MPID_nem_mx_send_conn_info (MPIDI_VC_t *vc);
-
-extern mx_endpoint_t MPID_nem_mx_local_endpoint;
-extern uint32_t MPID_NEM_MX_FILTER;
-extern uint64_t MPID_nem_mx_local_nic_id;
-extern uint32_t MPID_nem_mx_local_endpoint_id;
-
-/* The vc provides a generic buffer in which network modules can store
- private fields This removes all dependencies from the VC struction
- on the network module, facilitating dynamic module loading. */
-typedef struct
-{
- /* The following 2 are used to set-up the connection */
- uint32_t remote_endpoint_id;
- uint64_t remote_nic_id;
- uint16_t local_connected;
- uint16_t remote_connected;
- /* The following is used to actually send messages */
- mx_endpoint_addr_t remote_endpoint_addr;
- /* Poster recv pointer for anysource management*/
- /* int (* recv_posted)(MPID_Request *req, void *vc); */
- int pending_sends;
-} MPID_nem_mx_vc_area;
-
-/* accessor macro to private fields in VC */
-#define VC_FIELD(vcp, field) (((MPID_nem_mx_vc_area *)vcp->ch.netmod_area.padding)->field)
-
-/* The req provides a generic buffer in which network modules can store
- private fields This removes all dependencies from the req structure
- on the network module, facilitating dynamic module loading. */
-typedef struct
-{
- mx_request_t mx_request;
- int deltmpbuf;
-} MPID_nem_mx_req_area;
-
-#define TMP_DEL_VALUE 111
-/* accessor macro to private fields in REQ */
-#define REQ_FIELD(reqp, field) (((MPID_nem_mx_req_area *)((reqp)->ch.netmod_area.padding))->field)
-
-/* The begining of this structure is the same as MPID_Request */
-struct MPID_nem_mx_internal_req
-{
- MPIU_OBJECT_HEADER; /* adds (unused) handle and ref_count fields */
- MPID_Request_kind_t kind; /* used */
- MPIDI_CH3_Pkt_t pending_pkt;
- MPIDI_VC_t *vc;
- void *tmpbuf;
- MPIDI_msg_sz_t tmpbuf_sz;
- struct MPID_nem_mx_internal_req *next;
-} ;
-
-typedef struct MPID_nem_mx_internal_req MPID_nem_mx_internal_req_t;
-
-typedef union
-{
- MPID_nem_mx_internal_req_t nem_mx_req;
- MPID_Request mpi_req;
-}
-MPID_nem_mx_unified_req_t ;
-
-int MPID_nem_mx_internal_req_queue_init(void);
-int MPID_nem_mx_internal_req_queue_destroy(void);
-int MPID_nem_mx_internal_req_dequeue(MPID_nem_mx_internal_req_t **req);
-int MPID_nem_mx_internal_req_enqueue(MPID_nem_mx_internal_req_t *req);
-
-#if CH3_RANK_BITS == 16
-#ifdef USE_CTXT_AS_MARK
-#define NBITS_TAG 32
-#else /* USE_CTXT_AS_MARK */
-#define NBITS_TAG 31
-#endif /* USE_CTXT_AS_MARK */
-typedef int32_t Mx_Nem_tag_t;
-#elif CH3_RANK_BITS == 32
-#ifdef USE_CTXT_AS_MARK
-#define NBITS_TAG 16
-#else /* USE_CTXT_AS_MARK */
-#define NBITS_TAG 15
-#endif /* USE_CTXT_AS_MARK */
-typedef int16_t Mx_Nem_tag_t;
-#endif /* CH3_RANK_BITS */
-
-#ifdef USE_CTXT_AS_MARK
-#define NBITS_TYPE 0
-#else /* USE_CTXT_AS_MARK */
-#define NBITS_TYPE 1
-#endif /* USE_CTXT_AS_MARK */
-#define NBITS_RANK CH3_RANK_BITS
-#define NBITS_CTXT 16
-#define NBITS_PGRANK (sizeof(int)*8)
-
-#define NEM_MX_MATCHING_BITS (NBITS_TYPE+NBITS_TAG+NBITS_RANK+NBITS_CTXT)
-#define SHIFT_TYPE (NBITS_TAG+NBITS_RANK+NBITS_CTXT)
-#define SHIFT_TAG (NBITS_RANK+NBITS_CTXT)
-#define SHIFT_RANK (NBITS_CTXT)
-#define SHIFT_PGRANK (NBITS_CTXT)
-#define SHIFT_CTXT (0)
-
-#define NEM_MX_MAX_TYPE ((UINT64_C(1)<<NBITS_TYPE) -1)
-#define NEM_MX_MAX_TAG ((UINT64_C(1)<<NBITS_TAG) -1)
-#define NEM_MX_MAX_RANK ((UINT64_C(1)<<NBITS_RANK) -1)
-#define NEM_MX_MAX_CTXT ((UINT64_C(1)<<NBITS_CTXT) -1)
-#define NEM_MX_MAX_PGRANK ((UINT64_C(1)<<NBITS_PGRANK)-1)
-
-#define NEM_MX_TYPE_MASK (NEM_MX_MAX_TYPE<<SHIFT_TYPE)
-#define NEM_MX_TAG_MASK (NEM_MX_MAX_TAG <<SHIFT_TAG )
-#define NEM_MX_RANK_MASK (NEM_MX_MAX_RANK<<SHIFT_RANK)
-#define NEM_MX_CTXT_MASK (NEM_MX_MAX_CTXT<<SHIFT_CTXT)
-#define NEM_MX_PGRANK_MASK (NEM_MX_MAX_PGRANK<<SHIFT_PGRANK)
-
-#define NEM_MX_MATCH_INTRA (UINT64_C(0x8000000000000000))
-#define NEM_MX_MATCH_DIRECT (UINT64_C(0x0000000000000000))
-#define NEM_MX_MATCH_FULL_MASK (UINT64_C(0xffffffffffffffff))
-#define NEM_MX_MATCH_EMPTY_MASK (UINT64_C(0x0000000000000000))
-#define NEM_MX_MASK (UINT64_C(0x8000000000000000))
-
-#define NEM_MX_SET_TAG(_match, _tag) do { \
- MPIU_Assert((_tag >= 0)&&(_tag <= (NEM_MX_MAX_TAG))); \
- ((_match) |= (((uint64_t)((_tag)&(NEM_MX_MAX_TAG))) << SHIFT_TAG)); \
-}while(0)
-#define NEM_MX_SET_SRC(_match, _src) do { \
- MPIU_Assert((_src >= 0)&&(_src<=(NEM_MX_MAX_RANK))); \
- ((_match) |= (((uint64_t)(_src)) << SHIFT_RANK)); \
-}while(0)
-#define NEM_MX_SET_CTXT(_match, _ctxt) do { \
- MPIU_Assert((_ctxt >= 0)&&(_ctxt<=(NEM_MX_MAX_CTXT))); \
- ((_match) |= (((uint64_t)(_ctxt)) << SHIFT_CTXT)); \
-}while(0)
-#define NEM_MX_SET_PGRANK(_match, _pg_rank) do { \
- ((_match) |= (((uint64_t)(_pg_rank)) << SHIFT_PGRANK)); \
-}while(0)
-#define NEM_MX_SET_ANYSRC(_match) do{ \
- ((_match) &= ~NEM_MX_RANK_MASK);\
-}while(0)
-#define NEM_MX_SET_ANYTAG(_match) do{ \
- ((_match) &= ~NEM_MX_TAG_MASK); \
-}while(0)
-
-#define NEM_MX_MATCH_GET_TYPE(_match, _type) do{ \
- ((_type) = ((int16_t)(((_match) & NEM_MX_TYPE_MASK) >> SHIFT_TYPE))); \
-}while(0)
-#define NEM_MX_MATCH_GET_TAG(_match, _tag) do{ \
- ((_tag) = ((Mx_Nem_tag_t)(((_match) & NEM_MX_TAG_MASK) >> SHIFT_TAG)));\
-}while(0)
-#define NEM_MX_MATCH_GET_RANK(_match, _rank) do{ \
- ((_rank) = ((MPIR_Rank_t)(((_match) & NEM_MX_RANK_MASK) >> SHIFT_RANK)));\
-}while(0)
-#define NEM_MX_MATCH_GET_CTXT(_match, _ctxt) do{ \
-((_ctxt) = ((MPIR_Context_id_t)(((_match) & NEM_MX_CTXT_MASK) >> SHIFT_CTXT))); \
-}while(0)
-#define NEM_MX_MATCH_GET_PGRANK(_match, _pg_rank) do{ \
- ((_pg_rank) = ((int)(((_match) & NEM_MX_PGRANK_MASK) >> SHIFT_PGRANK))); \
-}while(0)
-
-#ifdef USE_CTXT_AS_MARK
-#define NEM_MX_INTRA_CTXT (0x0000000c)
-#define NEM_MX_SET_MATCH(_match,_tag,_rank,_context ) do{ \
- MPIU_Assert((_tag >= 0)&&(_tag <= (NEM_MX_MAX_TAG))); \
- MPIU_Assert((_rank >= 0)&&(_rank<=(NEM_MX_MAX_RANK))); \
- MPIU_Assert((_context >= 0)&&(_context<=(NEM_MX_MAX_CTXT)));\
- (_match)=((((uint64_t)(_tag)) << SHIFT_TAG) \
- |(((uint64_t)(_rank)) << SHIFT_RANK) \
- |(((uint64_t)(_context)) << SHIFT_CTXT)); \
-}while(0)
-#define NEM_MX_DIRECT_MATCH(_match,_tag,_rank,_context) NEM_MX_SET_MATCH(_match,_tag,_rank,_context)
-#define NEM_MX_ADI_MATCH(_match) NEM_MX_SET_MATCH(_match,0,0,NEM_MX_INTRA_CTXT)
-#else /* USE_CTXT_AS_MARK */
-#define NEM_MX_DIRECT_TYPE (0x0)
-#define NEM_MX_INTRA_TYPE (0x1)
-#define NEM_MX_SET_MATCH(_match,_type,_tag,_rank,_context ) do{ \
- MPIU_Assert((_tag >= 0)&&(_tag <= (NEM_MX_MAX_TAG))); \
- MPIU_Assert((_rank >= 0)&&(_rank<=(NEM_MX_MAX_RANK))); \
- MPIU_Assert((_context >= 0)&&(_context<=(NEM_MX_MAX_CTXT))); \
- (_match)=((((uint64_t) (_type)) << SHIFT_TYPE) \
- |(((uint64_t)((_tag)&(NEM_MX_MAX_TAG)))<< SHIFT_TAG) \
- |(((uint64_t) (_rank)) << SHIFT_RANK) \
- |(((uint64_t) (_context)) << SHIFT_CTXT));\
-}while(0)
-#define NEM_MX_DIRECT_MATCH(_match,_tag,_rank,_context) NEM_MX_SET_MATCH(_match,NEM_MX_DIRECT_TYPE,_tag,_rank,_context)
-#define NEM_MX_ADI_MATCH(_match) NEM_MX_SET_MATCH(_match,NEM_MX_INTRA_TYPE,0,0,0)
-#endif /* USE_CTXT_AS_MARK */
-
-#endif
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c
deleted file mode 100644
index 1daf18e..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c
+++ /dev/null
@@ -1,319 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mpid_nem_impl.h"
-#include "mx_impl.h"
-
-MPID_nem_netmod_funcs_t MPIDI_nem_mx_funcs = {
- MPID_nem_mx_init,
- MPID_nem_mx_finalize,
- MPID_nem_mx_poll,
- MPID_nem_mx_get_business_card,
- MPID_nem_mx_connect_to_root,
- MPID_nem_mx_vc_init,
- MPID_nem_mx_vc_destroy,
- MPID_nem_mx_vc_terminate,
- MPID_nem_mx_anysource_iprobe,
- MPID_nem_mx_anysource_improbe
-};
-
-static MPIDI_Comm_ops_t comm_ops = {
- MPID_nem_mx_directRecv, /* recv_posted */
-
- MPID_nem_mx_directSend, /* send */
- MPID_nem_mx_directSend, /* rsend */
- MPID_nem_mx_directSsend, /* ssend */
- MPID_nem_mx_directSend, /* isend */
- MPID_nem_mx_directSend, /* irsend */
- MPID_nem_mx_directSsend, /* issend */
-
- NULL, /* send_init */
- NULL, /* bsend_init */
- NULL, /* rsend_init */
- NULL, /* ssend_init */
- NULL, /* startall */
-
- MPID_nem_mx_cancel_send,/* cancel_send */
- MPID_nem_mx_cancel_recv, /* cancel_recv */
-
- MPID_nem_mx_probe, /* probe */
- MPID_nem_mx_iprobe, /* iprobe */
- MPID_nem_mx_improbe /* improbe */
-};
-
-
-#define MPIDI_CH3I_ENDPOINT_KEY "endpoint_id"
-#define MPIDI_CH3I_NIC_KEY "nic_id"
-
-uint32_t MPID_NEM_MX_FILTER = 0xabadbada;
-uint64_t MPID_nem_mx_local_nic_id;
-uint32_t MPID_nem_mx_local_endpoint_id;
-mx_endpoint_t MPID_nem_mx_local_endpoint;
-
-#undef FUNCNAME
-#define FUNCNAME init_mx
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int init_mx( MPIDI_PG_t *pg_p )
-{
- mx_endpoint_addr_t local_endpoint_addr;
- mx_return_t ret;
- mx_param_t param;
- int mpi_errno = MPI_SUCCESS;
- int r;
-
- r = MPL_putenv("MX_DISABLE_SHARED=1");
- MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
- r = MPL_putenv("MX_DISABLE_SELF=1");
- MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
-
- ret = mx_init();
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_init", "**mx_init %s", mx_strerror (ret));
-
- mx_set_error_handler(MX_ERRORS_RETURN);
-
- /*
- ret = mx_get_info(NULL, MX_NIC_COUNT, NULL, 0, &nic_count, sizeof(int));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
-
- count = ++nic_count;
- mx_nics = (uint64_t *)MPIU_Malloc(count*sizeof(uint64_t));
- ret = mx_get_info(NULL, MX_NIC_IDS, NULL, 0, mx_nics, count*sizeof(uint64_t));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
-
- do{
- ret = mx_nic_id_to_board_number(mx_nics[index],&mx_board_num);
- index++;
- }while(ret != MX_SUCCESS);
- */
-#ifndef USE_CTXT_AS_MARK
- param.key = MX_PARAM_CONTEXT_ID;
- param.val.context_id.bits = NEM_MX_MATCHING_BITS - SHIFT_TYPE;
- param.val.context_id.shift = SHIFT_TYPE;
- ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,¶m,1,&MPID_nem_mx_local_endpoint);
-#else
- ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,NULL,0,&MPID_nem_mx_local_endpoint);
-#endif
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_open_endpoint", "**mx_open_endpoint %s", mx_strerror (ret));
-
- mx_register_unexp_handler(MPID_nem_mx_local_endpoint,MPID_nem_mx_get_adi_msg,NULL);
-
- ret = mx_get_endpoint_addr(MPID_nem_mx_local_endpoint,&local_endpoint_addr);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_endpoint_addr", "**mx_get_endpoint_addr %s", mx_strerror (ret));
-
- ret = mx_decompose_endpoint_addr(local_endpoint_addr,&MPID_nem_mx_local_nic_id,&MPID_nem_mx_local_endpoint_id);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_decompose_endpoint_addr", "**mx_decompose_endpoint_addr %s", mx_strerror (ret));
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/*
- int
- MPID_nem_mx_init(MPID_nem_queue_ptr_t proc_recv_queue, MPID_nem_queue_ptr_t proc_free_queue, MPID_nem_cell_ptr_t proc_elements, int num_proc_elements,
- MPID_nem_cell_ptr_t module_elements, int num_module_elements,
- MPID_nem_queue_ptr_t *module_free_queue)
-
- IN
- proc_recv_queue -- main recv queue for the process
- proc_free_queue -- main free queueu for the process
- proc_elements -- pointer to the process' queue elements
- num_proc_elements -- number of process' queue elements
- module_elements -- pointer to queue elements to be used by this module
- num_module_elements -- number of queue elements for this module
- OUT
- free_queue -- pointer to the free queue for this module. The process will return elements to
- this queue
-*/
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_init (MPIDI_PG_t *pg_p, int pg_rank,
- char **bc_val_p, int *val_max_sz_p)
-{
- int mpi_errno = MPI_SUCCESS ;
-
- MPID_nem_mx_internal_req_queue_init();
-
- mpi_errno = init_mx(pg_p);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_mx_get_business_card (pg_rank, bc_val_p, val_max_sz_p);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_mx_anysource_posted,
- MPID_nem_mx_anysource_matched);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_get_business_card
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
-{
- int mpi_errno = MPI_SUCCESS;
- int str_errno = MPIU_STR_SUCCESS;
-
- str_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_ENDPOINT_KEY, MPID_nem_mx_local_endpoint_id);
- if (str_errno) {
- MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
- }
-
- str_errno = MPIU_Str_add_binary_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_NIC_KEY, (char *)&MPID_nem_mx_local_nic_id, sizeof(uint64_t));
- if (str_errno) {
- MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_get_from_bc
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_get_from_bc(const char *business_card, uint32_t *remote_endpoint_id, uint64_t *remote_nic_id)
-{
- int mpi_errno = MPI_SUCCESS;
- int str_errno = MPIU_STR_SUCCESS;
- int len;
- int tmp_endpoint_id;
-
- mpi_errno = MPIU_Str_get_int_arg(business_card, MPIDI_CH3I_ENDPOINT_KEY, &tmp_endpoint_id);
- /* FIXME: create a real error string for this */
- MPIU_ERR_CHKANDJUMP(str_errno, mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
- *remote_endpoint_id = (uint32_t)tmp_endpoint_id;
-
- mpi_errno = MPIU_Str_get_binary_arg (business_card, MPIDI_CH3I_NIC_KEY, (char *)remote_nic_id, sizeof(uint64_t), &len);
- /* FIXME: create a real error string for this */
- MPIU_ERR_CHKANDJUMP(str_errno || len != sizeof(uint64_t), mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_connect_to_root
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
-{
- return MPI_SUCCESS;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_vc_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_vc_init (MPIDI_VC_t *vc)
-{
- uint32_t threshold;
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- int mpi_errno = MPI_SUCCESS;
-
- /* first make sure that our private fields in the vc fit into the area provided */
- MPIU_Assert(sizeof(MPID_nem_mx_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
-
-#ifdef ONDEMAND
- VC_FIELD(vc, local_connected) = 0;
- VC_FIELD(vc, remote_connected) = 0;
-#else
- {
- char *business_card;
- int val_max_sz;
- int ret;
-#ifdef USE_PMI2_API
- val_max_sz = PMI2_MAX_VALLEN;
-#else
- mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
-#endif
- business_card = (char *)MPIU_Malloc(val_max_sz);
- mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card,val_max_sz, vc->pg);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- MPIU_Free(business_card);
-
- ret = mx_connect(MPID_nem_mx_local_endpoint,VC_FIELD(vc, remote_nic_id),VC_FIELD(vc, remote_endpoint_id),
- MPID_NEM_MX_FILTER,MX_INFINITE,&(VC_FIELD(vc, remote_endpoint_addr)));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_connect", "**mx_connect %s", mx_strerror (ret));
- mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
-
- MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
- }
-#endif
- mx_get_info(MPID_nem_mx_local_endpoint, MX_COPY_SEND_MAX, NULL, 0, &threshold, sizeof(uint32_t));
- VC_FIELD(vc,pending_sends) = 0;
-
- vc->eager_max_msg_sz = threshold;
- vc->ready_eager_max_msg_sz = threshold;
- vc->rndvSend_fn = NULL;
- vc->sendNoncontig_fn = MPID_nem_mx_SendNoncontig;
- vc->comm_ops = &comm_ops;
-
- vc_ch->iStartContigMsg = MPID_nem_mx_iStartContigMsg;
- vc_ch->iSendContig = MPID_nem_mx_iSendContig;
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_vc_destroy
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_vc_destroy(MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- /* free any resources associated with this VC here */
-
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_vc_terminate
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_vc_terminate (MPIDI_VC_t *vc)
-{
- /* FIXME: Check to make sure that it's OK to terminate the
- connection without making sure that all sends have been sent */
- while((VC_FIELD(vc,pending_sends)) > 0)
- MPID_nem_mx_poll(FALSE);
-
- return MPIDI_CH3U_Handle_connection (vc, MPIDI_VC_EVENT_TERMINATED);
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c
deleted file mode 100644
index 8b39b7e..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c
+++ /dev/null
@@ -1,825 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mx_impl.h"
-#include "my_papi_defs.h"
-#include "uthash.h"
-
-typedef struct mpid_nem_mx_hash_struct {
- MPID_Request *mpid_req_ptr;
- MPIDI_VC_t *vc;
- mx_request_t *mx_req_ptr;
- UT_hash_handle hh1,hh2;
-}mpid_nem_mx_hash_t;
-
-static mpid_nem_mx_hash_t *mpid_nem_mx_asreqs = NULL;
-#define MPID_MEM_MX_ADD_REQ_IN_HASH(_mpi_req,_mx_req) do{ \
- mpid_nem_mx_hash_t *s; \
- s = MPIU_Malloc(sizeof(mpid_nem_mx_hash_t)); \
- s->mpid_req_ptr = (_mpi_req); \
- s->mx_req_ptr = (_mx_req); \
- HASH_ADD(hh1, mpid_nem_mx_asreqs, mpid_req_ptr, sizeof(MPID_Request*), s); \
-}while(0)
-#define MPID_NEM_MX_GET_REQ_FROM_HASH(_mpi_req_ptr,_mx_req) do{ \
- mpid_nem_mx_hash_t *s; \
- HASH_FIND(hh1, mpid_nem_mx_asreqs, &(_mpi_req_ptr), sizeof(MPID_Request*), s); \
- if(s){HASH_DELETE(hh1, mpid_nem_mx_asreqs, s); (_mx_req) = s->mx_req_ptr; MPIU_Free(s);} else {(_mx_req) = NULL;} \
-}while(0)
-
-static mpid_nem_mx_hash_t *mpid_nem_mx_connreqs ATTRIBUTE((unused, used))= NULL;
-#define MPID_MEM_MX_ADD_VC_IN_HASH(_vc,_mx_req) do{ \
- mpid_nem_mx_hash_t *s; \
- s = MPIU_Malloc(sizeof(mpid_nem_mx_hash_t)); \
- s->vc = (_vc); \
- s->mx_req_ptr = (_mx_req); \
- HASH_ADD(hh2, mpid_nem_mx_connreqs, mpid_req_ptr, sizeof(MPID_Request*), s); \
-}while(0)
-#define MPID_NEM_MX_REM_VC_FROM_HASH(_vc,_mx_req) do{ \
- mpid_nem_mx_hash_t *s; \
- HASH_FIND(hh2, mpid_nem_mx_connreqs, &(_mpi_req_ptr), sizeof(MPID_Request*), s); \
- if(s){HASH_DELETE(hh2, mpid_nem_mx_connreqs, s); (_mx_req) = s->mx_req_ptr; } else {(_mx_req) = NULL;} \
-}while(0)
-#define MPID_NEM_MX_IS_VC_IN_HASH(_vc,_ret) do{ \
- mpid_nem_mx_hash_t *s; \
- HASH_FIND(hh2, mpid_nem_mx_connreqs, &(_mpi_req_ptr), sizeof(MPID_Request*), s); \
- if(s){ (_ret) = 1; } else {(_ret) = 0;} \
-}while(0)
-
-static int MPID_nem_mx_handle_sreq(MPID_Request *sreq);
-static int MPID_nem_mx_handle_rreq(MPID_Request *rreq, mx_status_t status);
-
-/* This routine cannot manipulate MPI message queues or request queues */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_get_adi_msg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-mx_unexp_handler_action_t MPID_nem_mx_get_adi_msg(void *context, mx_endpoint_addr_t source, uint64_t match_info,
- uint32_t length, void *data)
-{
- MPIDI_VC_t *vc;
-#ifdef USE_CTXT_AS_MARK
- MPIR_Context_id_t ctxt;
- NEM_MX_MATCH_GET_CTXT(match_info, ctxt);
-#else
- int16_t type;
- NEM_MX_MATCH_GET_TYPE(match_info,type);
-#endif
-
-#ifdef ONDEMAND
- mx_get_endpoint_addr_context(source,(void **)(&vc));
-
- if (vc == NULL)
- {
-#ifdef USE_CTXT_AS_MARK
- if(ctxt == NEM_MX_INTRA_CTXT)
-#else
- if(type == NEM_MX_INTRA_TYPE)
-#endif
- {
- mx_return_t ret;
- uint32_t result;
- uint64_t remote_nic_id;
- uint32_t remote_endpoint_id;
- int pg_rank;
- char pg_id[MPID_NEM_MAX_NETMOD_STRING_LEN];
- MPIDI_PG_t *pg;
- mx_request_t mx_request;
- mx_status_t status;
-
- remote_nic_id = *((uint64_t *)data);
- remote_endpoint_id = *((uint32_t *)((char *)data+sizeof(uint64_t)));
- MPIU_Memcpy(pg_id,(char *)data+sizeof(uint64_t)+sizeof(uint32_t),length-sizeof(uint64_t)-sizeof(uint32_t));
-
- MPIDI_PG_Find (pg_id, &pg);
- NEM_MX_MATCH_GET_PGRANK(match_info,pg_rank);
- MPIDI_PG_Get_vc_set_active(pg, pg_rank, &vc);
-
- fprintf(stdout,"[%i]=== NULL VC : Receiver Unex Got infos (%lx) from Sender (in data) ..%li %i %s %p\n",
- MPID_nem_mem_region.rank,match_info,remote_nic_id,remote_endpoint_id,pg_id,vc);
- ret = mx_iconnect(MPID_nem_mx_local_endpoint, remote_nic_id,remote_endpoint_id ,
- MPID_NEM_MX_FILTER,match_info,NULL,&mx_request);
- MPIU_Assert(ret == MX_SUCCESS);
-
- /*fprintf(stdout,"[%i]=== NULL VC : Receiver IConnect posted \n", MPID_nem_mem_region.rank); */
-
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,&mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
-
- VC_FIELD(vc, remote_connected) = 1;
- VC_FIELD(vc, local_connected) = 1;
- VC_FIELD(vc, remote_endpoint_addr) = status.source;
- mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
- fprintf(stdout,"[%i]=== NULL VC : Receiver IConnect done \n", MPID_nem_mem_region.rank);
-
-
- return MX_RECV_FINISHED;
- }
- else
-#ifndef USE_CTXT_AS_MARK
-if(type == NEM_MX_DIRECT_TYPE)
-#endif
- {
- fprintf(stdout,"[%i]=== NULL VC : callback for direct %lx \n", MPID_nem_mem_region.rank,match_info);
- return MX_RECV_CONTINUE;
- }
-#ifndef USE_CTXT_AS_MARK
- else
- {
- fprintf(stdout,"Unknown Message Type :%i, aborting ...\n",type);
- MPIU_Assert(0);
- abort();
- }
-#endif
- }
- else {
-#endif
-#ifdef USE_CTXT_AS_MARK
- if(ctxt == NEM_MX_INTRA_CTXT)
-#else
- if (type == NEM_MX_INTRA_TYPE)
-#endif
- {
- MPID_nem_mx_internal_req_t *rreq;
- mx_request_t mx_request;
- mx_segment_t iov;
- mx_return_t ret;
-
-#ifdef ONDEMAND
- if (VC_FIELD(vc, remote_connected) == 0)
- {
- mx_return_t ret;
- uint64_t remote_nic_id;
- uint32_t remote_endpoint_id;
- int pg_rank;
- char pg_id[MPID_NEM_MAX_NETMOD_STRING_LEN];
- MPIDI_PG_t *pg;
- uint32_t result;
-
- remote_nic_id = *((uint64_t *)data);
- remote_endpoint_id = *((uint32_t *)((char *)data+sizeof(uint64_t)));
- MPIU_Memcpy(pg_id,(char *)data+sizeof(uint64_t)+sizeof(uint32_t),length-sizeof(uint64_t)-sizeof(uint32_t));
-
- fprintf(stdout,"[%i]=== NOT NULL VC : Receiver Unex Got infos from Sender (in data) ..%li %i %s vc is %p\n",
- MPID_nem_mem_region.rank,remote_nic_id,remote_endpoint_id,pg_id,vc);
-
- MPIDI_PG_Find (pg_id, &pg);
- NEM_MX_MATCH_GET_PGRANK(match_info,pg_rank);
- MPIDI_PG_Get_vc_set_active(pg, pg_rank, &vc);
-
- if(VC_FIELD(vc, local_connected) == 0)
- {
- mx_request_t mx_request;
- mx_status_t status;
- ret = mx_iconnect(MPID_nem_mx_local_endpoint,remote_nic_id,remote_endpoint_id,
- MPID_NEM_MX_FILTER,match_info,NULL,&mx_request);
- MPIU_Assert(ret == MX_SUCCESS);
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,&mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
-
- VC_FIELD(vc, remote_endpoint_addr) = status.source;
- mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
- VC_FIELD(vc, local_connected) = 1;
- }
- VC_FIELD(vc, remote_connected) = 1;
- fprintf(stdout,"[%i]=== NOT NULL VC : Receiver FULLY Connected with peer \n", MPID_nem_mem_region.rank);
-
- return MX_RECV_FINISHED;
- }
-#endif
-
- MPID_nem_mx_internal_req_dequeue(&rreq);
- rreq->kind = MPID_REQUEST_RECV;
- mx_get_endpoint_addr_context(source,(void **)(&vc));
- rreq->vc = vc;
-
- if(length <= sizeof(MPIDI_CH3_Pkt_t)) {
- iov.segment_ptr = (char*)&(rreq->pending_pkt);
- }
- else{
- rreq->tmpbuf = MPIU_Malloc(length);
- MPIU_Assert(rreq->tmpbuf);
- rreq->tmpbuf_sz = length;
- iov.segment_ptr = (char*)(rreq->tmpbuf);
- }
- iov.segment_length = length;
-
- ret = mx_irecv(MPID_nem_mx_local_endpoint,&iov,1,match_info,NEM_MX_MATCH_FULL_MASK,(void *)rreq,&mx_request);
- MPIU_Assert(ret == MX_SUCCESS);
-
- return MX_RECV_CONTINUE;
- }
- else
-#ifndef USE_CTXT_AS_MARK
-if (type == NEM_MX_DIRECT_TYPE)
-#endif
- {
- /* Do nothing: */
- /* This shall be eventually matched in recv */
- return MX_RECV_CONTINUE;
- }
-#ifndef USE_CTXT_AS_MARK
- else
- {
- fprintf(stdout,"Unknown Message Type :%i, aborting ...\n",type);
- MPIU_Assert(0);
- abort();
- }
-#endif
-#ifdef ONDEMAND
- }
-#endif
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_directRecv
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_directRecv(MPIDI_VC_t *vc, MPID_Request *rreq)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_DIRECTRECV);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_DIRECTRECV);
-
- if (!vc->ch.is_local)
- {
- uint64_t match_info = NEM_MX_MATCH_DIRECT;
- uint64_t match_mask = NEM_MX_MATCH_FULL_MASK;
- MPIR_Rank_t source = rreq->dev.match.parts.rank;
- MPIR_Context_id_t context = rreq->dev.match.parts.context_id;
- Mx_Nem_tag_t tag = rreq->dev.match.parts.tag;
- mx_segment_t mx_iov[MX_MAX_SEGMENTS];
- mx_return_t ret;
- uint32_t num_seg = 1;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype *dt_ptr;
- /*int threshold = (vc->eager_max_msg_sz - sizeof(MPIDI_CH3_Pkt_t));*/
-
- NEM_MX_DIRECT_MATCH(match_info,0,source,context);
- if (tag == MPI_ANY_TAG)
- {
- NEM_MX_SET_ANYTAG(match_info);
- NEM_MX_SET_ANYTAG(match_mask);
- }
- else
- NEM_MX_SET_TAG(match_info,tag);
-
- MPIDI_Datatype_get_info(rreq->dev.user_count,rreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
- rreq->dev.OnDataAvail = NULL;
-
- if (dt_contig)
- {
- mx_iov[0].segment_ptr = (char *)(rreq->dev.user_buf) + dt_true_lb;
- mx_iov[0].segment_length = data_sz;
- }
- else
- MPID_nem_mx_process_rdtype(&rreq,dt_ptr,data_sz,mx_iov,&num_seg);
- ret = mx_irecv(MPID_nem_mx_local_endpoint,mx_iov,num_seg,match_info,match_mask,(void *)rreq, &(REQ_FIELD(rreq,mx_request)));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_irecv", "**mx_irecv %s", mx_strerror (ret));
- }
- else
- {
- /* Fixme : this might not work in the case of multiple netmods */
- memset((&(REQ_FIELD(rreq,mx_request))),0,sizeof(mx_request_t));
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_DIRECTRECV);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#ifndef USE_CTXT_AS_MARK
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_poll
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_poll(int in_blocking_poll)
-{
- int mpi_errno = MPI_SUCCESS;
- mx_status_t status;
- mx_return_t ret;
- uint32_t result;
-
- /* first check ADI msgs */
- ret = mx_test_any(MPID_nem_mx_local_endpoint,NEM_MX_MATCH_INTRA,NEM_MX_MASK,&status,&result);
- MPIU_Assert(ret == MX_SUCCESS);
- if ((ret == MX_SUCCESS) && (result > 0))
- {
- MPID_nem_mx_unified_req_t *myreq = (MPID_nem_mx_unified_req_t *)(status.context);
- MPID_Request *req = &(myreq->mpi_req);
- MPID_Request_kind_t kind = req->kind;
-
- if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
- {
- MPID_nem_mx_handle_sreq(req);
- }
- else if (kind == MPID_REQUEST_RECV)
- {
- MPID_nem_mx_internal_req_t *adi_req = &(myreq->nem_mx_req);
- MPIU_Assert(status.code != MX_STATUS_TRUNCATED);
- if(adi_req->vc == NULL)
- mx_get_endpoint_addr_context(status.source,(void **)(&(adi_req->vc)));
- if (status.msg_length <= sizeof(MPIDI_CH3_Pkt_t))
- {
- MPID_nem_handle_pkt(adi_req->vc,(char *)&(adi_req->pending_pkt),(MPIDI_msg_sz_t)(status.msg_length));
- }
- else
- {
- MPID_nem_handle_pkt(adi_req->vc,(char *)(adi_req->tmpbuf),(MPIDI_msg_sz_t)(adi_req->tmpbuf_sz));
- MPIU_Free(adi_req->tmpbuf);
- }
- mx_disable_progression(MPID_nem_mx_local_endpoint);
- MPID_nem_mx_internal_req_enqueue(adi_req);
- mx_reenable_progression(MPID_nem_mx_local_endpoint);
- }
- else
- {
- /* Error : unknown REQ type */
- MPIU_ERR_CHKINTERNAL(TRUE, mpi_errno, "unknown REQ type");
- }
- }
-
- /* Then check Direct MPI msgs */
- ret = mx_test_any(MPID_nem_mx_local_endpoint,NEM_MX_MATCH_DIRECT,NEM_MX_MASK,&status,&result);
- MPIU_Assert(ret == MX_SUCCESS);
- if ((ret == MX_SUCCESS) && (result > 0))
- {
- MPID_nem_mx_unified_req_t *myreq = (MPID_nem_mx_unified_req_t *)(status.context);
- MPID_Request *req = &(myreq->mpi_req);
- MPID_Request_kind_t kind = req->kind;
-
- if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
- {
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- mpi_errno = MPID_nem_mx_handle_sreq(req);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
- else if ((kind == MPID_REQUEST_RECV) || (kind == MPID_PREQUEST_RECV))
- {
- int found = FALSE;
- mx_request_t *mx_request = NULL;
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
- MPID_NEM_MX_GET_REQ_FROM_HASH(req,mx_request);
- if(mx_request != NULL)
- {
- MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
- MPIU_Free(mx_request);
- }
- found = MPIDI_CH3U_Recvq_DP(req);
- if(found){
- mpi_errno = MPID_nem_mx_handle_rreq(req, status);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
- MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
- }
- else
- {
- /* Error : unknown REQ type */
- MPIU_ERR_CHKINTERNAL(TRUE, mpi_errno, "unknown REQ type");
- }
- }
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#else /*USE_CTXT_AS_MARK */
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_poll
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_poll(int in_blocking_progress)
-{
- int mpi_errno = MPI_SUCCESS;
- mx_status_t status;
- mx_return_t ret;
- uint32_t result;
-
- ret = mx_test_any(MPID_nem_mx_local_endpoint,NEM_MX_MATCH_EMPTY_MASK,NEM_MX_MATCH_EMPTY_MASK,&status,&result);
- MPIU_Assert(ret == MX_SUCCESS);
- if ((ret == MX_SUCCESS) && (result > 0))
- {
- MPID_nem_mx_unified_req_t *myreq = (MPID_nem_mx_unified_req_t *)(status.context);
- MPID_Request *req = &(myreq->mpi_req);
- MPID_Request_kind_t kind = req->kind;
- MPIR_Context_id_t ctxt;
-
- NEM_MX_MATCH_GET_CTXT(status.match_info, ctxt);
-
- if(ctxt == NEM_MX_INTRA_CTXT)
- {
- if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
- {
-
- MPID_nem_mx_handle_sreq(req);
- }
- else if (kind == MPID_REQUEST_RECV)
- {
- MPID_nem_mx_internal_req_t *adi_req = &(myreq->nem_mx_req);
- MPIU_Assert(status.code != MX_STATUS_TRUNCATED);
- if(adi_req->vc == NULL)
- mx_get_endpoint_addr_context(status.source,(void **)(&(adi_req->vc)));
- if (status.msg_length <= sizeof(MPIDI_CH3_Pkt_t))
- {
- MPID_nem_handle_pkt(adi_req->vc,(char *)&(adi_req->pending_pkt),(MPIDI_msg_sz_t)(status.msg_length));
- }
- else
- {
- MPID_nem_handle_pkt(adi_req->vc,(char *)(adi_req->tmpbuf),(MPIDI_msg_sz_t)(adi_req->tmpbuf_sz));
- MPIU_Free(adi_req->tmpbuf);
- }
- mx_disable_progression(MPID_nem_mx_local_endpoint);
- MPID_nem_mx_internal_req_enqueue(adi_req);
- mx_reenable_progression(MPID_nem_mx_local_endpoint);
- }
- else
- {
- MPIU_Assert(0);
- }
- }
- else
- {
- if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
- {
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- mpi_errno = MPID_nem_mx_handle_sreq(req);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
- else if ((kind == MPID_REQUEST_RECV) || (kind == MPID_PREQUEST_RECV))
- {
- int found = FALSE;
- mx_request_t *mx_request = NULL;
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
- MPID_NEM_MX_GET_REQ_FROM_HASH(req,mx_request);
- if(mx_request != NULL)
- {
- MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
- MPIU_Free(mx_request);
- }
- found = MPIDI_CH3U_Recvq_DP(req);
- if(found){
- mpi_errno = MPID_nem_mx_handle_rreq(req, status);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
- MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
- }
- else
- {
- MPIU_Assert(0);
- }
- }
- }
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-#endif /*USE_CTXT_AS_MARK */
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_handle_sreq
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int
-MPID_nem_mx_handle_sreq(MPID_Request *req)
-{
- int mpi_errno = MPI_SUCCESS;
- int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
-
- (VC_FIELD(req->ch.vc,pending_sends)) -= 1;
- if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))
- || ((REQ_FIELD(req,deltmpbuf)) == TMP_DEL_VALUE))
- {
- MPIU_Free(req->dev.tmpbuf);
- REQ_FIELD(req,deltmpbuf) = 0;
- }
- reqFn = req->dev.OnDataAvail;
- if (!reqFn){
- MPIDI_CH3U_Request_complete(req);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- }
- else{
- MPIDI_VC_t *vc = req->ch.vc;
- int complete = 0;
- mpi_errno = reqFn(vc, req, &complete);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- if(!complete)
- {
- /* FIXME */
- /* GM: enqueue the not complete sreq somewhere and test it again later */
- MPIU_Assert(complete == TRUE);
- }
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_handle_rreq
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int
-MPID_nem_mx_handle_rreq(MPID_Request *req, mx_status_t status)
-{
- int mpi_errno = MPI_SUCCESS;
- uint64_t match_info = status.match_info;
- int complete = FALSE;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPIDI_msg_sz_t userbuf_sz;
- MPID_Datatype *dt_ptr;
- MPIDI_msg_sz_t data_sz;
- MPIDI_VC_t *vc;
-
- NEM_MX_MATCH_GET_RANK(match_info,req->status.MPI_SOURCE);
- NEM_MX_MATCH_GET_TAG(match_info,req->status.MPI_TAG);
- MPIR_STATUS_SET_COUNT(req->status, status.xfer_length);
- req->dev.recv_data_sz = status.xfer_length;
-
- MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr, dt_true_lb);
- /*fprintf(stdout," ===> userbuf_size is %i, msg_length is %i, xfer_length is %i\n",userbuf_sz,status.msg_length,status.xfer_length); */
-
- if (status.msg_length <= userbuf_sz)
- {
- data_sz = status.xfer_length;
- /* the sent message was truncated */
- if (status.msg_length != status.xfer_length )
- {
- MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
- "message truncated on receiver side, real_msg_sz="
- MPIDI_MSG_SZ_FMT ", expected_msg_sz="
- MPIDI_MSG_SZ_FMT,
- status.xfer_length, status.msg_length));
- req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
- MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
- "**truncate", "**truncate %d %d %d %d",
- req->status.MPI_SOURCE, req->status.MPI_TAG,
- req->dev.recv_data_sz, userbuf_sz );
- }
- }
- else
- {
- MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
- "receive buffer too small; message truncated, msg_sz="
- MPIDI_MSG_SZ_FMT ", userbuf_sz="
- MPIDI_MSG_SZ_FMT,
- req->dev.recv_data_sz, userbuf_sz));
- req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
- MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
- "**truncate", "**truncate %d %d %d %d",
- req->status.MPI_SOURCE, req->status.MPI_TAG,
- req->dev.recv_data_sz, userbuf_sz );
- MPIR_STATUS_SET_COUNT(req->status, userbuf_sz);
- data_sz = userbuf_sz;
- }
-
- if ((!dt_contig)&&(req->dev.tmpbuf != NULL))
- {
- MPIDI_msg_sz_t last;
- last = req->dev.recv_data_sz;
- MPID_Segment_unpack( req->dev.segment_ptr, 0, &last, req->dev.tmpbuf );
- MPIU_Free(req->dev.tmpbuf);
- if (last != data_sz) {
- MPIR_STATUS_SET_COUNT(req->status, last);
- if (req->dev.recv_data_sz <= userbuf_sz) {
- MPIU_ERR_SETSIMPLE(req->status.MPI_ERROR,MPI_ERR_TYPE,"**dtypemismatch");
- }
- }
- }
-
- mx_get_endpoint_addr_context(status.source,(void **)(&vc));
-#ifdef ONDEMAND
- if( vc == NULL)
- {
- char business_card[MPID_NEM_MAX_NETMOD_STRING_LEN];
- mx_return_t ret;
- mx_request_t mx_request;
- mx_status_t status;
- uint32_t result;
-
- MPIDI_Comm_get_vc_set_active(req->comm, req->status.MPI_SOURCE, &vc);
- mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, MPID_NEM_MAX_NETMOD_STRING_LEN, vc->pg);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- ret = mx_iconnect(MPID_nem_mx_local_endpoint,VC_FIELD(vc, remote_nic_id),
- VC_FIELD(vc, remote_endpoint_id),MPID_NEM_MX_FILTER,match_info,NULL,&mx_request);
- MPIU_Assert(ret == MX_SUCCESS);
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,&mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
-
- fprintf(stdout,"[%i]=== Connected on recv with %i ... %p \n", MPID_nem_mem_region.rank,vc->lpid,vc);
- VC_FIELD(vc, remote_endpoint_addr) = status.source;
- VC_FIELD(vc, local_connected) = 1;
- VC_FIELD(vc, remote_connected) = 1;
- mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
- fprintf(stdout,"[%i]=== Connected 2 on recv with %i ... %p \n", MPID_nem_mem_region.rank,vc->lpid,vc);
- }
-#endif
-
- MPIDI_CH3U_Handle_recv_req(vc, req, &complete);
- MPIU_Assert(complete == TRUE);
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_anysource_posted
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-void MPID_nem_mx_anysource_posted(MPID_Request *rreq)
-{
- /* This function is called whenever an anyource request has been
- posted to the posted receive queue. */
- MPIR_Context_id_t context;
- Mx_Nem_tag_t tag;
- uint64_t match_info = 0;
- uint64_t match_mask = NEM_MX_MATCH_FULL_MASK;
- mx_request_t *mx_request = MPIU_Malloc(sizeof(mx_request_t));
- mx_segment_t mx_iov[MX_MAX_SEGMENTS];
- uint32_t num_seg = 1;
- mx_return_t ret;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype *dt_ptr;
-
- MPIDI_Datatype_get_info(rreq->dev.user_count,rreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
-
- tag = rreq->dev.match.parts.tag;
- context = rreq->dev.match.parts.context_id;
- NEM_MX_DIRECT_MATCH(match_info,0,0,context);
- if (tag == MPI_ANY_TAG)
- {
- NEM_MX_SET_ANYTAG(match_info);
- NEM_MX_SET_ANYTAG(match_mask);
- }
- else
- NEM_MX_SET_TAG(match_info,tag);
- NEM_MX_SET_ANYSRC(match_info);
- NEM_MX_SET_ANYSRC(match_mask);
-
- if (dt_contig)
- {
- mx_iov[0].segment_ptr = (char *)(rreq->dev.user_buf) + dt_true_lb;
- mx_iov[0].segment_length = data_sz;
- }
- else
- MPID_nem_mx_process_rdtype(&rreq,dt_ptr,data_sz,mx_iov,&num_seg);
- ret = mx_irecv(MPID_nem_mx_local_endpoint,mx_iov,num_seg,match_info,match_mask,(void *)rreq,mx_request);
- /* FIXME: this function can't return an error because it's called
- from a recvq function that doesn't check for errors. For now,
- I'm replacing the chkandjump with an assertp. */
- /* MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_irecv", "**mx_irecv %s", mx_strerror (ret)); */
- MPIU_Assertp(ret == MX_SUCCESS);
-
- MPID_MEM_MX_ADD_REQ_IN_HASH(rreq,mx_request);
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_anysource_matched
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_anysource_matched(MPID_Request *rreq)
-{
- /* This function is called when an anysource request in the posted
- receive queue is matched and dequeued. It returns 0 if the req
- was not matched by mx; non-zero otherwise. */
- mx_request_t *mx_request = NULL;
- mx_return_t ret;
- uint32_t result;
- int matched = FALSE;
- int mpi_errno = MPI_SUCCESS;
-
- MPID_NEM_MX_GET_REQ_FROM_HASH(rreq,mx_request);
- if(mx_request != NULL)
- {
- ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
- if (ret == MX_SUCCESS)
- {
- if (result != 1)
- {
- mx_status_t status;
- MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
- mpi_errno = MPID_nem_mx_handle_rreq(rreq, status);
- /* FIXME : how can we report MPI_ERR_TRUNC in this case?*/
- matched = TRUE;
- }
- else
- {
- MPID_Segment_free(rreq->dev.segment_ptr);
- }
- MPIU_Free(mx_request);
- }
- }
- return matched;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_process_rdtype
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_process_rdtype(MPID_Request **rreq_p, MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, mx_segment_t *mx_iov, uint32_t *num_seg)
-{
- MPID_Request *rreq =*rreq_p;
- MPID_IOV *iov;
- MPIDI_msg_sz_t last;
- int num_entries = MX_MAX_SEGMENTS;
- int n_iov = 0;
- int mpi_errno = MPI_SUCCESS;
- int index;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_PROCESS_RDTYPE);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_PROCESS_RDTYPE);
-
- if (rreq->dev.segment_ptr == NULL)
- {
- rreq->dev.segment_ptr = MPID_Segment_alloc( );
- MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_allo\
-c");
- }
- MPID_Segment_init(rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.segment_ptr, 0);
- rreq->dev.segment_first = 0;
- rreq->dev.segment_size = data_sz;
- last = rreq->dev.segment_size;
- MPID_Segment_count_contig_blocks(rreq->dev.segment_ptr,rreq->dev.segment_first,&last,&n_iov);
- MPIU_Assert(n_iov > 0);
- iov = MPIU_Malloc(n_iov*sizeof(MPID_IOV));
-
- MPID_Segment_unpack_vector(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, iov, &n_iov);
- MPIU_Assert(last == rreq->dev.segment_size);
-
-#ifdef DEBUG_IOV
- fprintf(stdout,"=============== %i entries (free slots : %i)\n",n_iov,num_entries);
- for(index = 0; index < n_iov; index++)
- fprintf(stdout,"[%i]======= Recv iov[%i] = ptr : %p, len : %i \n",
- MPID_nem_mem_region.rank,index,iov[index].MPID_IOV_BUF,iov[index].MPID_IOV_LEN);
-#endif
- if(n_iov <= num_entries)
- {
- for(index = 0; index < n_iov ; index++)
- {
- (mx_iov)[index].segment_ptr = iov[index].MPID_IOV_BUF;
- (mx_iov)[index].segment_length = iov[index].MPID_IOV_LEN;
- }
- rreq->dev.tmpbuf = NULL;
- *num_seg = n_iov;
- }
- else
- {
- int packsize = 0;
- MPIR_Pack_size_impl(rreq->dev.user_count, rreq->dev.datatype, &packsize);
- rreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
- MPIU_Assert(rreq->dev.tmpbuf);
- rreq->dev.tmpbuf_sz = packsize;
- mx_iov[0].segment_ptr = (char *) rreq->dev.tmpbuf;
- mx_iov[0].segment_length = (uint32_t) packsize;
- *num_seg = 1 ;
- }
- MPIU_Free(iov);
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_PROCESS_RDTYPE);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c
deleted file mode 100644
index 237f7e2..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mx_impl.h"
-#include "my_papi_defs.h"
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_probe
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_probe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, MPI_Status *status)
-{
- uint64_t match_info = NEM_MX_MATCH_DIRECT;
- uint64_t match_mask = NEM_MX_MATCH_FULL_MASK;
- int mpi_errno = MPI_SUCCESS;
- mx_return_t ret;
- mx_status_t mx_status;
- uint32_t result;
-
- NEM_MX_DIRECT_MATCH(match_info,0,source,comm->context_id + context_offset);
- if (tag == MPI_ANY_TAG)
- {
- NEM_MX_SET_ANYTAG(match_info);
- NEM_MX_SET_ANYTAG(match_mask);
- }
- else
- NEM_MX_SET_TAG(match_info,tag);
-
-
- ret = mx_probe(MPID_nem_mx_local_endpoint,MX_INFINITE,match_info,match_mask,&mx_status,&result);
- MPIU_Assert(ret == MX_SUCCESS);
- MPIU_Assert(result != 0);
-
- NEM_MX_MATCH_GET_RANK(mx_status.match_info,status->MPI_SOURCE);
- NEM_MX_MATCH_GET_TAG(mx_status.match_info,status->MPI_TAG);
- MPIR_STATUS_SET_COUNT(*status, mx_status.xfer_length);
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_iprobe
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_iprobe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, int *flag, MPI_Status *status)
-{
- uint64_t match_info = NEM_MX_MATCH_DIRECT;
- uint64_t match_mask = NEM_MX_MATCH_FULL_MASK;
- int mpi_errno = MPI_SUCCESS;
- mx_return_t ret;
- mx_status_t mx_status;
- uint32_t result;
-
- NEM_MX_SET_CTXT(match_info,comm->context_id + context_offset);
- if( source == MPI_ANY_SOURCE)
- {
- NEM_MX_SET_ANYSRC(match_info);
- NEM_MX_SET_ANYSRC(match_mask);
- }
- else
- NEM_MX_SET_SRC(match_info,source);
- if (tag == MPI_ANY_TAG)
- {
- NEM_MX_SET_ANYTAG(match_info);
- NEM_MX_SET_ANYTAG(match_mask);
- }
- else
- NEM_MX_SET_TAG(match_info,tag);
-
- ret = mx_iprobe(MPID_nem_mx_local_endpoint,match_info,match_mask,&mx_status,&result);
- MPIU_Assert(ret == MX_SUCCESS);
-
- if (result != 0)
- {
- NEM_MX_MATCH_GET_RANK(mx_status.match_info,status->MPI_SOURCE);
- NEM_MX_MATCH_GET_TAG(mx_status.match_info,status->MPI_TAG);
- MPIR_STATUS_SET_COUNT(*status, mx_status.xfer_length);
- *flag = TRUE;
- }
- else
- *flag = FALSE;
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_improbe
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_improbe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, int context_offset, int *flag, MPID_Request **message, MPI_Status *status)
-{
- int mpi_errno = MPI_SUCCESS;
-
- /* not currently implemented for MX */
- MPIU_ERR_SET(mpi_errno, MPI_ERR_INTERN, "**nomprobe");
-
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_anysource_iprobe
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPID_nem_mx_anysource_iprobe(int tag, MPID_Comm *comm, int context_offset, int *flag, MPI_Status *status)
-{
- return MPID_nem_mx_iprobe(NULL, MPI_ANY_SOURCE, tag, comm, context_offset, flag, status);
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_anysource_iprobe
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPID_nem_mx_anysource_improbe(int tag, MPID_Comm *comm, int context_offset, int *flag, MPID_Request **message, MPI_Status *status)
-{
- return MPID_nem_mx_improbe(NULL, MPI_ANY_SOURCE, tag, comm, context_offset, flag, message, status);
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c
deleted file mode 100644
index 2d4e6ee..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c
+++ /dev/null
@@ -1,561 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- *
- * Copyright © 2006-2011 Guillaume Mercier, Institut Polytechnique de
- * Bordeaux. All rights reserved. Permission is hereby granted to use,
- * reproduce, prepare derivative works, and to redistribute to others.
- */
-
-
-
-#include "mx_impl.h"
-#include "my_papi_defs.h"
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_iSendContig
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
-{
- int mpi_errno = MPI_SUCCESS;
- mx_request_t mx_request;
- mx_segment_t mx_iov[2];
- uint32_t num_seg = 1;
- mx_return_t ret;
- uint64_t match_info = 0;
- /*MPIDI_CH3_Pkt_type_t type = ((MPIDI_CH3_Pkt_t *)(hdr))->type; */
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_ISENDCONTIGMSG);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_ISENDCONTIGMSG);
-
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mx_iSendContig");
- MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)hdr);
-
-#ifdef ONDEMAND
- if( VC_FIELD(vc, local_connected) == 0)
- {
- MPID_nem_mx_send_conn_info(vc);
- }
-#endif
-
- NEM_MX_ADI_MATCH(match_info);
- MPIU_Memcpy(&(sreq->dev.pending_pkt),(char *)hdr,sizeof(MPIDI_CH3_Pkt_t));
- mx_iov[0].segment_ptr = (char *)&(sreq->dev.pending_pkt);
- mx_iov[0].segment_length = sizeof(MPIDI_CH3_Pkt_t);
- num_seg = 1;
- if(data_sz)
- {
- mx_iov[1].segment_ptr = data;
- mx_iov[1].segment_length = data_sz;
- num_seg += 1;
- }
-
- ret = mx_isend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),match_info,(void*)sreq,&mx_request);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
- (VC_FIELD(vc,pending_sends)) += 1;
- sreq->ch.vc = vc;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_ISENDCONTIGMSG);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_iStartContigMsg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz, MPID_Request **sreq_ptr)
-{
- MPID_Request *sreq = NULL;
- int mpi_errno = MPI_SUCCESS;
- mx_request_t mx_request;
- mx_segment_t mx_iov[2];
- uint32_t num_seg = 1;
- mx_return_t ret;
- uint64_t match_info = 0;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_ISTARTCONTIGMSG);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_ISTARTCONTIGMSG);
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mx_iSendContig");
- MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)hdr);
-
-#ifdef ONDEMAND
- if( VC_FIELD(vc, local_connected) == 0)
- {
- MPID_nem_mx_send_conn_info(vc);
- }
-#endif
- /* create a request */
- sreq = MPID_Request_create();
- MPIU_Assert (sreq != NULL);
- MPIU_Object_set_ref (sreq, 2);
- sreq->kind = MPID_REQUEST_SEND;
- sreq->dev.OnDataAvail = 0;
-
- NEM_MX_ADI_MATCH(match_info);
- /*fprintf(stdout,"[%i]=== Startcontigmsg sending (%lx) to %i... \n",MPID_nem_mem_region.rank,match_info,vc->lpid); */
-
- MPIU_Memcpy(&(sreq->dev.pending_pkt),(char *)hdr,sizeof(MPIDI_CH3_Pkt_t));
- mx_iov[0].segment_ptr = (char *)&(sreq->dev.pending_pkt);
- mx_iov[0].segment_length = sizeof(MPIDI_CH3_Pkt_t);
- num_seg = 1;
- if (data_sz)
- {
- mx_iov[1].segment_ptr = (char *)data;
- mx_iov[1].segment_length = data_sz;
- num_seg += 1;
- }
-
- ret = mx_isend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),match_info,(void *)sreq,&mx_request);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
- (VC_FIELD(vc,pending_sends)) += 1;
- sreq->ch.vc = vc;
-
- fn_exit:
- *sreq_ptr = sreq;
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_ISTARTCONTIGMSG);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_SendNoncontig
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz)
-{
- mx_segment_t mx_iov[2];
- uint32_t num_seg = 1;
- int mpi_errno = MPI_SUCCESS;
- mx_request_t mx_request;
- mx_return_t ret;
- uint64_t match_info;
- MPIDI_msg_sz_t last;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_SENDNONCONTIGMSG);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_SENDNONCONTIGMSG);
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
- MPIU_Assert(MPID_IOV_LIMIT < MX_MAX_SEGMENTS);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "MPID_nem_mx_iSendNoncontig");
-
-#ifdef ONDEMAND
- if( VC_FIELD(vc, local_connected) == 0)
- {
- MPID_nem_mx_send_conn_info(vc);
- }
-#endif
-
- NEM_MX_ADI_MATCH(match_info);
- MPIU_Memcpy(&(sreq->dev.pending_pkt),(char *)header,sizeof(MPIDI_CH3_Pkt_t));
- mx_iov[0].segment_ptr = (char *)&(sreq->dev.pending_pkt);
- mx_iov[0].segment_length = sizeof(MPIDI_CH3_Pkt_t);
- num_seg = 1;
-
- MPIU_Assert(sreq->dev.segment_first == 0);
- last = sreq->dev.segment_size;
- if (last > 0)
- {
- sreq->dev.tmpbuf = MPIU_Malloc((size_t)sreq->dev.segment_size);
- MPID_Segment_pack(sreq->dev.segment_ptr,sreq->dev.segment_first, &last,(char *)(sreq->dev.tmpbuf));
- MPIU_Assert(last == sreq->dev.segment_size);
- mx_iov[1].segment_ptr = (char *)(sreq->dev.tmpbuf);
- mx_iov[1].segment_length = (uint32_t)last;
- num_seg++;
- (REQ_FIELD(sreq,deltmpbuf)) = TMP_DEL_VALUE;
- }
-
- /*
- MPIDI_Datatype_get_info(sreq->dev.user_count,sreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
- if(data_sz)
- {
- if( data_sz <= vc->eager_max_msg_sz)
- {
- MPID_nem_mx_process_sdtype(&sreq,sreq->dev.datatype,dt_ptr,sreq->dev.user_buf,sreq->dev.user_count,
- data_sz, mx_iov,&num_seg,1);
- }
- else
- {
- int packsize = 0;
- MPI_Aint last;
- MPIU_Assert(sreq->dev.segment_ptr == NULL);
- sreq->dev.segment_ptr = MPID_Segment_alloc( );
- MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPIR_Pack_size_impl(sreq->dev.user_count, sreq->dev.datatype, &packsize);
- sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
- MPIU_Assert(sreq->dev.tmpbuf);
- MPID_Segment_init(sreq->dev.user_buf, sreq->dev.user_count, sreq->dev.datatype, sreq->dev.segment_ptr, 0);
- last = data_sz;
- MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
- mx_iov[1].segment_ptr = (char *) sreq->dev.tmpbuf;
- mx_iov[1].segment_length = (uint32_t) last;
- num_seg++;
- }
- }
- */
-
- MPIU_Assert(num_seg <= MX_MAX_SEGMENTS);
- ret = mx_isend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),match_info,(void *)sreq,&mx_request);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
-
- (VC_FIELD(vc,pending_sends)) += 1;
- sreq->ch.vc = vc;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_SENDNONCONTIGMSG);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_directSend
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_directSend(MPIDI_VC_t *vc, const void * buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset,MPID_Request **request)
-{
- MPID_Request *sreq = NULL;
- mx_segment_t mx_iov[MX_MAX_SEGMENTS];
- uint32_t num_seg = 1;
- mx_return_t ret;
- uint64_t mx_matching;
- int mpi_errno = MPI_SUCCESS;
- MPID_Datatype *dt_ptr;
- int dt_contig;
- MPIDI_msg_sz_t data_sz;
- MPI_Aint dt_true_lb;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_DIRECTSEND);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_DIRECTSEND);
-
- MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
- MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
- sreq->partner_request = NULL;
-
-#ifdef ONDEMAND
- if( VC_FIELD(vc, local_connected) == 0)
- {
- MPID_nem_mx_send_conn_info(vc);
- fprintf(stdout,"[%i]=== DirectSend info Sender FULLY connected with %i %p \n", MPID_nem_mem_region.rank,vc->lpid,vc);
- }
-#endif
-
- MPIDI_VC_FAI_send_seqnum(vc, seqnum);
- MPIDI_Request_set_seqnum(sreq, seqnum);
- sreq->ch.vc = vc;
- sreq->dev.OnDataAvail = NULL;
-
- NEM_MX_DIRECT_MATCH(mx_matching,tag,comm->rank,comm->context_id + context_offset);
- if(data_sz)
- {
- if (dt_contig)
- {
- mx_iov[0].segment_ptr = (char*)buf + dt_true_lb;
- mx_iov[0].segment_length = data_sz;
- }
- else
- {
- if( data_sz <= vc->eager_max_msg_sz)
- {
- MPID_nem_mx_process_sdtype(&sreq,datatype,dt_ptr,buf,count,data_sz,mx_iov,&num_seg,0);
-#ifdef DEBUG_IOV
- {
- int index;
- fprintf(stdout,"==========================\n");
- for(index = 0; index < num_seg; index++)
- fprintf(stdout,"[%i]======= MX iov[%i] = ptr : %p, len : %i \n",
- MPID_nem_mem_region.rank,index,mx_iov[index].segment_ptr,mx_iov[index].segment_length);
- }
-#endif
- }
- else
- {
- int packsize = 0;
- MPI_Aint last;
- sreq->dev.segment_ptr = MPID_Segment_alloc( );
- MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPIR_Pack_size_impl(count, datatype, &packsize);
- sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
- MPIU_Assert(sreq->dev.tmpbuf);
- (REQ_FIELD(sreq,deltmpbuf)) = TMP_DEL_VALUE;
- MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
- last = data_sz;
- MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
- mx_iov[0].segment_ptr = (char *) sreq->dev.tmpbuf;
- mx_iov[0].segment_length = (uint32_t) last;
- }
- }
- }
- else
- {
- mx_iov[0].segment_ptr = NULL;
- mx_iov[0].segment_length = 0;
- }
-
- ret = mx_isend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),
- mx_matching,(void *)sreq,&(REQ_FIELD(sreq,mx_request)));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
- (VC_FIELD(vc,pending_sends)) += 1;
-
- fn_exit:
- *request = sreq;
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_DIRECTSEND);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_directSsend
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_directSsend(MPIDI_VC_t *vc, const void * buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset,MPID_Request **request)
-{
- MPID_Request *sreq = NULL;
- uint32_t num_seg = 1;
- mx_segment_t mx_iov[MX_MAX_SEGMENTS];
- mx_return_t ret;
- uint64_t mx_matching;
- int mpi_errno = MPI_SUCCESS;
- MPID_Datatype *dt_ptr;
- int dt_contig;
- MPIDI_msg_sz_t data_sz;
- MPI_Aint dt_true_lb;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_DIRECTSSEND);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_DIRECTSSEND);
-
- MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
- MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
- sreq->partner_request = NULL;
-
-#ifdef ONDEMAND
- if( VC_FIELD(vc, local_connected) == 0)
- {
- MPID_nem_mx_send_conn_info(vc);
- fprintf(stdout,"[%i]=== DirectSsend info Sender FULLY connected with %i %p \n", MPID_nem_mem_region.rank,vc->lpid,vc);
- }
-#endif
- MPIDI_VC_FAI_send_seqnum(vc, seqnum);
- MPIDI_Request_set_seqnum(sreq, seqnum);
- sreq->ch.vc = vc;
- sreq->dev.OnDataAvail = NULL;
-
- NEM_MX_DIRECT_MATCH(mx_matching,tag,comm->rank,comm->context_id + context_offset);
-
- if(data_sz)
- {
- if (dt_contig)
- {
- mx_iov[0].segment_ptr = (char*)(buf) + dt_true_lb;
- mx_iov[0].segment_length = data_sz;
- }
- else
- {
- if( data_sz <= vc->eager_max_msg_sz)
- {
- MPID_nem_mx_process_sdtype(&sreq,datatype,dt_ptr,buf,count,data_sz,mx_iov,&num_seg,0);
- }
- else
- {
- int packsize = 0;
- MPI_Aint last;
- sreq->dev.segment_ptr = MPID_Segment_alloc( );
- MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPIR_Pack_size_impl(count, datatype, &packsize);
- sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
- MPIU_Assert(sreq->dev.tmpbuf);
- (REQ_FIELD(sreq,deltmpbuf)) = TMP_DEL_VALUE;
- MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
- last = data_sz;
- MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
- mx_iov[0].segment_ptr = (char *) sreq->dev.tmpbuf;
- mx_iov[0].segment_length = (uint32_t) last;
- }
- }
- }
- else
- {
- mx_iov[0].segment_ptr = NULL;
- mx_iov[0].segment_length = 0;
- }
-
- ret = mx_issend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),
- mx_matching,(void *)sreq,&(REQ_FIELD(sreq,mx_request)));
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
- (VC_FIELD(vc,pending_sends)) += 1;
-
- fn_exit:
- *request = sreq;
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_DIRECTSSEND);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_process_sdtype
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_mx_process_sdtype(MPID_Request **sreq_p, MPI_Datatype datatype, MPID_Datatype * dt_ptr, const void *buf, int count, MPIDI_msg_sz_t data_sz, mx_segment_t *mx_iov, uint32_t *num_seg,int first_free_slot)
-{
- MPID_Request *sreq =*sreq_p;
- MPID_IOV *iov;
- MPIDI_msg_sz_t last;
- int num_entries = MX_MAX_SEGMENTS - first_free_slot;
- int n_iov = 0;
- int mpi_errno = MPI_SUCCESS;
- int index;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_PROCESS_SDTYPE);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_PROCESS_SDTYPE);
-
- sreq->dev.segment_ptr = MPID_Segment_alloc( );
- MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
- MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
- sreq->dev.segment_first = 0;
- sreq->dev.segment_size = data_sz;
- last = sreq->dev.segment_size;
-
- MPID_Segment_count_contig_blocks(sreq->dev.segment_ptr,sreq->dev.segment_first,&last,&n_iov);
- MPIU_Assert(n_iov > 0);
- iov = MPIU_Malloc(n_iov*sizeof(MPID_IOV));
-
- MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, iov, &n_iov);
- MPIU_Assert(last == sreq->dev.segment_size);
-
-#ifdef DEBUG_IOV
- fprintf(stdout,"=============== %i entries (free slots : %i)\n",n_iov,num_entries);
- for(index = 0; index < n_iov; index++)
- fprintf(stdout,"[%i]======= Send iov[%i] = ptr : %p, len : %i \n",
- MPID_nem_mem_region.rank,index,iov[index].MPID_IOV_BUF,iov[index].MPID_IOV_LEN);
-#endif
-
- if(n_iov <= num_entries)
- {
- for(index = 0; index < n_iov ; index++)
- {
- (mx_iov)[first_free_slot+index].segment_ptr = iov[index].MPID_IOV_BUF;
- (mx_iov)[first_free_slot+index].segment_length = iov[index].MPID_IOV_LEN;
- }
- *num_seg = n_iov;
- }
- else
- {
- int size_to_copy = 0;
- int offset = 0;
- int last_entry = num_entries - 1;
- for(index = 0; index < n_iov ; index++)
- {
- if (index <= (last_entry-1))
- {
- (mx_iov)[first_free_slot+index].segment_ptr = iov[index].MPID_IOV_BUF;
- (mx_iov)[first_free_slot+index].segment_length = iov[index].MPID_IOV_LEN;
- }
- else
- {
- size_to_copy += iov[index].MPID_IOV_LEN;
- }
- }
- sreq->dev.tmpbuf = MPIU_Malloc(size_to_copy);
- MPIU_Assert(sreq->dev.tmpbuf);
- (REQ_FIELD(sreq,deltmpbuf)) = TMP_DEL_VALUE;
- for(index = last_entry; index < n_iov; index++)
- {
- MPIU_Memcpy((char *)(sreq->dev.tmpbuf) + offset, iov[index].MPID_IOV_BUF, iov[index].MPID_IOV_LEN);
- offset += iov[index].MPID_IOV_LEN;
- }
- (mx_iov)[MX_MAX_SEGMENTS-1].segment_ptr = sreq->dev.tmpbuf;
- (mx_iov)[MX_MAX_SEGMENTS-1].segment_length = size_to_copy;
- *num_seg = MX_MAX_SEGMENTS ;
- }
- MPIU_Free(iov);
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_PROCESS_SDTYPE);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_mx_send_conn_info
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int
-MPID_nem_mx_send_conn_info (MPIDI_VC_t *vc)
-{
- char business_card[MPID_NEM_MAX_NETMOD_STRING_LEN];
- uint64_t match_info;
- uint32_t num_seg = 3;
- mx_segment_t mx_iov[3];
- mx_request_t mx_request;
- mx_return_t ret;
- mx_status_t status;
- uint32_t result;
- int mpi_errno = MPI_SUCCESS;
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MX_SEND_CONN_INFO);
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MX_SEND_CONN_INFO);
-
- mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, MPID_NEM_MAX_NETMOD_STRING_LEN, vc->pg);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- fprintf(stdout,"[%i]=== Sender connecting to %i \n", MPID_nem_mem_region.rank,vc->lpid);
-
- /* FIXME: match_info is used uninitialized */
- ret = mx_iconnect(MPID_nem_mx_local_endpoint,VC_FIELD(vc, remote_nic_id),
- VC_FIELD(vc, remote_endpoint_id),MPID_NEM_MX_FILTER,match_info,NULL,&mx_request);
- MPIU_Assert(ret == MX_SUCCESS);
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,&mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
- VC_FIELD(vc, remote_endpoint_addr) = status.source;
- VC_FIELD(vc, local_connected) = 1;
- mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
-
- fprintf(stdout,"[%i]=== Sending conn info connection to %i \n", MPID_nem_mem_region.rank,vc->lpid);
-
- NEM_MX_ADI_MATCH(match_info);
- NEM_MX_SET_PGRANK(match_info,MPIDI_Process.my_pg_rank);
-
- mx_iov[0].segment_ptr = (char*)&MPID_nem_mx_local_nic_id;
- mx_iov[0].segment_length = sizeof(uint64_t);
- mx_iov[1].segment_ptr = (char*)&MPID_nem_mx_local_endpoint_id;
- mx_iov[1].segment_length = sizeof(uint32_t);
- mx_iov[2].segment_ptr = MPIDI_Process.my_pg->id;
- mx_iov[2].segment_length = strlen(MPIDI_Process.my_pg->id) + 1;
-
- ret = mx_isend(MPID_nem_mx_local_endpoint,mx_iov,num_seg,VC_FIELD(vc,remote_endpoint_addr),match_info,NULL,&mx_request);
- MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_isend", "**mx_isend %s", mx_strerror (ret));
- do{
- ret = mx_test(MPID_nem_mx_local_endpoint,&mx_request,&status,&result);
- }while((result == 0) && (ret == MX_SUCCESS));
- MPIU_Assert(ret == MX_SUCCESS);
- fprintf(stdout,"[%i]=== Send conn info to %i (%Lx) (%Li %i %s) ... \n",
- MPID_nem_mem_region.rank,vc->lpid,match_info,MPID_nem_mx_local_nic_id,MPID_nem_mx_local_endpoint_id,(char *)MPIDI_Process.my_pg->id);
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MX_SEND_CONN_INFO);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4
deleted file mode 100644
index f2169e5..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4
+++ /dev/null
@@ -1,41 +0,0 @@
-[#] start of __file__
-dnl MPICH_SUBCFG_AFTER=src/mpid/ch3/channels/nemesis
-
-AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[
- AM_COND_IF([BUILD_CH3_NEMESIS],[
- for net in $nemesis_networks ; do
- AS_CASE([$net],[mx],[build_nemesis_netmod_mx=yes])
- done
- ])
- AM_CONDITIONAL([BUILD_NEMESIS_NETMOD_MX],[test "X$build_nemesis_netmod_mx" = "Xyes"])
-])dnl
-
-AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[
-AM_COND_IF([BUILD_NEMESIS_NETMOD_MX],[
- AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:mx])
-
- PAC_SET_HEADER_LIB_PATH(mx)
- PAC_PUSH_FLAG(LIBS)
- PAC_CHECK_HEADER_LIB_FATAL(mx, myriexpress.h, myriexpress, mx_finalize)
- PAC_POP_FLAG(LIBS)
- AC_CHECK_HEADER([mx_extensions.h], , [
- AC_MSG_ERROR(['mx_extensions.h not found. Are you running a recent version of MX (at least 1.2.1)?'])
- ])
- AC_TRY_COMPILE([
- #include "myriexpress.h"
- #include "mx_extensions.h"
- #if MX_API < 0x301
- #error You need at least MX 1.2.1 (MX_API >= 0x301)
- #endif],
- [int a=0;],
- mx_api_version=yes,
- mx_api_version=no)
- if test "$mx_api_version" = no ; then
- AC_MSG_ERROR(['MX API version Problem. Are you running a recent version of MX (at least 1.2.1)?'])
- fi;
- AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions])
- PAC_APPEND_FLAG([-lmx],[EXTERNAL_LIBS])
-
-])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_MX,...)
-])dnl end _BODY
-[#] end of __file__
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h b/src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h
deleted file mode 100644
index 2a9b67d..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h
+++ /dev/null
@@ -1,612 +0,0 @@
-/*
-Copyright (c) 2003-2009, Troy D. Hanson http://uthash.sourceforge.net
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#include <string.h> /* memcmp,strlen */
-#include <stddef.h> /* ptrdiff_t */
-
-#ifndef UTHASH_H
-#define UTHASH_H
-
-#define UTHASH_VERSION 1.5
-
-/* C++ requires extra stringent casting */
-#if defined __cplusplus
-#define TYPEOF(x) (typeof(x))
-#else
-#define TYPEOF(x)
-#endif
-
-
-#define uthash_fatal(msg) MPID_Abort(NULL, MPI_ERR_OTHER, -1, msg) /* fatal error (out of memory,etc) */
-#define uthash_bkt_malloc(sz) MPIU_Malloc(sz) /* malloc fcn for UT_hash_bucket's */
-#define uthash_bkt_free(ptr) MPIU_Free(ptr) /* free fcn for UT_hash_bucket's */
-#define uthash_tbl_malloc(sz) MPIU_Malloc(sz) /* malloc fcn for UT_hash_table */
-#define uthash_tbl_free(ptr) MPIU_Free(ptr) /* free fcn for UT_hash_table */
-
-#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */
-#define uthash_expand_fyi(tbl) /* can be defined to log expands */
-
-/* initial number of buckets */
-#define HASH_INITIAL_NUM_BUCKETS 32 /* initial number of buckets */
-#define HASH_INITIAL_NUM_BUCKETS_LOG2 5 /* lg2 of initial number of buckets */
-#define HASH_BKT_CAPACITY_THRESH 10 /* expand when bucket count reaches */
-
-/* calculate the element whose hash handle address is hhe */
-#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)hhp) - (tbl)->hho))
-
-#define HASH_FIND(hh,head,keyptr,keylen,out) \
-do { \
- unsigned _hf_bkt,_hf_hashv; \
- out=TYPEOF(out)head; \
- if (head) { \
- HASH_FCN(keyptr,keylen, (head)->hh.tbl->num_buckets, _hf_hashv, _hf_bkt); \
- HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], \
- keyptr,keylen,out); \
- } \
-} while (0)
-
-#define HASH_ADD(hh,head,fieldname,keylen_in,add) \
- HASH_ADD_KEYPTR(hh,head,&add->fieldname,keylen_in,add)
-
-#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \
-do { \
- unsigned _ha_bkt; \
- add->hh.next = NULL; \
- add->hh.key = (char*)keyptr; \
- add->hh.keylen = keylen_in; \
- if (!(head)) { \
- head = add; \
- (head)->hh.prev = NULL; \
- (head)->hh.tbl = (UT_hash_table*)uthash_tbl_malloc( \
- sizeof(UT_hash_table)); \
- if (!((head)->hh.tbl)) { uthash_fatal( "out of memory"); } \
- memset((head)->hh.tbl, 0, sizeof(UT_hash_table)); \
- (head)->hh.tbl->tail = &(add->hh); \
- (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \
- (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \
- (head)->hh.tbl->hho = (char*)(&add->hh) - (char*)(add); \
- (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_bkt_malloc( \
- HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
- if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); } \
- memset((head)->hh.tbl->buckets, 0, \
- HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
- } else { \
- (head)->hh.tbl->tail->next = add; \
- add->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \
- (head)->hh.tbl->tail = &(add->hh); \
- } \
- (head)->hh.tbl->num_items++; \
- add->hh.tbl = (head)->hh.tbl; \
- HASH_FCN(keyptr,keylen_in, (head)->hh.tbl->num_buckets, \
- (add)->hh.hashv, _ha_bkt); \
- HASH_ADD_TO_BKT(hh,(head)->hh.tbl->buckets[_ha_bkt],add); \
- HASH_EMIT_KEY(hh,head,keyptr,keylen_in); \
- HASH_FSCK(hh,head); \
-} while(0)
-
-#define HASH_TO_BKT( hashv, num_bkts, bkt ) bkt = ((hashv) & ((num_bkts) - 1))
-
-/* delete "delptr" from the hash table.
- * "the usual" patch-up process for the app-order doubly-linked-list.
- * The use of _hd_hh_del below deserves special explanation.
- * These used to be expressed using (delptr) but that led to a bug
- * if someone used the same symbol for the head and deletee, like
- * HASH_DELETE(hh,users,users);
- * We want that to work, but by changing the head (users) below
- * we were forfeiting our ability to further refer to the deletee (users)
- * in the patch-up process. Solution: use scratch space in the table to
- * copy the deletee pointer, then the latter references are via that
- * scratch pointer rather than through the repointed (users) symbol.
- */
-#define HASH_DELETE(hh,head,delptr) \
-do { \
- unsigned _hd_bkt; \
- struct UT_hash_handle *_hd_hh_del; \
- if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) ) { \
- uthash_bkt_free((head)->hh.tbl->buckets ); \
- uthash_tbl_free((head)->hh.tbl); \
- head = NULL; \
- } else { \
- _hd_hh_del = &((delptr)->hh); \
- if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) { \
- (head)->hh.tbl->tail = \
- (UT_hash_handle*)((char*)((delptr)->hh.prev) + \
- (head)->hh.tbl->hho); \
- } \
- if ((delptr)->hh.prev) { \
- ((UT_hash_handle*)((char*)((delptr)->hh.prev) + \
- (head)->hh.tbl->hho))->next = (delptr)->hh.next; \
- } else { \
- head = TYPEOF(head)((delptr)->hh.next); \
- } \
- if (_hd_hh_del->next) { \
- ((UT_hash_handle*)((char*)_hd_hh_del->next + \
- (head)->hh.tbl->hho))->prev = \
- _hd_hh_del->prev; \
- } \
- HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \
- HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \
- (head)->hh.tbl->num_items--; \
- } \
- HASH_FSCK(hh,head); \
-} while (0)
-
-
-/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */
-#define HASH_FIND_STR(head,findstr,out) \
- HASH_FIND(hh,head,findstr,strlen(findstr),out)
-#define HASH_ADD_STR(head,strfield,add) \
- HASH_ADD(hh,head,strfield,strlen(add->strfield),add)
-#define HASH_FIND_INT(head,findint,out) \
- HASH_FIND(hh,head,findint,sizeof(int),out)
-#define HASH_ADD_INT(head,intfield,add) \
- HASH_ADD(hh,head,intfield,sizeof(int),add)
-#define HASH_DEL(head,delptr) \
- HASH_DELETE(hh,head,delptr)
-
-/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined.
- * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined.
- */
-#ifdef HASH_DEBUG
-#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0)
-#define HASH_FSCK(hh,head) \
-do { \
- unsigned _bkt_i; \
- unsigned _count, _bkt_count; \
- char *_prev; \
- struct UT_hash_handle *_thh; \
- if (head) { \
- _count = 0; \
- for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) { \
- _bkt_count = 0; \
- _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \
- _prev = NULL; \
- while (_thh) { \
- if (_prev != (char*)(_thh->hh_prev)) { \
- HASH_OOPS("invalid hh_prev %p, actual %p\n", \
- _thh->hh_prev, _prev ); \
- } \
- _bkt_count++; \
- _prev = (char*)(_thh); \
- _thh = _thh->hh_next; \
- } \
- _count += _bkt_count; \
- if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \
- HASH_OOPS("invalid bucket count %d, actual %d\n", \
- (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \
- } \
- } \
- if (_count != (head)->hh.tbl->num_items) { \
- HASH_OOPS("invalid hh item count %d, actual %d\n", \
- (head)->hh.tbl->num_items, _count ); \
- } \
- /* traverse hh in app order; check next/prev integrity, count */ \
- _count = 0; \
- _prev = NULL; \
- _thh = &(head)->hh; \
- while (_thh) { \
- _count++; \
- if (_prev !=(char*)(_thh->prev)) { \
- HASH_OOPS("invalid prev %p, actual %p\n", \
- _thh->prev, _prev ); \
- } \
- _prev = ELMT_FROM_HH((head)->hh.tbl, _thh); \
- _thh = ( _thh->next ? (UT_hash_handle*)((char*)(_thh->next) + \
- (head)->hh.tbl->hho) : NULL ); \
- } \
- if (_count != (head)->hh.tbl->num_items) { \
- HASH_OOPS("invalid app item count %d, actual %d\n", \
- (head)->hh.tbl->num_items, _count ); \
- } \
- } \
-} while (0)
-#else
-#define HASH_FSCK(hh,head)
-#endif
-
-/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to
- * the descriptor to which this macro is defined for tuning the hash function.
- * The app can #include <unistd.h> to get the prototype for write(2). */
-#ifdef HASH_EMIT_KEYS
-#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \
-do { \
- unsigned _klen = fieldlen; \
- write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \
- write(HASH_EMIT_KEYS, keyptr, fieldlen); \
-} while (0)
-#else
-#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)
-#endif
-
-/* default to Jenkins unless specified e.g. DHASH_FUNCTION=HASH_SAX */
-#ifdef HASH_FUNCTION
-#define HASH_FCN HASH_FUNCTION
-#else
-#define HASH_FCN HASH_JEN
-#endif
-
-/* The Bernstein hash function, used in Perl prior to v5.6 */
-#define HASH_BER(key,keylen,num_bkts,hashv,bkt) \
-do { \
- unsigned _hb_keylen=keylen; \
- char *_hb_key=(char*)key; \
- (hashv) = 0; \
- while (_hb_keylen--) { (hashv) = ((hashv) * 33) + *_hb_key++; } \
- bkt = (hashv) & (num_bkts-1); \
-} while (0)
-
-
-/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at
- * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
-#define HASH_SAX(key,keylen,num_bkts,hashv,bkt) \
-do { \
- unsigned _sx_i; \
- hashv = 0; \
- for(_sx_i=0; _sx_i < keylen; _sx_i++) \
- hashv ^= (hashv << 5) + (hashv >> 2) + key[_sx_i]; \
- bkt = hashv & (num_bkts-1); \
-} while (0)
-
-#define HASH_FNV(key,keylen,num_bkts,hashv,bkt) \
-do { \
- unsigned _fn_i; \
- hashv = 2166136261UL; \
- for(_fn_i=0; _fn_i < keylen; _fn_i++) \
- hashv = (hashv * 16777619) ^ key[_fn_i]; \
- bkt = hashv & (num_bkts-1); \
-} while(0);
-
-#define HASH_OAT(key,keylen,num_bkts,hashv,bkt) \
-do { \
- unsigned _ho_i; \
- hashv = 0; \
- for(_ho_i=0; _ho_i < keylen; _ho_i++) { \
- hashv += key[_ho_i]; \
- hashv += (hashv << 10); \
- hashv ^= (hashv >> 6); \
- } \
- hashv += (hashv << 3); \
- hashv ^= (hashv >> 11); \
- hashv += (hashv << 15); \
- bkt = hashv & (num_bkts-1); \
-} while(0)
-
-#define HASH_JEN_MIX(a,b,c) \
-do { \
- a -= b; a -= c; a ^= ( c >> 13 ); \
- b -= c; b -= a; b ^= ( a << 8 ); \
- c -= a; c -= b; c ^= ( b >> 13 ); \
- a -= b; a -= c; a ^= ( c >> 12 ); \
- b -= c; b -= a; b ^= ( a << 16 ); \
- c -= a; c -= b; c ^= ( b >> 5 ); \
- a -= b; a -= c; a ^= ( c >> 3 ); \
- b -= c; b -= a; b ^= ( a << 10 ); \
- c -= a; c -= b; c ^= ( b >> 15 ); \
-} while (0)
-
-#define HASH_JEN(key,keylen,num_bkts,hashv,bkt) \
-do { \
- unsigned _hj_i,_hj_j,_hj_k; \
- char *_hj_key=(char*)key; \
- hashv = 0xfeedbeef; \
- _hj_i = _hj_j = 0x9e3779b9; \
- _hj_k = keylen; \
- while (_hj_k >= 12) { \
- _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \
- + ( (unsigned)_hj_key[2] << 16 ) \
- + ( (unsigned)_hj_key[3] << 24 ) ); \
- _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \
- + ( (unsigned)_hj_key[6] << 16 ) \
- + ( (unsigned)_hj_key[7] << 24 ) ); \
- hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \
- + ( (unsigned)_hj_key[10] << 16 ) \
- + ( (unsigned)_hj_key[11] << 24 ) ); \
- \
- HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
- \
- _hj_key += 12; \
- _hj_k -= 12; \
- } \
- hashv += keylen; \
- switch ( _hj_k ) { \
- case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); \
- case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); \
- case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); \
- case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); \
- case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); \
- case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); \
- case 5: _hj_j += _hj_key[4]; \
- case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); \
- case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); \
- case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); \
- case 1: _hj_i += _hj_key[0]; \
- } \
- HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
- bkt = hashv & (num_bkts-1); \
-} while(0)
-
-
-/* key comparison function; return 0 if keys equal */
-#define HASH_KEYCMP(a,b,len) memcmp(a,b,len)
-
-/* iterate over items in a known bucket to find desired item */
-#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,out) \
-out = TYPEOF(out)((head.hh_head) ? ELMT_FROM_HH(tbl,head.hh_head) : NULL); \
-while (out) { \
- if (out->hh.keylen == keylen_in) { \
- if ((HASH_KEYCMP(out->hh.key,keyptr,keylen_in)) == 0) break; \
- } \
- out= TYPEOF(out)((out->hh.hh_next) ? \
- ELMT_FROM_HH(tbl,out->hh.hh_next) : NULL); \
-}
-
-/* add an item to a bucket */
-#define HASH_ADD_TO_BKT(hh,head,add) \
- head.count++; \
- add->hh.hh_next = head.hh_head; \
- add->hh.hh_prev = NULL; \
- if (head.hh_head) head.hh_head->hh_prev = &add->hh; \
- head.hh_head=&add->hh; \
- if (head.count >= ((head.expand_mult+1) * HASH_BKT_CAPACITY_THRESH) \
- && add->hh.tbl->noexpand != 1) { \
- HASH_EXPAND_BUCKETS(add->hh.tbl); \
- }
-
-/* remove an item from a given bucket */
-#define HASH_DEL_IN_BKT(hh,head,hh_del) \
- (head).count--; \
- if ((head).hh_head == hh_del) { \
- (head).hh_head = hh_del->hh_next; \
- } \
- if (hh_del->hh_prev) { \
- hh_del->hh_prev->hh_next = hh_del->hh_next; \
- } \
- if (hh_del->hh_next) { \
- hh_del->hh_next->hh_prev = hh_del->hh_prev; \
- }
-
-/* Bucket expansion has the effect of doubling the number of buckets
- * and redistributing the items into the new buckets. Ideally the
- * items will distribute more or less evenly into the new buckets
- * (the extent to which this is true is a measure of the quality of
- * the hash function as it applies to the key domain).
- *
- * With the items distributed into more buckets, the chain length
- * (item count) in each bucket is reduced. Thus by expanding buckets
- * the hash keeps a bound on the chain length. This bounded chain
- * length is the essence of how a hash provides constant time lookup.
- *
- * The calculation of tbl->ideal_chain_maxlen below deserves some
- * explanation. First, keep in mind that we're calculating the ideal
- * maximum chain length based on the *new* (doubled) bucket count.
- * In fractions this is just n/b (n=number of items,b=new num buckets).
- * Since the ideal chain length is an integer, we want to calculate
- * ceil(n/b). We don't depend on floating point arithmetic in this
- * hash, so to calculate ceil(n/b) with integers we could write
- *
- * ceil(n/b) = (n/b) + ((n%b)?1:0)
- *
- * and in fact a previous version of this hash did just that.
- * But now we have improved things a bit by recognizing that b is
- * always a power of two. We keep its base 2 log handy (call it lb),
- * so now we can write this with a bit shift and logical AND:
- *
- * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0)
- *
- */
-#define HASH_EXPAND_BUCKETS(tbl) \
-do { \
- unsigned _he_bkt; \
- unsigned _he_bkt_i; \
- struct UT_hash_handle *_he_thh, *_he_hh_nxt; \
- UT_hash_bucket *_he_new_buckets, *_he_newbkt; \
- _he_new_buckets = (UT_hash_bucket*)uthash_bkt_malloc( \
- 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
- if (!_he_new_buckets) { uthash_fatal( "out of memory"); } \
- memset(_he_new_buckets, 0, \
- 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
- tbl->ideal_chain_maxlen = \
- (tbl->num_items >> (tbl->log2_num_buckets+1)) + \
- ((tbl->num_items & ((tbl->num_buckets*2)-1)) ? 1 : 0); \
- tbl->nonideal_items = 0; \
- for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++) \
- { \
- _he_thh = tbl->buckets[ _he_bkt_i ].hh_head; \
- while (_he_thh) { \
- _he_hh_nxt = _he_thh->hh_next; \
- HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2, _he_bkt); \
- _he_newbkt = &(_he_new_buckets[ _he_bkt ]); \
- if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) { \
- tbl->nonideal_items++; \
- _he_newbkt->expand_mult = _he_newbkt->count / \
- tbl->ideal_chain_maxlen; \
- } \
- _he_thh->hh_prev = NULL; \
- _he_thh->hh_next = _he_newbkt->hh_head; \
- if (_he_newbkt->hh_head) _he_newbkt->hh_head->hh_prev = \
- _he_thh; \
- _he_newbkt->hh_head = _he_thh; \
- _he_thh = _he_hh_nxt; \
- } \
- } \
- tbl->num_buckets *= 2; \
- tbl->log2_num_buckets++; \
- uthash_bkt_free( tbl->buckets ); \
- tbl->buckets = _he_new_buckets; \
- tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ? \
- (tbl->ineff_expands+1) : 0; \
- if (tbl->ineff_expands > 1) { \
- tbl->noexpand=1; \
- uthash_noexpand_fyi(tbl); \
- } \
- uthash_expand_fyi(tbl); \
-} while(0)
-
-
-/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */
-/* Note that HASH_SORT assumes the hash handle name to be hh.
- * HASH_SRT was added to allow the hash handle name to be passed in. */
-#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn)
-#define HASH_SRT(hh,head,cmpfcn) \
-do { \
- unsigned _hs_i; \
- unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \
- struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \
- if (head) { \
- _hs_insize = 1; \
- _hs_looping = 1; \
- _hs_list = &((head)->hh); \
- while (_hs_looping) { \
- _hs_p = _hs_list; \
- _hs_list = NULL; \
- _hs_tail = NULL; \
- _hs_nmerges = 0; \
- while (_hs_p) { \
- _hs_nmerges++; \
- _hs_q = _hs_p; \
- _hs_psize = 0; \
- for ( _hs_i = 0; _hs_i < _hs_insize; _hs_i++ ) { \
- _hs_psize++; \
- _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
- ((void*)((char*)(_hs_q->next) + \
- (head)->hh.tbl->hho)) : NULL); \
- if (! (_hs_q) ) break; \
- } \
- _hs_qsize = _hs_insize; \
- while ((_hs_psize > 0) || ((_hs_qsize > 0) && _hs_q )) { \
- if (_hs_psize == 0) { \
- _hs_e = _hs_q; \
- _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
- ((void*)((char*)(_hs_q->next) + \
- (head)->hh.tbl->hho)) : NULL); \
- _hs_qsize--; \
- } else if ( (_hs_qsize == 0) || !(_hs_q) ) { \
- _hs_e = _hs_p; \
- _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
- ((void*)((char*)(_hs_p->next) + \
- (head)->hh.tbl->hho)) : NULL); \
- _hs_psize--; \
- } else if (( \
- cmpfcn(TYPEOF(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \
- TYPEOF(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q))) \
- ) <= 0) { \
- _hs_e = _hs_p; \
- _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
- ((void*)((char*)(_hs_p->next) + \
- (head)->hh.tbl->hho)) : NULL); \
- _hs_psize--; \
- } else { \
- _hs_e = _hs_q; \
- _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
- ((void*)((char*)(_hs_q->next) + \
- (head)->hh.tbl->hho)) : NULL); \
- _hs_qsize--; \
- } \
- if ( _hs_tail ) { \
- _hs_tail->next = ((_hs_e) ? \
- ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL); \
- } else { \
- _hs_list = _hs_e; \
- } \
- _hs_e->prev = ((_hs_tail) ? \
- ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL); \
- _hs_tail = _hs_e; \
- } \
- _hs_p = _hs_q; \
- } \
- _hs_tail->next = NULL; \
- if ( _hs_nmerges <= 1 ) { \
- _hs_looping=0; \
- (head)->hh.tbl->tail = _hs_tail; \
- (head) = TYPEOF(head)ELMT_FROM_HH((head)->hh.tbl, _hs_list); \
- } \
- _hs_insize *= 2; \
- } \
- HASH_FSCK(hh,head); \
- } \
-} while (0)
-
-/* obtain a count of items in the hash */
-#define HASH_COUNT(head) HASH_CNT(hh,head)
-#define HASH_CNT(hh,head) (head?(head->hh.tbl->num_items):0)
-
-typedef struct UT_hash_bucket {
- struct UT_hash_handle *hh_head;
- unsigned count;
-
- /* expand_mult is normally set to 0. In this situation, the max chain length
- * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If
- * the bucket's chain exceeds this length, bucket expansion is triggered).
- * However, setting expand_mult to a non-zero value delays bucket expansion
- * (that would be triggered by additions to this particular bucket)
- * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH.
- * (The multiplier is simply expand_mult+1). The whole idea of this
- * multiplier is to reduce bucket expansions, since they are expensive, in
- * situations where we know that a particular bucket tends to be overused.
- * It is better to let its chain length grow to a longer yet-still-bounded
- * value, than to do an O(n) bucket expansion too often.
- */
- unsigned expand_mult;
-
-} UT_hash_bucket;
-
-typedef struct UT_hash_table {
- UT_hash_bucket *buckets;
- unsigned num_buckets, log2_num_buckets;
- unsigned num_items;
- struct UT_hash_handle *tail; /* tail hh in app order, for fast append */
- ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */
-
- /* in an ideal situation (all buckets used equally), no bucket would have
- * more than ceil(#items/#buckets) items. that's the ideal chain length. */
- unsigned ideal_chain_maxlen;
-
- /* nonideal_items is the number of items in the hash whose chain position
- * exceeds the ideal chain maxlen. these items pay the penalty for an uneven
- * hash distribution; reaching them in a chain traversal takes >ideal steps */
- unsigned nonideal_items;
-
- /* ineffective expands occur when a bucket doubling was performed, but
- * afterward, more than half the items in the hash had nonideal chain
- * positions. If this happens on two consecutive expansions we inhibit any
- * further expansion, as it's not helping; this happens when the hash
- * function isn't a good fit for the key domain. When expansion is inhibited
- * the hash will still work, albeit no longer in constant time. */
- unsigned ineff_expands, noexpand;
-
-
-} UT_hash_table;
-
-
-typedef struct UT_hash_handle {
- struct UT_hash_table *tbl;
- void *prev; /* prev element in app order */
- void *next; /* next element in app order */
- struct UT_hash_handle *hh_prev; /* previous hh in bucket order */
- struct UT_hash_handle *hh_next; /* next hh in bucket order */
- void *key; /* ptr to enclosing struct's key */
- unsigned keylen; /* enclosing struct's key len */
- unsigned hashv; /* result of hash-fcn(key) */
-} UT_hash_handle;
-
-#endif /* UTHASH_H */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/Makefile.sm b/src/mpid/ch3/channels/nemesis/netmod/wintcp/Makefile.sm
deleted file mode 100644
index 2333e01..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/Makefile.sm
+++ /dev/null
@@ -1,5 +0,0 @@
-INCLUDES = -I. -I../../include -I${master_top_srcdir}/src/ch3/channels/nemesis/include -I${master_top_srcdir}/src/include \
- -I${top_builddir}/src/include
-lib${MPILIBNAME}_a_SOURCES = \
- wintcp_finalize.c wintcp_init.c wintcp_poll.c wintcp_send.c wintcp_utility.c socksm.c
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/README b/src/mpid/ch3/channels/nemesis/netmod/wintcp/README
deleted file mode 100644
index 8618bf1..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/README
+++ /dev/null
@@ -1,5 +0,0 @@
-This directory contains the network module for windows. Eventually this network module should be merged with the new_tcpmodule.
-Since WSAPoll() is available on windows ver > Vista/Srv2008 we modify socksm.c in newtcp to use select() instead of poll() for now and later merge windows tcp module with the unix tcp module once WSAPoll() becomes the default polling method.
-Creating an abstraction for select() and poll() is not easy due to the difference in the usage model for the funcs. Some differences which make the performance sensitive abstraction difficult are given below,
-1) select() overwrites the input arg/fd_sets => If one needs to select() on the same set of fds the fd_set has to be saved and copied back each time (and is there a reliable way to do that considering the fact that fd_set is opaque)
-2) select() returns the number of available events while poll() returns the number of fds which have events
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.c
deleted file mode 100644
index fec9e89..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.c
+++ /dev/null
@@ -1,2824 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#define SOCKSM_H_DEFGLOBALS_
-
-#include "wintcp_impl.h"
-#include "socksm.h"
-
-typedef struct freenode {
- struct sockconn_t *sc;
- struct freenode* next;
-} freenode_t;
-
-/* freeq contains the list of sockconns no longer in use.
- * Use sockconns from this list before allocating a new sockconn
- */
-static struct {
- freenode_t *head, *tail;
-} freeq = {NULL, NULL};
-
-#define SOCKCONN_TMP_BUF_SZ MPID_NEM_NEWTCP_MODULE_RECV_MAX_PKT_LEN
-
-static int g_sc_tbl_size = 0;
-static int g_sc_tbl_capacity = CONN_TBL_INIT_SIZE;
-static int g_sc_tbl_grow_size = CONN_TBL_GROW_SIZE;
-
-/* Current global socket conn table
- * - This table contains the latest set of sockconn s allocated
- * - To get a free sockconn, first check the freeq and then check
- * the global sockconn table. If no sockconns available then expand
- * the global sockconn table.
- */
-static sockconn_t *g_sc_tbl = NULL;
-
-/* Each node of the global sc table list contains,
- * sc_tbl ==> sc_tbl allocated
- * sc_tbl_sz ==> size of sc_tbl
- * next ==> next sc_tbl
- */
-typedef struct sc_tbl_list_node{
- sockconn_t *sc_tbl;
- int sc_tbl_sz;
- struct sc_tbl_list_node *next;
-} sc_tbl_list_node_t;
-
-/* Global sc table list containing the list of sc tables allocated
- * Whenever an sc table is allocated, add it to this list/queue. This
- * list is used for browsing through all sockconns - for get/free
- */
-struct {
- sc_tbl_list_node_t *head, *tail;
-} g_sc_tbl_list = { NULL, NULL};
-
-/* Forward iterator for the global sc table list */
-typedef struct sc_tbl_fw_iterator{
- sc_tbl_list_node_t *cur_sc_tbl_list_node;
- int g_sc_tbl_index;
-} sc_tbl_fw_iterator_t;
-
-/* Initialize the sc_tbl forward iterator */
-static inline void sc_tbl_fw_iterator_init(sc_tbl_fw_iterator_t *iter)
-{
- MPIU_Assert(iter != NULL);
-
- iter->cur_sc_tbl_list_node = g_sc_tbl_list.head;
- iter->g_sc_tbl_index = 0;
-}
-
-/* Finalize the sc_tbl iterator */
-static inline void sc_tbl_fw_iterator_finalize(sc_tbl_fw_iterator_t *iter)
-{
- sc_tbl_fw_iterator_init(iter);
-}
-/* Returns 1 is sc_tbl has more elements, 0 otherwise */
-static inline int sc_tbl_has_next(sc_tbl_fw_iterator_t *iter)
-{
- MPIU_Assert(iter != NULL);
-
- return( (iter->cur_sc_tbl_list_node != NULL) &&
- ( (iter->g_sc_tbl_index < iter->cur_sc_tbl_list_node->sc_tbl_sz) ||
- (iter->cur_sc_tbl_list_node->next != NULL) ) );
-}
-
-/* Get the next sc from sc_tbl. This iterator searches all the
- * sc_tbls registered with the global sc tbl list
- */
-static inline sockconn_t * sc_tbl_get_next(sc_tbl_fw_iterator_t *iter)
-{
- int cur_ind = 0;
- MPIU_Assert(iter != NULL);
- if(iter->g_sc_tbl_index >= iter->cur_sc_tbl_list_node->sc_tbl_sz){
- /* Move to the next sc tbl registered with global sc tbl list */
- iter->cur_sc_tbl_list_node = iter->cur_sc_tbl_list_node->next;
- iter->g_sc_tbl_index = 0;
- }
- MPIU_Assert(iter->cur_sc_tbl_list_node != NULL);
- cur_ind = iter->g_sc_tbl_index++;
- return (iter->cur_sc_tbl_list_node->sc_tbl + cur_ind);
-}
-
-/* Init global sc tbl list */
-static inline int init_sc_tbl_list(void )
-{
- Q_EMPTY(g_sc_tbl_list);
- return MPI_SUCCESS;
-}
-
-/* Add sc table to the global sc table list */
- static inline int add_sc_tbl_to_list(sockconn_t *sc_tbl, int sc_tbl_sz)
-{
- sc_tbl_list_node_t *node = NULL;
- int mpi_errno = MPI_SUCCESS;
-
- node = MPIU_Malloc(sizeof(sc_tbl_list_node_t));
- MPIU_ERR_CHKANDJUMP(node == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem");
-
- node->sc_tbl = sc_tbl;
- node->sc_tbl_sz = sc_tbl_sz;
- node->next = NULL;
-
- Q_ENQUEUE(&g_sc_tbl_list, node);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/* Free sc tables in the global sc table list */
-static inline int free_sc_tbls_in_list(void )
-{
- while(!Q_EMPTY(g_sc_tbl_list)){
- sc_tbl_list_node_t *node;
- Q_DEQUEUE(&g_sc_tbl_list, &node);
- MPIU_Free(node->sc_tbl);
- }
- return MPI_SUCCESS;
-}
-
-/* Free sc tbl list */
-static inline int free_sc_tbl_list(void )
-{
- while(!Q_EMPTY(g_sc_tbl_list)){
- sc_tbl_list_node_t *node;
- Q_DEQUEUE(&g_sc_tbl_list, &node);
- MPIU_Free(node);
- }
- /* Sanity reset of the queue */
- Q_EMPTY(g_sc_tbl_list);
- return MPI_SUCCESS;
-}
-
-/* FIXME: Why should the listen sock conn be global ? */
-sockconn_t MPID_nem_newtcp_module_g_lstn_sc;
-
-/* newtcp module executive set handle */
-MPIU_ExSetHandle_t MPID_nem_newtcp_module_ex_set_hnd = MPIU_EX_INVALID_SET;
-
-/* We define this in order to trick the compiler into including
- information about the MPID_nem_newtcp_module_vc_area type. This is
- needed to easily expand the VC_FIELD macro in a debugger. The
- 'unused' attribute keeps the compiler from complaining. The 'used'
- attribute makes sure the symbol is added in the lib, even if it's
- unused */
-static MPID_nem_newtcp_module_vc_area *dummy_vc_area ATTRIBUTE((unused, used)) = NULL;
-
-/* FIXME: Tune the skip poll values since we now have a
- * fast executive (instead of select)
- */
-#define MIN_SKIP_POLLS_INACTIVE (512)
-#define SKIP_POLLS_INC_CNT (512)
-#define MAX_SKIP_POLLS_INACTIVE (1024)
-#define SKIP_POLLS_INC(cnt) ( \
- (cnt >= MAX_SKIP_POLLS_INACTIVE) \
- ? (cnt = MIN_SKIP_POLLS_INACTIVE) \
- : (cnt += SKIP_POLLS_INC_CNT) \
-)
-
-/* There is very less overhead in handling req using iocp. Do we
- * need to skip at all ?
- */
-#define MAX_SKIP_POLLS_ACTIVE (16)
-static int MPID_nem_tcp_skip_polls = MIN_SKIP_POLLS_INACTIVE;
-
-/* Debug function to dump the sockconn table. This is intended to be
- called from a debugger. The 'unused' attribute keeps the compiler
- from complaining. The 'used' attribute makes sure the function is
- added in the lib, despite the fact that it's unused. */
-static void dbg_print_sc_tbl(FILE *stream, int print_free_entries) ATTRIBUTE((unused, used));
-
-typedef struct sc_state_info{
- /* FIXME: We won't need a common place holder for these handlers. Remove it eventually. */
- handler_func_t sc_state_rd_success_handler;
- handler_func_t sc_state_rd_fail_handler;
- handler_func_t sc_state_wr_success_handler;
- handler_func_t sc_state_wr_fail_handler;
-} sc_state_info_t;
-
-sc_state_info_t sc_state_info[SOCK_STATE_SIZE];
-
-#define IS_SAME_PGID(id1, id2) \
- (strcmp(id1, id2) == 0)
-
-/* Will evaluate to false if either one of these sc's does not have valid pg data */
-#define IS_SAME_CONNECTION(sc1, sc2) \
- (sc1->pg_is_set && sc2->pg_is_set && \
- sc1->pg_rank == sc2->pg_rank && \
- ((sc1->is_same_pg && sc2->is_same_pg) || \
- (!sc1->is_same_pg && !sc2->is_same_pg && \
- IS_SAME_PGID(sc1->pg_id, sc2->pg_id))))
-
-/* Initialize an sc entry - Initialize sc before using it for the
- * first time, or before reusing it
- */
-#define INIT_SC_ENTRY(sc, ind) \
- do { \
- (sc)->fd = MPIU_SOCKW_SOCKFD_INVALID; \
- MPIU_ExInitOverlapped(&((sc)->rd_ov), NULL, NULL); \
- MPIU_ExInitOverlapped(&((sc)->wr_ov), NULL, NULL); \
- memset(&((sc)->read), 0x0, sizeof(sock_read_context_t)); \
- memset(&((sc)->write), 0x0, sizeof(sock_write_context_t)); \
- memset(&((sc)->close), 0x0, sizeof(sock_close_context_t)); \
- (sc)->tmp_buf = (char *)MPIU_Malloc(SOCKCONN_TMP_BUF_SZ); \
- (sc)->tmp_buf_len = SOCKCONN_TMP_BUF_SZ; \
- (sc)->index = ind; \
- (sc)->vc = NULL; \
- (sc)->pg_is_set = FALSE; \
- (sc)->is_tmpvc = FALSE; \
- (sc)->state = CONN_STATE_TS_CLOSED; \
- } while (0)
-
-/* Finalize an sc */
-#define FINALIZE_SC_ENTRY(sc, ind) \
- do { \
- (sc)->fd = MPIU_SOCKW_SOCKFD_INVALID; \
- if((sc)->tmp_buf) MPIU_Free((sc)->tmp_buf); \
- (sc)->tmp_buf_len = 0; \
- (sc)->index = -1; \
- (sc)->vc = NULL; \
- (sc)->pg_is_set = FALSE; \
- (sc)->is_tmpvc = FALSE; \
- (sc)->state = CONN_STATE_TS_CLOSED; \
- } while (0)
-
-/* --BEGIN ERROR HANDLING-- */
-/* This function can be called from a debugger to dump the contents of the
- g_sc_tbl to the given stream. If print_free_entries is true entries
- 0..g_tbl_capacity will be printed. Otherwise, only 0..g_tbl_size will be
- shown. */
-static void dbg_print_sc_tbl(FILE *stream, int print_free_entries)
-{
- int i;
- sockconn_t *sc;
-
- fprintf(stream, "========================================\n");
- for (i = 0; i < (print_free_entries ? g_sc_tbl_capacity : g_sc_tbl_size); ++i) {
- sc = &g_sc_tbl[i];
-#define TF(_b) ((_b) ? "TRUE" : "FALSE")
- fprintf(stream, "[%d] ptr=%p idx=%d fd=%d state=%s\n",
- i, sc, sc->index, sc->fd, SOCK_STATE_TO_STRING(sc->state));
- fprintf(stream, "....pg_is_set=%s is_same_pg=%s is_tmpvc=%s pg_rank=%d pg_id=%s\n",
- TF(sc->pg_is_set), TF(sc->is_same_pg), TF(sc->is_tmpvc), sc->pg_rank, sc->pg_id);
-#undef TF
- }
- fprintf(stream, "========================================\n");
-}
-/* --END ERROR HANDLING-- */
-
-/* Reset the rd context on a sock conn */
-static inline int MPID_nem_newtcp_module_reset_readv_ex(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_Assert(sc != NULL);
- sc->read.n_iov = -1;
-
- return mpi_errno;
-}
-
-/* Reset the wr context on a sock conn */
-static inline int MPID_nem_newtcp_module_reset_writev_ex(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_Assert(sc != NULL);
- sc->write.n_iov = -1;
-
- return mpi_errno;
-}
-
-/* Post a read on a sock conn with an iov
- * This function assumes that the read handlers are set prior to
- * calling this func. Note: Don't post multiple simultaneous reads on the same sc
- * Input:
- * sc - ptr to sock connection
- * iov - ptr to the device IOV to read data.
- * n_iov - num of iovs
- */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_readv_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_post_readv_ex(sockconn_t *sc, MPID_IOV *iov, int n_iov)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_READV_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_READV_EX);
-
- MPIU_Assert(sc != NULL);
-
- if(iov != NULL){
- /* A fresh post on sc */
- MPIU_Assert(n_iov > 0);
- sc->read.iov = iov;
- sc->read.n_iov = n_iov;
- sc->read.nb = 0;
- }
- else{
- /* We are continuing a read on the iov */
- MPIU_Assert(sc->read.iov != NULL);
- /* Make sure that we have some space left in the iov */
- MPIU_Assert((sc->read.iov)->MPID_IOV_LEN > 0);
- }
-
- mpi_errno = MPIU_SOCKW_Readv_ex(sc->fd, sc->read.iov, sc->read.n_iov, NULL, &(sc->rd_ov));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_READV_EX);
- return mpi_errno;
-fn_fail:
- goto fn_exit;
-}
-
-/* Post a read on a sock conn without an iov (eg: a char buf)
- * This function assumes that the read handlers are set prior to
- * calling this func.
- * Note: Don't post multiple simultaneous reads on the same sc
- * Input:
- * sc - ptr to sock conn
- * buf - buffer to read data
- * nb - size of buffer, buf
- */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_read_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_post_read_ex(sockconn_t *sc, void *buf, int nb)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_READ_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_READ_EX);
-
- MPIU_Assert(sc != NULL);
- MPIU_Assert(buf != NULL);
- MPIU_Assert(nb > 0);
-
- sc->read.tmp_iov.MPID_IOV_BUF = buf;
- sc->read.tmp_iov.MPID_IOV_LEN = nb;
-
- sc->read.iov = &(sc->read.tmp_iov);
- sc->read.n_iov = 1;
- sc->read.nb = 0;
-
- mpi_errno = MPIU_SOCKW_Readv_ex(sc->fd, sc->read.iov, sc->read.n_iov, NULL, &(sc->rd_ov));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_READ_EX);
- return mpi_errno;
-fn_fail:
- goto fn_exit;
-}
-
-/* Post a dummy read on a sock conn */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_dummy_read_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_post_dummy_read_ex(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_DUMMY_READ_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_DUMMY_READ_EX);
-
- MPIU_Assert(sc != NULL);
-
- mpi_errno = MPID_nem_newtcp_module_reset_readv_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- MPIU_ExPostOverlapped(MPID_nem_newtcp_module_ex_set_hnd, MPIU_EX_WIN32_COMP_PROC_KEY, &(sc->rd_ov));
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_DUMMY_READ_EX);
- return mpi_errno;
-fn_fail:
- goto fn_exit;
-}
-
-/* Post a write on socket with iov
- * This function assumes that the write handlers are set prior to
- * calling this func. Note: Don't post multiple simultaneous writes on the same sc
- * Input:
- * sc - ptr to sock connection
- * iov - ptr to the device IOV containing the data to write.
- * n_iov - num of iovs
- */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_writev_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_post_writev_ex(sockconn_t *sc, MPID_IOV *iov, int n_iov)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_WRITEV_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_WRITEV_EX);
-
- MPIU_Assert(sc != NULL);
-
- if(iov != NULL){
- MPIU_Assert(n_iov > 0);
- sc->write.iov = iov;
- sc->write.n_iov = n_iov;
- sc->write.nb = 0;
- }
- else{
- /* Continue writing on the write iov */
- MPIU_Assert(sc->write.iov != NULL);
- /* Sanity check that there is something to write */
- MPIU_Assert(sc->write.iov->MPID_IOV_LEN > 0);
- MPIU_Assert(sc->write.n_iov > 0);
- }
-
- mpi_errno = MPIU_SOCKW_Writev_ex(sc->fd, sc->write.iov, sc->write.n_iov, NULL, &(sc->wr_ov));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_WRITEV_EX);
- return mpi_errno;
-fn_fail:
- goto fn_exit;
-}
-
-/* Post a write on socket without iov
- * This function assumes that the write handlers are set prior to
- * calling this func.
- * Note: Don't post multiple simultaneous writes on the same sc
- * Input:
- * sc - ptr to sock conn
- * buf - buffer to write data from
- * nb - size of buffer, buf
- */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_write_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_post_write_ex(sockconn_t *sc, void *buf, int nb)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_WRITE_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_WRITE_EX);
- MPIU_Assert(sc != NULL);
- MPIU_Assert(buf != NULL);
- MPIU_Assert(nb > 0);
-
- sc->write.tmp_iov.MPID_IOV_BUF = buf;
- sc->write.tmp_iov.MPID_IOV_LEN = nb;
-
- sc->write.iov = &(sc->write.tmp_iov);
- sc->write.n_iov = 1;
- sc->write.nb = 0;
-
- mpi_errno = MPIU_SOCKW_Writev_ex(sc->fd, sc->write.iov, sc->write.n_iov, NULL, &(sc->wr_ov));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_WRITE_EX);
- return mpi_errno;
-fn_fail:
- goto fn_exit;
-}
-
-/* Post a connect on sc */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_connect_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPID_nem_newtcp_module_post_connect_ex(sockconn_t *sc, struct sockaddr_in *sin)
-{
- int mpi_errno = MPI_SUCCESS;
- /* ConnectEx() requires the connect sock to be bound */
- struct sockaddr_in sockaddr_bindany;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_CONNECT_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_CONNECT_EX);
-
- MPIU_Assert(sc != NULL);
- MPIU_Assert(sin != NULL);
-
- memset(&(sc->connect), 0x0, sizeof(sock_connect_context_t));
- /* Required when we retry connect */
- memcpy(&(sc->connect.sin), sin, sizeof(struct sockaddr_in));
-
- /* Bind the connect socket */
- memset(&sockaddr_bindany, 0x0, sizeof(struct sockaddr_in));
- sockaddr_bindany.sin_family = AF_INET;
- sockaddr_bindany.sin_addr.s_addr = INADDR_ANY;
- sockaddr_bindany.sin_port = htons(0);
-
- mpi_errno = MPIU_SOCKW_Bind(sc->fd, &sockaddr_bindany, sizeof(struct sockaddr_in));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPIU_SOCKW_Connect_ex(sc->fd, sin, sizeof(struct sockaddr_in), &(sc->wr_ov));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_CONNECT_EX);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/* Post an accept on sc */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_accept_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPID_nem_newtcp_module_post_accept_ex(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_ACCEPT_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_ACCEPT_EX);
- MPIU_Assert(sc != NULL);
-
- memset(&(sc->accept), 0x0, sizeof(sock_listen_context_t));
- mpi_errno = MPIU_SOCKW_Sock_open(AF_INET, SOCK_STREAM, IPPROTO_TCP, &(sc->accept.connfd));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPIU_SOCKW_Accept_ex(sc->fd, sc->accept.accept_buffer,
- sizeof(sc->accept.accept_buffer), &(sc->accept.connfd), &(sc->rd_ov), &(sc->accept.nb));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_ACCEPT_EX);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/* Post an close on sc */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_post_close_ex
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static inline int MPID_nem_newtcp_module_post_close_ex(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_POST_CLOSE_EX);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_POST_CLOSE_EX);
- MPIU_Assert(sc != NULL);
-
- /* Flush write sock buffers - do we need to check the retval ?*/
- FlushFileBuffers((HANDLE )sc->fd);
-
- sc->close.conn_closing = 1;
-
- mpi_errno = MPIU_SOCKW_Sock_close(sc->fd);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- MPIU_ExPostOverlapped(MPID_nem_newtcp_module_ex_set_hnd, MPIU_EX_WIN32_COMP_PROC_KEY, &(sc->wr_ov));
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_POST_CLOSE_EX);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-/* This function trims the iov array, *iov_p, of size *n_iov_p
- * assuming nb bytes are transferred
- * Side-effect : *iov_p, *n_iov_p, buf & len of (*iov_p)
- * could be modified by this function.
- */
-static void trim_iov(MPID_IOV **iov_p, int *n_iov_p, unsigned int nb)
-{
- MPID_IOV *cur_iov;
- int cur_n_iov;
-
- MPIU_Assert(iov_p);
- MPIU_Assert(*iov_p);
- MPIU_Assert(n_iov_p);
- MPIU_Assert((*n_iov_p) > 0);
-
- cur_iov = *iov_p;
- cur_n_iov = *n_iov_p;
-
- while(nb > 0){
- if(nb < cur_iov->MPID_IOV_LEN){
- cur_iov->MPID_IOV_BUF += nb;
- cur_iov->MPID_IOV_LEN -= nb;
- nb = 0;
- }
- else{
- nb -= cur_iov->MPID_IOV_LEN;
- cur_iov->MPID_IOV_LEN = 0;
- cur_n_iov--;
- if(cur_n_iov > 0){
- cur_iov++;
- }
- else{
- /* We should never have read more data
- * than available in the IOV
- */
- MPIU_Assert(nb == 0);
- break;
- }
- }
- }
-
- *iov_p = cur_iov;
- *n_iov_p = cur_n_iov;
-}
-
-/* Returns the total number of bytes read on sc for the posted read
- * Output:
- * complete - (0/1) indicates whether the current read is complete
- * Note: (complete == 1) if sock is closed or when using EX bypass
- * Note: On a timeout (complete == 0)
- */
-static int ex_read_progress_update(sockconn_t *sc, int *complete)
-{
- int nb = 0;
-
- MPIU_Assert(sc != NULL);
- MPIU_Assert(complete != NULL);
-
- *complete = 0;
- if(sc->read.n_iov < 0){
- /* Bail out if no iov to read from - Useful for handlers
- * called explicitly by the user - EX bypass
- */
- sc->read.nb = 0;
- *complete = 1;
- goto fn_exit;
- }
-
- nb = MPIU_ExGetBytesTransferred(&(sc->rd_ov));
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "EX Read (sc = %p) = %d", sc, nb));
-
- if(nb <= 0){
- *complete = 1;
- goto fn_exit;
- }
- else{
- /* Update IOV */
- trim_iov(&(sc->read.iov), &(sc->read.n_iov), nb);
- sc->read.nb += nb;
- if(sc->read.n_iov == 0) *complete = 1;
- }
-
- fn_exit:
- return sc->read.nb;
-}
-
-/* Returns the total number of bytes written on sc for the posted wr
- * Output:
- * complete - (0/1) indicates whether the current write is complete
- * Note: (complete == 1) if sock is closed or when using EX bypass
- * Note: On a timeout (complete == 0)
- */
-static int ex_write_progress_update(sockconn_t *sc, int *complete)
-{
- int nb = 0;
-
- MPIU_Assert(sc != NULL);
- MPIU_Assert(complete != NULL);
-
- *complete = 0;
- if(sc->write.n_iov < 0){
- sc->write.nb = 0;
- *complete = 1;
- goto fn_exit;
- }
-
- nb = MPIU_ExGetBytesTransferred(&(sc->wr_ov));
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "EX Write (sc = %p) = %d", sc, nb));
-
- if(nb <= 0){
- *complete = 1;
- goto fn_exit;
- }
- else{
- /* Update IOV */
- trim_iov(&(sc->write.iov), &(sc->write.n_iov), nb);
- sc->write.nb += nb;
- if(sc->write.n_iov == 0) *complete = 1;
- }
-
- fn_exit:
- return sc->write.nb;
-}
-
-static int find_free_sc(sockconn_t **sc_p);
-
-/* Allocate a global sc table */
-#undef FUNCNAME
-#define FUNCNAME alloc_sc_tbl
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int alloc_sc_tbl(void)
-{
- int i, mpi_errno = MPI_SUCCESS;
- MPIU_CHKPMEM_DECL (2);
-
- MPIU_Assert(g_sc_tbl == NULL);
- MPIU_CHKPMEM_MALLOC (g_sc_tbl, sockconn_t *, g_sc_tbl_capacity * sizeof(sockconn_t),
- mpi_errno, "connection table");
-
- for (i = 0; i < g_sc_tbl_capacity; i++) {
- INIT_SC_ENTRY(((sockconn_t *)&g_sc_tbl[i]), i);
- }
- MPIU_CHKPMEM_COMMIT();
-
- /* Add sc tbl to global list of sc tbls */
- mpi_errno = add_sc_tbl_to_list(g_sc_tbl, g_sc_tbl_capacity);
- if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP (mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* Free all sc tables allocated */
-#undef FUNCNAME
-#define FUNCNAME free_sc_tbls
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int free_sc_tbls (void)
-{
- int mpi_errno = MPI_SUCCESS;
-
- mpi_errno = free_sc_tbls_in_list();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/*
- Reason for not doing realloc for both sc and plfd tables :
- Either both the tables have to be expanded or both should remain the same size, if
- enough memory could not be allocated, as we have only one set of variables to control
- the size of the tables. Also, it is not useful to expand one table and leave the other
- at the same size, 'coz of memory allocation failures.
-*/
-/*
- Since we no longer use pollfd tables (hence no more same index for pollfd and sc
- tables) when expanding sc table we should just allocate a new sc table and add
- the sc table to the global list of sc tables.
-*/
-#undef FUNCNAME
-#define FUNCNAME expand_sc_tbl
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int expand_sc_tbl (void)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *new_sc_tbl = NULL;
- /* FIXME: We need to check whether a non-linear growth of the sc_tbl
- * will be better for performance for large jobs
- */
- int new_capacity = g_sc_tbl_capacity + g_sc_tbl_grow_size;
- MPIU_CHKPMEM_DECL (2);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "expand_sc_tbl Entry"));
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "expand_sc_tbl b4 g_sc_tbl[0].fd=%d", g_sc_tbl[0].fd));
-
- MPIU_CHKPMEM_MALLOC (new_sc_tbl, sockconn_t *, new_capacity * sizeof(sockconn_t),
- mpi_errno, "expanded connection table");
-
- g_sc_tbl = new_sc_tbl;
- g_sc_tbl_capacity = new_capacity;
- g_sc_tbl_size = 0;
-
- /* Add the new sc table to global sc table list */
- mpi_errno = add_sc_tbl_to_list(g_sc_tbl, new_capacity);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- MPIU_CHKPMEM_COMMIT();
- fn_exit:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "expand_sc_plfd_tbls Exit"));
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/*
- ==== OLD ====
- Finds the first free entry in the connection table/pollfd table. Note that both the
- tables are indexed the same. i.e. each entry in one table corresponds to the
- entry of the same index in the other table. If an entry is valid in one table, then
- it is valid in the other table too.
- ==== OLD ====
-
- This function finds the first free entry in both the tables by checking the queue of
- free elements. If the free queue is empty, then it returns the next available slot
- in the tables. If the size of the slot is already full, then this expands the table
- and then returns the next available slot
-*/
-#undef FUNCNAME
-#define FUNCNAME find_free_sc
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int find_free_sc(sockconn_t **sc_p)
-{
- int mpi_errno = MPI_SUCCESS;
- freenode_t *node;
-
- MPIU_Assert(sc_p != NULL);
- /* FIXME: Use if-else and get rid of goto */
- if (!Q_EMPTY(freeq)) {
- Q_DEQUEUE(&freeq, ((freenode_t **)&node));
- *sc_p = node->sc;
- MPIU_Free(node);
- INIT_SC_ENTRY(*sc_p, -1);
- goto fn_exit;
- }
-
- /* Look into the latest global sc table allocated for an entry */
- if (g_sc_tbl_size == g_sc_tbl_capacity) {
- mpi_errno = expand_sc_tbl();
- if (mpi_errno != MPI_SUCCESS){ MPIU_ERR_POP(mpi_errno); }
- }
-
- MPIU_Assert(g_sc_tbl_capacity > g_sc_tbl_size);
-
- *sc_p = &g_sc_tbl[g_sc_tbl_size];
- INIT_SC_ENTRY(*sc_p, g_sc_tbl_size);
- ++g_sc_tbl_size;
-
- fn_exit:
- /* This function is the closest thing we have to a constructor, so we throw
- in a couple of initializers here in case the caller is sloppy about his
- assumptions. */
- /* FIXME: The index could be invalid in the case of an error */
- /* INIT_SC_ENTRY(&g_sc_tbl[*index], *index); */
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* Note:
- fnd_sc is returned only for certain states. If it is not returned for a state,
- the handler function can simply pass NULL as the second argument.
- */
-#undef FUNCNAME
-#define FUNCNAME found_better_sc
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int found_better_sc(sockconn_t *sc, sockconn_t **fnd_sc)
-{
- int found = FALSE;
- sc_tbl_fw_iterator_t iter;
- MPIDI_STATE_DECL(MPID_STATE_FOUND_BETTER_SC);
-
- MPIDI_FUNC_ENTER(MPID_STATE_FOUND_BETTER_SC);
-
- /* tmpvc's can never match a better sc */
- if (sc->is_tmpvc) {
- found = FALSE;
- goto fn_exit;
- }
-
- /* if we don't know our own pg info, how can we look for a better SC? */
- MPIU_Assert(sc->pg_is_set);
-
- sc_tbl_fw_iterator_init(&iter);
-
- while(sc_tbl_has_next(&iter)){
- sockconn_t *iter_sc = sc_tbl_get_next(&iter);
- MPID_nem_newtcp_module_sock_state_t istate = iter_sc->state;
- if (iter_sc != sc && iter_sc->fd != MPIU_SOCKW_SOCKFD_INVALID
- && IS_SAME_CONNECTION(iter_sc, sc)){
- switch (sc->state){
- case CONN_STATE_TC_C_CNTD:
- MPIU_Assert(fnd_sc == NULL);
- if (istate == CONN_STATE_TS_COMMRDY ||
- istate == CONN_STATE_TA_C_RANKRCVD ||
- istate == CONN_STATE_TC_C_TMPVCSENT){
- found = TRUE;
- }
- break;
- case CONN_STATE_TA_C_RANKRCVD:
- MPIU_Assert(fnd_sc != NULL);
- if (istate == CONN_STATE_TS_COMMRDY ||
- istate == CONN_STATE_TC_C_RANKSENT) {
- found = TRUE;
- *fnd_sc = iter_sc;
- }
- break;
- case CONN_STATE_TA_C_TMPVCRCVD:
- MPIU_Assert(fnd_sc != NULL);
- if (istate == CONN_STATE_TS_COMMRDY ||
- istate == CONN_STATE_TC_C_TMPVCSENT) {
- found = TRUE;
- *fnd_sc = iter_sc;
- }
- break;
-
- /* Add code for other states here, if need be. */
- default:
- /* FIXME: need to handle error condition better */
- MPIU_Assert (0);
- break;
- }
- }
- }
- sc_tbl_fw_iterator_finalize(&iter);
-
-fn_exit:
- if (found) {
- if (fnd_sc) {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE,
- (MPIU_DBG_FDEST, "found_better_sc(sc=%p (%s), *fnd_sc=%p (%s)) found=TRUE",
- sc, SOCK_STATE_STR[sc->state],
- *fnd_sc, (*fnd_sc ? SOCK_STATE_STR[(*fnd_sc)->state] : "N/A")));
- }
- else {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE,
- (MPIU_DBG_FDEST, "found_better_sc(sc=%p (%s), fnd_sc=(nil)) found=TRUE",
- sc, SOCK_STATE_STR[sc->state]));
- }
- }
- else {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE,
- (MPIU_DBG_FDEST, "found_better_sc(sc=%p (%s), *fnd_sc=N/A) found=FALSE",
- sc, SOCK_STATE_STR[sc->state]));
- }
- MPIDI_FUNC_EXIT(MPID_STATE_FOUND_BETTER_SC);
- return found;
-}
-
-#undef FUNCNAME
-#define FUNCNAME vc_is_in_shutdown
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int vc_is_in_shutdown(MPIDI_VC_t *vc)
-{
- int retval = FALSE;
- MPIU_Assert(vc != NULL);
- if (vc->state == MPIDI_VC_STATE_REMOTE_CLOSE ||
- vc->state == MPIDI_VC_STATE_CLOSE_ACKED ||
- vc->state == MPIDI_VC_STATE_CLOSED ||
- vc->state == MPIDI_VC_STATE_LOCAL_CLOSE ||
- vc->state == MPIDI_VC_STATE_INACTIVE ||
- vc->state == MPIDI_VC_STATE_INACTIVE_CLOSED ||
- vc->state == MPIDI_VC_STATE_MORIBUND)
- {
- retval = TRUE;
- }
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "vc_is_in_shutdown(%p)=%s", vc, (retval ? "TRUE" : "FALSE")));
- return retval;
-}
-
-#undef FUNCNAME
-#define FUNCNAME send_id_info
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_id_info(const sockconn_t *const sc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_nem_newtcp_module_idinfo_t id_info;
- MPIDI_nem_newtcp_module_header_t hdr;
- MPID_IOV iov[3];
- int pg_id_len = 0, offset, buf_size, iov_cnt = 2;
- MPIDI_STATE_DECL(MPID_STATE_SEND_ID_INFO);
-
- MPIDI_FUNC_ENTER(MPID_STATE_SEND_ID_INFO);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "my->pg_rank=%d, sc->pg_rank=%d"
- , MPIDI_Process.my_pg_rank, sc->pg_rank));
- if (!sc->is_same_pg)
- pg_id_len = strlen(MPIDI_Process.my_pg->id) + 1;
-/* store ending NULL also */
-/* FIXME better keep pg_id_len itself as part of MPIDI_Process.my_pg structure to */
-/* avoid computing the length of string everytime this function is called. */
-
- hdr.pkt_type = MPIDI_NEM_NEWTCP_MODULE_PKT_ID_INFO;
- hdr.datalen = sizeof(MPIDI_nem_newtcp_module_idinfo_t) + pg_id_len;
- id_info.pg_rank = MPIDI_Process.my_pg_rank;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST )&hdr;
- iov[0].MPID_IOV_LEN = sizeof(hdr);
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST )&id_info;
- iov[1].MPID_IOV_LEN = sizeof(id_info);
- buf_size = sizeof(hdr) + sizeof(id_info);
-
- if (!sc->is_same_pg) {
- iov[2].MPID_IOV_BUF = MPIDI_Process.my_pg->id;
- iov[2].MPID_IOV_LEN = pg_id_len;
- buf_size += pg_id_len;
- ++iov_cnt;
- }
-
- /* FIXME: Post a write instead of assuming a write would succeed
- * Looks like we should be *usually* ok here since the conn is just
- * established and socket send buffer > sizeof iov
- */
- MPIU_OSW_RETRYON_INTR((offset == -1),
- (mpi_errno = MPIU_SOCKW_Writev(sc->fd, iov, iov_cnt, &offset)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- MPIU_ERR_CHKANDJUMP1 (offset != buf_size, mpi_errno, MPI_ERR_OTHER,
- "**write", "**write %s", MPIU_OSW_Strerror(MPIU_OSW_Get_errno()));
-
-/* FIXME log appropriate error */
-/* FIXME-Z1 socket is just connected and we are sending a few bytes. So, there should not */
-/* be a problem of partial data only being written to. If partial data written, */
-/* handle this. */
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_SEND_ID_INFO);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d, offset=%d, errno=%d %s",
- mpi_errno, offset, MPIU_OSW_Get_errno(), MPIU_OSW_Strerror(MPIU_OSW_Get_errno())));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME send_tmpvc_info
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_tmpvc_info(const sockconn_t *const sc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_nem_newtcp_module_portinfo_t port_info;
- MPIDI_nem_newtcp_module_header_t hdr;
- MPID_IOV iov[3];
- int offset, buf_size, iov_cnt = 2;
- MPIDI_STATE_DECL(MPID_STATE_SEND_TMPVC_INFO);
-
- MPIDI_FUNC_ENTER(MPID_STATE_SEND_TMPVC_INFO);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "my->pg_rank=%d, sc->pg_rank=%d"
- , MPIDI_Process.my_pg_rank, sc->pg_rank));
-
-/* store ending NULL also */
-/* FIXME better keep pg_id_len itself as part of MPIDI_Process.my_pg structure to */
-/* avoid computing the length of string everytime this function is called. */
-
- hdr.pkt_type = MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_INFO;
- hdr.datalen = sizeof(MPIDI_nem_newtcp_module_portinfo_t);
- port_info.port_name_tag = sc->vc->port_name_tag;
-
- iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST )&hdr;
- iov[0].MPID_IOV_LEN = sizeof(hdr);
- iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST )&port_info;
- iov[1].MPID_IOV_LEN = sizeof(port_info);
- buf_size = sizeof(hdr) + sizeof(port_info);
-
- /* FIXME: Post a write instead of assuming a write would succeed
- * Looks like we should be *usually* ok here since the conn is just
- * established and socket send buffer > sizeof iov
- */
- MPIU_OSW_RETRYON_INTR((offset == -1),
- (mpi_errno = MPIU_SOCKW_Writev(sc->fd, iov, iov_cnt, &offset)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
- MPIU_ERR_CHKANDJUMP1 (offset != buf_size, mpi_errno, MPI_ERR_OTHER,
- "**write", "**write %s", MPIU_OSW_Strerror(MPIU_OSW_Get_errno()));
-/* FIXME log appropriate error */
-/* FIXME-Z1 socket is just connected and we are sending a few bytes. So, there should not */
-/* be a problem of partial data only being written to. If partial data written, */
-/* handle this. */
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_SEND_TMPVC_INFO);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d, offset=%d, errno=%d %s", mpi_errno, offset, errno, strerror(errno)));
- goto fn_exit;
-}
-
-static int cleanup_sc_vc(sockconn_t *sc);
-
-#undef FUNCNAME
-#define FUNCNAME gen_read_fail_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int gen_read_fail_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIU_Ex_status_t ex_status;
- MPIDI_STATE_DECL(MPID_STATE_STATE_GEN_RD_FAIL_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_GEN_RD_FAIL_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- ex_status = MPIU_ExGetStatus(rd_ov);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "Read failed (ex_status = %x)", ex_status));
- if((ex_status == MPIU_EX_STATUS_IO_ABORT) && !(IS_SOCKCONN_CLOSING(sc))){
- /* Thread aborted */
- int nb, complete;
-
- nb = ex_read_progress_update(sc, &complete);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "THREAD ABORT: Reposting read (already read %d bytes) on sc=%p", nb, sc));
- /* Try to repost the read */
- mpi_errno = MPID_nem_newtcp_module_post_readv_ex(sc, NULL, 0);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- /* FIXME: We should ideally do a shutdown() not close () */
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
-
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**read",
- "**read %s", MPIU_OSW_Strerror(MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(rd_ov))));
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_GEN_RD_FAIL_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME gen_write_fail_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int gen_write_fail_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIU_Ex_status_t ex_status;
- MPIDI_STATE_DECL(MPID_STATE_STATE_GEN_WR_FAIL_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_GEN_WR_FAIL_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- ex_status = MPIU_ExGetStatus(wr_ov);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "Write failed (ex_status = %x)", ex_status));
- if((ex_status == MPIU_EX_STATUS_IO_ABORT) && !(IS_SOCKCONN_CLOSING(sc))){
- /* Thread abort */
- int nb, complete;
-
- nb = ex_write_progress_update(sc, &complete);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "THREAD ABORT: Reposting write (already written %d bytes) on sc=%p", nb, sc));
-
- /* Try reposting write */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(sc, NULL, 0);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPIDI_CH3U_Handle_connection(sc->vc, MPIDI_VC_EVENT_TERMINATED);
- /* FIXME: We should ideally do a shutdown() not close () */
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
-
- MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**write",
- "**write %s", MPIU_OSW_Strerror(MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(wr_ov))));
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_GEN_WR_FAIL_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#define recv_id_or_tmpvc_info_fail_handler gen_read_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME recv_id_or_tmpvc_info_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int recv_id_or_tmpvc_info_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- int nb = 0, complete=0;
- MPIDI_nem_newtcp_module_header_t *hdr = NULL;
- int hdr_len = sizeof(MPIDI_nem_newtcp_module_header_t);
- MPID_IOV iov[2];
- int pg_id_len = 0, iov_cnt = 1;
- char *pg_id = NULL;
-
- MPIU_CHKPMEM_DECL (1);
- MPIU_CHKLMEM_DECL (1);
- MPIDI_STATE_DECL(MPID_STATE_RECV_ID_OR_TMPVC_INFO_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_RECV_ID_OR_TMPVC_INFO_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_read_progress_update(sc, &complete);
-
- /* FIXME: Remove the assumption that we will be able to read
- * the complete id/tmp_vc info in a single read...
- */
- MPIU_Assert(complete);
-
- if(nb == 0){
- /* Sock conn closed - possibly because of a head-to-head conn resoln */
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- goto fn_exit;
- }
-
- hdr = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- /* FIXME: Just retry if we don't read the complete header */
- MPIU_ERR_CHKANDJUMP1 (nb != hdr_len, mpi_errno, MPI_ERR_OTHER,
- "**read", "**read %s", strerror (errno));
- MPIU_Assert(hdr->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_INFO ||
- hdr->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_INFO);
- MPIU_Assert(hdr->datalen != 0);
-
- if (hdr->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_INFO) {
- iov[0].MPID_IOV_BUF = (void *) &(sc->pg_rank);
- iov[0].MPID_IOV_LEN = sizeof(sc->pg_rank);
- pg_id_len = hdr->datalen - sizeof(MPIDI_nem_newtcp_module_idinfo_t);
- if (pg_id_len != 0) {
- /* FIXME: We might want to use sc->tmp_buf - after hdr for
- * reading the pg_id
- */
- MPIU_CHKLMEM_MALLOC (pg_id, char *, pg_id_len, mpi_errno, "sockconn pg_id");
- iov[1].MPID_IOV_BUF = (void *)pg_id;
- iov[1].MPID_IOV_LEN = pg_id_len;
- ++iov_cnt;
- }
- MPIU_OSW_RETRYON_INTR((nb == -1),
- (mpi_errno = MPIU_SOCKW_Readv(sc->fd, iov, iov_cnt, &nb)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- /* FIXME: Post a read if read fails - instead of aborting */
- /* Also make sure that you have only one read posted at a time */
- MPIU_ERR_CHKANDJUMP1 (nb != hdr->datalen, mpi_errno, MPI_ERR_OTHER,
- "**read", "**read %s", strerror (errno)); /* FIXME-Z1 */
- if (pg_id_len == 0) {
- sc->is_same_pg = TRUE;
- mpi_errno = MPID_nem_newtcp_module_get_vc_from_conninfo (MPIDI_Process.my_pg->id,
- sc->pg_rank, &sc->vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- sc->pg_id = NULL;
- }
- else {
- sc->is_same_pg = FALSE;
- mpi_errno = MPID_nem_newtcp_module_get_vc_from_conninfo (pg_id, sc->pg_rank, &sc->vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- sc->pg_id = sc->vc->pg->id;
- }
-
- MPIU_Assert(sc->vc != NULL);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "about to incr sc_ref_count sc=%p sc->vc=%p sc_ref_count=%d", sc, sc->vc, VC_FIELD(sc->vc, sc_ref_count)));
- ++VC_FIELD(sc->vc, sc_ref_count);
-
- /* very important, without this IS_SAME_CONNECTION will always fail */
- sc->pg_is_set = TRUE;
-
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PKT_ID_INFO (pg_rank = %d, pg_id_len = %d): sc->fd=%d, sc->vc=%p, sc=%p", sc->pg_rank, pg_id_len, sc->fd, sc->vc, sc));
- }
- else if (hdr->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_INFO) {
- MPIDI_VC_t *vc;
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "PKT_TMPVC_INFO: sc->fd=%d", sc->fd));
- /* create a new VC */
- MPIU_CHKPMEM_MALLOC (vc, MPIDI_VC_t *, sizeof(MPIDI_VC_t), mpi_errno, "real vc from tmp vc");
- /* --BEGIN ERROR HANDLING-- */
- if (vc == NULL) {
- mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", NULL);
- goto fn_fail;
- }
- /* --END ERROR HANDLING-- */
- MPIDI_VC_Init(vc, NULL, 0);
- VC_FIELD(vc, state) = MPID_NEM_NEWTCP_MODULE_VC_STATE_CONNECTED; /* FIXME: is it needed ? */
- sc->vc = vc;
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "about to incr sc_ref_count sc=%p sc->vc=%p sc_ref_count=%d", sc, sc->vc, VC_FIELD(sc->vc, sc_ref_count)));
- ++VC_FIELD(vc, sc_ref_count);
-
- ASSIGN_SC_TO_VC(vc, sc);
-
- /* get the port's tag from the packet and stash it in the VC */
- iov[0].MPID_IOV_BUF = (void *) &(sc->vc->port_name_tag);
- iov[0].MPID_IOV_LEN = sizeof(sc->vc->port_name_tag);
-
- MPIU_OSW_RETRYON_INTR((nb == -1),
- (mpi_errno = MPIU_SOCKW_Readv(sc->fd, iov, iov_cnt, &nb)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- /* FIXME: Don't assume read will read the whole pg_id/tmpvc */
- MPIU_ERR_CHKANDJUMP1 (nb != hdr->datalen, mpi_errno, MPI_ERR_OTHER,
- "**read", "**read %s", strerror (errno)); /* FIXME-Z1 */
- sc->is_same_pg = FALSE;
- sc->pg_id = NULL;
- sc->is_tmpvc = TRUE;
-
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "enqueuing on acceptq vc=%p, sc->fd=%d, tag=%d", vc, sc->fd, sc->vc->port_name_tag));
- MPIDI_CH3I_Acceptq_enqueue(vc, sc->vc->port_name_tag);
- }
-
- if(!sc->is_tmpvc){
- CHANGE_STATE(sc, CONN_STATE_TA_C_RANKRCVD);
- }
- else{
- CHANGE_STATE(sc, CONN_STATE_TA_C_TMPVCRCVD);
- }
-
- MPIU_CHKPMEM_COMMIT();
-
- /* Explicitly call the read handlers */
- mpi_errno = MPID_nem_newtcp_module_reset_readv_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = CALL_RD_HANDLER(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIU_CHKLMEM_FREEALL();
-
- MPIDI_FUNC_EXIT(MPID_STATE_RECV_ID_OR_TMPVC_INFO_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define send_cmd_pkt(sc_, pkt_type_) ( \
- send_cmd_pkt_func(sc_, pkt_type_) \
-)
-
-/*
- This function is used to send commands that don't have data but just only
- the header.
- */
-#undef FUNCNAME
-#define FUNCNAME send_cmd_pkt
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int send_cmd_pkt_func(sockconn_t *sc, MPIDI_nem_newtcp_module_pkt_type_t pkt_type)
-{
- int mpi_errno = MPI_SUCCESS, offset;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int pkt_len = sizeof(MPIDI_nem_newtcp_module_header_t);
-
- MPIU_Assert(sc != NULL);
- MPIU_Assert(pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_ACK ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_NAK ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_REQ ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_ACK ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_NAK ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_ACK ||
- pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_NAK);
-
- /* FIXME: We are assuming the command packets are only sent during
- * connection phase - enough sock buffer for the calls to succeed
- */
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- pkt->pkt_type = pkt_type;
- pkt->datalen = 0;
-
- MPIU_OSW_RETRYON_INTR((offset == -1),
- (mpi_errno = MPIU_SOCKW_Write(sc->fd, (char *)pkt, pkt_len, &offset)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- MPIU_ERR_CHKANDJUMP1 (offset != pkt_len, mpi_errno, MPI_ERR_OTHER,
- "**write", "**write %s", strerror (errno)); /* FIXME-Z1 */
- fn_exit:
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_connect
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_connect (struct MPIDI_VC *const vc)
-{
- sockconn_t *sc = NULL;
- int mpi_errno = MPI_SUCCESS;
- freenode_t *node;
- MPIU_CHKLMEM_DECL(1);
-
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT);
-
- MPIU_Assert(vc != NULL);
-
- /* We have an active connection, start polling more often */
- MPID_nem_tcp_skip_polls = MAX_SKIP_POLLS_ACTIVE;
-
- MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
-
- if(VC_FIELD(vc, state) == MPID_NEM_NEWTCP_MODULE_VC_STATE_DISCONNECTED){
- struct sockaddr_in *sock_addr;
- struct in_addr addr;
-
- MPIU_Assert(VC_FIELD(vc, sc) == NULL);
-
- mpi_errno = find_free_sc(&sc);
- if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP (mpi_errno);
-
- /* FIXME:
- We need to set addr and port using bc.
- If a process is dynamically spawned, vc->pg is NULL.
- In that case, same procedure is done
- in MPID_nem_newtcp_module_connect_to_root()
- */
- if (vc->pg != NULL) { /* VC is not a temporary one */
- char *bc;
- int pmi_errno;
- int val_max_sz;
-
- pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
- MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
- MPIU_CHKLMEM_MALLOC(bc, char *, val_max_sz, mpi_errno, "bc");
-
- sc->is_tmpvc = FALSE;
-
- mpi_errno = vc->pg->getConnInfo(vc->pg_rank, bc, val_max_sz, vc->pg);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_newtcp_module_get_addr_port_from_bc(bc, &addr, &(VC_FIELD(vc, sock_id).sin_port));
- VC_FIELD(vc, sock_id).sin_addr.s_addr = addr.s_addr;
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
- else {
- sc->is_tmpvc = TRUE;
- }
-
- sock_addr = &(VC_FIELD(vc, sock_id));
-
- MPIU_OSW_RETRYON_INTR((sc->fd == -1),
- (mpi_errno = MPIU_SOCKW_Sock_open(AF_INET, SOCK_STREAM, 0, &(sc->fd))));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- mpi_errno = MPID_nem_newtcp_module_set_sockopts(sc->fd);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- MPIU_ExAttachHandle(MPID_nem_newtcp_module_ex_set_hnd, MPIU_EX_WIN32_COMP_PROC_KEY, (HANDLE )(sc->fd));
-
- CHANGE_STATE(sc, CONN_STATE_TC_C_CNTING);
-
- mpi_errno = MPID_nem_newtcp_module_post_connect_ex(sc, sock_addr);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- VC_FIELD(vc, state) = MPID_NEM_NEWTCP_MODULE_VC_STATE_CONNECTED;
- sc->pg_rank = vc->pg_rank;
- if (vc->pg != NULL) { /* normal (non-dynamic) connection */
- if (IS_SAME_PGID(vc->pg->id, MPIDI_Process.my_pg->id)) {
- sc->is_same_pg = TRUE;
- sc->pg_id = NULL;
- }
- else {
- sc->is_same_pg = FALSE;
- sc->pg_id = vc->pg->id;
- }
- }
- else { /* (vc->pg == NULL), dynamic proc connection - temp vc */
- sc->is_same_pg = FALSE;
- sc->pg_id = NULL;
- }
- /* very important, without this IS_SAME_CONNECTION will always fail */
- sc->pg_is_set = TRUE;
-
- ASSIGN_SC_TO_VC(vc, sc);
- sc->vc = vc;
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "about to incr sc_ref_count sc=%p sc->vc=%p sc_ref_count=%d", sc, sc->vc, VC_FIELD(sc->vc, sc_ref_count)));
- ++VC_FIELD(vc, sc_ref_count);
- }
- else if(VC_FIELD(vc, state) == MPID_NEM_NEWTCP_MODULE_VC_STATE_CONNECTED){
- sc = VC_FIELD(vc, sc);
- MPIU_Assert(sc != NULL);
- /* Do nothing here, the caller just needs to wait for the connection
- state machine to work its way through the states. Doing something at
- this point will almost always just mess up any head-to-head
- resolution. */
- }
- else{
- MPIU_Assert(0);
- }
-
- fn_exit:
- MPIU_CHKLMEM_FREEALL();
- /* MPID_nem_newtcp_module_connpoll(); FIXME-Imp should be called? */
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT);
- return mpi_errno;
- fn_fail:
- if (sc != NULL) {
- if (sc->fd != MPIU_SOCKW_SOCKFD_INVALID) {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "MPID_nem_newtcp_module_connect(). closing fd = %d", sc->fd));
-
- /* FIXME: Post a close and let the quiescent handler take care of adding sc to freeq */
- mpi_errno = MPIU_SOCKW_Sock_close(sc->fd);
- }
- FINALIZE_SC_ENTRY(sc, -1);
- node = MPIU_Malloc(sizeof(freenode_t));
- MPIU_ERR_CHKANDSTMT(node == NULL, mpi_errno, MPI_ERR_OTHER, goto fn_exit, "**nomem");
- node->sc = sc;
-/* Note: MPIU_ERR_CHKANDJUMP should not be used here as it will be recursive */
-/* within fn_fail */
- Q_ENQUEUE(&freeq, node);
- }
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* Cleanup vc related stuff in sc
- * When vc->state becomes CLOSED, MPID_nem_newtcp_module_cleanup() calls this function
- * to cleanup vc related fields in sc
- * We cleanup sc only after the sock close succeeds
- */
-static int cleanup_sc_vc(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- if (sc == NULL){
- goto fn_exit;
- }
- if (sc->vc) {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "about to decr sc_ref_count sc=%p sc->vc=%p sc_ref_count=%d", sc, sc->vc, VC_FIELD(sc->vc, sc_ref_count)));
- MPIU_Assert(VC_FIELD(sc->vc, sc_ref_count) > 0);
- --VC_FIELD(sc->vc, sc_ref_count);
- }
- if (sc->vc && VC_FIELD(sc->vc, sc) == sc) /* this vc may be connecting/accepting with another sc e.g., this sc lost
-the tie-breaker */
- {
- VC_FIELD(sc->vc, state) = MPID_NEM_NEWTCP_MODULE_VC_STATE_DISCONNECTED;
- ASSIGN_SC_TO_VC(sc->vc, NULL);
- }
- sc->vc = NULL;
-
- fn_exit:
- return mpi_errno;
-}
-
-/* Called to transition an sc to CLOSED. This might be done as part of a ch3
- close protocol or it might be done because the sc is in a quiescent state. */
-static int cleanup_sc(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
- freenode_t *node;
- MPIU_CHKPMEM_DECL(1);
- if (sc == NULL)
- goto fn_exit;
-
- CHANGE_STATE(sc, CONN_STATE_TS_CLOSED);
-
- /* FIXME: How can the sc->fd be invalid here ? */
- if(sc->fd != MPIU_SOCKW_SOCKFD_INVALID){
- /* There is no way to detach a handle from the Executive */
- sc->fd = MPIU_SOCKW_SOCKFD_INVALID;
- }
-
- FINALIZE_SC_ENTRY(sc, -1);
-
- MPIU_CHKPMEM_MALLOC (node, freenode_t *, sizeof(freenode_t), mpi_errno, "free node");
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- node->sc = sc;
- Q_ENQUEUE(&freeq, node);
-
- MPIU_CHKPMEM_COMMIT();
- fn_exit:
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* this function is called when vc->state becomes CLOSED */
-/* FIXME XXX DJG do we need to do anything here to ensure that the final
- close(TRUE) packet has made it into a writev call? The code might have a
- race for queued messages. */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_cleanup
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_cleanup (struct MPIDI_VC *const vc)
-{
- int mpi_errno = MPI_SUCCESS;
- sc_tbl_fw_iterator_t iter;
- sockconn_t *sc = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CLEANUP);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CLEANUP);
-
- MPIU_Assert(vc->state == MPIDI_VC_STATE_CLOSED);
- sc = VC_FIELD(vc, sc);
-
- if (sc != NULL) {
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- sc_tbl_fw_iterator_init(&iter);
- while(sc_tbl_has_next(&iter) && (VC_FIELD(vc, sc_ref_count) > 0)){
- sockconn_t *iter_sc = sc_tbl_get_next(&iter);
- if((iter_sc->fd != sc->fd) && (iter_sc->vc == vc)){
- mpi_errno = cleanup_sc_vc(iter_sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(iter_sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(iter_sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- }
- sc_tbl_fw_iterator_finalize(&iter);
-
- /* cleanup_sc can technically cause a reconnect on a per-sc basis, but I
- don't think that it can happen when _module_cleanup is called. Let's
- assert this for now and remove it if we prove that it can happen. */
- /* FIXME: WINTCP ASYNC MPIU_Assert(VC_FIELD(vc, sc_ref_count) == 0); */
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CLEANUP);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME state_tc_c_cnting_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_tc_c_cnting_success_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc;
- MPIDI_STATE_DECL(MPID_STATE_STATE_TC_C_CNTING_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_TC_C_CNTING_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- CHANGE_STATE(sc, CONN_STATE_TC_C_CNTD);
-
- /* FIXME: We are calling the cntd success handler explicitly here.
- * Get rid of one of these states
- */
- mpi_errno = MPID_nem_newtcp_module_reset_writev_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CALL_WR_HANDLER(sc);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_TC_C_CNTING_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME gen_cnting_fail_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int gen_cnting_fail_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc;
- MPIU_Ex_status_t ex_status;
- MPIDI_STATE_DECL(MPID_STATE_GEN_CNTING_FAIL_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_GEN_CNTING_FAIL_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- ex_status = MPIU_ExGetStatus(wr_ov);
- if((ex_status == MPIU_EX_STATUS_IO_ABORT) && !(IS_SOCKCONN_CLOSING(sc))){
- /* Thread aborted - repost connect */
- mpi_errno = MPID_nem_newtcp_module_post_connect_ex(sc, &(sc->connect.sin));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
-
- MPIU_ERR_SETANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**sock_connect",
- "**sock_connect %s %d", MPIU_OSW_Strerror(MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(wr_ov))),
- MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(wr_ov)));
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_GEN_CNTING_FAIL_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_tc_c_cnting_fail_handler gen_cnting_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_tc_c_cntd_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_tc_c_cntd_success_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
-
- MPIDI_STATE_DECL(MPID_STATE_STATE_TC_C_CNTD_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_TC_C_CNTD_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- if (found_better_sc(sc, NULL)) {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_tc_c_cntd_handler(): changing to "
- "quiescent"));
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- goto fn_exit;
- }
-
- if (!sc->is_tmpvc) { /* normal connection */
- if (send_id_info(sc) == MPI_SUCCESS) {
- CHANGE_STATE(sc, CONN_STATE_TC_C_RANKSENT);
- }
- else {
- mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME,
- __LINE__, MPI_ERR_OTHER,
- "**fail", 0);
- /* FIXME-Danger add error string actual string : "**cannot send idinfo" */
- goto fn_fail;
- }
- }
- else { /* temp VC */
- if (send_tmpvc_info(sc) == MPI_SUCCESS) {
- CHANGE_STATE(sc, CONN_STATE_TC_C_TMPVCSENT);
- }
- else {
- mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME,
- __LINE__, MPI_ERR_OTHER,
- "**fail", 0);
- goto fn_fail;
- }
- }
- /* Call the handlers explicitly since we don't post a write
- * for id and tmp_vc
- * Since we are calling the EX handlers explicitly we need
- * to reset the write context
- */
- mpi_errno = MPID_nem_newtcp_module_reset_writev_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = CALL_WR_HANDLER(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_TC_C_CNTD_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- /* Explicitly call the fail handlers */
- mpi_errno = MPID_nem_newtcp_module_reset_writev_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CALL_WR_FAIL_HANDLER(sc);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_tc_c_cntd_fail_handler gen_cnting_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_c_ack_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_c_ack_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int nb, complete=0;
- MPIDI_STATE_DECL(MPID_STATE_STATE_C_ACKRECV_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_C_ACKRECV_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_read_progress_update(sc, &complete);
- /* FIXME: Don't assume that the whole hdr is read */
- MPIU_Assert(complete);
- /* Sanity check that the conn is not closed */
- MPIU_Assert(nb > 0);
-
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
-
- /* We are just expecting ACK/NACK command packets - no data */
- MPIU_Assert(pkt->datalen == 0);
- MPIU_Assert(pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_ACK ||
- pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_NAK);
-
- if (pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_ID_ACK) {
- CHANGE_STATE(sc, CONN_STATE_TS_COMMRDY);
- ASSIGN_SC_TO_VC(sc->vc, sc);
- MPID_nem_newtcp_module_conn_est(sc->vc);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_c_ack_success_handler(): connection established: sc=%p, sc->vc=%p, sc->fd=%d, is_same_pg=%s, pg_rank=%d", sc, sc->vc, sc->fd, (sc->is_same_pg ? "TRUE" : "FALSE"), sc->pg_rank));
- }
- else {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_c_ack_success_handler() 2: changing to "
- "quiescent"));
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_C_ACKRECV_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#define state_c_ack_fail_handler gen_read_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_c_ranksent_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_c_ranksent_success_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- int nb, complete;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int pkt_len;
- MPIDI_STATE_DECL(MPID_STATE_STATE_C_RANKSENT_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_C_RANKSENT_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_write_progress_update(sc, &complete);
- MPIU_Assert(complete);
-
- SOCKCONN_EX_RD_HANDLERS_SET(sc, state_c_ack_success_handler, state_c_ack_fail_handler);
-
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- pkt_len = sizeof(MPIDI_nem_newtcp_module_header_t);
- MPIU_Assert(sc->tmp_buf_len >= pkt_len);
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(sc, pkt, pkt_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_C_RANKSENT_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_c_ranksent_fail_handler gen_write_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_c_tmpvcack_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_c_tmpvcack_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int nb, complete;
- MPIDI_STATE_DECL(MPID_STATE_STATE_C_TMPVCACK_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_C_TMPVCACK_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_read_progress_update(sc, &complete);
- /* FIXME : Remove this restriction */
- MPIU_Assert(complete);
- /* Make sure that conn is not closed */
- MPIU_Assert(nb > 0);
-
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- MPIU_Assert(pkt->datalen == 0);
-
- MPIU_Assert(pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_ACK ||
- pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_NAK);
-
- if (pkt->pkt_type == MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_ACK) {
- CHANGE_STATE(sc, CONN_STATE_TS_COMMRDY);
- ASSIGN_SC_TO_VC(sc->vc, sc);
- MPID_nem_newtcp_module_conn_est (sc->vc);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "c_tmpvcsent_handler(): connection established (fd=%d, sc=%p, sc->vc=%p)", sc->fd, sc, sc->vc));
- }
- else {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_c_tmpvcsent_handler() 2: changing to quiescent"));
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_C_TMPVCACK_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#define state_c_tmpvcack_fail_handler gen_read_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_c_tmpvcsent_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_c_tmpvcsent_success_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int pkt_len, nb, complete;
- MPIDI_STATE_DECL(MPID_STATE_STATE_C_TMPVCSENT_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_C_TMPVCSENT_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_write_progress_update(sc, &complete);
- MPIU_Assert(complete);
-
- SOCKCONN_EX_RD_HANDLERS_SET(sc, state_c_tmpvcack_success_handler, state_c_tmpvcack_fail_handler);
-
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- pkt_len = sizeof(MPIDI_nem_newtcp_module_header_t);
-
- /* Post a read for TMPVC ACK/NACK */
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(sc, pkt, pkt_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_C_TMPVCSENT_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_c_tmpvcsent_fail_handler gen_write_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_l_cntd_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_l_cntd_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_nem_newtcp_module_header_t *pkt;
- int pkt_len;
- MPIDI_STATE_DECL(MPID_STATE_STATE_L_CNTD_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_L_CNTD_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- /* We have an active connection, start polling more often */
- MPID_nem_tcp_skip_polls = MAX_SKIP_POLLS_ACTIVE;
-
- pkt = (MPIDI_nem_newtcp_module_header_t *) (sc->tmp_buf);
- pkt_len = sizeof(MPIDI_nem_newtcp_module_header_t);
-
- /* FIXME: Add more states instead of setting handlers explicitly..
- * OR get rid of all states
- */
- SOCKCONN_EX_RD_HANDLERS_SET(sc, recv_id_or_tmpvc_info_success_handler, recv_id_or_tmpvc_info_fail_handler);
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(sc, pkt, pkt_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_L_CNTD_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* FIXME: Is this reqd ? - there is a *accept_fail_handler*/
-#undef FUNCNAME
-#define FUNCNAME state_l_cntd_fail_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_l_cntd_fail_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_STATE_DECL(MPID_STATE_STATE_L_CNTD_FAIL_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_L_CNTD_FAIL_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_l_cntd_handler() 1: changing to "
- "quiescent"));
-
- /* FIXME: Implement connect retry ...*/
- /* FIXME : Is this needed ? On accept side the sc won't have a vc right now */
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- MPID_nem_newtcp_module_post_close_ex(sc);
- MPIU_ERR_SETANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**sock_accept",
- "**sock_accept %s %d", MPIU_OSW_Strerror(MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(rd_ov))),
- MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(rd_ov)));
-
-fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_L_CNTD_FAIL_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/*
- Returns TRUE, if the process(self) wins against the remote process
- FALSE, otherwise
- */
-#undef FUNCNAME
-#define FUNCNAME do_i_win
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int do_i_win(sockconn_t *rmt_sc)
-{
- int win = FALSE;
- MPIDI_STATE_DECL(MPID_STATE_DO_I_WIN);
-
- MPIDI_FUNC_ENTER(MPID_STATE_DO_I_WIN);
-
- MPIU_Assert(rmt_sc->pg_is_set);
-
- if (rmt_sc->is_same_pg) {
- if (MPIDI_Process.my_pg_rank > rmt_sc->pg_rank)
- win = TRUE;
- }
- else {
- if (strcmp(MPIDI_Process.my_pg->id, rmt_sc->pg_id) > 0)
- win = TRUE;
- }
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE,
- (MPIU_DBG_FDEST, "do_i_win(rmt_sc=%p (%s)) win=%s is_same_pg=%s my_pg_rank=%d rmt_pg_rank=%d",
- rmt_sc, SOCK_STATE_STR[rmt_sc->state],
- (win ? "TRUE" : "FALSE"),(rmt_sc->is_same_pg ? "TRUE" : "FALSE"), MPIDI_Process.my_pg_rank,
- rmt_sc->pg_rank));
- MPIDI_FUNC_EXIT(MPID_STATE_DO_I_WIN);
- return win;
-}
-
-#undef FUNCNAME
-#define FUNCNAME state_l_rankrcvd_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_l_rankrcvd_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- sockconn_t *fnd_sc;
- int snd_nak = FALSE;
- int nb, complete;
- MPIDI_STATE_DECL(MPID_STATE_STATE_L_RANKRCVD_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_L_RANKRCVD_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_read_progress_update(sc, &complete);
- MPIU_Assert(complete);
- /* FIXME: Re-introduce after we get rid of explicit calls to handlers
- MPIU_Assert(nb > 0);
- */
-
- /* FIXME: Get rid of found_better_sc() in the C_CNTD state.
- * Always have the connection ACK/NACK by the listen side
- * There are 2 cases here:
- * 1) A connect() is in progress - Send a NACK - Let the other
- * connection succeed
- * 2) A connection already exists - Send a NACK
- */
- if (found_better_sc(sc, &fnd_sc)) {
- if (fnd_sc->state == CONN_STATE_TS_COMMRDY)
- snd_nak = TRUE;
- else if (fnd_sc->state == CONN_STATE_TC_C_RANKSENT)
- snd_nak = do_i_win(sc);
- }
-
- if (snd_nak) {
- if (send_cmd_pkt(sc, MPIDI_NEM_NEWTCP_MODULE_PKT_ID_NAK) == MPI_SUCCESS) {
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "state_l_rankrcvd_handler() 2: changing to "
- "quiescent"));
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- }
- else {
- /* The following line is _crucial_ to correct operation. We need to
- * ensure that all head-to-head resolution has completed before we
- * move to COMMRDY and send any pending messages. If we don't this
- * connection could shut down before the other connection has a
- * chance to finish the connect protocol. That can lead to all
- * kinds of badness, including zombie connections, segfaults, and
- * accessing PG/VC info that is no longer present. */
- if (VC_FIELD(sc->vc, sc_ref_count) > 1){
- /* Post a dummy read so that EX does not lose track of this sock conn */
- mpi_errno = MPID_nem_newtcp_module_post_dummy_read_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- goto fn_exit;
- }
-
- if (send_cmd_pkt(sc, MPIDI_NEM_NEWTCP_MODULE_PKT_ID_ACK) == MPI_SUCCESS) {
- CHANGE_STATE(sc, CONN_STATE_TS_COMMRDY);
- ASSIGN_SC_TO_VC(sc->vc, sc);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "connection established: sc=%p, sc->vc=%p, sc->fd=%d, is_same_pg=%s, pg_rank=%d", sc, sc->vc, sc->fd, (sc->is_same_pg ? "TRUE" : "FALSE"), sc->pg_rank));
- MPID_nem_newtcp_module_conn_est(sc->vc);
- }
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_L_RANKRCVD_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_l_rankrcvd_fail_handler gen_read_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_l_tmpvcrcvd_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_l_tmpvcrcvd_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- int snd_nak = FALSE;
- int nb, complete;
- MPIDI_STATE_DECL(MPID_STATE_STATE_L_TMPVCRCVD_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_L_TMPVCRCVD_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_read_progress_update(sc, &complete);
- MPIU_Assert(complete);
- MPIU_Assert(nb > 0);
-
- /* FIXME: When is snd_nak true ? */
- if (snd_nak) {
- if (send_cmd_pkt(sc, MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_NAK) == MPI_SUCCESS) {
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_dummy_read_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- }
- else {
- if (send_cmd_pkt(sc, MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_ACK) == MPI_SUCCESS) {
- CHANGE_STATE(sc, CONN_STATE_TS_COMMRDY);
- ASSIGN_SC_TO_VC(sc->vc, sc);
- MPID_nem_newtcp_module_conn_est (sc->vc);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "fd=%d: TMPVC_ACK sent, connection established!", sc->fd));
- }
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_L_TMPVCRCVD_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_l_tmpvcrcvd_fail_handler gen_read_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_recv_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int MPID_nem_newtcp_module_recv_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- int complete;
- ssize_t bytes_recvd;
- MPID_Request *rreq = NULL;
- MPID_IOV *iov = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_RECV_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_RECV_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(sc != NULL);
-
- bytes_recvd = ex_read_progress_update(sc, &complete);
- rreq = sc->vc->ch.recv_active;
-
- if (rreq == NULL){
- /* Received a new message on scratch pad, sc->tmp_buf */
- if (bytes_recvd == 0){
- MPIU_Assert(sc != NULL);
- MPIU_Assert(sc->vc != NULL);
- /* sc->vc->sc will be NULL if sc->vc->state == _INACTIVE */
- MPIU_Assert(VC_FIELD(sc->vc, sc) == NULL || VC_FIELD(sc->vc, sc) == sc);
- if (vc_is_in_shutdown(sc->vc)){
- /* there's currently no hook for CH3 to tell nemesis/newtcp
- that we are in the middle of a disconnection dance. So
- if we don't check to see if we are currently
- disconnecting, then we end up with a potential race where
- the other side performs a tcp close() before we do and we
- blow up here. */
- mpi_errno = cleanup_sc_vc(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CHANGE_STATE(sc, CONN_STATE_TS_D_QUIESCENT);
-
- mpi_errno = MPID_nem_newtcp_module_post_close_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- goto fn_exit;
- }
- else{
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "ERROR: sock (fd=%d) is closed: bytes_recvd == 0", sc->fd );
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**sock_closed");
- }
- }
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "New recv %d (fd=%d, vc=%p, sc=%p)", bytes_recvd, sc->fd, sc->vc, sc));
-
- mpi_errno = MPID_nem_handle_pkt(sc->vc, sc->tmp_buf, bytes_recvd);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- /* The pkt handlers can sever the conn btw sc and vc - eg: when vc is terminated*/
- if(sc->vc != NULL){
- rreq = sc->vc->ch.recv_active;
- if (rreq == NULL){
- /* The packets were completely consumed - post read on scratch pad recv buf */
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "Packets (bytes = %d) on (fd=%d, vc=%p, sc=%p) completely consumed", bytes_recvd, sc->fd, sc->vc, sc));
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(sc, sc->tmp_buf, sc->tmp_buf_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- /* The packet was not completely consumed - post read for remaining data */
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "Packets (bytes = %d) on (fd=%d, vc=%p, sc=%p) NOT completely consumed", bytes_recvd, sc->fd, sc->vc, sc));
- iov = &rreq->dev.iov[rreq->dev.iov_offset];
- mpi_errno = MPID_nem_newtcp_module_post_readv_ex(sc, iov, rreq->dev.iov_count);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- }
- }
- else{
- /* Continuing recv of an old message */
- /* There is a pending receive, and we are receiving
- * data directly into the user buffer
- */
- int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
-
- MPIU_Assert(rreq->dev.iov_count > 0);
- MPIU_Assert(rreq->dev.iov_offset >= 0);
- MPIU_Assert(rreq->dev.iov_offset < MPID_IOV_LIMIT);
- MPIU_Assert(rreq->dev.iov_count <= MPID_IOV_LIMIT);
-
- iov = &rreq->dev.iov[rreq->dev.iov_offset];
- if (bytes_recvd == 0){
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**sock_closed");
- }
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "Cont recv, total bytes = %d", bytes_recvd);
-
- if(!complete){
- /* IOV is not filled completely. Continue read on the prev posted IOV. */
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Cont recv, IOV not filled completely - cont recv on same iov");
- mpi_errno = MPID_nem_newtcp_module_post_readv_ex(sc, NULL, 0);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- /* the whole iov has been received */
- int req_complete=0;
-
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Cont recv, IOV filled completely...");
- reqFn = rreq->dev.OnDataAvail;
- if (!reqFn){
- MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(rreq);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Req complete...");
- sc->vc->ch.recv_active = NULL;
- req_complete = 1;
- }
- else{
- req_complete = 0;
-
- mpi_errno = reqFn(sc->vc, rreq, &req_complete);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- if (req_complete){
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Req complete...");
- sc->vc->ch.recv_active = NULL;
- }
- else{
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Req not complete...");
- }
- }
- if(req_complete){
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(sc, sc->tmp_buf, sc->tmp_buf_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- iov = &rreq->dev.iov[rreq->dev.iov_offset];
-
- mpi_errno = MPID_nem_newtcp_module_post_readv_ex(sc, iov, rreq->dev.iov_count);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- } /* the whole iov has been received */
- } /* processing recv on an old message */
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_RECV_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#define MPID_nem_newtcp_module_recv_fail_handler gen_read_fail_handler
-
-#define state_commrdy_rd_success_handler MPID_nem_newtcp_module_recv_success_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_commrdy_wr_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_commrdy_wr_success_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- int nb, complete = 0;
-
- MPIDI_STATE_DECL(MPID_STATE_STATE_COMMRDY_WR_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_COMMRDY_WR_SUCCESS_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- nb = ex_write_progress_update(sc, &complete);
-
- if(complete){
- /* Call req handler */
- mpi_errno = MPID_nem_newtcp_module_handle_sendq_head_req(sc->vc, &complete);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- /* Send queued data, if any */
- mpi_errno = MPID_nem_newtcp_module_send_queued(sc->vc);
- if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP (mpi_errno);
- }
- else{
- /* Continue writing from the posted iov */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(sc, NULL, 0);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_COMMRDY_WR_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-
-}
-
-#define state_commrdy_rd_fail_handler gen_read_fail_handler
-#define state_commrdy_wr_fail_handler gen_write_fail_handler
-
-#undef FUNCNAME
-#define FUNCNAME state_d_quiescent_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int state_d_quiescent_handler(MPIU_EXOVERLAPPED *wr_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIDI_STATE_DECL(MPID_STATE_STATE_D_QUIESCENT_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_STATE_D_QUIESCENT_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_WR_OV(wr_ov);
- MPIU_Assert(sc != NULL);
-
- mpi_errno = cleanup_sc_vc(sc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = cleanup_sc(sc);
- if(mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_STATE_D_QUIESCENT_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#define state_d_quiescent_success_handler state_d_quiescent_handler
-#define state_d_quiescent_fail_handler state_d_quiescent_handler
-
-static inline int sc_state_init_handler(sc_state_info_t *sc_state_info,
- handler_func_t rd_success_fn, handler_func_t rd_fail_fn,
- handler_func_t wr_success_fn, handler_func_t wr_fail_fn)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_Assert(sc_state_info != NULL);
- sc_state_info->sc_state_rd_success_handler = rd_success_fn;
- sc_state_info->sc_state_rd_fail_handler = rd_fail_fn;
- sc_state_info->sc_state_wr_success_handler = wr_success_fn;
- sc_state_info->sc_state_wr_fail_handler = wr_fail_fn;
-
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_sm_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_sm_init()
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
-
-#ifdef MPICH_IS_THREADED
- MPIU_THREAD_CHECK_BEGIN;
- /* FIXME: Create post thread here */
- MPIU_THREAD_CHECK_END;
-#endif
- /* Set the appropriate handlers */
- sc_state_init_handler(&sc_state_info[CONN_STATE_TS_CLOSED],
- state_d_quiescent_success_handler,
- state_d_quiescent_fail_handler,
- state_d_quiescent_success_handler,
- state_d_quiescent_fail_handler);
-
- sc_state_init_handler(&sc_state_info[CONN_STATE_TC_C_CNTING],
- NULL, NULL,
- state_tc_c_cnting_success_handler,
- state_tc_c_cnting_fail_handler);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TC_C_CNTD],
- NULL, NULL,
- state_tc_c_cntd_success_handler,
- state_tc_c_cntd_fail_handler);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TC_C_RANKSENT],
- NULL, NULL,
- state_c_ranksent_success_handler,
- state_c_ranksent_fail_handler);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TC_C_TMPVCSENT],
- NULL, NULL,
- state_c_tmpvcsent_success_handler,
- state_c_tmpvcsent_fail_handler);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TA_C_CNTD],
- state_l_cntd_success_handler,
- state_l_cntd_fail_handler,
- NULL, NULL);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TA_C_RANKRCVD],
- state_l_rankrcvd_success_handler,
- state_l_rankrcvd_fail_handler,
- NULL, NULL);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TA_C_TMPVCRCVD],
- state_l_tmpvcrcvd_success_handler,
- state_l_tmpvcrcvd_fail_handler,
- NULL, NULL);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TS_COMMRDY],
- state_commrdy_rd_success_handler,
- state_commrdy_rd_fail_handler,
- state_commrdy_wr_success_handler,
- state_commrdy_wr_fail_handler);
- sc_state_init_handler(&sc_state_info[CONN_STATE_TS_D_QUIESCENT],
- NULL, NULL,
- state_d_quiescent_success_handler,
- state_d_quiescent_fail_handler);
-
- /* Init the sc tbls list */
- mpi_errno = init_sc_tbl_list();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- /* Create & init the sock conn table */
- mpi_errno = alloc_sc_tbl();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- /* Post the first accept on the listening sock */
- sc = &MPID_nem_newtcp_module_g_lstn_sc;
- SOCKCONN_EX_RD_HANDLERS_SET(sc, MPID_nem_newtcp_module_state_accept_success_handler, MPID_nem_newtcp_module_state_accept_fail_handler);
- SOCKCONN_EX_WR_HANDLERS_SET(sc, NULL, NULL);
-
- mpi_errno = MPID_nem_newtcp_module_post_accept_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- return mpi_errno;
-fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_sm_finalize
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_sm_finalize()
-{
- freenode_t *node;
- int mpi_errno = MPI_SUCCESS;
-
-#ifdef MPICH_IS_THREADED
- MPIU_THREAD_CHECK_BEGIN;
- /* FIXME: Kill post thread here */
- MPIU_THREAD_CHECK_END;
-#endif
-
- /* walk the freeq and free all the elements */
- while (!Q_EMPTY(freeq)) {
- Q_DEQUEUE(&freeq, ((freenode_t **)&node));
- MPIU_Free(node);
- }
-
- mpi_errno = free_sc_tbls_in_list();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = free_sc_tbl_list();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
- return mpi_errno;
-fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-
-/*
- N1: create a new listener fd?? While doing so, if we bind it to the same port used befor,
-then it is ok. Else,the new port number(and thus the business card) has to be communicated
-to the other processes (in same and different pg's), which is not quite simple to do.
-Evaluate the need for it by testing and then do it, if needed.
-
-*/
-#define SOCK_PROGRESS_EVENTS_MAX 100
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_connpoll
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_connpoll(int in_blocking_poll)
-{
- int mpi_errno = MPI_SUCCESS;
- static int num_skipped_polls = 0;
- int nevents = 0;
- /* BOOL wait_for_event_and_status = (in_blocking_poll) ? TRUE : FALSE; */
- BOOL wait_for_event_and_status = FALSE;
-
- /* To improve shared memory performance, we don't call the poll()
- * systemcall every time. The MPID_nem_tcp_skip_polls value is
- * changed depending on whether we have any active connections. */
- if (in_blocking_poll && num_skipped_polls++ < MPID_nem_tcp_skip_polls){
- goto fn_exit;
- }
- SKIP_POLLS_INC(MPID_nem_tcp_skip_polls);
- num_skipped_polls = 0;
-
- mpi_errno = MPIU_ExProcessCompletions(MPID_nem_newtcp_module_ex_set_hnd, &wait_for_event_and_status);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- /* FIXME: We also need to tune the number of times we need to
- * wait for an event.
- * eg: SOCK_PROGRESS_EVENTS_MAX = 2 x No_of_fds_in_ex_set
- */
- while((wait_for_event_and_status == TRUE) &&
- (++nevents < SOCK_PROGRESS_EVENTS_MAX)){
- /* Don't block for an event to complete */
- wait_for_event_and_status = FALSE;
- /* On return, if (wait_for_event_and_status == FALSE) then
- * there are no more events in EX queue at the moment
- */
- mpi_errno = MPIU_ExProcessCompletions(MPID_nem_newtcp_module_ex_set_hnd, &wait_for_event_and_status);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-static int complete_connection(sockconn_t *sc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_Assert(sc != NULL);
-
- MPID_nem_newtcp_module_set_sockopts(sc->fd); /* (N2) */
-
- sc->pg_rank = CONN_INVALID_RANK;
- sc->pg_is_set = FALSE;
- sc->is_tmpvc = 0;
-
- MPIU_ExAttachHandle(MPID_nem_newtcp_module_ex_set_hnd, MPIU_EX_WIN32_COMP_PROC_KEY, (HANDLE )sc->fd);
-
- CHANGE_STATE(sc, CONN_STATE_TA_C_CNTD);
-
- /* FIXME: Try to avoid calling the rd handler explicitly */
- /* Explicitly call the read handlers */
- mpi_errno = MPID_nem_newtcp_module_reset_readv_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- CALL_RD_HANDLER(sc);
-
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "accept success, added to table, connfd=%d", sc->accept.connfd));
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/*
- FIXME
- 1.Check for accept error.
- 2.If listening socket dies, create a new listening socket so that future connects
- can use that. In that case, how to tell other processes about the new listening port. It's
- probably a design issue at MPI level.
-
- N1: There might be a timing window where poll on listen_fd was successful and there was
- a connection in the queue. Assume there was only one connection in the listen queue and
- before we called accept, the peer had reset the connection. On receiving a TCP RST,
- some implementations remove the connection from the listen queue. So, accept on
- a non-blocking fd would return error with EWOULDBLOCK.
-
- N2: The peer might have reset or closed the connection. In some implementations,
- even if the connection is reset by the peer, accept is successful. By POSIX standard,
- if the connection is closed by the peer, accept will still be successful. So, soon
- after accept, check whether the new fd is really connected (i.e neither reset nor
- EOF received from peer).
- Now, it is decided not to check for this condition at this point. After all, in the next
- state, anyhow we can close the socket, if we receive an EOF.
-
- N3: find_free_sc is called within the while loop. It may cause the table to expand. So,
- the arguments passed for this callback function may get invalidated. So, it is imperative
- that we obtain sc pointer and plfd pointer everytime within the while loop.
- Accordingly, the parameters are named unused1 and unused2 for clarity.
-*/
-#undef FUNCNAME
-#define FUNCNAME state_listening_success_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_state_accept_success_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *l_sc = NULL;
- int len;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_SUCCESS_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_SUCCESS_HANDLER);
-
- /* Accept succeeded... */
- l_sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
- MPIU_Assert(l_sc != NULL);
-
- /* Process all connections waiting to be accepted & post the next accept */
- while (1) {
- if(l_sc->accept.connfd == MPIU_SOCKW_SOCKFD_INVALID){
- /* Post next accept and quit */
- mpi_errno = MPID_nem_newtcp_module_post_accept_ex(l_sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- break;
- }
- else {
- sockconn_t *sc;
- mpi_errno = find_free_sc(&sc);
- if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP (mpi_errno); }
-
- sc->fd = l_sc->accept.connfd;
-
- mpi_errno = complete_connection(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- len = sizeof(SA_IN);
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "before accept"));
- MPIU_OSW_RETRYON_INTR((mpi_errno != MPI_SUCCESS),
- (mpi_errno = MPIU_SOCKW_Accept(l_sc->fd, (SA *) &l_sc->accept.accept_buffer, &len, &(l_sc->accept.connfd))));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_SUCCESS_HANDLER);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_state_accept_fail_handler
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_state_accept_fail_handler(MPIU_EXOVERLAPPED *rd_ov)
-{
- int mpi_errno = MPI_SUCCESS;
- sockconn_t *sc = NULL;
- MPIU_Ex_status_t ex_status;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_FAIL_HANDLER);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_FAIL_HANDLER);
-
- sc = GET_SOCKCONN_FROM_EX_RD_OV(rd_ov);
-
- MPIU_Assert(sc != NULL);
-
- ex_status = MPIU_ExGetStatus(rd_ov);
- if((ex_status == MPIU_EX_STATUS_IO_ABORT) && !(IS_SOCKCONN_CLOSING(sc))){
- /* Thread aborted - try to repost accept */
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "THREAD ABORT: Reposting accept on sc=%p", sc));
- mpi_errno = MPID_nem_newtcp_module_post_accept_ex(sc);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- else{
- /* FIXME: Get rid of static allocn of listen sock and post a close */
- mpi_errno = MPIU_SOCKW_Sock_close(sc->fd);
- MPIU_ERR_SETANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**sock_accept",
- "**sock_accept %s %d", MPIU_OSW_Strerror(MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(rd_ov))),
- MPIU_EX_STATUS_TO_ERRNO(MPIU_ExGetStatus(rd_ov)));
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_STATE_LISTENING_FAIL_HANDLER);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.h b/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.h
deleted file mode 100644
index fac0aee..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.h
+++ /dev/null
@@ -1,292 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#ifndef SOCKSM_H_INCLUDED
-#define SOCKSM_H_INCLUDED
-
-#ifdef HAVE_SYS_POLL_H
- #include <sys/poll.h>
-#endif
-#include <stddef.h>
-#ifdef HAVE_WINNT_H
- #include <winnt.h>
-#endif
-#include "mpiu_os_wrappers.h"
-#include "mpiu_ex.h"
-
-enum SOCK_CONSTS { /* more type safe than #define's */
- LISTENQLEN = 10,
- POLL_CONN_TIME = 2
-};
-
-typedef struct sockaddr_in SA_IN;
-typedef struct sockaddr SA;
-
-enum CONSTS {
- CONN_TBL_INIT_SIZE = 20,
- CONN_TBL_GROW_SIZE = 10,
- CONN_WAITSET_INIT_SIZE = 20,
- CONN_INVALID_RANK = -1,
- NEGOMSG_DATA_LEN = 4, /* Length of data during negotiatiion message exchanges. */
- SLEEP_INTERVAL = 1500,
- PROGSM_TIMES = 20
-};
-
-
-typedef enum {
- MPID_NEM_NEWTCP_MODULE_SOCK_ERROR_EOF, /* either a socket error or EOF received from peer */
- MPID_NEM_NEWTCP_MODULE_SOCK_CONNECTED,
- MPID_NEM_NEWTCP_MODULE_SOCK_NOEVENT /* No poll event on socket */
-}MPID_NEM_NEWTCP_MODULE_SOCK_STATUS_t;
-
-#define M_(x) x
-#define CONN_TYPE_ M_(TYPE_CONN), M_(TYPE_ACPT)
-
-/* Note : '_' denotes sub-states */
-/* For example, CSO_DISCCONNECTING_DISCREQSENT, CSO_DISCONNECTING_DISCRSPRCVD are sub-states */
-/* of CSO_DISCONNECTING */
-/* LSO - Listening SOcket states */
-#define LISTEN_STATE_ \
- M_(LISTEN_STATE_CLOSED), \
- M_(LISTEN_STATE_LISTENING)
-
-/*
- CONN_STATE - Connection states of socket
- TC = Type connected (state of a socket that was issued a connect on)
- TA = Type Accepted (state of a socket returned by accept)
- TS = Type Shared (state of either TC or TA)
-
- C - Connection sub-states
- D - Disconnection sub-states
-*/
-
-#define CONN_STATE_ \
- M_(CONN_STATE_TS_CLOSED), \
- M_(CONN_STATE_TC_C_CNTING), \
- M_(CONN_STATE_TC_C_CNTD), \
- M_(CONN_STATE_TC_C_RANKSENT), \
- M_(CONN_STATE_TC_C_TMPVCSENT), \
- M_(CONN_STATE_TC_C_UB_), \
- M_(CONN_STATE_TA_C_CNTD), \
- M_(CONN_STATE_TA_C_RANKRCVD), \
- M_(CONN_STATE_TA_C_TMPVCRCVD), \
- M_(CONN_STATE_TA_C_UB_), \
- M_(CONN_STATE_TS_COMMRDY), \
- M_(CONN_STATE_TS_D_QUIESCENT)
-
-#define SOCK_STATE_ \
- LISTEN_STATE_, \
- CONN_STATE_
-
-#define IS_LISTEN_CONN_STATE(_state) ((CONN_STATE_TC_C_UB_ < (_state)) && \
- ((_state) < CONN_STATE_TA_C_UB_) )
-
-/* REQ - Request, RSP - Response */
-
-typedef enum CONN_TYPE {CONN_TYPE_, CONN_TYPE_SIZE} Conn_type_t;
-
-typedef enum MPID_nem_newtcp_module_sock_state {SOCK_STATE_LB_, SOCK_STATE_, SOCK_STATE_SIZE}
- MPID_nem_newtcp_module_sock_state_t;
-
-#define SOCK_STATE_TO_STRING(_state) \
- (( (_state) > SOCK_STATE_LB_ && (_state) < SOCK_STATE_SIZE ) ? SOCK_STATE_STR[_state] : "out_of_range")
-/*
- Note: event numbering starts from 1, as 0 is assumed to be the state of all-events cleared
- */
-typedef enum sockconn_event {EVENT_CONNECT = 1, EVENT_DISCONNECT}
- sockconn_event_t;
-
-#undef M_
-#define M_(x) #x
-
-#if defined(SOCKSM_H_DEFGLOBALS_)
-const char *const CONN_TYPE_STR[CONN_TYPE_SIZE] = {CONN_TYPE_};
-const char *const SOCK_STATE_STR[SOCK_STATE_SIZE] = {SOCK_STATE_};
-#elif defined(SOCKSM_H_EXTERNS_)
-extern const char *const CONN_TYPE_STR[];
-extern const char *const SOCK_STATE_STR[];
-#endif
-
-#undef M_
-
-#define SOCKCONN_EX_RD_HANDLERS_SET(sc, succ_fn, fail_fn) \
- MPIU_ExInitOverlapped(&(sc->rd_ov), succ_fn, fail_fn)
-#define SOCKCONN_EX_WR_HANDLERS_SET(sc, succ_fn, fail_fn) \
- MPIU_ExInitOverlapped(&(sc->wr_ov), succ_fn, fail_fn)
-
-#define CHANGE_STATE(_sc, _state) do { \
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "CHANGE_STATE %p (state= %s to %s)", _sc, SOCK_STATE_TO_STRING((_sc)->state), SOCK_STATE_TO_STRING(_state))); \
- (_sc)->state = _state; \
- SOCKCONN_EX_RD_HANDLERS_SET((_sc), sc_state_info[_state].sc_state_rd_success_handler, sc_state_info[_state].sc_state_rd_fail_handler); \
- SOCKCONN_EX_WR_HANDLERS_SET((_sc), sc_state_info[_state].sc_state_wr_success_handler, sc_state_info[_state].sc_state_wr_fail_handler); \
-} while(0)
-
-/* We might need to explicitly call the handlers when we don't post
- * a req to EX . This will help us compare perf with and without
- * posting all req to EX.
- */
-#define CALL_RD_HANDLER(_sc_p) (sc_state_info[_sc_p->state].sc_state_rd_success_handler(&(_sc_p->rd_ov)))
-#define CALL_RD_FAIL_HANDLER(_sc_p) (sc_state_info[_sc_p->state].sc_state_rd_fail_handler(&(_sc_p->rd_ov)))
-#define CALL_WR_HANDLER(_sc_p) (sc_state_info[_sc_p->state].sc_state_wr_success_handler(&(_sc_p->wr_ov)))
-#define CALL_WR_FAIL_HANDLER(_sc_p) (sc_state_info[_sc_p->state].sc_state_wr_fail_handler(&(_sc_p->wr_ov)))
-
-struct MPID_nem_new_tcp_module_sockconn;
-typedef struct MPID_nem_new_tcp_module_sockconn sockconn_t;
-
-typedef int (*handler_func_t) (MPIU_EXOVERLAPPED *ov);
-
-/* FIXME: This structure can be moved to sock wrappers once MPIU_Sock_fd_t
- * becomes more opaque OR we replace MPIU_Sock_fd_t with an opaque MPIU_Sock_t
- * Also, look into the MSMPI code on how to organize this private struct members into
- * different states, listen_state/accept_state/read_state/write_state,
- * depending on the type of socket.
- */
-typedef struct sock_read_context{
- /* tmp iov is required for supporting reads posted with ex without iov */
- MPID_IOV tmp_iov;
- /* iov points to the current IOV being read */
- MPID_IOV *iov;
- /* tmp_iov and iov are not used at the same time */
- /* n_iov is the length of tmp_iov/iov */
- int n_iov;
- /* nb shows the number of bytes read */
- int nb;
-} sock_read_context_t;
-
-typedef struct sock_write_context{
- /* tmp iov is required for supporting writes posted with ex without iov */
- MPID_IOV tmp_iov;
- /* iov points to the current IOV being written */
- MPID_IOV *iov;
- /* tmp_iov and iov are not used at the same time */
- /* n_iov is the current length of tmp_iov/iov */
- int n_iov;
- /* nb shows the number of bytes written */
- int nb;
-} sock_write_context_t;
-
-typedef struct sock_listen_context{
- /* Accept buffer is used for storing local address and remote address
- * As per msdn docs, when using
- * sizeof buffer for local address >= 16 + sizeof(local address of server)
- * & sizeof buffer for remote address >= 16 + sizeof(remote address)
- */
- char accept_buffer[sizeof(struct sockaddr_in)*2+32];
- /* connfd is the newly connected sock */
- MPIU_SOCKW_Sockfd_t connfd;
- /* nb = bytes read from the AcceptEx() call */
- int nb;
-} sock_listen_context_t;
-
-/* Max retries to connect to a host */
-#define MAX_CONN_RETRIES 5
-typedef struct sock_connect_context{
- /* Host to connect to ...*/
- const char *host;
- /* Port to connect to ...*/
- int port;
- /* Currently instead of storing host and port we just store
- * the pointer to sockaddr struct
- */
- struct sockaddr_in sin;
- /* No of retries */
- int retry_cnt;
-} sock_connect_context_t;
-
-typedef struct sock_close_context{
- /* 0/1 - 1 if sock conn is closing */
- int conn_closing;
-} sock_close_context_t;
-
-/* 1 if sockconn is closing, 0 otherwise */
-#define IS_SOCKCONN_CLOSING(scp) (scp->close.conn_closing == 1)
-/* FIXME: Check whether the elements of sockconn is aligned correctly */
-typedef struct MPID_nem_new_tcp_module_sockconn{
- MPIU_SOCKW_Sockfd_t fd;
- /* Executive overlapped struct for read/listen */
- MPIU_EXOVERLAPPED rd_ov;
- /* Executive overlapped struct for write/connect */
- MPIU_EXOVERLAPPED wr_ov;
-
- sock_read_context_t read;
- sock_write_context_t write;
- union{
- sock_connect_context_t connect;
- sock_listen_context_t accept;
- };
- sock_close_context_t close;
- /* Used during posting read/write for id/tmp_vc & reading command headers */
- char *tmp_buf;
- int tmp_buf_len;
-
- /* FIXME: Remove the index. We no longer need it */
- int index;
-
- /* Used to prevent the usage of uninitialized pg info. is_same_pg, pg_rank,
- * and pg_id are _ONLY VALID_ if this (pg_is_set) is true */
- int pg_is_set;
- int is_same_pg; /* TRUE/FALSE - */
-/* FIXME: see whether this can be removed, by using only pg_id = NULL or non-NULL */
-/* NULL = if same_pg and valid pointer if different pgs. */
-
- int is_tmpvc;
- int pg_rank; /* rank and id cached here to avoid chasing pointers in vc and vc->pg */
- char *pg_id; /* MUST be used only if is_same_pg == FALSE */
-
- MPID_nem_newtcp_module_sock_state_t state;
- MPIDI_VC_t *vc;
- /* Conn_type_t conn_type; Probably useful for debugging/analyzing purposes. */
-} MPID_nem_new_tcp_module_sockconn_t;
-
-/* FIXME: CONTAINING_RECORD is only defined for windows
- * - Use offsetof() to define CONTAINING_RECORD() on unix
- */
-#define GET_SOCKCONN_FROM_EX_RD_OV(ov_addr) \
- CONTAINING_RECORD(ov_addr, MPID_nem_new_tcp_module_sockconn_t, rd_ov)
-#define GET_SOCKCONN_FROM_EX_WR_OV(ov_addr) \
- CONTAINING_RECORD(ov_addr, MPID_nem_new_tcp_module_sockconn_t, wr_ov)
-
-
-typedef enum MPIDI_nem_newtcp_module_pkt_type {
- MPIDI_NEM_NEWTCP_MODULE_PKT_ID_INFO, /* ID = rank + pg_id */
- MPIDI_NEM_NEWTCP_MODULE_PKT_ID_ACK,
- MPIDI_NEM_NEWTCP_MODULE_PKT_ID_NAK,
- MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_REQ,
- MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_ACK,
- MPIDI_NEM_NEWTCP_MODULE_PKT_DISC_NAK,
- MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_INFO,
- MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_ACK,
- MPIDI_NEM_NEWTCP_MODULE_PKT_TMPVC_NAK
-} MPIDI_nem_newtcp_module_pkt_type_t;
-
-typedef struct MPIDI_nem_newtcp_module_header {
- MPIDI_nem_newtcp_module_pkt_type_t pkt_type;
- int datalen;
-} MPIDI_nem_newtcp_module_header_t;
-
-typedef struct MPIDI_nem_newtcp_module_idinfo {
- int pg_rank;
-/* Commented intentionally */
-/* char pg_id[pg_id_len+1]; Memory is dynamically allocated for pg_id_len+1 */
-/* Also, this is optional. Sent only, if the recipient belongs to a different pg. */
-/* As long as another variable length field needs to be sent across(if at all required */
-/* in the future), datalen of header itself is enough to find the offset of pg_id */
-/* in the packet to be sent. */
-} MPIDI_nem_newtcp_module_idinfo_t;
-
-/* FIXME: bc actually contains port_name info */
-typedef struct MPIDI_nem_newtcp_module_portinfo {
- int port_name_tag;
-} MPIDI_nem_newtcp_module_portinfo_t;
-
-#define MPID_nem_newtcp_module_vc_is_connected(vc) (VC_FIELD(vc, sc) && VC_FIELD(vc, sc)->state == CONN_STATE_TS_COMMRDY)
-int MPID_nem_newtcp_module_state_accept_success_handler(MPIU_EXOVERLAPPED *rd_ov);
-int MPID_nem_newtcp_module_state_accept_fail_handler(MPIU_EXOVERLAPPED *rd_ov);
-int MPID_nem_newtcp_module_post_readv_ex(sockconn_t *sc, MPID_IOV *iov, int n_iov);
-int MPID_nem_newtcp_module_post_read_ex(sockconn_t *sc, void *buf, int nb);
-int MPID_nem_newtcp_module_post_writev_ex(sockconn_t *sc, MPID_IOV *iov, int n_iov);
-int MPID_nem_newtcp_module_post_write_ex(sockconn_t *sc, void *buf, int nb);
-#endif
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_finalize.c
deleted file mode 100644
index 8531cba..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_finalize.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-
-extern sockconn_t MPID_nem_newtcp_module_g_lstn_sc;
-/* FIXME: Move all externs to say socksm_globals.h */
-extern MPIU_ExSetHandle_t MPID_nem_newtcp_module_ex_set_hnd;
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_finalize
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_finalize()
-{
- int mpi_errno = MPI_SUCCESS;
-
- /* FIXME: Why don't we have a finalize for sm - MPID_nem_newtcp_module_finalize_sm() - ? */
- /* FIXME: Shouldn't the order of finalize() be the reverse order of init() ?
- * i.e., *finalize_sm(); *poll_finalize(); *send_finalize();
- */
- mpi_errno = MPID_nem_newtcp_module_send_finalize();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- mpi_errno = MPID_nem_newtcp_module_poll_finalize();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- mpi_errno = MPID_nem_newtcp_module_sm_finalize();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- if(MPIU_SOCKW_Sockfd_is_valid(MPID_nem_newtcp_module_g_lstn_sc.fd))
- {
- /* Since the listen sc is global we don't need to post a close and
- * invoke the EX handlers
- */
- MPIU_OSW_RETRYON_INTR((mpi_errno != MPI_SUCCESS), (mpi_errno = MPIU_SOCKW_Sock_close(MPID_nem_newtcp_module_g_lstn_sc.fd)));
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
- }
-
- /* Close the newtcp module executive set */
- MPIU_ExCloseSet(MPID_nem_newtcp_module_ex_set_hnd);
-
- mpi_errno = MPIU_ExFinalize();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- mpi_errno = MPIU_SOCKW_Finalize();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_ckpt_shutdown
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_ckpt_shutdown()
-{
- return MPID_nem_newtcp_module_finalize();
-}
-
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_impl.h b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_impl.h
deleted file mode 100644
index 606e588..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_impl.h
+++ /dev/null
@@ -1,175 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#ifndef NEWTCP_MODULE_IMPL_H
-#define NEWTCP_MODULE_IMPL_H
-
-#include "mpid_nem_impl.h"
-/* #include "newtcp_module.h" */
-#ifdef HAVE_SYS_TYPES_H
- #include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
- #include <sys/socket.h>
-#endif
-#ifdef HAVE_ERRNO_H
- #include <errno.h>
-#endif
-#ifdef HAVE_NETINET_IN_H
- #include <netinet/in.h>
-#endif
-#ifdef HAVE_NETINET_TCP_H
- #include <netinet/tcp.h>
-#endif
-#ifdef HAVE_NETDB_H
- #include <netdb.h>
-#endif
-
-#include "mpiu_ex.h"
-#include "socksm.h"
-#include "pmi.h"
-
-/* globals */
-extern MPID_nem_queue_ptr_t MPID_nem_newtcp_module_free_queue;
-extern MPID_nem_queue_ptr_t MPID_nem_process_recv_queue;
-extern MPID_nem_queue_ptr_t MPID_nem_process_free_queue;
-extern int MPID_nem_newtcp_module_listen_fd;
-
-typedef enum{
- MPID_NEM_NEWTCP_MODULE_VC_STATE_DISCONNECTED=0,
- MPID_NEM_NEWTCP_MODULE_VC_STATE_CONNECTED
-} MPID_Nem_newtcp_module_vc_state_t;
-
-extern char *MPID_nem_newtcp_module_recv_buf;
-#define MPID_NEM_NEWTCP_MODULE_RECV_MAX_PKT_LEN 2048
-#define MPID_NEM_NEWTCP_MODULE_RCVBUF_SZ (128*1024)
-#define MPID_NEM_NEWTCP_MODULE_SNDBUF_SZ (128*1024)
-
-/* The vc provides a generic buffer in which network modules can store
- private fields This removes all dependencies from the VC struction
- on the network module, facilitating dynamic module loading. */
-typedef struct
-{
- struct sockaddr_in sock_id;
- struct MPID_nem_new_tcp_module_sockconn *sc;
- struct
- {
- struct MPID_Request *head;
- struct MPID_Request *tail;
- } send_queue;
- /* this is a count of how many sc objects refer to this vc */
- MPID_Nem_newtcp_module_vc_state_t state;
- int sc_ref_count;
-} MPID_nem_newtcp_module_vc_area;
-
-/* accessor macro to private fields in VC */
-#define VC_FIELD(vc, field) (((MPID_nem_newtcp_module_vc_area *)((vc)->ch.netmod_area.padding)->field)
-
-#define ASSIGN_SC_TO_VC(vc_, sc_) do { \
- VC_FIELD((vc_), sc) = (sc_); \
- } while (0)
-
-/* functions */
-int MPID_nem_newtcp_module_init (MPIDI_PG_t *pg_p, int pg_rank,
- char **bc_val_p, int *val_max_sz_p);
-int MPID_nem_newtcp_module_finalize (void);
-int MPID_nem_newtcp_module_ckpt_shutdown (void);
-int MPID_nem_newtcp_module_poll(int);
-int MPID_nem_newtcp_module_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p);
-int MPID_nem_newtcp_module_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc);
-int MPID_nem_newtcp_module_vc_init (MPIDI_VC_t *vc);
-int MPID_nem_newtcp_module_vc_destroy(MPIDI_VC_t *vc);
-int MPID_nem_newtcp_module_vc_terminate (MPIDI_VC_t *vc);
-
-/* completion counter is atomically decremented when operation completes */
-int MPID_nem_newtcp_module_get (void *target_p, void *source_p, int source_node, int len, int *completion_ctr);
-int MPID_nem_newtcp_module_put (void *target_p, int target_node, void *source_p, int len, int *completion_ctr);
-
-int MPID_nem_newtcp_module_send_init (void);
-int MPID_nem_newtcp_module_handle_sendq_head_req(MPIDI_VC_t *vc, int *req_complete);
-int MPID_nem_newtcp_module_send_queued (MPIDI_VC_t *vc);
-int MPID_nem_newtcp_module_poll_init (void);
-int MPID_nem_newtcp_module_connect (struct MPIDI_VC *const vc);
-/* FIXME: WINTCP ASYNC
-int MPID_nem_newtcp_module_conn_wr_enable (struct MPIDI_VC *const vc);
-int MPID_nem_newtcp_module_conn_wr_disable (struct MPIDI_VC *const vc);
-*/
-int MPID_nem_newtcp_module_connpoll (int );
-int MPID_nem_newtcp_module_sm_init (void);
-int MPID_nem_newtcp_module_sm_finalize (void);
-int MPID_nem_newtcp_module_set_sockopts (MPIU_SOCKW_Sockfd_t fd);
-/* FIXME: WINTCP ASYNC
-MPID_NEM_NEWTCP_MODULE_SOCK_STATUS_t MPID_nem_newtcp_module_check_sock_status(MPIU_SOCKW_Waitset_sock_hnd_t fd_ws_hnd);
-*/
-int MPID_nem_newtcp_module_poll_finalize (void);
-int MPID_nem_newtcp_module_send_finalize (void);
-int MPID_nem_newtcp_module_bind (MPIU_SOCKW_Sockfd_t sockfd);
-/* FIXME: WINTCP ASYNC
-int MPID_nem_newtcp_module_recv_handler (MPIU_SOCKW_Waitset_sock_hnd_t fd_ws_hnd, sockconn_t *sc);
-*/
-int MPID_nem_newtcp_module_recv_handler (MPIU_EXOVERLAPPED *ov);
-int MPID_nem_newtcp_module_conn_est (MPIDI_VC_t *vc);
-int MPID_nem_newtcp_module_get_conninfo (struct MPIDI_VC *vc, struct sockaddr_in *addr, char **pg_id, int *pg_rank);
-int MPID_nem_newtcp_module_get_vc_from_conninfo (char *pg_id, int pg_rank, struct MPIDI_VC **vc);
-int MPID_nem_newtcp_module_get_addr_port_from_bc (const char *business_card, struct in_addr *addr, in_port_t *port);
-int MPID_nem_newtcp_module_is_sock_connected(int fd);
-int MPID_nem_newtcp_module_disconnect (struct MPIDI_VC *const vc);
-int MPID_nem_newtcp_module_cleanup (struct MPIDI_VC *const vc);
-/* FIXME: WINTCP ASYNC
-int MPID_nem_newtcp_module_state_listening_handler(MPIU_SOCKW_Waitset_sock_hnd_t fd_ws_hnd, sockconn_t *const l_sc);
-*/
-int MPID_nem_newtcp_module_state_listening_handler(MPIU_EXOVERLAPPED *ov);
-
-int MPID_nem_newtcp_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz);
-int MPID_nem_newtcp_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz,
- MPID_Request **sreq_ptr);
-int MPID_nem_newtcp_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz);
-
-/* Macros */
-
-/* system call wrapper -- This retries the syscall each time it is interrupted.
- Example usage: instead of writing "ret = write(fd, buf, len);"
- use: "CHECK_EINTR(ret, write(fd, buf, len));
- Caution:
- (1) Some of the system calls have value-result parameters. Those system calls
- should not be used within CHECK_EINTR macro or should be used with CARE.
- For eg. accept, the last parameter (addrlen) is a value-result one. So, even if the
- system call is interrupted, addrlen should be initialized to appropriate value before
- calling it again.
-
- (2) connect should not be called within a loop. In case, the connect is interrupted after
- the TCP handshake is initiated, calling connect again will only fail. So, select/poll
- should be called to check the status of the socket.
- I don't know what will happen, if a connect is interrupted even before the system call
- tries to initiate TCP handshake. No book/manual doesn't seem to explain this scenario.
-*/
-/* CHECK_EINTR is now deprecated. Use MPIU_OSW_RETRYON_EINTR() instead*/
-#define CHECK_EINTR(var, func) do { \
- (var) = (func); \
- } while ((var) == -1 && errno == EINTR)
-
-/* Send queue macros */
-#define Q_EMPTY(q) GENERIC_Q_EMPTY (q)
-#define Q_HEAD(q) GENERIC_Q_HEAD (q)
-#define Q_ENQUEUE_EMPTY(qp, ep) GENERIC_Q_ENQUEUE_EMPTY (qp, ep, next)
-#define Q_ENQUEUE(qp, ep) GENERIC_Q_ENQUEUE (qp, ep, next)
-#define Q_ENQUEUE_EMPTY_MULTIPLE(qp, ep0, ep1) GENERIC_Q_ENQUEUE_EMPTY_MULTIPLE (qp, ep0, ep1, next)
-#define Q_ENQUEUE_MULTIPLE(qp, ep0, ep1) GENERIC_Q_ENQUEUE_MULTIPLE (qp, ep0, ep1, next)
-#define Q_DEQUEUE(qp, ep) GENERIC_Q_DEQUEUE (qp, ep, next)
-#define Q_REMOVE_ELEMENTS(qp, ep0, ep1) GENERIC_Q_REMOVE_ELEMENTS (qp, ep0, ep1, next)
-
-/* VC list macros */
-#define VC_L_EMPTY(q) GENERIC_L_EMPTY (q)
-#define VC_L_HEAD(q) GENERIC_L_HEAD (q)
-
-/* stack macros */
-#define S_EMPTY(s) GENERIC_S_EMPTY (s)
-#define S_TOP(s) GENERIC_S_TOP (s)
-#define S_PUSH(sp, ep) GENERIC_S_PUSH (sp, ep, next)
-#define S_PUSH_MULTIPLE(sp, ep0, ep1) GENERIC_S_PUSH_MULTIPLE (sp, ep0, ep1, next)
-#define S_POP(sp, ep) GENERIC_S_POP (sp, ep, next)
-
-#endif /* NEWTCP_MODULE_IMPL_H */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
deleted file mode 100644
index f42d7b2..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
+++ /dev/null
@@ -1,668 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-#ifdef HAVE_SYS_TYPES_H
- #include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
- #include <sys/socket.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
- #include <arpa/inet.h>
-#endif
-
-/*S
- MPIDU_Sock_ifaddr_t - Structure to hold an Internet address.
-
-+ len - Length of the address. 4 for IPv4, 16 for IPv6.
-- ifaddr - Address bytes (as bytes, not characters)
-
-S*/
-typedef struct MPIDU_Sock_ifaddr_t {
- int len, type;
- unsigned char ifaddr[16];
-} MPIDU_Sock_ifaddr_t;
-
-/* FIXME: Move all externs to say socksm_globals.h */
-extern MPIU_ExSetHandle_t MPID_nem_newtcp_module_ex_set_hnd;
-
-extern sockconn_t MPID_nem_newtcp_module_g_lstn_sc;
-extern struct pollfd g_lstn_plfd;
-
-static MPID_nem_queue_t _free_queue;
-
-static int dbg_ifname = 0;
-
-static int get_addr_port_from_bc (const char *business_card, struct in_addr *addr, in_port_t *port);
-static int GetIPInterface( MPIDU_Sock_ifaddr_t *, int * );
-
-MPID_nem_netmod_funcs_t MPIDI_nem_newtcp_module_funcs = {
- MPID_nem_newtcp_module_init,
- MPID_nem_newtcp_module_finalize,
- MPID_nem_newtcp_module_poll,
- MPID_nem_newtcp_module_get_business_card,
- MPID_nem_newtcp_module_connect_to_root,
- MPID_nem_newtcp_module_vc_init,
- MPID_nem_newtcp_module_vc_destroy,
- MPID_nem_newtcp_module_vc_terminate
-};
-
-#define MPIDI_CH3I_PORT_KEY "port"
-#define MPIDI_CH3I_HOST_DESCRIPTION_KEY "description"
-#define MPIDI_CH3I_IFNAME_KEY "ifname"
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_init (MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_INIT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_INIT);
-
- MPIU_UNREFERENCED_ARG(pg_p);
-
- /* first make sure that our private fields in the vc fit into the area provided */
- MPIU_Assert(sizeof(MPID_nem_newtcp_module_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
-
- /* Initialize sock wrapper */
- mpi_errno = MPIU_SOCKW_Init();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- /* Initialize the Executive progress engine */
- mpi_errno = MPIU_ExInitialize();
- if(mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
-
- /* Create an Executive set */
- /* FIXME: All MPI util funcs should return an MPI error code */
- MPID_nem_newtcp_module_ex_set_hnd = MPIU_ExCreateSet();
- MPIU_ERR_CHKANDJUMP(MPID_nem_newtcp_module_ex_set_hnd == MPIU_EX_INVALID_SET,
- mpi_errno, MPI_ERR_OTHER, "**ex_create_set");
-
- /* Setup listen socket */
- mpi_errno = MPIU_SOCKW_Sock_open(AF_INET, SOCK_STREAM, IPPROTO_TCP, &(MPID_nem_newtcp_module_g_lstn_sc.fd));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_newtcp_module_set_sockopts(MPID_nem_newtcp_module_g_lstn_sc.fd);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPID_nem_newtcp_module_bind (MPID_nem_newtcp_module_g_lstn_sc.fd);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- mpi_errno = MPIU_SOCKW_Listen(MPID_nem_newtcp_module_g_lstn_sc.fd, SOMAXCONN);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- /* Associate the listen sock with the newtcp module executive set
- * - Key 0 command processor */
- /* FIXME: No error code returned ! */
- MPIU_ExAttachHandle(MPID_nem_newtcp_module_ex_set_hnd, MPIU_EX_WIN32_COMP_PROC_KEY, (HANDLE )MPID_nem_newtcp_module_g_lstn_sc.fd);
-
- /* Register the listening handlers with executive */
- /* We also need to post the first accept here */
-
- MPID_nem_newtcp_module_g_lstn_sc.state = LISTEN_STATE_LISTENING;
- /* FIXME: WINTCP ASYNC
- MPID_nem_newtcp_module_g_lstn_sc.handler = MPID_nem_newtcp_module_state_listening_handler;
- */
-
- /* create business card */
- mpi_errno = MPID_nem_newtcp_module_get_business_card (pg_rank, bc_val_p, val_max_sz_p);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- mpi_errno = MPID_nem_newtcp_module_sm_init();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- mpi_errno = MPID_nem_newtcp_module_send_init();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- mpi_errno = MPID_nem_newtcp_module_poll_init();
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_INIT);
- return mpi_errno;
- fn_fail:
-/* fprintf(stdout, "failure. mpi_errno = %d\n", mpi_errno); */
- goto fn_exit;
-}
-
-/*
- * Get a description of the network interface to use for socket communication
- *
- * Here are the steps. This order of checks is used to provide the
- * user control over the choice of interface and to avoid, where possible,
- * the use of non-scalable services, such as centeralized name servers.
- *
- * MPICH_INTERFACE_HOSTNAME
- * MPICH_INTERFACE_HOSTNAME_R%d
- * a single (non-localhost) available IP address, if possible
- * gethostbyname(gethostname())
- *
- * We return the following items:
- *
- * ifname - name of the interface. This may or may not be the same
- * as the name returned by gethostname (in Unix)
- * ifaddr - This structure includes the interface IP address (as bytes),
- * and the type (e.g., AF_INET or AF_INET6). Only
- * ipv4 (AF_INET) is used so far.
- */
-
-static int GetSockInterfaceAddr(int myRank, char *ifname, int maxIfname,
- MPIDU_Sock_ifaddr_t *ifaddr)
-{
- char *ifname_string;
- int mpi_errno = MPI_SUCCESS;
- int ifaddrFound = 0;
-
- /* Set "not found" for ifaddr */
- ifaddr->len = 0;
-
- /* Check for the name supplied through an environment variable */
- ifname_string = getenv("MPICH_INTERFACE_HOSTNAME");
- if (!ifname_string) {
- /* See if there is a per-process name for the interfaces (e.g.,
- the process manager only delievers the same values for the
- environment to each process */
- char namebuf[1024];
- MPIU_Snprintf( namebuf, sizeof(namebuf),
- "MPICH_INTERFACE_HOSTNAME_R%d", myRank );
- ifname_string = getenv( namebuf );
- if (dbg_ifname && ifname_string) {
- fprintf( stdout, "Found interface name %s from %s\n",
- ifname_string, namebuf );
- fflush( stdout );
- }
- }
- else if (dbg_ifname) {
- fprintf( stdout,
- "Found interface name %s from MPICH_INTERFACE_HOSTNAME\n",
- ifname_string );
- fflush( stdout );
- }
-
- if (!ifname_string) {
- int len;
-
- /* If we have nothing, then use the host name */
- mpi_errno = MPID_Get_processor_name(ifname, maxIfname, &len );
- ifname_string = ifname;
-
- /* If we didn't find a specific name, then try to get an IP address
- directly from the available interfaces, if that is supported on
- this platform. Otherwise, we'll drop into the next step that uses
- the ifname */
- mpi_errno = GetIPInterface( ifaddr, &ifaddrFound );
- }
- else {
- /* Copy this name into the output name */
- MPIU_Strncpy( ifname, ifname_string, maxIfname );
- }
-
- /* If we don't have an IP address, try to get it from the name */
- if (!ifaddrFound) {
- struct hostent *info;
- info = gethostbyname( ifname_string );
- if (info && info->h_addr_list) {
- /* Use the primary address */
- ifaddr->len = info->h_length;
- ifaddr->type = info->h_addrtype;
- if (ifaddr->len > sizeof(ifaddr->ifaddr)) {
- /* If the address won't fit in the field, reset to
- no address */
- ifaddr->len = 0;
- ifaddr->type = -1;
- }
- else
- MPIU_Memcpy( ifaddr->ifaddr, info->h_addr_list[0], ifaddr->len );
- }
- }
-
- return 0;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_get_business_card
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
-{
- int mpi_errno = MPI_SUCCESS;
- int str_errno = MPIU_STR_SUCCESS;
- MPIDU_Sock_ifaddr_t ifaddr;
- char ifname[MAX_HOST_DESCRIPTION_LEN];
- int ret;
- struct sockaddr_in sock_id;
- socklen_t len;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_GET_BUSINESS_CARD);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_GET_BUSINESS_CARD);
-
- mpi_errno = GetSockInterfaceAddr(my_rank, ifname, sizeof(ifname), &ifaddr);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-
- str_errno = MPIU_Str_add_string_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_HOST_DESCRIPTION_KEY, ifname);
- if (str_errno) {
- MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
- }
-
- len = sizeof(sock_id);
- ret = getsockname (MPID_nem_newtcp_module_g_lstn_sc.fd, (struct sockaddr *)&sock_id, &len);
- MPIU_ERR_CHKANDJUMP1 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**getsockname", "**getsockname %s", strerror (errno));
-
- str_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_PORT_KEY, sock_id.sin_port);
- if (str_errno) {
- MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
- }
-
- {
- char ifname[256];
- unsigned char *p;
- if (ifaddr.len > 0 && ifaddr.type == AF_INET)
- {
- p = (unsigned char *)(ifaddr.ifaddr);
- MPIU_Snprintf( ifname, sizeof(ifname), "%u.%u.%u.%u", p[0], p[1], p[2], p[3] );
- MPIU_DBG_MSG_S(CH3_CONNECT,VERBOSE,"ifname = %s",ifname );
- str_errno = MPIU_Str_add_string_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_IFNAME_KEY, ifname);
- if (str_errno) {
- MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
- MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
- }
- }
- }
-
- /* printf("MPID_nem_newtcp_module_get_business_card. port=%d\n", sock_id.sin_port); */
-
- fn_exit:
-/* fprintf(stdout, "MPID_nem_newtcp_module_get_business_card Exit, mpi_errno=%d\n", mpi_errno); fflush(stdout); */
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_GET_BUSINESS_CARD);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_connect_to_root
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
-{
- int mpi_errno = MPI_SUCCESS;
- struct in_addr addr;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT_TO_ROOT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT_TO_ROOT);
-
- /* vc is already allocated before reaching this point */
-
- mpi_errno = MPID_nem_newtcp_module_get_addr_port_from_bc(business_card, &addr, &(VC_FIELD(new_vc, sock_id).sin_port));
- VC_FIELD(new_vc, sock_id).sin_addr.s_addr = addr.s_addr;
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPIDI_GetTagFromPort(business_card, &new_vc->port_name_tag);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- MPID_nem_newtcp_module_connect(new_vc);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONNECT_TO_ROOT);
- return mpi_errno;
-
- fn_fail:
- goto fn_exit;}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_vc_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_vc_init (MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_VC_INIT);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_VC_INIT);
-
- VC_FIELD(vc, state) = MPID_NEM_NEWTCP_MODULE_VC_STATE_DISCONNECTED;
-
- vc->sendNoncontig_fn = MPID_nem_newtcp_SendNoncontig;
- vc_ch->iStartContigMsg = MPID_nem_newtcp_iStartContigMsg;
- vc_ch->iSendContig = MPID_nem_newtcp_iSendContig;
- memset(&VC_FIELD(vc, sock_id), 0, sizeof(VC_FIELD(vc, sock_id)));
- VC_FIELD(vc, sock_id).sin_family = AF_INET;
-
- vc_ch->next = NULL;
- vc_ch->prev = NULL;
- VC_FIELD(vc, sc) = NULL;
- VC_FIELD(vc, sc_ref_count) = 0;
- VC_FIELD(vc, send_queue).head = VC_FIELD(vc, send_queue).tail = NULL;
-
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_VC_INIT);
- return mpi_errno;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_vc_destroy
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_vc_destroy(MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
-
- /* free any resources associated with this VC here */
- MPIU_UNREFERENCED_ARG(vc);
-
- return mpi_errno;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_get_addr_port_from_bc
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_get_addr_port_from_bc (const char *business_card, struct in_addr *addr, in_port_t *port)
-{
- int mpi_errno = MPI_SUCCESS;
- int ret;
- /* char desc_str[256]; */
- char ifname[256];
- MPIDI_STATE_DECL(MPID_STATE_NEWTCP_MODULE_GET_ADDR_PORT_FROM_BC);
-
- MPIDI_FUNC_ENTER(MPID_STATE_NEWTCP_MODULE_GET_ADDR_PORT_FROM_BC);
-
- /* desc_str is only used for debugging
- ret = MPIU_Str_get_string_arg (business_card, MPIDI_CH3I_HOST_DESCRIPTION_KEY, desc_str, sizeof(desc_str));
- MPIU_ERR_CHKANDJUMP (ret != MPIU_STR_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
- */
-
- ret = MPIU_Str_get_int_arg (business_card, MPIDI_CH3I_PORT_KEY, (int *)port);
- MPIU_ERR_CHKANDJUMP (ret != MPIU_STR_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**argstr_missingport");
- /* fprintf(stdout, "get_addr_port_from_bc buscard=%s desc=%s port=%d\n",business_card, desc_str, *port); fflush(stdout); */
-
- ret = MPIU_Str_get_string_arg(business_card, MPIDI_CH3I_IFNAME_KEY, ifname, sizeof(ifname));
- MPIU_ERR_CHKANDJUMP (ret != MPIU_STR_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**argstr_missingifname");
-
- /*
- ret = inet_pton (AF_INET, (const char *)ifname, addr);
- MPIU_ERR_CHKANDJUMP(ret == 0, mpi_errno,MPI_ERR_OTHER,"**ifnameinvalid");
- MPIU_ERR_CHKANDJUMP(ret < 0, mpi_errno, MPI_ERR_OTHER, "**afinetinvalid");
- */
- mpi_errno = MPIU_SOCKW_Inet_addr(ifname, &(addr->s_addr));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_NEWTCP_MODULE_GET_ADDR_PORT_FROM_BC);
- return mpi_errno;
- fn_fail:
-/* fprintf(stdout, "failure. mpi_errno = %d\n", mpi_errno); */
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-/* MPID_nem_newtcp_module_bind -- if MPICH_PORT_RANGE is set, this
- binds the socket to an available port number in the range.
- Otherwise, it binds it to any addr and any port */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_bind
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_bind (MPIU_SOCKW_Sockfd_t sockfd)
-{
- int mpi_errno = MPI_SUCCESS;
- struct sockaddr_in sin;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_BIND);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_BIND);
-
- MPIU_ERR_CHKANDJUMP(MPIR_CVAR_CH3_PORT_RANGE.low < 0 || MPIR_CVAR_CH3_PORT_RANGE.low > MPIR_CVAR_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
-
- memset((void *)&sin, 0, sizeof(sin));
- sin.sin_family = AF_INET;
- sin.sin_addr.s_addr = htonl(INADDR_ANY);
-
- mpi_errno = MPIU_SOCKW_Bind_port_range(sockfd, &sin, MPIR_CVAR_CH3_PORT_RANGE.low, MPIR_CVAR_CH3_PORT_RANGE.high);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
-/* if (ret == 0) */
-/* fprintf(stdout, "sockfd=%d port=%d bound\n", sockfd, port); */
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_BIND);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_vc_terminate
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_vc_terminate (MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_NEM_NEWTCP_MODULE_VC_TERMINATE);
-
- MPIDI_FUNC_ENTER(MPID_NEM_NEWTCP_MODULE_VC_TERMINATE);
-
- mpi_errno = MPID_nem_newtcp_module_cleanup(vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_NEM_NEWTCP_MODULE_VC_TERMINATE);
- return mpi_errno;
- fn_fail:
- MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno));
- goto fn_exit;
-}
-
-
-/* These includes are here because they're used just for getting the interface
- * names
- */
-
-
-#include <sys/types.h>
-
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_NET_IF_H
-#include <net/if.h>
-#endif
-#ifdef HAVE_SYS_SOCKIO_H
-/* Needed for SIOCGIFCONF */
-#include <sys/sockio.h>
-#endif
-
-#if defined(SIOCGIFCONF) && defined(HAVE_STRUCT_IFCONF)
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <sys/ioctl.h>
-#include <errno.h>
-
-/* We can only access the interfaces if we have a number of features.
- Test for these, otherwise define this routine to return false in the
- "found" variable */
-
-#define NUM_IFREQS 10
-
-static int GetIPInterface( MPIDU_Sock_ifaddr_t *ifaddr, int *found )
-{
- char *buf_ptr, *ptr;
- int buf_len, buf_len_prev;
- int fd;
- MPIDU_Sock_ifaddr_t myifaddr;
- int nfound = 0, foundLocalhost = 0;
- /* We predefine the LSB and MSB localhost addresses */
- unsigned int localhost = 0x0100007f;
-#ifdef WORDS_BIGENDIAN
- unsigned int MSBlocalhost = 0x7f000001;
-#endif
-
- fd = socket(AF_INET, SOCK_DGRAM, 0);
- if (fd < 0) {
- fprintf( stderr, "Unable to open an AF_INET socket\n" );
- return 1;
- }
-
- /* Use MSB localhost if necessary */
-#ifdef WORDS_BIGENDIAN
- localhost = MSBlocalhost;
-#endif
-
-
- /*
- * Obtain the interface information from the operating system
- *
- * Note: much of this code is borrowed from W. Richard Stevens' book
- * entitled "UNIX Network Programming", Volume 1, Second Edition. See
- * section 16.6 for details.
- */
- buf_len = NUM_IFREQS * sizeof(struct ifreq);
- buf_len_prev = 0;
-
- for(;;)
- {
- struct ifconf ifconf;
- int rc;
-
- buf_ptr = (char *) MPIU_Malloc(buf_len);
- if (buf_ptr == NULL) {
- fprintf( stderr, "Unable to allocate %d bytes\n", buf_len );
- return 1;
- }
-
- ifconf.ifc_buf = buf_ptr;
- ifconf.ifc_len = buf_len;
-
- rc = ioctl(fd, SIOCGIFCONF, &ifconf);
- if (rc < 0) {
- if (errno != EINVAL || buf_len_prev != 0) {
- fprintf( stderr, "Error from ioctl = %d\n", errno );
- perror(" Error is: ");
- return 1;
- }
- }
- else {
- if (ifconf.ifc_len == buf_len_prev) {
- buf_len = ifconf.ifc_len;
- break;
- }
-
- buf_len_prev = ifconf.ifc_len;
- }
-
- MPIU_Free(buf_ptr);
- buf_len += NUM_IFREQS * sizeof(struct ifreq);
- }
-
- /*
- * Now that we've got the interface information, we need to run through
- * the interfaces and check out the ip addresses. If we find a
- * unique, non-lcoal host (127.0.0.1) address, return that, otherwise
- * return nothing.
- */
- ptr = buf_ptr;
-
- while(ptr < buf_ptr + buf_len) {
- struct ifreq * ifreq;
-
- ifreq = (struct ifreq *) ptr;
-
- if (dbg_ifname) {
- fprintf( stdout, "%10s\t", ifreq->ifr_name );
- }
-
- if (ifreq->ifr_addr.sa_family == AF_INET) {
- struct in_addr addr;
-
- addr = ((struct sockaddr_in *) &(ifreq->ifr_addr))->sin_addr;
- if (dbg_ifname) {
- fprintf( stdout, "IPv4 address = %08x (%s)\n", addr.s_addr,
- inet_ntoa( addr ) );
- }
-
- if (addr.s_addr == localhost && dbg_ifname) {
- fprintf( stdout, "Found local host\n" );
- }
- /* Save localhost if we find it. Let any new interface
- overwrite localhost. However, if we find more than
- one non-localhost interface, then we'll choose none for the
- interfaces */
- if (addr.s_addr == localhost) {
- foundLocalhost = 1;
- if (nfound == 0) {
- myifaddr.type = AF_INET;
- myifaddr.len = 4;
- MPIU_Memcpy( myifaddr.ifaddr, &addr.s_addr, 4 );
- }
- }
- else {
- nfound++;
- myifaddr.type = AF_INET;
- myifaddr.len = 4;
- MPIU_Memcpy( myifaddr.ifaddr, &addr.s_addr, 4 );
- }
- }
- else {
- if (dbg_ifname) {
- fprintf( stdout, "\n" );
- }
- }
-
- /*
- * Increment pointer to the next ifreq; some adjustment may be
- * required if the address is an IPv6 address
- */
- /* This is needed for MAX OSX */
-#ifdef _SIZEOF_ADDR_IFREQ
- ptr += _SIZEOF_ADDR_IFREQ(*ifreq);
-#else
- ptr += sizeof(struct ifreq);
-
-# if defined(AF_INET6)
- {
- if (ifreq->ifr_addr.sa_family == AF_INET6)
- {
- ptr += sizeof(struct sockaddr_in6) - sizeof(struct sockaddr);
- }
- }
-# endif
-#endif
- }
-
- MPIU_Free(buf_ptr);
- close(fd);
-
- /* If we found a unique address, use that */
- if (nfound == 1 || (nfound == 0 && foundLocalhost == 1)) {
- *ifaddr = myifaddr;
- *found = 1;
- }
- else {
- *found = 0;
- }
-
- return 0;
-}
-
-#else /* things needed to find the interfaces */
-
-/* In this case, just return false for interfaces found */
-static int GetIPInterface( MPIDU_Sock_ifaddr_t *ifaddr, int *found )
-{
- *found = 0;
- MPIU_UNREFERENCED_ARG(ifaddr);
- return 0;
-}
-#endif
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_lmt.c
deleted file mode 100644
index 8a26db0..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_lmt.c
+++ /dev/null
@@ -1,565 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-extern int h_errno;
-
-static int create_s_cookie (int data_sz, char **cookie, int *len);
-static int read_s_cookie (MPID_IOV cookie, int *data_sz);
-static int create_r_cookie (char *hostname, int port, int data_sz, char **cookie, int *len);
-static int read_r_cookie (MPID_IOV cookie, char **hostname, int *port, int *data_sz);
-static void free_cookie (void *c);
-static int set_sockopts (int fd);
-
-//#define TESTING_CHUNKING
-#ifdef TESTING_CHUNKING
-#define CHUNK 6299651//(32*1024)
-#endif
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_pre_send
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_pre_send (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV *cookie)
-{
- int mpi_errno = MPI_SUCCESS;
- int len;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype * dt_ptr;
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
-
- MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- mpi_errno = create_s_cookie (data_sz, &vc_ch->net.tcp.lmt_cookie, &len);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- cookie->MPID_IOV_BUF = vc_ch->net.tcp.lmt_cookie;
- cookie->MPID_IOV_LEN = len;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_SEND);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_pre_recv
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_pre_recv (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV s_cookie, MPID_IOV *r_cookie, int *send_cts)
-{
- int mpi_errno = MPI_SUCCESS;
- int ret;
- unsigned int len;
- struct sockaddr_in saddr;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype * dt_ptr;
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
-
- mpi_errno = read_s_cookie (s_cookie, &vc_ch->net.tcp.lmt_s_len);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- memset (&saddr, sizeof(saddr), 0);
-
- if (!vc_ch->net.tcp.lmt_connected)
- {
- vc_ch->net.tcp.lmt_desc = socket (AF_INET, SOCK_STREAM, 0);
- MPIU_ERR_CHKANDJUMP2 (vc_ch->net.tcp.lmt_desc == -1, mpi_errno, MPI_ERR_OTHER, "**sock_create", "**sock_create %s %d", strerror (errno), errno);
-
- // ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
- // MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- saddr.sin_family = AF_INET;
- saddr.sin_addr.s_addr = htonl (INADDR_ANY);
- saddr.sin_port = htons (0);
-
- ret = bind (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, sizeof (saddr));
- MPIU_ERR_CHKANDJUMP3 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**sock|poll|bind", "**sock|poll|bind %d %d %s", ntohs (saddr.sin_port), errno, strerror (errno));
-
- len = sizeof (saddr);
- ret = getsockname (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, &len);
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- set_sockopts (vc_ch->net.tcp.lmt_desc);
-
- ret = listen (vc_ch->net.tcp.lmt_desc, SOMAXCONN);
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**listen", "**listen %s %d", errno, strerror (errno));
- }
-
- MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- mpi_errno = create_r_cookie (MPID_nem_hostname, ntohs (saddr.sin_port), data_sz, &vc_ch->net.tcp.lmt_cookie, &len);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- r_cookie->MPID_IOV_BUF = vc_ch->net.tcp.lmt_cookie;
- r_cookie->MPID_IOV_LEN = len;
-
- *send_cts = 1;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_start_send
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_start_send (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV r_cookie)
-{
- int mpi_errno = MPI_SUCCESS;
- int ret;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype * dt_ptr;
- MPIDI_msg_sz_t last;
- int nb;
- int s_len = 0;
- int r_len;
- int r_port;
- char *r_hostname;
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
-
- mpi_errno = read_r_cookie (r_cookie, &r_hostname, &r_port, &r_len);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- free_cookie (vc_ch->net.tcp.lmt_cookie);
-
- if (!vc_ch->net.tcp.lmt_connected)
- {
- struct sockaddr_in saddr;
- struct hostent *hp;
-
- vc_ch->net.tcp.lmt_desc = socket (AF_INET, SOCK_STREAM, 0);
- MPIU_ERR_CHKANDJUMP2 (vc_ch->net.tcp.lmt_desc == -1, mpi_errno, MPI_ERR_OTHER, "**sock_create", "**sock_create %s %d", strerror (errno), errno);
-
- // ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
- // MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- hp = gethostbyname (r_hostname);
- MPIU_ERR_CHKANDJUMP2 (hp == NULL, mpi_errno, MPI_ERR_OTHER, "**gethostbyname", "**gethostbyname %s %d", hstrerror (h_errno), h_errno);
-
- memset (&saddr, sizeof(saddr), 0);
- saddr.sin_family = AF_INET;
- saddr.sin_port = htons (r_port);
- MPIU_Memcpy (&saddr.sin_addr, hp->h_addr, hp->h_length);
-
- set_sockopts (vc_ch->net.tcp.lmt_desc);
-
- ret = connect (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, sizeof(saddr));
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- vc_ch->net.tcp.lmt_connected = 1;
- }
-
- MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- if (r_len < data_sz)
- {
- /* message will be truncated */
- s_len = data_sz;
- data_sz = r_len;
- req->status.MPI_ERROR = MPIU_ERR_SET2 (mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", s_len, r_len);
- }
-
- MPID_Segment_init (req->dev.user_buf, req->dev.user_count, req->dev.datatype, &req->dev.segment, 0);
- req->dev.segment_first = 0;
- req->dev.segment_size = data_sz;
- req->dev.iov_count = MPID_IOV_LIMIT;
- req->dev.iov_offset = 0;
- last = data_sz;
-
- do
- {
- int iov_offset;
- int left_to_send;
- MPID_Segment_pack_vector (&req->dev.segment, req->dev.segment_first, &last, req->dev.iov, &req->dev.iov_count);
-
- left_to_send = last - req->dev.segment_first;
- iov_offset = 0;
-
-#ifdef TESTING_CHUNKING
- {
- char *buf = req->dev.iov[0].MPID_IOV_BUF;
- int l;
- while (left_to_send)
- {
- if (left_to_send > CHUNK)
- l = CHUNK;
- else
- l = left_to_send;
-
- do
- nb = write (vc_ch->net.tcp.lmt_desc, buf, l);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
-
- left_to_send -= nb;
- buf += nb;
- }
-
- MPIDI_CH3U_Request_complete (req);
- goto fn_exit;
- }
-#endif
-
- do
- nb = writev (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
-
- left_to_send -= nb;
- while (left_to_send)
- { /* send rest of iov */
- while (nb >= req->dev.iov[iov_offset].MPID_IOV_LEN)
- { /* update iov to reflect sent bytes */
- nb -= req->dev.iov[iov_offset].MPID_IOV_LEN;
- ++iov_offset;
- }
- req->dev.iov[iov_offset].MPID_IOV_BUF = (char *)req->dev.iov[iov_offset].MPID_IOV_BUF + nb;
- req->dev.iov[iov_offset].MPID_IOV_LEN -= nb;
-
- do
- nb = writev (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
- left_to_send -= nb;
- }
- }
- while (last < data_sz);
-
- MPIDI_CH3U_Request_complete (req);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_start_recv
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_start_recv (MPIDI_VC_t *vc, MPID_Request *req)
-{
- int mpi_errno = MPI_SUCCESS;
- int ret;
- MPIDI_msg_sz_t data_sz;
- int dt_contig;
- MPI_Aint dt_true_lb;
- MPID_Datatype * dt_ptr;
- MPIDI_msg_sz_t last;
- int nb;
- int r_len;
- MPIDI_CH3I_VC *vc_ch = &vc->ch;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
-
- free_cookie (vc_ch->net.tcp.lmt_cookie);
-
- if (!vc_ch->net.tcp.lmt_connected)
- {
- int len;
- struct sockaddr_in saddr;
- int connfd;
-
- len = sizeof (saddr);
- connfd = accept (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, &len);
- MPIU_ERR_CHKANDJUMP2 (connfd == -1, mpi_errno, MPI_ERR_OTHER, "**sock|poll|accept", "**sock|poll|accept %d %s", errno, strerror (errno));
-
- /* close listen fd */
- do
- ret = close (vc_ch->net.tcp.lmt_desc);
- while (ret == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**closesocket", "**closesocket %s %d", strerror (errno), errno);
-
- /* set lmt_desc to new connected fd */
- vc_ch->net.tcp.lmt_desc = connfd;
- vc_ch->net.tcp.lmt_connected = 1;
-
- // ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
- // MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
- }
-
- MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
-
- if (data_sz > vc_ch->net.tcp.lmt_s_len)
- {
- data_sz = vc_ch->net.tcp.lmt_s_len;
- }
- else if (data_sz < vc_ch->net.tcp.lmt_s_len)
- {
- /* message will be truncated */
- r_len = data_sz;
- req->status.MPI_ERROR = MPIU_ERR_SET2 (mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", vc_ch->net.tcp.lmt_s_len, r_len);
- }
-
- MPID_Segment_init (req->dev.user_buf, req->dev.user_count, req->dev.datatype, &req->dev.segment, 0);
- req->dev.segment_first = 0;
- req->dev.segment_size = data_sz;
- req->dev.iov_count = MPID_IOV_LIMIT;
- req->dev.iov_offset = 0;
- last = data_sz;
-
- do
- {
- int iov_offset;
- int left_to_recv;
-
- MPID_Segment_unpack_vector (&req->dev.segment, req->dev.segment_first, &last, req->dev.iov, &req->dev.iov_count);
-
- left_to_recv = last - req->dev.segment_first;
- iov_offset = 0;
-
-#ifdef TESTING_CHUNKING
- {
- char *buf = req->dev.iov[0].MPID_IOV_BUF;
- int l;
- while (left_to_recv)
- {
- if (left_to_recv > CHUNK)
- l = CHUNK;
- else
- l = left_to_recv;
-
- do
- nb = read (vc_ch->net.tcp.lmt_desc, buf, l);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
-
- left_to_recv -= nb;
- buf += nb;
- }
- MPIDI_CH3U_Request_complete (req);
- goto fn_exit;
- }
-#endif
-
- do
- nb = readv (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP2 (nb == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
- MPIU_ERR_CHKANDJUMP (nb == 0, mpi_errno, MPI_ERR_OTHER, "**fail");
-
- left_to_recv -= nb;
- while (left_to_recv)
- { /* recv rest of iov */
- while (nb >= req->dev.iov[iov_offset].MPID_IOV_LEN)
- { /* update iov to reflect sent bytes */
- nb -= req->dev.iov[iov_offset].MPID_IOV_LEN;
- ++iov_offset;
- }
- req->dev.iov[iov_offset].MPID_IOV_BUF = (char *)req->dev.iov[iov_offset].MPID_IOV_BUF + nb;
- req->dev.iov[iov_offset].MPID_IOV_LEN -= nb;
-
- do
- nb = readv (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
- while (nb == -1 && errno == EINTR);
- MPIU_ERR_CHKANDJUMP2 (nb == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
- MPIU_ERR_CHKANDJUMP (nb == 0, mpi_errno, MPI_ERR_OTHER, "**fail");
- left_to_recv -= nb;
- }
- }
- while (last < data_sz);
-
- MPIDI_CH3U_Request_complete (req);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_post_send
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_post_send (MPIDI_VC_t *vc, MPID_Request *req)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_SEND);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_SEND);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_SEND);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_tcp_module_lmt_post_recv
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_tcp_module_lmt_post_recv (MPIDI_VC_t *vc, MPID_Request *req)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME create_s_cookie
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int create_s_cookie (int data_sz, char **cookie, int *len)
-{
- int mpi_errno = MPI_SUCCESS;
- int *int_cookie;
-
- int_cookie = MPIU_Malloc (sizeof (data_sz));
- MPIU_ERR_CHKANDJUMP (int_cookie == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem");
- *int_cookie = data_sz;
-
- *cookie = (char *)int_cookie;
- *len = sizeof (data_sz);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME read_s_cookie
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int read_s_cookie (MPID_IOV cookie, int *data_sz)
-{
- int mpi_errno = MPI_SUCCESS;
-
- MPIU_ERR_CHKANDJUMP (cookie.MPID_IOV_LEN != sizeof (data_sz), mpi_errno, MPI_ERR_OTHER, "**fail");
-
- *data_sz = *(int *)cookie.MPID_IOV_BUF;
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-typedef struct r_cookie
-{
- int port;
- int data_sz;
- char hostname[1];
-} r_cookie_t;
-
-#undef FUNCNAME
-#define FUNCNAME create_r_cookie
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int create_r_cookie (char *hostname, int port, int data_sz, char **cookie, int *len)
-{
- int mpi_errno = MPI_SUCCESS;
- int hostname_len;
- int cookie_len;
- r_cookie_t *c;
-
- hostname_len = strnlen (hostname, MAX_HOSTNAME_LEN) + 1;
-
- cookie_len = sizeof (r_cookie_t) - 1 + hostname_len;
-
- c = MPIU_Malloc (cookie_len);
- MPIU_ERR_CHKANDJUMP (c == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem");
-
- c->port = port;
- c->data_sz = data_sz;
- MPIU_Strncpy (c->hostname, hostname, hostname_len);
-
- *cookie = (char *)c;
- *len = sizeof (r_cookie_t) - 1 + hostname_len;
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-/* read_r_cookie - extracts hostname, port and data size from a recv cookie
- data pointed to by hostname is valid only as long as the packet containing the cookie is valid
- */
-#undef FUNCNAME
-#define FUNCNAME read_r_cookie
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int read_r_cookie (MPID_IOV cookie, char **hostname, int *port, int *data_sz)
-{
- int mpi_errno = MPI_SUCCESS;
- r_cookie_t *c;
-
- MPIU_ERR_CHKANDJUMP (cookie.MPID_IOV_LEN < sizeof (r_cookie_t), mpi_errno, MPI_ERR_OTHER, "**fail");
-
- c = (r_cookie_t *)cookie.MPID_IOV_BUF;
-
- *hostname = c->hostname;
- *port = c->port;
- *data_sz = c->data_sz;
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-static void free_cookie (void *c)
-{
- MPIU_Free (c);
-}
-
-#undef FUNCNAME
-#define FUNCNAME set_sockopts
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int set_sockopts (int fd)
-{
- int mpi_errno = MPI_SUCCESS;
- int option;
- int ret;
-
- option = 0;
- ret = setsockopt (fd, IPPROTO_TCP, TCP_NODELAY, &option, sizeof(int));
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- option = 128*1024;
- setsockopt (fd, SOL_SOCKET, SO_RCVBUF, &option, sizeof(int));
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
- setsockopt (fd, SOL_SOCKET, SO_SNDBUF, &option, sizeof(int));
- MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_poll.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_poll.c
deleted file mode 100644
index 1b7978e..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_poll.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-#ifdef HAVE_ERRNO_H
- #include <errno.h>
-#endif
-
-/* FIXME : Get rid of these dummy funcs */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_poll_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_poll_init()
-{
- return MPI_SUCCESS;
-}
-
-
-int MPID_nem_newtcp_module_poll_finalize()
-{
- return MPI_SUCCESS;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_poll
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_poll(int in_blocking_poll)
-{
- int mpi_errno = MPI_SUCCESS;
-
- mpi_errno = MPID_nem_newtcp_module_connpoll(in_blocking_poll);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_queue.h b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_queue.h
deleted file mode 100644
index fbf097a..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_queue.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#ifndef TCP_MODULE_QUEUE_H
-#define TCP_MODULE_QUEUE_H
-
-/* Generic queue macros -- "next_field" should be set to the name of
- the next pointer field in the element (e.g., "ch.newtcp_sendq_next") */
-
-#define PRINT_QUEUE(qp, next_field) do { \
- } while(0)
-
-
-#define GENERIC_Q_EMPTY(q) ((q).head == NULL)
-
-#define GENERIC_Q_HEAD(q) ((q).head)
-
-#define GENERIC_Q_ENQUEUE_EMPTY(qp, ep, next_field) do { \
- MPIU_Assert (GENERIC_Q_EMPTY (*(qp))); \
- (qp)->head = (qp)->tail = ep; \
- (ep)->next_field = NULL; \
- PRINT_QUEUE (qp, next_field); \
- } while (0)
-
-#define GENERIC_Q_ENQUEUE(qp, ep, next_field) do { \
- if (GENERIC_Q_EMPTY (*(qp))) \
- GENERIC_Q_ENQUEUE_EMPTY (qp, ep, next_field); \
- else \
- { \
- (qp)->tail->next_field = (qp)->tail = ep; \
- (ep)->next_field = NULL; \
- } \
- PRINT_QUEUE (qp, next_field); \
- } while (0)
-
-/* the _MULTIPLE routines assume that ep0 is the head and ep1 is the
- tail of a linked list of elements. The list is inserted on the end
- of the queue. */
-#define GENERIC_Q_ENQUEUE_EMPTY_MULTIPLE(qp, ep0, ep1, next_field) do { \
- MPIU_Assert (GENERIC_Q_EMPTY (*(qp))); \
- (qp)->head = ep0; \
- (qp)->tail = ep1; \
- (ep1)->next_field = NULL; \
- } while (0)
-
-#define GENERIC_Q_ENQUEUE_MULTIPLE(qp, ep0, ep1, next_field) do { \
- if (GENERIC_Q_EMPTY (*(qp))) \
- GENERIC_Q_ENQUEUE_EMPTY_MULTIPLE (qp, ep0, ep1, next_field); \
- else \
- { \
- (qp)->tail->next_field = ep0; \
- (qp)->tail = ep1; \
- (ep1)->next_field = NULL; \
- } \
- } while (0)
-
-
-#define GENERIC_Q_DEQUEUE(qp, epp, next_field) do { \
- MPIU_Assert (!GENERIC_Q_EMPTY (*(qp))); \
- *(epp) = (qp)->head; \
- (qp)->head = (*(epp))->next_field; \
- if ((qp)->head == NULL) \
- (qp)->tail = NULL; \
- } while (0)
-
-/* remove the elements from the top of the queue starting with ep0 through ep1 */
-#define GENERIC_Q_REMOVE_ELEMENTS(qp, ep0, ep1, next_field) do { \
- MPIU_Assert (GENERIC_Q_HEAD (*(qp)) == (ep0)); \
- (qp)->head = (ep1)->next_field; \
- if ((qp)->head == NULL) \
- (qp)->tail = NULL; \
- } while (0)
-
-
-
-/* Generic list macros */
-#define GENERIC_L_EMPTY(q) ((q).head == NULL)
-
-#define GENERIC_L_HEAD(q) ((q).head)
-
-#define GENERIC_L_ADD_EMPTY(qp, ep, next_field, prev_field) do { \
- MPIU_Assert (GENERIC_L_EMPTY (*(qp))); \
- (qp)->head = ep; \
- (ep)->next_field = (ep)->prev_field = NULL; \
- } while (0)
-
-#define GENERIC_L_ADD(qp, ep, next_field, prev_field) do { \
- if (GENERIC_L_EMPTY (*(qp))) \
- GENERIC_L_ADD_EMPTY (qp, ep, next_field, prev_field); \
- else \
- { \
- (ep)->prev_field = NULL; \
- (ep)->next_field = (qp)->head; \
- (qp)->head->prev_field = ep; \
- (qp)->head = ep; \
- } \
- } while (0)
-
-#define GENERIC_L_REMOVE(qp, ep, next_field, prev_field) do { \
- MPIU_Assert (!GENERIC_L_EMPTY (*(qp))); \
- if ((ep)->prev_field) \
- ((ep)->prev_field)->next_field = (ep)->next_field; \
- else \
- (qp)->head = (ep)->next_field; \
- if ((ep)->next_field) \
- ((ep)->next_field)->prev_field = (ep)->prev_field; \
- } while (0)
-
-
-/* Generic stack macros */
-#define GENERIC_S_EMPTY(s) ((s).top == NULL)
-
-#define GENERIC_S_TOP(s) ((s).top)
-
-#define GENERIC_S_PUSH(sp, ep, next_field) do { \
- (ep)->next_field = (sp)->top; \
- (sp)->top = ep; \
- } while (0)
-
-/* PUSH_MULTIPLE pushes a linked list of elements onto the stack. It
- assumes that ep0 is the head of the linked list and ep1 is at the tail */
-#define GENERIC_S_PUSH_MULTIPLE(sp, ep0, ep1, next_field) do { \
- (ep1)->next_field = (sp)->top; \
- (sp)->top = ep0; \
- } while (0)
-
-#define GENERIC_S_POP(sp, ep, next_field) do { \
- *(ep) = (sp)->top; \
- (sp)->top = (*(ep))->next_field; \
- } while (0)
-#endif /* TCP_MODULE_QUEUE_H */
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_send.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_send.c
deleted file mode 100644
index 1cc7c18..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_send.c
+++ /dev/null
@@ -1,621 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-
-#define NUM_PREALLOC_SENDQ 10
-#define MAX_SEND_IOV 10
-
-#define SENDQ_EMPTY(q) GENERIC_Q_EMPTY (q)
-#define SENDQ_HEAD(q) GENERIC_Q_HEAD (q)
-#define SENDQ_ENQUEUE(qp, ep) GENERIC_Q_ENQUEUE (qp, ep, dev.next)
-#define SENDQ_DEQUEUE(qp, ep) GENERIC_Q_DEQUEUE (qp, ep, dev.next)
-
-
-typedef struct MPID_nem_newtcp_module_send_q_element
-{
- struct MPID_nem_newtcp_module_send_q_element *next;
- size_t len; /* number of bytes left to send */
- char *start; /* pointer to next byte to send */
- MPID_nem_cell_ptr_t cell;
- /* char buf[MPID_NEM_MAX_PACKET_LEN];*/ /* data to be sent */
-} MPID_nem_newtcp_module_send_q_element_t;
-
-static struct {MPID_nem_newtcp_module_send_q_element_t *top;} free_buffers = {0};
-
-#define ALLOC_Q_ELEMENT(e) do { \
- if (S_EMPTY (free_buffers)) \
- { \
- MPIU_CHKPMEM_MALLOC (*(e), MPID_nem_newtcp_module_send_q_element_t *, sizeof(MPID_nem_newtcp_module_send_q_element_t), \
- mpi_errno, "send queue element"); \
- } \
- else \
- { \
- S_POP (&free_buffers, e); \
- } \
- } while (0)
-
-/* FREE_Q_ELEMENTS() frees a list if elements starting at e0 through e1 */
-#define FREE_Q_ELEMENTS(e0, e1) S_PUSH_MULTIPLE (&free_buffers, e0, e1)
-#define FREE_Q_ELEMENT(e) S_PUSH (&free_buffers, e)
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_send_init
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_send_init()
-{
- int mpi_errno = MPI_SUCCESS;
- int i;
- MPIU_CHKPMEM_DECL (NUM_PREALLOC_SENDQ);
-
- /* preallocate sendq elements */
- for (i = 0; i < NUM_PREALLOC_SENDQ; ++i)
- {
- MPID_nem_newtcp_module_send_q_element_t *e;
-
- MPIU_CHKPMEM_MALLOC (e, MPID_nem_newtcp_module_send_q_element_t *,
- sizeof(MPID_nem_newtcp_module_send_q_element_t), mpi_errno, "send queue element");
- S_PUSH (&free_buffers, e);
- }
-
- MPIU_CHKPMEM_COMMIT();
- return mpi_errno;
- fn_fail:
- MPIU_CHKPMEM_REAP();
- return mpi_errno;
-}
-
-/* Handle the request at the head of vc's sendq */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_handle_sendq_head_req
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_handle_sendq_head_req(MPIDI_VC_t *vc, int *req_complete)
-{
- int (*req_handler)(MPIDI_VC_t *, MPID_Request *, int *);
- int mpi_errno = MPI_SUCCESS;
- MPID_Request *req = NULL;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_HANDLE_SENDQ_HEAD_REQ);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_HANDLE_SENDQ_HEAD_REQ);
-
- MPIU_Assert(req_complete != NULL);
-
- MPIU_Assert(!SENDQ_EMPTY(VC_FIELD(vc, send_queue)));
- req = SENDQ_HEAD(VC_FIELD(vc, send_queue));
- MPIU_Assert(req != NULL);
-
- req_handler = req->dev.OnDataAvail;
- if (!req_handler){
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(req);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- SENDQ_DEQUEUE(&VC_FIELD(vc, send_queue), &req);
- *req_complete = 1;
- }
- else{
- *req_complete = 0;
- mpi_errno = req_handler(vc, req, req_complete);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- if (*req_complete){
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- SENDQ_DEQUEUE(&VC_FIELD(vc, send_queue), &req);
- }
- }
- req->dev.iov_offset = 0;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_HANDLE_SENDQ_HEAD_REQ);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_send_queued
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_send_queued (MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPID_Request *sreq;
- MPIDI_msg_sz_t offset;
- MPID_IOV *iov;
- int complete;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_SEND_QUEUED);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_SEND_QUEUED);
-
- MPIU_Assert(vc != NULL);
-
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)))
- goto fn_exit;
-
- while (!SENDQ_EMPTY(VC_FIELD(vc, send_queue)))
- {
- sreq = SENDQ_HEAD(VC_FIELD(vc, send_queue));
-
- iov = &sreq->dev.iov[sreq->dev.iov_offset];
-
-/* printf("sreq = %p sreq->dev.iov = %p iov = %p\n", sreq, sreq->dev.iov, iov); */
-/* printf("iov[0].MPID_IOV_BUF = %p iov[0].MPID_IOV_LEN = %d iov_count = %d\n", iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN, sreq->dev.iov_count);//DARIUS */
-/* printf("&iov[0].MPID_IOV_LEN = %p sreq->dev.iov_offset = %d\n", &iov[0].MPID_IOV_LEN, sreq->dev.iov_offset);//DARIUS */
-
- MPIU_OSW_RETRYON_INTR((offset < 0),
- (mpi_errno = MPIU_SOCKW_Writev(VC_FIELD(vc, sc)->fd, iov, sreq->dev.iov_count, &offset)));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- if(offset < 0){
- offset = 0;
- }
-
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write %d", offset);
-
- complete = 1;
- for (iov = &sreq->dev.iov[sreq->dev.iov_offset]; iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov)
- {
- if (offset < iov->MPID_IOV_LEN)
- {
- iov->MPID_IOV_BUF = (char *)iov->MPID_IOV_BUF + offset;
- iov->MPID_IOV_LEN -= offset;
- /* iov_count should be equal to the number of iov's remaining */
- sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset);
- sreq->dev.iov_offset = iov - sreq->dev.iov;
- complete = 0;
- break;
- }
- offset -= iov->MPID_IOV_LEN;
- }
- if (!complete)
- {
- MPID_IOV *iov = &sreq->dev.iov[sreq->dev.iov_offset];
- int n_iov = sreq->dev.iov_count;
- /* writev couldn't write the entire iov,
- * post a write for the rem data and exit */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(VC_FIELD(vc, sc), iov, n_iov);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- break;
- }
- else
- {
- /* sent whole message */
- complete = 0;
- mpi_errno = MPID_nem_newtcp_module_handle_sendq_head_req(vc, &complete);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
- }
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_SEND_QUEUED);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_send_finalize
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_send_finalize()
-{
- int mpi_errno = MPI_SUCCESS;
-
- while (!S_EMPTY (free_buffers))
- {
- MPID_nem_newtcp_module_send_q_element_t *e;
- S_POP (&free_buffers, &e);
- MPIU_Free (e);
- }
- return mpi_errno;
-}
-
-/* MPID_nem_newtcp_module_conn_est -- this function is called when the
- * connection is finally extablished to send any pending sends
- * & post the first recv
- */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_conn_est
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_conn_est (MPIDI_VC_t *vc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONN_EST);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONN_EST);
-
- MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
-
- if (!SENDQ_EMPTY (VC_FIELD(vc, send_queue)))
- {
- mpi_errno = MPID_nem_newtcp_module_send_queued (vc);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
- }
-
- /* Post a read on the module scrap recv buf */
- mpi_errno = MPID_nem_newtcp_module_post_read_ex(VC_FIELD(vc, sc),
- VC_FIELD(vc, sc)->tmp_buf, VC_FIELD(vc, sc)->tmp_buf_len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_fail:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONN_EST);
- return mpi_errno;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_iStartContigMsg
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz,
- MPID_Request **sreq_ptr)
-{
- int mpi_errno = MPI_SUCCESS;
- MPID_Request * sreq = NULL;
- MPIDI_msg_sz_t offset = 0;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_ISTARTCONTIGMSG);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_ISTARTCONTIGMSG);
-
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
-
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "newtcp_iStartContigMsg");
- MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)hdr);
- if (MPID_nem_newtcp_module_vc_is_connected(vc))
- {
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)))
- {
- MPID_IOV iov[2];
-
- iov[0].MPID_IOV_BUF = hdr;
- iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
- iov[1].MPID_IOV_BUF = data;
- iov[1].MPID_IOV_LEN = data_sz;
-
- MPIU_OSW_RETRYON_INTR((offset < 0),
- (mpi_errno = MPIU_SOCKW_Writev(VC_FIELD(vc, sc)->fd, iov, 2, &offset)));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- if(offset < 0){
- offset = 0;
- }
-
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write %d", offset);
-
- if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz)
- {
- /* sent whole message */
- *sreq_ptr = NULL;
- goto fn_exit;
- }
- }
- }
- else
- {
- mpi_errno = MPID_nem_newtcp_module_connect(vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
-
- /* Data is partially written OR vc is still connecting...
- * create and enqueue request
- */
- MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, "enqueuing");
-
- /* create a request */
- sreq = MPID_Request_create();
- MPIU_Assert (sreq != NULL);
- MPIU_Object_set_ref (sreq, 2);
- sreq->kind = MPID_REQUEST_SEND;
-
- sreq->dev.OnDataAvail = 0;
- sreq->ch.vc = vc;
- sreq->dev.iov_offset = 0;
-
-/* printf("&sreq->dev.pending_pkt = %p sizeof(MPIDI_CH3_Pkt_t) = %d\n", &sreq->dev.pending_pkt, sizeof(MPIDI_CH3_Pkt_t));//DARIUS */
-/* printf("offset = %d\n", offset);//DARIUS */
-
- if (offset < sizeof(MPIDI_CH3_Pkt_t))
- {
- sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)hdr;
- sreq->dev.iov[0].MPID_IOV_BUF = (char *)&sreq->dev.pending_pkt + offset;
- sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset ;
- if (data_sz)
- {
- sreq->dev.iov[1].MPID_IOV_BUF = data;
- sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
- sreq->dev.iov_count = 2;
- }
- else
- sreq->dev.iov_count = 1;
- }
- else
- {
- sreq->dev.iov[0].MPID_IOV_BUF = (char *)data + (offset - sizeof(MPIDI_CH3_Pkt_t));
- sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
- sreq->dev.iov_count = 1;
- }
-
- MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0);
-
-/* printf("sreq = %p sreq->dev.iov = %p\n", sreq, sreq->dev.iov); */
-/* printf("sreq->dev.iov[0].MPID_IOV_BUF = %p\n", sreq->dev.iov[0].MPID_IOV_BUF);//DARIUS */
-/* printf("sreq->dev.iov[0].MPID_IOV_LEN = %d\n", sreq->dev.iov[0].MPID_IOV_LEN);//DARIUS */
-/* printf("&sreq->dev.iov[0].MPID_IOV_LEN = %p\n", &sreq->dev.iov[0].MPID_IOV_LEN);//DARIUS */
-
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)) && MPID_nem_newtcp_module_vc_is_connected(vc)){
- /* Data was partially written... Post a write for the rem data */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(VC_FIELD(vc, sc),
- sreq->dev.iov, sreq->dev.iov_count);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- SENDQ_ENQUEUE(&VC_FIELD(vc, send_queue), sreq);
-
- *sreq_ptr = sreq;
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_ISTARTCONTIGMSG);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_iSendContig
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
- void *data, MPIDI_msg_sz_t data_sz)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_msg_sz_t offset = 0;
- MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWTCP_ISENDCONTIGMSG);
-
- MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_ISENDCONTIGMSG);
-
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
-
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "newtcp_iSendContig");
-
- MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)hdr);
- if (MPID_nem_newtcp_module_vc_is_connected(vc))
- {
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)))
- {
- MPID_IOV iov[2];
- int iov_cnt;
-
- iov[0].MPID_IOV_BUF = hdr;
- iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
- if(data_sz > 0){
- iov[1].MPID_IOV_BUF = data;
- iov[1].MPID_IOV_LEN = data_sz;
- iov_cnt = 2;
- }
- else{
- iov_cnt = 1;
- }
-
- MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "writing hdr (sz = %d) and data (sz = %d)", sizeof(MPIDI_CH3_Pkt_t), data_sz));
-
- MPIU_OSW_RETRYON_INTR((offset < 0),
- (mpi_errno = MPIU_SOCKW_Writev(VC_FIELD(vc, sc)->fd, iov, iov_cnt, &offset)));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- if(offset < 0){
- offset = 0;
- }
-
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write %d", offset);
-
- if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz)
- {
- /* sent whole message */
- int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
-
- reqFn = sreq->dev.OnDataAvail;
- if (!reqFn)
- {
- MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIDI_CH3U_Request_complete(sreq);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- goto fn_exit;
- }
- else
- {
- int complete = 0;
-
- mpi_errno = reqFn(vc, sreq, &complete);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- if (complete)
- {
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- goto fn_exit;
- }
-
- /* not completed: more to send */
- goto enqueue_request;
- }
- }
- }
- /* If sendq is not empty we default to enqueueing the request */
- }
- else
- {
- mpi_errno = MPID_nem_newtcp_module_connect(vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
-
- /* save iov */
- if (offset < sizeof(MPIDI_CH3_Pkt_t))
- {
- sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)hdr;
- sreq->dev.iov[0].MPID_IOV_BUF = (char *)&sreq->dev.pending_pkt + offset;
- sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset;
- if (data_sz)
- {
- sreq->dev.iov[1].MPID_IOV_BUF = data;
- sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
- sreq->dev.iov_count = 2;
- }
- else
- sreq->dev.iov_count = 1;
- }
- else
- {
- sreq->dev.iov[0].MPID_IOV_BUF = (char *)data + (offset - sizeof(MPIDI_CH3_Pkt_t));
- sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
- sreq->dev.iov_count = 1;
- }
-
- enqueue_request:
- /* enqueue request */
- MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, "enqueuing");
- MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0);
-
- sreq->ch.vc = vc;
- sreq->dev.iov_offset = 0;
-
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)) && MPID_nem_newtcp_module_vc_is_connected(vc)){
- /* Post a write for the iov */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(VC_FIELD(vc, sc),
- sreq->dev.iov, sreq->dev.iov_count);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- SENDQ_ENQUEUE(&VC_FIELD(vc, send_queue), sreq);
-
- fn_exit:
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWTCP_ISENDCONTIGMSG);
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_SendNoncontig
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz)
-{
- int mpi_errno = MPI_SUCCESS;
- int iov_n;
- MPID_IOV iov[MPID_IOV_LIMIT];
- MPID_IOV *iov_p;
- MPIDI_msg_sz_t offset;
- int complete;
-
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "newtcp_SendNoncontig");
- MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
-
- iov[0].MPID_IOV_BUF = header;
- iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
-
- iov_n = MPID_IOV_LIMIT - 1;
- /* On the initial load of a send iov req, set the OnFinal action (null
- for point-to-point) */
- mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n);
- MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|loadsendiov");
-
- iov_n += 1;
- offset = 0;
-
- if (MPID_nem_newtcp_module_vc_is_connected(vc))
- {
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)))
- {
- MPIU_OSW_RETRYON_INTR((offset < 0),
- (mpi_errno = MPIU_SOCKW_Writev(VC_FIELD(vc, sc)->fd, iov, iov_n, &offset)));
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- if(offset < 0){
- offset = 0;
- }
-
- MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write noncontig %d", offset);
- }
- }
- else
- {
- mpi_errno = MPID_nem_newtcp_module_connect(vc);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- }
-
- if (offset < iov[0].MPID_IOV_LEN)
- {
- /* header was not yet sent, save it in req */
- sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
- iov[0].MPID_IOV_BUF = &sreq->dev.pending_pkt;
- iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
- }
-
- /* check if whole iov was sent, and save any unsent portion of iov */
- sreq->dev.iov_count = 0;
- complete = 1;
- for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p)
- {
- if (offset < iov_p->MPID_IOV_LEN)
- {
- sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF = (char *)iov_p->MPID_IOV_BUF + offset;
- sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset;
- offset = 0;
- ++sreq->dev.iov_count;
- complete = 0;
- }
- else
- offset -= iov_p->MPID_IOV_LEN;
- }
-
- if (complete){
- /* sent whole iov */
- int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
-
- reqFn = sreq->dev.OnDataAvail;
- if (!reqFn)
- {
- MPIDI_CH3U_Request_complete(sreq);
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- goto fn_exit;
- }
-
- complete = 0;
- mpi_errno = reqFn(vc, sreq, &complete);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
- if (complete)
- {
- MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
- goto fn_exit;
- }
- }
-
- /* enqueue request */
- MPIU_DBG_MSG (CH3_CHANNEL, VERBOSE, "enqueuing");
- MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0);
-
- sreq->ch.vc = vc;
- sreq->dev.iov_offset = 0;
-
- if (SENDQ_EMPTY(VC_FIELD(vc, send_queue)) && MPID_nem_newtcp_module_vc_is_connected(vc)){
- /* Post a write for the iov */
- mpi_errno = MPID_nem_newtcp_module_post_writev_ex(VC_FIELD(vc, sc),
- sreq->dev.iov, sreq->dev.iov_count);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
- }
-
- SENDQ_ENQUEUE(&VC_FIELD(vc, send_queue), sreq);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- MPID_Request_release(sreq);
- goto fn_exit;
-}
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_utility.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_utility.c
deleted file mode 100644
index 759c936..0000000
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_utility.c
+++ /dev/null
@@ -1,147 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- * (C) 2006 by Argonne National Laboratory.
- * See COPYRIGHT in top-level directory.
- */
-
-#include "wintcp_impl.h"
-
-/* MPID_nem_newtcp_module_get_conninfo -- This function takes a VC
- pointer as input and outputs the sockaddr, pg_id, and pg_rank of
- the remote process associated with this VC. [NOTE: I'm not sure
- yet, if the pg_id parameters will be char* or char**. I'd like to
- avoid a copy on this.] */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_get_conninfo
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_get_conninfo (struct MPIDI_VC *vc, struct sockaddr_in *addr, char **pg_id, int *pg_rank)
-{
- int mpi_errno = MPI_SUCCESS;
-
- *addr = VC_FIELD(vc, sock_id);
- *pg_id = (char *)vc->pg->id;
- *pg_rank = vc->pg_rank;
-
- return mpi_errno;
-}
-
-/* MPID_nem_newtcp_module_get_vc_from_conninfo -- This function takes
- the pg_id and pg_rank and returns the corresponding VC. */
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_get_vc_from_conninfo
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_get_vc_from_conninfo (char *pg_id, int pg_rank, struct MPIDI_VC **vc)
-{
- int mpi_errno = MPI_SUCCESS;
- MPIDI_PG_t *pg;
-
- mpi_errno = MPIDI_PG_Find (pg_id, &pg);
- if (mpi_errno) MPIU_ERR_POP (mpi_errno);
-
- MPIU_ERR_CHKANDJUMP1 (pg == NULL, mpi_errno, MPI_ERR_OTHER, "**intern", "**intern %s", "invalid PG");
- MPIU_ERR_CHKANDJUMP1 (pg_rank < 0 || pg_rank > MPIDI_PG_Get_size (pg), mpi_errno, MPI_ERR_OTHER, "**intern", "**intern %s", "invalid pg_rank");
-
- MPIDI_PG_Get_vc_set_active (pg, pg_rank, vc);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-#undef FUNCNAME
-#define FUNCNAME set_sockopts
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-int MPID_nem_newtcp_module_set_sockopts (MPIU_SOCKW_Sockfd_t fd)
-{
- int mpi_errno = MPI_SUCCESS;
- int option;
- socklen_t len;
-
- /* I heard you have to read the options after setting them in some implementations */
-
- option = 1;
- len = sizeof(int);
-
- mpi_errno = MPIU_SOCKW_Sock_setopt(fd, IPPROTO_TCP, TCP_NODELAY, &option, len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- option = MPID_NEM_NEWTCP_MODULE_RCVBUF_SZ;
- len = sizeof(int);
-
- mpi_errno = MPIU_SOCKW_Sock_setopt(fd, SOL_SOCKET, SO_RCVBUF, &option, len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- option = MPID_NEM_NEWTCP_MODULE_SNDBUF_SZ;
- len = sizeof(int);
-
- mpi_errno = MPIU_SOCKW_Sock_setopt(fd, SOL_SOCKET, SO_SNDBUF, &option, len);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- option = 1;
- mpi_errno = MPIU_SOCKW_Sock_cntrl_nb(fd, option);
- if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
-
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
-}
-
-
-/*
- MPID_NEM_NEWTCP_MODULE_SOCK_ERROR_EOF : connection failed
- MPID_NEM_NEWTCP_MODULE_SOCK_CONNECTED : socket connected (connection success)
- MPID_NEM_NEWTCP_MODULE_SOCK_NOEVENT : No event on socket
-
-N1: some implementations do not set POLLERR when there is a pending error on socket.
-So, solution is to check for readability/writeablility and then call getsockopt.
-Again, getsockopt behaves differently in different implementations which is handled
-safely here (per Stevens-Unix Network Programming)
-
-N2: As far as the socket code is concerned, it doesn't really differentiate whether
-there is an error in the socket or whether the peer has closed it (i.e we have received
-EOF and hence recv returns 0). Either way, we deccide the socket fd is not usable any
-more. So, same return code is used.
-A design decision is not to write also, if the peer has closed the socket. Please note that
-write will still be succesful, even if the peer has sent us FIN. Only the subsequent
-write will fail. So, this function is made tight enough and this should be called
-before doing any read/write at least in the connection establishment state machine code.
-
-N3: return code MPID_NEM_NEWTCP_MODULE_SOCK_NOEVENT is used only by the code that wants to
-know whether the connect is still not complete after a non-blocking connect is issued.
-
-TODO: Make this a macro for performance, if needed based on the usage.
-FIXME : Above comments are inconsistent now with the changes. No check for EOF is
-actually done now in this function.
-*/
-
-
-#undef FUNCNAME
-#define FUNCNAME MPID_nem_newtcp_module_check_sock_status
-#undef FCNAME
-#define FCNAME MPIDI_QUOTE(FUNCNAME)
-MPID_NEM_NEWTCP_MODULE_SOCK_STATUS_t
-MPID_nem_newtcp_module_check_sock_status(MPIU_SOCKW_Waitset_sock_hnd_t fd_ws_hnd)
-{
- int rc = MPID_NEM_NEWTCP_MODULE_SOCK_NOEVENT;
-
- /* FIXME: At least on windows there is no guarantee that a successful socket call
- * resets the socket error code
- * We will assume that sock is connected for now and let the state machine handle
- * errors
- */
- if(MPIU_SOCKW_Waitset_is_sock_readable(fd_ws_hnd) ||
- MPIU_SOCKW_Waitset_is_sock_writeable(fd_ws_hnd)){
- rc = MPID_NEM_NEWTCP_MODULE_SOCK_CONNECTED;
- }
- else{
- /* Should we return no event for OOB data - EX ? */
- rc = MPID_NEM_NEWTCP_MODULE_SOCK_NOEVENT;
- }
-
- return rc;
-}
http://git.mpich.org/mpich.git/commitdiff/2df2063305764000b8cdeaba5735a43de4511d46
commit 2df2063305764000b8cdeaba5735a43de4511d46
Author: Pavan Balaji <balaji at anl.gov>
Date: Thu Oct 16 18:14:06 2014 -0500
Remove unnecessary arbitrary code.
Signed-off-by: Wesley Bland <wbland at anl.gov>
diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
index e183cac..755c327 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
@@ -65,29 +65,6 @@ int MPID_nem_mx_cancel_send(MPIDI_VC_t *vc, MPID_Request *sreq)
}
-/* code in cancel_recv */
-/* FIXME: The vc is only needed to find which function to call*/
-/* This is otherwise any_source ready */
-/*
-#ifdef ENABLE_COMM_OVERRIDES
- {
- MPIDI_VC_t *vc;
- MPIU_Assert(rreq->dev.match.parts.rank != MPI_ANY_SOURCE);
- MPIDI_Comm_get_vc_set_active(rreq->comm, rreq->dev.match.parts.rank, &vc);
- if (vc->comm_ops && vc->comm_ops->cancel_recv)
- {
- int handled;
- handled = vc->comm_ops->cancel_recv(NULL, rreq);
- if (handled)
- {
- MPIDI_FUNC_EXIT(MPID_STATE_MPID_CANCEL_RECV);
- return MPI_SUCCESS;
- }
- }
- }
- #endif
-*/
-
#undef FUNCNAME
#define FUNCNAME MPID_nem_mx_cancel_recv
#undef FCNAME
-----------------------------------------------------------------------
Summary of changes:
src/mpid/ch3/channels/nemesis/netmod/Makefile.mk | 1 -
.../ch3/channels/nemesis/netmod/mx/Makefile.mk | 24 -
src/mpid/ch3/channels/nemesis/netmod/mx/TODO.txt | 23 -
src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c | 134 -
.../ch3/channels/nemesis/netmod/mx/mx_cancel.c | 134 -
.../ch3/channels/nemesis/netmod/mx/mx_finalize.c | 39 -
src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h | 258 --
src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c | 319 ---
src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c | 825 ------
src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c | 140 -
src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c | 561 ----
.../ch3/channels/nemesis/netmod/mx/subconfigure.m4 | 41 -
src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h | 612 -----
.../ch3/channels/nemesis/netmod/wintcp/Makefile.sm | 5 -
src/mpid/ch3/channels/nemesis/netmod/wintcp/README | 5 -
.../ch3/channels/nemesis/netmod/wintcp/socksm.c | 2824 --------------------
.../ch3/channels/nemesis/netmod/wintcp/socksm.h | 292 --
.../nemesis/netmod/wintcp/wintcp_finalize.c | 66 -
.../channels/nemesis/netmod/wintcp/wintcp_impl.h | 175 --
.../channels/nemesis/netmod/wintcp/wintcp_init.c | 668 -----
.../channels/nemesis/netmod/wintcp/wintcp_lmt.c | 565 ----
.../channels/nemesis/netmod/wintcp/wintcp_poll.c | 43 -
.../channels/nemesis/netmod/wintcp/wintcp_queue.h | 134 -
.../channels/nemesis/netmod/wintcp/wintcp_send.c | 621 -----
.../nemesis/netmod/wintcp/wintcp_utility.c | 147 -
25 files changed, 0 insertions(+), 8656 deletions(-)
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/Makefile.mk
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/TODO.txt
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_alloc.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_cancel.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_finalize.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_impl.h
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_init.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_poll.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_probe.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/mx_send.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/mx/uthash.h
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/Makefile.sm
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/README
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/socksm.h
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_finalize.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_impl.h
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_lmt.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_poll.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_queue.h
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_send.c
delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_utility.c
hooks/post-receive
--
MPICH primary repository
More information about the commits
mailing list