0	0	2b8fd52bfc60[2073]	256	84.881108	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	152	decr 0x2b8fd3156790 (0xac000002 kind=REQUEST) refcount to 2
0	0	2b8fd52bfc60[2073]	16384	84.881115	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	152	just incremented MPIDI_CH3I_progress_completion_count=957
0	0	2b8fd52bfc60[2073]	32768	84.881122	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	153	.... complete
0	0	2b8fd52bfc60[2073]	32768	84.881130	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	154	MPIDI_CH3I_Sendq_dequeuereq=0x2b8fd3156790 (handle=0xac000002), queue=0x3ab9ea60
0	0	2b8fd52bfc60[2073]	256	84.881138	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	154	decr 0x2b8fd3156790 (0xac000002 kind=REQUEST) refcount to 1
0	0	2b8fd52bfc60[2073]	32	84.881144	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	176	Leaving MPID_STATE_MPID_NEM_TCP_SEND_QUEUED
0	0	2b8fd52bfc60[2073]	32	84.881151	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1696	Leaving MPID_STATE_STATE_COMMRDY_HANDLER
0	0	2b8fd52bfc60[2073]	32	84.881159	src/mpid/ch3/channels/nemesis/src/ch3_progress.c	527	Leaving MPID_STATE_MPIDI_CH3I_PROGRESS
0	0	2b8fd52bfc60[2073]	256	84.881168	src/mpi/pt2pt/send.c	158	decr 0x2b8fd3156790 (0xac000002 kind=REQUEST) refcount to 0
0	0	2b8fd52bfc60[2073]	16	84.881175	src/mpid/ch3/src/ch3u_request.c	117	Entering MPID_STATE_MPIDI_CH3_REQUEST_DESTROY
0	0	2b8fd52bfc60[2073]	32768	84.881181	src/mpid/ch3/src/ch3u_request.c	119	freeing request, handle=0xac000002
0	0	2b8fd52bfc60[2073]	256	84.881189	./src/include/mpiimpl.h	1280	decr 0x2b8fd3131ee0 (0x44000000 kind=COMM) refcount to 1
0	0	2b8fd52bfc60[2073]	256	84.881198	src/util/mem/handlemem.c	472	Freeing object ptr 0x2b8fd3156790 (0xac000002 kind=REQUEST) refcount=0
0	0	2b8fd52bfc60[2073]	32	84.881204	src/mpid/ch3/src/ch3u_request.c	168	Leaving MPID_STATE_MPIDI_CH3_REQUEST_DESTROY
0	0	2b8fd52bfc60[2073]	32	84.881211	src/mpi/pt2pt/send.c	165	Leaving MPID_STATE_MPI_SEND
0	0	2b8fd52bfc60[2073]	16	84.929776	src/mpi/pt2pt/recv.c	79	Entering MPID_STATE_MPI_RECV
0	0	2b8fd52bfc60[2073]	16	84.929808	src/mpid/ch3/src/mpid_recv.c	29	Entering MPID_STATE_MPID_RECV
0	0	2b8fd52bfc60[2073]	65536	84.929823	src/mpid/ch3/src/mpid_recv.c	33	rank=1, tag=14131, context=0
0	0	2b8fd52bfc60[2073]	16	84.929843	src/mpid/ch3/src/ch3u_recvq.c	503	Entering MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP
0	0	2b8fd52bfc60[2073]	256	84.929853	src/util/mem/handlemem.c	444	Allocating object ptr 0x2b8fd3156790 (handle val 0xac000002)
0	0	2b8fd52bfc60[2073]	32768	84.929860	src/mpid/ch3/src/ch3u_recvq.c	601	allocated request, handle=0xac000002
0	0	2b8fd52bfc60[2073]	256	84.929869	src/mpid/ch3/src/ch3u_recvq.c	601	set 0x2b8fd3156790 (0xac000002 kind=REQUEST) refcount to 2
0	0	2b8fd52bfc60[2073]	256	84.929878	src/mpid/ch3/src/ch3u_recvq.c	619	incr 0x2b8fd3131ee0 (0x44000000 kind=COMM) refcount to 2
0	0	2b8fd52bfc60[2073]	16	84.929885	src/mpid/ch3/channels/nemesis/src/ch3_progress.c	1151	Entering MPID_STATE_MPIDI_CH3I_POSTED_RECV_ENQUEUED
0	0	2b8fd52bfc60[2073]	32	84.929892	src/mpid/ch3/channels/nemesis/src/ch3_progress.c	1205	Leaving MPID_STATE_MPIDI_CH3I_POSTED_RECV_ENQUEUED
0	0	2b8fd52bfc60[2073]	32	84.929898	src/mpid/ch3/src/ch3u_recvq.c	661	Leaving MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP
0	0	2b8fd52bfc60[2073]	65536	84.929905	src/mpid/ch3/src/mpid_recv.c	157	request allocated in posted queue
0	0	2b8fd52bfc60[2073]	65536	84.929911	src/mpid/ch3/src/mpid_recv.c	189	request allocated, handle=0xac000002
0	0	2b8fd52bfc60[2073]	32	84.929918	src/mpid/ch3/src/mpid_recv.c	192	Leaving MPID_STATE_MPID_RECV
0	0	2b8fd52bfc60[2073]	16	84.929925	src/mpid/ch3/channels/nemesis/src/ch3_progress.c	297	Entering MPID_STATE_MPIDI_CH3I_PROGRESS
0	0	2b8fd52bfc60[2073]	16	92.449157	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1676	Entering MPID_STATE_STATE_COMMRDY_HANDLER
0	0	2b8fd52bfc60[2073]	32768	92.449191	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1678	vc = 0x3ab9e730
0	0	2b8fd52bfc60[2073]	16	92.449201	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1524	Entering MPID_STATE_MPID_NEM_TCP_RECV_HANDLER
0	0	2b8fd52bfc60[2073]	524288	92.449214	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	448	vc_is_in_shutdown(0x3ab9e730)=FALSE
0	0	2b8fd52bfc60[2073]	32768	92.449221	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1558	ERROR: sock (fd=57) is closed: bytes_recvd == 0
0	0	2b8fd52bfc60[2073]	8388608	92.449241	src/mpi/errhan/errutil.c	867	Error created: last=0000000000 class=0x0000000f MPID_nem_tcp_recv_handler(1559) **sock_closed
0	0	2b8fd52bfc60[2073]	8388608	92.449291	src/mpi/errhan/errutil.c	1081	New ErrorRing[0]
0	0	2b8fd52bfc60[2073]	8388608	92.449300	src/mpi/errhan/errutil.c	1082	    id         = 0x0c034a0f
0	0	2b8fd52bfc60[2073]	8388608	92.449307	src/mpi/errhan/errutil.c	1083	    prev_error = 0000000000
0	0	2b8fd52bfc60[2073]	8388608	92.449314	src/mpi/errhan/errutil.c	1084	    user=0
0	0	2b8fd52bfc60[2073]	8388608	92.449324	src/mpi/errhan/errutil.c	867	Error created: last=0x0c034a0f class=0x00000036 MPID_nem_tcp_recv_handler(1651) **comm_fail
0	0	2b8fd52bfc60[2073]	8388608	92.449345	src/mpi/errhan/errutil.c	1883	code=0x0c034a0f ring_idx=0 ring_id=0x0c034a0f generic_idx=841
0	0	2b8fd52bfc60[2073]	32768	92.449362	src/mpi/errhan/errutil.c	1579	memcpy(0x2b8fd3139744, 0x3abc3d60, 30)
0	0	2b8fd52bfc60[2073]	8388608	92.449373	src/mpi/errhan/errutil.c	1081	New ErrorRing[1]
0	0	2b8fd52bfc60[2073]	8388608	92.449380	src/mpi/errhan/errutil.c	1082	    id         = 0x3800a536
0	0	2b8fd52bfc60[2073]	8388608	92.449387	src/mpi/errhan/errutil.c	1083	    prev_error = 0x0c034a0f
0	0	2b8fd52bfc60[2073]	8388608	92.449393	src/mpi/errhan/errutil.c	1084	    user=0
0	0	2b8fd52bfc60[2073]	16	92.449402	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1986	Entering MPID_STATE_MPID_NEM_TCP_CLEANUP_ON_ERROR
0	0	2b8fd52bfc60[2073]	16	92.449410	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1029	Entering MPID_STATE_MPID_NEM_TCP_CLEANUP
0	0	2b8fd52bfc60[2073]	16	92.449417	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	992	Entering MPID_STATE_CLOSE_CLEANUP_AND_FREE_SC_PLFD
0	0	2b8fd52bfc60[2073]	524288	92.449424	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	994	vc=0x3ab9e730, sc=0x3aba0e80, closing fd=57
0	0	2b8fd52bfc60[2073]	16	92.449448	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	941	Entering MPID_STATE_CLEANUP_AND_FREE_SC_PLFD
0	0	2b8fd52bfc60[2073]	524288	92.449465	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	947	about to decr sc_ref_count sc=0x3aba0e80 sc->vc=0x3ab9e730 sc_ref_count=1
0	0	2b8fd52bfc60[2073]	524288	92.449474	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	958	CHANGE_STATE(_sc=0x3aba0e80, _cstate=1 (CONN_STATE_TS_CLOSED)) - old_state=CONN_STATE_TS_COMMRDY sc->vc=0x3ab9e730
0	0	2b8fd52bfc60[2073]	524288	92.449482	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	960	CHANGE_STATE(_sc=0x3aba0e80, _cstate=1 (CONN_STATE_TS_CLOSED)) - old_state=out_of_range sc->vc=(nil)
0	0	2b8fd52bfc60[2073]	32	92.449489	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	969	Leaving MPID_STATE_CLEANUP_AND_FREE_SC_PLFD
0	0	2b8fd52bfc60[2073]	32	92.449496	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1007	Leaving MPID_STATE_CLOSE_CLEANUP_AND_FREE_SC_PLFD
0	0	2b8fd52bfc60[2073]	32	92.449502	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1055	Leaving MPID_STATE_MPID_NEM_TCP_CLEANUP
0	0	2b8fd52bfc60[2073]	16	92.449509	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1956	Entering MPID_STATE_ERROR_CLOSED
0	0	2b8fd52bfc60[2073]	16	92.449517	src/mpid/ch3/src/ch3u_handle_connection.c	52	Entering MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION
0	0	2b8fd52bfc60[2073]	8192	92.449524	src/mpid/ch3/src/ch3u_handle_connection.c	98	Connection closed prematurely.
0	0	2b8fd52bfc60[2073]	16	92.449532	src/mpid/ch3/src/ch3u_recvq.c	936	Entering MPID_STATE_MPIDU_COMPLETE_POSTED_WITH_ERROR
0	0	2b8fd52bfc60[2073]	32	92.449539	src/mpid/ch3/src/ch3u_recvq.c	956	Leaving MPID_STATE_MPIDU_COMPLETE_POSTED_WITH_ERROR
0	0	2b8fd52bfc60[2073]	4096	92.449547	src/mpid/ch3/src/ch3u_handle_connection.c	104	vc=0x3ab9e730: Setting state (vc) from MPIDI_VC_STATE_ACTIVE to VC_STATE_MORIBUND, vcchstate is N/A
0	0	2b8fd52bfc60[2073]	32	92.449554	src/mpid/ch3/src/ch3u_handle_connection.c	192	Leaving MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION
0	0	2b8fd52bfc60[2073]	16	92.449562	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	823	Entering MPID_STATE_MPID_NEM_TCP_ERROR_OUT_SEND_QUEUE
0	0	2b8fd52bfc60[2073]	32	92.449569	src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c	845	Leaving MPID_STATE_MPID_NEM_TCP_ERROR_OUT_SEND_QUEUE
0	0	2b8fd52bfc60[2073]	32	92.449576	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1967	Leaving MPID_STATE_ERROR_CLOSED
0	0	2b8fd52bfc60[2073]	32	92.449588	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1995	Leaving MPID_STATE_MPID_NEM_TCP_CLEANUP_ON_ERROR
0	0	2b8fd52bfc60[2073]	32	92.449598	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1646	Leaving MPID_STATE_MPID_NEM_TCP_RECV_HANDLER
0	0	2b8fd52bfc60[2073]	32	92.449606	src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c	1696	Leaving MPID_STATE_STATE_COMMRDY_HANDLER