SLURM_JOB_NODELIST = n[001-020] num_proc = 20 host: n001 host: n002 host: n003 host: n004 host: n005 host: n006 host: n007 host: n008 host: n009 host: n010 host: n011 host: n012 host: n013 host: n014 host: n015 host: n016 host: n017 host: n018 host: n019 host: n020 ================================================================================================== mpiexec options: ---------------- Base path: /home/kmccall/mpich-slurm-patch-install-4.0/bin/ Launcher: ssh Debug level: 1 Enable X: -1 Global environment: ------------------- LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64 LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36: ODBG= SLURM_NODEID=0 SLURM_TASK_PID=288151 NEEDLES_NUM_MANAGERS=20 SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22 SLURM_PRIO_PROCESS=0 n_vars=-n_vars 54 CDC_PREW2KHOST=rocci all_orders=-all_orders 0 MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD LANG=en_US.UTF-8 SLURM_SUBMIT_DIR=/home/kmccall/Needles2 HISTCONTROL=ignoredups NEEDLES_INPUT_FILE=haystack_out.bin HOSTNAME=n001 OLDPWD=/home/kmccall/Needles2 NEEDLES_OUTPUT_FILE=needles_out.txt ENVIRONMENT=BATCH PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1 LOADEDMODULES_modshare=intel/intelmpi:1 CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov SLURM_PROCID=0 SLURM_JOB_GID=513 CDC_LOCALHOST=rocci.ndc.nasa.gov work_dir=/home/kmccall/Needles2 SLURMD_NODENAME=n001 FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov SLURM_TASKS_PER_NODE=1(x20) met_name=-metric m_disp_contin which_declare=declare -f LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1 CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar XDG_SESSION_ID=59181 MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl SLURM_NNODES=20 USER=kmccall XTERM_SHELL=/bin/bash man_mas_cmd=NeedlesMpiMM job_id_cmd=-job_id 654 PWD=/home/kmccall/Needles2 SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass SLURM_JOB_NODELIST=n[001-020] HOME=/home/kmccall SLURM_CLUSTER_NAME=cluster SLURM_NODELIST=n[001-020] SSH_CLIENT=156.68.206.221 58978 22 SLURM_NTASKS=20 SLURM_JOB_CPUS_PER_NODE=24(x20) XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share SLURM_TOPOLOGY_ADDR=n001 _LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1 debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101 SLURM_JOB_NAME=Ndls_963.914579087035 TMPDIR=/tmp LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib SLURM_JOBID=123654 NEEDLES_METRIC_NAME=m_disp_contin SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf n_ndxs=-n_ndxs 2 LOADEDMODULES=intel/intelmpi XTERM_VERSION=XTerm(331) SLURM_NODE_ALIASES=(null) SLURM_JOB_QOS=normal SLURM_TOPOLOGY_ADDR_PATTERN=node DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7 MAIL=/var/spool/mail/kmccall SLURM_CPUS_ON_NODE=24 SLURM_JOB_NUM_NODES=20 epoch_arg=-epoch 1645128985 SHELL=/bin/bash TERM=xterm EPOCH=1645128985 SLURM_JOB_UID=384580260 TC_LIB_DIR=/usr/lib64/tc NEEDLES_NUM_NDXS=2 STRACE= output_file=-output_file needles_out.txt SLURM_JOB_PARTITION=normal num_proc=20 NEEDLES_NUM_WORKERS=23 job_id=654 SLURM_JOB_USER=kmccall SLURM_NPROCS=20 CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov NEEDLES_ALL_ORDERS=0 SHLVL=5 SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov SLURM_JOB_ACCOUNT=users CDC_JOINED_DOMAIN=ndc.nasa.gov MANPATH=:: worker_cmd=-worker_cmd NeedlesMpiWork WINDOWID=18874402 workers_per_manager=-works_per_man 23 manager_cmd=-manager_cmd NeedlesMpiMM GDK_BACKEND=x11 MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles SLURM_GTIDS=0 LOGNAME=kmccall DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f XDG_RUNTIME_DIR=/run/user/384580260 MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1 PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin SLURM_JOB_ID=123654 _LMFILES_=/opt/modulefiles/intel/intelmpi in_file=-input_file haystack_out.bin MODULESHOME=/usr/share/Modules I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi HISTSIZE=1000 USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV NEEDLES_NUM_VARS=54 XTERM_LOCALE=en_US.UTF-8 SLURM_LOCALID=0 CDC_JOINED_SITE=MSFCPrivate LESSOPEN=||/usr/bin/lesspipe.sh %s BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" } BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 } BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus } BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi } BASH_FUNC_ml%%=() { module ml "$@" } _=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec Hydra internal environment: --------------------------- GFORTRAN_UNBUFFERED_PRECONNECTED=y Proxy information: ********************* [1] proxy: n001 (1 cores) Exec list: NeedlesMpiMM (1 processes); [2] proxy: n002 (1 cores) Exec list: NeedlesMpiMM (1 processes); [3] proxy: n003 (1 cores) Exec list: NeedlesMpiMM (1 processes); [4] proxy: n004 (1 cores) Exec list: NeedlesMpiMM (1 processes); [5] proxy: n005 (1 cores) Exec list: NeedlesMpiMM (1 processes); [6] proxy: n006 (1 cores) Exec list: NeedlesMpiMM (1 processes); [7] proxy: n007 (1 cores) Exec list: NeedlesMpiMM (1 processes); [8] proxy: n008 (1 cores) Exec list: NeedlesMpiMM (1 processes); [9] proxy: n009 (1 cores) Exec list: NeedlesMpiMM (1 processes); [10] proxy: n010 (1 cores) Exec list: NeedlesMpiMM (1 processes); [11] proxy: n011 (1 cores) Exec list: NeedlesMpiMM (1 processes); [12] proxy: n012 (1 cores) Exec list: NeedlesMpiMM (1 processes); [13] proxy: n013 (1 cores) Exec list: NeedlesMpiMM (1 processes); [14] proxy: n014 (1 cores) Exec list: NeedlesMpiMM (1 processes); [15] proxy: n015 (1 cores) Exec list: NeedlesMpiMM (1 processes); [16] proxy: n016 (1 cores) Exec list: NeedlesMpiMM (1 processes); [17] proxy: n017 (1 cores) Exec list: NeedlesMpiMM (1 processes); [18] proxy: n018 (1 cores) Exec list: NeedlesMpiMM (1 processes); [19] proxy: n019 (1 cores) Exec list: NeedlesMpiMM (1 processes); [20] proxy: n020 (1 cores) Exec list: NeedlesMpiMM (1 processes); ================================================================================================== [mpiexec@n001.cluster.pssclabs.com] Timeout set to -1 (-1 means infinite) [mpiexec@n001.cluster.pssclabs.com] Got a control port string of n001.cluster.pssclabs.com:36667 Proxy launch args: /home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id Arguments being passed to proxy 0: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n001 --global-core-map 0,1,20 --pmi-id-map 0,0 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 1: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n002 --global-core-map 0,1,20 --pmi-id-map 0,1 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 2: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n003 --global-core-map 0,1,20 --pmi-id-map 0,2 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 3: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n004 --global-core-map 0,1,20 --pmi-id-map 0,3 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 4: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n005 --global-core-map 0,1,20 --pmi-id-map 0,4 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 5: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n006 --global-core-map 0,1,20 --pmi-id-map 0,5 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 6: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n007 --global-core-map 0,1,20 --pmi-id-map 0,6 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 7: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n008 --global-core-map 0,1,20 --pmi-id-map 0,7 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 8: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n009 --global-core-map 0,1,20 --pmi-id-map 0,8 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 9: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n010 --global-core-map 0,1,20 --pmi-id-map 0,9 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 10: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n011 --global-core-map 0,1,20 --pmi-id-map 0,10 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 11: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n012 --global-core-map 0,1,20 --pmi-id-map 0,11 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 12: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n013 --global-core-map 0,1,20 --pmi-id-map 0,12 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 13: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n014 --global-core-map 0,1,20 --pmi-id-map 0,13 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 14: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n015 --global-core-map 0,1,20 --pmi-id-map 0,14 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 15: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n016 --global-core-map 0,1,20 --pmi-id-map 0,15 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 16: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n017 --global-core-map 0,1,20 --pmi-id-map 0,16 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 17: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n018 --global-core-map 0,1,20 --pmi-id-map 0,17 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 18: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n019 --global-core-map 0,1,20 --pmi-id-map 0,18 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 Arguments being passed to proxy 19: --version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n020 --global-core-map 0,1,20 --pmi-id-map 0,19 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_288154_0_844984536_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=288151' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.206.221 58978 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars 54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders 0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=59181' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 654' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.206.221 58978 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm -hold -display localhost:14.0 -e gdb -x /home/kmccall/Needles2/gdb_cmds -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_963.914579087035' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123654' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs 2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=109f47f4-23db-7745-9547-5cc070bcaca7' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645128985' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645128985' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=654' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc12.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-SzYbPGD22S,guid=0f5c68d8df7cd3cc3b10c7b9620ea37f' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123654' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() { ( alias; eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@" }' 'BASH_FUNC_module%%=() { _module_raw "$@" 2>&1 }' 'BASH_FUNC__module_raw%%=() { unset _mlshdbg; if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then case "$-" in *v*x*) set +vx; _mlshdbg='vx' ;; *v*) set +v; _mlshdbg='v' ;; *x*) set +x; _mlshdbg='x' ;; *) _mlshdbg='' ;; esac; fi; unset _mlre _mlIFS; if [ -n "${IFS+x}" ]; then _mlIFS=$IFS; fi; IFS=' '; for _mlv in ${MODULES_RUN_QUARANTINE:-}; do if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' "; fi; _mlrv="MODULES_RUNENV_${_mlv}"; _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' "; fi; done; if [ -n "${_mlre:-}" ]; then eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`; else eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`; fi; _mlstatus=$?; if [ -n "${_mlIFS+x}" ]; then IFS=$_mlIFS; else unset IFS; fi; unset _mlre _mlv _mlrv _mlIFS; if [ -n "${_mlshdbg:-}" ]; then set -$_mlshdbg; fi; unset _mlshdbg; return $_mlstatus }' 'BASH_FUNC_switchml%%=() { typeset swfound=1; if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then typeset swname='main'; if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then typeset swfound=0; unset MODULES_USE_COMPAT_VERSION; fi; else typeset swname='compatibility'; if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then typeset swfound=0; MODULES_USE_COMPAT_VERSION=1; export MODULES_USE_COMPAT_VERSION; fi; fi; if [ $swfound -eq 0 ]; then echo "Switching to Modules $swname version"; source /usr/share/Modules/init/bash; else echo "Cannot switch to Modules $swname version, command not found"; return 1; fi }' 'BASH_FUNC_ml%%=() { module ml "$@" }' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 1 'DISPLAY=localhost:10.0' --exec-wdir /home/kmccall/Needles2 --exec-args 23 NeedlesMpiMM -input_file haystack_out.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 54 -n_ndxs 2 -metric m_disp_contin -all_orders 0 -output_file needles_out.txt -epoch 1645128985 -job_id 654 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n001 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 0 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n002 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 1 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n003 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 2 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n004 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 3 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n005 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 4 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n006 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 5 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n007 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 6 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n008 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 7 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n009 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 8 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n010 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 9 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n011 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 10 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n012 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 11 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n013 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 12 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n014 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 13 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n015 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 14 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n016 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 15 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n017 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 16 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n018 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 17 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n019 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 18 [mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n020 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:36667 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 19 (gnome-ssh-askpass:288177): Gtk-WARNING **: 14:16:25.949: cannot open display: localhost:14.0 (gnome-ssh-askpass:288178): Gtk-WARNING **: 14:16:25.949: cannot open display: localhost:14.0 (gnome-ssh-askpass:288179): Gtk-WARNING **: 14:16:25.949: cannot open display: localhost:14.0 (gnome-ssh-askpass:288180): Gtk-WARNING **: 14:16:25.949: cannot open display: localhost:14.0 (gnome-ssh-askpass:288181): Gtk-WARNING **: 14:16:25.950: cannot open display: localhost:14.0 (gnome-ssh-askpass:288182): Gtk-WARNING **: 14:16:25.950: cannot open display: localhost:14.0 (gnome-ssh-askpass:288183): Gtk-WARNING **: 14:16:25.950: cannot open display: localhost:14.0 (gnome-ssh-askpass:288184): Gtk-WARNING **: 14:16:25.951: cannot open display: localhost:14.0 Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. (gnome-ssh-askpass:288185): Gtk-WARNING **: 14:16:25.951: cannot open display: localhost:14.0 (gnome-ssh-askpass:288187): Gtk-WARNING **: 14:16:25.951: cannot open display: localhost:14.0 (gnome-ssh-askpass:288186): Gtk-WARNING **: 14:16:25.951: cannot open display: localhost:14.0 (gnome-ssh-askpass:288188): Gtk-WARNING **: 14:16:25.951: cannot open display: localhost:14.0 Host key verification failed. Host key verification failed. Host key verification failed. (gnome-ssh-askpass:288189): Gtk-WARNING **: 14:16:25.953: cannot open display: localhost:14.0 Host key verification failed. Host key verification failed. (gnome-ssh-askpass:288190): Gtk-WARNING **: 14:16:25.953: cannot open display: localhost:14.0 (gnome-ssh-askpass:288191): Gtk-WARNING **: 14:16:25.953: cannot open display: localhost:14.0 (gnome-ssh-askpass:288192): Gtk-WARNING **: 14:16:25.953: cannot open display: localhost:14.0 (gnome-ssh-askpass:288193): Gtk-WARNING **: 14:16:25.954: cannot open display: localhost:14.0 Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. Host key verification failed. (gnome-ssh-askpass:288194): Gtk-WARNING **: 14:16:25.955: cannot open display: localhost:14.0 Host key verification failed. By accessing and using this information system, you acknowledge and consent to the following: You are accessing a U.S. Government information system, which includes: (1) this computer";" (2) this computer network";" (3) all computers connected to this network including end user systems";" (4) all devices and storage media attached to this network or to any computer on this network";" and (5) cloud and remote information services. This information system is provided for U.S. Government-authorized use only. You have no reasonable expectation of privacy regarding any communication transmitted through or data stored on this information system. At any time, and for any lawful purpose, the U.S. Government may monitor, intercept, search, and seize any communication or data transiting, stored on, or traveling to or from this information system. You are NOT authorized to process classified information on this information system. Unauthorized or improper use of this system may result in suspension or loss of access privileges, disciplinary action, and civil and/or criminal penalties. By accessing and using this information system, you acknowledge and consent to the following: You are accessing a U.S. Government information system, which includes: (1) this computer";" (2) this computer network";" (3) all computers connected to this network including end user systems";" (4) all devices and storage media attached to this network or to any computer on this network";" and (5) cloud and remote information services. This information system is provided for U.S. Government-authorized use only. You have no reasonable expectation of privacy regarding any communication transmitted through or data stored on this information system. At any time, and for any lawful purpose, the U.S. Government may monitor, intercept, search, and seize any communication or data transiting, stored on, or traveling to or from this information system. You are NOT authorized to process classified information on this information system. Unauthorized or improper use of this system may result in suspension or loss of access privileges, disciplinary action, and civil and/or criminal penalties. [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): init pmi_version=1 pmi_subversion=1 [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=response_to_init pmi_version=1 pmi_subversion=1 rc=0 [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): get_maxes [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=maxes kvsname_max=256 keylen_max=64 vallen_max=1024 [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): get_appnum [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=appnum appnum=0 [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): get_my_kvsname [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=my_kvsname kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): get kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com key=PMI_process_mapping [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=get_result rc=0 msg=success value=(vector,(0,20,1)) [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): get_my_kvsname [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=my_kvsname kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): put kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com key=P0-businesscard value=description#n001$port#33403$ifname#172.16.56.1$ [proxy:0:0@n001.cluster.pssclabs.com] cached command: P0-businesscard=description#n001$port#33403$ifname#172.16.56.1$ [proxy:0:0@n001.cluster.pssclabs.com] PMI response: cmd=put_result rc=0 msg=success [mpiexec@n001.cluster.pssclabs.com] [pgid: 0] got PMI command: cmd=put P0-businesscard=description#n001$port#33403$ifname#172.16.56.1$ [proxy:0:0@n001.cluster.pssclabs.com] got pmi command (from 4): barrier_in [proxy:0:0@n001.cluster.pssclabs.com] flushing 1 put command(s) out [proxy:0:0@n001.cluster.pssclabs.com] forwarding command (cmd=put P0-businesscard=description#n001$port#33403$ifname#172.16.56.1$) upstream [proxy:0:0@n001.cluster.pssclabs.com] forwarding command (cmd=barrier_in) upstream [mpiexec@n001.cluster.pssclabs.com] [pgid: 0] got PMI command: cmd=barrier_in [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): init pmi_version=1 pmi_subversion=1 [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=response_to_init pmi_version=1 pmi_subversion=1 rc=0 [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): get_maxes [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=maxes kvsname_max=256 keylen_max=64 vallen_max=1024 [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): get_appnum [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=appnum appnum=0 [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): get_my_kvsname [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=my_kvsname kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): get kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com key=PMI_process_mapping [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=get_result rc=0 msg=success value=(vector,(0,20,1)) [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): get_my_kvsname [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=my_kvsname kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): put kvsname=kvs_288154_0_844984536_n001.cluster.pssclabs.com key=P1-businesscard value=description#n002$port#53555$ifname#172.16.56.2$ [proxy:0:1@n002.cluster.pssclabs.com] cached command: P1-businesscard=description#n002$port#53555$ifname#172.16.56.2$ [proxy:0:1@n002.cluster.pssclabs.com] PMI response: cmd=put_result rc=0 msg=success [proxy:0:1@n002.cluster.pssclabs.com] got pmi command (from 4): barrier_in [proxy:0:1@n002.cluster.pssclabs.com] flushing 1 put command(s) out [proxy:0:1@n002.cluster.pssclabs.com] forwarding command (cmd=put P1-businesscard=description#n002$port#53555$ifname#172.16.56.2$) upstream [proxy:0:1@n002.cluster.pssclabs.com] forwarding command (cmd=barrier_in) upstream [mpiexec@n001.cluster.pssclabs.com] [pgid: 0] got PMI command: cmd=put P1-businesscard=description#n002$port#53555$ifname#172.16.56.2$ [mpiexec@n001.cluster.pssclabs.com] [pgid: 0] got PMI command: cmd=barrier_in