SLURM_JOB_NODELIST =  n[001-020]
from RunNeedles.slurm.bash
workdir= /home/kmccall/Needles2
nodefile=
num_proc= 20
DISPLAY= localhost:19.0
man_mas_cmd= ./NeedlesMpiMM
job_id_cmd= -job_id 027
head_node_cmd=
workers_per_manager= -works_per_man 23
manager_cmd= -manager_cmd NeedlesMpiMM
worker_cmd= -worker_cmd NeedlesMpiWork
epoch= -epoch 1655139691
 
host: n001
host: n002
host: n003
host: n004
host: n005
host: n006
host: n007
host: n008
host: n009
host: n010
host: n011
host: n012
host: n013
host: n014
host: n015
host: n016
host: n017
host: n018
host: n019
host: n020

==================================================================================================
mpiexec options:
----------------
  Base path: /opt/mpich/bin/
  Launcher: ssh
  Debug level: 1
  Enable X: -1

  Global environment:
  -------------------
    HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
    LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64
    LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:
    ODBG=
    SLURM_NODEID=0
    SLURM_TASK_PID=1080190
    NEEDLES_NUM_MANAGERS=20
    SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22
    SLURM_PRIO_PROCESS=0
    n_vars=-n_vars  1917
    CDC_PREW2KHOST=rocci
    all_orders=-all_orders  0
    MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD
    LANG=en_US.UTF-8
    SLURM_SUBMIT_DIR=/home/kmccall/Needles2
    HISTCONTROL=ignoredups
    NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin
    HOSTNAME=n001
    OLDPWD=/home/kmccall/Needles2
    NEEDLES_OUTPUT_FILE=needles_out.txt
    ENVIRONMENT=BATCH
    PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1
    LOADEDMODULES_modshare=intel/intelmpi:1
    CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov
    NEEDLES_USE_VALGRIND=0
    SLURM_PROCID=0
    SLURM_JOB_GID=513
    CDC_LOCALHOST=rocci.ndc.nasa.gov
    work_dir=/home/kmccall/Needles2
    SLURMD_NODENAME=n001
    FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov
    SLURM_TASKS_PER_NODE=24(x20)
    met_name=-metric m_stable
    which_declare=declare -f
    LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1
    CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar
    XDG_SESSION_ID=79266
    MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl
    SLURM_NNODES=20
    USER=kmccall
    XTERM_SHELL=/bin/bash
    man_mas_cmd=./NeedlesMpiMM
    VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes
    job_id_cmd=-job_id 027
    PWD=/home/kmccall/Needles2
    use_valgrind=0
    SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass
    SLURM_JOB_NODELIST=n[001-020]
    HOME=/home/kmccall
    SLURM_CLUSTER_NAME=cluster
    SLURM_NODELIST=n[001-020]
    SSH_CLIENT=156.68.204.55 57280 22
    SLURM_NTASKS=480
    SLURM_JOB_CPUS_PER_NODE=24(x20)
    XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share
    SLURM_TOPOLOGY_ADDR=n001
    _LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1
    debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args 
    SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101
    SLURM_JOB_NAME=Ndls_157.272990866662
    TMPDIR=/tmp
    LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib
    SLURM_JOBID=1088027
    NEEDLES_METRIC_NAME=m_stable
    SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf
    n_ndxs=-n_ndxs  1
    LOADEDMODULES=intel/intelmpi
    XTERM_VERSION=XTerm(331)
    SLURM_NODE_ALIASES=(null)
    SLURM_JOB_QOS=normal
    SLURM_TOPOLOGY_ADDR_PATTERN=node
    DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503
    MAIL=/var/spool/mail/kmccall
    SLURM_CPUS_ON_NODE=24
    SLURM_JOB_NUM_NODES=20
    epoch_arg=-epoch 1655139691
    NEEDLES_NUM_CORES_PER_NODE=24
    SHELL=/bin/bash
    TERM=xterm
    EPOCH=1655139691
    SLURM_JOB_UID=384580260
    NEEDLES_NUM_NDXS=1
    STRACE=
    output_file=-output_file needles_out.txt
    SLURM_JOB_PARTITION=normal
    num_proc=20
    NEEDLES_NUM_WORKERS=23
    job_id=027
    USE_MPICH=1
    SLURM_JOB_USER=kmccall
    SLURM_NPROCS=480
    CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov
    NEEDLES_ALL_ORDERS=0
    SHLVL=5
    SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov
    SLURM_JOB_ACCOUNT=users
    CDC_JOINED_DOMAIN=ndc.nasa.gov
    MANPATH=::
    worker_cmd=-worker_cmd NeedlesMpiWork
    WINDOWID=27263010
    workers_per_manager=-works_per_man 23
    manager_cmd=-manager_cmd NeedlesMpiMM
    GDK_BACKEND=x11
    MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles
    SLURM_GTIDS=0
    LOGNAME=kmccall
    DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2
    XDG_RUNTIME_DIR=/run/user/384580260
    MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1
    PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin
    SLURM_JOB_ID=1088027
    _LMFILES_=/opt/modulefiles/intel/intelmpi
    in_file=-input_file haystack_out.2000.misalign.bin
    MODULESHOME=/usr/share/Modules
    I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi
    HISTSIZE=1000
    USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV
    NEEDLES_NUM_VARS=1917
    XTERM_LOCALE=en_US.UTF-8
    SLURM_LOCALID=0
    CDC_JOINED_SITE=MSFCPrivate
    LESSOPEN=||/usr/bin/lesspipe.sh %s
    BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}
    BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}
    BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}
    BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}
    BASH_FUNC_ml%%=() {  module ml "$@"
}
    _=/opt/mpich/bin/mpiexec

  Hydra internal environment:
  ---------------------------
    GFORTRAN_UNBUFFERED_PRECONNECTED=y


    Proxy information:
    *********************
      [1] proxy: n001 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [2] proxy: n002 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [3] proxy: n003 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [4] proxy: n004 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [5] proxy: n005 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [6] proxy: n006 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [7] proxy: n007 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [8] proxy: n008 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [9] proxy: n009 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [10] proxy: n010 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [11] proxy: n011 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [12] proxy: n012 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [13] proxy: n013 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [14] proxy: n014 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [15] proxy: n015 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [16] proxy: n016 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [17] proxy: n017 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [18] proxy: n018 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [19] proxy: n019 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 

      [20] proxy: n020 (1 cores)
      Exec list: ./NeedlesMpiMM (1 processes); 


==================================================================================================

[mpiexec@n001.cluster.pssclabs.com] Timeout set to -1 (-1 means infinite)
[mpiexec@n001.cluster.pssclabs.com] Got a control port string of n001.cluster.pssclabs.com:37533

Proxy launch args: /opt/mpich/bin/hydra_pmi_proxy --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 

Arguments being passed to proxy 0:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n001 --global-core-map 0,1,20 --pmi-id-map 0,0 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 1:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n002 --global-core-map 0,1,20 --pmi-id-map 0,1 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 2:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n003 --global-core-map 0,1,20 --pmi-id-map 0,2 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 3:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n004 --global-core-map 0,1,20 --pmi-id-map 0,3 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 4:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n005 --global-core-map 0,1,20 --pmi-id-map 0,4 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 5:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n006 --global-core-map 0,1,20 --pmi-id-map 0,5 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 6:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n007 --global-core-map 0,1,20 --pmi-id-map 0,6 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 7:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n008 --global-core-map 0,1,20 --pmi-id-map 0,7 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 8:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n009 --global-core-map 0,1,20 --pmi-id-map 0,8 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 9:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n010 --global-core-map 0,1,20 --pmi-id-map 0,9 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 10:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n011 --global-core-map 0,1,20 --pmi-id-map 0,10 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 11:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n012 --global-core-map 0,1,20 --pmi-id-map 0,11 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 12:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n013 --global-core-map 0,1,20 --pmi-id-map 0,12 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 13:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n014 --global-core-map 0,1,20 --pmi-id-map 0,13 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 14:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n015 --global-core-map 0,1,20 --pmi-id-map 0,14 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 15:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n016 --global-core-map 0,1,20 --pmi-id-map 0,15 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 16:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n017 --global-core-map 0,1,20 --pmi-id-map 0,16 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 17:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n018 --global-core-map 0,1,20 --pmi-id-map 0,17 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 18:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n019 --global-core-map 0,1,20 --pmi-id-map 0,18 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

Arguments being passed to proxy 19:
--version 4.0.1 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n020 --global-core-map 0,1,20 --pmi-id-map 0,19 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_1080193_0_1900895546_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 128 'HYDRA_LAUNCHER_EXTRA_ARGS=-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 'LD_LIBRARY_PATH=/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=1080190' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.204.55 57280 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  1917' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.2000.misalign.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'NEEDLES_USE_VALGRIND=0' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=24(x20)' 'met_name=-metric m_stable' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=79266' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=./NeedlesMpiMM' 'VALGRIND=/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes  --verbose --log-file=valgrind_out.txt --read-var-info=yes' 'job_id_cmd=-job_id 027' 'PWD=/home/kmccall/Needles2' 'use_valgrind=0' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.204.55 57280 22' 'SLURM_NTASKS=480' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:19.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_157.272990866662' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=1088027' 'NEEDLES_METRIC_NAME=m_stable' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  1' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=cbe144e4-5299-f44d-b8c8-3efb2dd30503' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1655139691' 'NEEDLES_NUM_CORES_PER_NODE=24' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1655139691' 'SLURM_JOB_UID=384580260' 'NEEDLES_NUM_NDXS=1' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=027' 'USE_MPICH=1' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=480' 'CDC_JOINED_DC=ndmsadc11.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=27263010' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-GBy6xwkfxy,guid=8094130c434f1c1a4fd3cf9462a76ac2' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=1088027' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.2000.misalign.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=1917' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/opt/mpich/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 23 ./NeedlesMpiMM -input_file haystack_out.2000.misalign.bin -works_per_man 23 -manager_cmd NeedlesMpiMM -worker_cmd NeedlesMpiWork -n_vars 1917 -n_ndxs 1 -metric m_stable -all_orders 0 -output_file needles_out.txt -epoch 1655139691 -job_id 027 

[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n001 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 0 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n002 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 1 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n003 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 2 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n004 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 3 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n005 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 4 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n006 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 5 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n007 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 6 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n008 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 7 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n009 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 8 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n010 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 9 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n011 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 10 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n012 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 11 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n013 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 12 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n014 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 13 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n015 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 14 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n016 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 15 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n017 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 16 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n018 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 17 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n019 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 18 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no n020 "/opt/mpich/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:37533 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 19 
slurmstepd: error: *** JOB 1088027 ON n001 CANCELLED AT 2022-06-13T12:02:20 ***