Skip to content

Commit 16ead71

Browse files
heather999heather999
committed
bring stable setup up to date
1 parent 6c05352 commit 16ead71

1 file changed

Lines changed: 71 additions & 70 deletions

File tree

‎nersc/setup_td.sh‎

Lines changed: 71 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -60,92 +60,76 @@ export TD_SN=${TD}/SN
6060
export TD_SOFTWARE=${TD}/SOFTWARE
6161
export TD_PUBLIC=/global/cfs/cdirs/lsst/www/DESC_TD_PUBLIC
6262

63-
export PYSYN_CDBS=${TD_SOFTWARE}/bayeSN/synphot/grp/redcat/trds
64-
export VERSION_LIBPYTHON=3.10
63+
#export PYSYN_CDBS=${TD_SOFTWARE}/bayeSN/synphot/grp/redcat/trds
64+
#export VERSION_LIBPYTHON=3.10
6565

6666
if [[ -z "$keepenv" ]] && [[ -z "$gpuenv" ]] && [[ -z $SHIFTER_RUNTIME ]];
6767
then
6868
module purge
6969
fi
7070

71-
# setup without LSST Science Pipelines
72-
# Broken since March 2022 Cori OS Upgrade
73-
if [[ $nolsst ]];
71+
if [ $shifterenv ] || [ $SHIFTER_RUNTIME ]
7472
then
75-
module unload python
76-
module unload PrgEnv-intel/6.0.5
77-
module load PrgEnv-gnu/6.0.5
78-
module swap gcc gcc/9.3.0
79-
module rm craype-network-aries
80-
module rm cray-libsci
81-
module unload craype
82-
module load cfitsio/3.47
83-
module load gsl
84-
module load root/6.18.00-py3
85-
module load intel/19.1.3.304 # for CosmoMC (Mar 5 2021)
86-
export CC=gcc
87-
88-
export COSMOMC_DIR="$SN_GROUP/CosmoMCBBC"
89-
export PATH=$PATH:${COSMOMC_DIR}
90-
91-
# Set up SN python
92-
export LSST_INST_DIR=/global/common/software/lsst/common/miniconda
93-
export SN_PYTHON_VER=sn-py
94-
module unload python
95-
unset PYTHONHOME
96-
unset PYTHONPATH
97-
export PYTHONNOUSERSITE=' '
98-
99-
# Just in case GCRCatalogs is installed
100-
export DESC_GCR_SITE='nersc'
101-
102-
source $LSST_INST_DIR/$SN_PYTHON_VER/etc/profile.d/conda.sh
103-
conda activate root
104-
OUTPUTPY="$(which python)"
105-
echo Now using "${OUTPUTPY}"
106-
107-
# Aug 24 2020 RK - silly hack for CFITSIO
108-
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CFITSIO_DIR/lib
109-
110-
elif [ $shifterenv ] || [ $SHIFTER_RUNTIME ]
111-
then
112-
unset LSST_HOME EUPS_PATH LSST_DEVEL EUPS_PKGROOT REPOSITORY_PATH PYTHONPATH
113-
# SHIFTER LSST Sci Pipelines env does not have the "-exact" suffice,
114-
# while local NERSC builds do (mystery)
115-
export LSST_CONDA_ENV_NAME=lsst-scipipe-4.1.0
116-
source /opt/lsst/software/stack/loadLSST.bash
117-
setup lsst_distrib
118-
#
119-
# For cosmosis and firecrown. Should try to find a better way to set these
120-
export CSL_DIR=$CONDA_PREFIX/lib/python3.10/site-packages/cosmosis/cosmosis-standard-library
121-
export FIRECROWN_SITE_PACKAGES=$CONDA_PREFIX/lib/python3.10/site-packages
122-
export FIRECROWN_DIR=/opt/lsst/software/stack/firecrown
123-
export FIRECROWN_EXAMPLES_DIR=$FIRECROWN_DIR/examples
124-
125-
# Fixes missing support in the Perlmutter libfabric:
126-
# https://docs.nersc.gov/development/languages/python/using-python-perlmutter/ #missing-support-for-matched-proberecv
127-
export MPI4PY_RC_RECV_MPROBE=0
128-
129-
# Tries to prevent cosmosis from launching any subprocesses, since that is
130-
# not allowed on Perlmutter.
131-
export COSMOSIS_NO_SUBPROCESS=1
132-
73+
if [ $gpuenv ]
74+
then
75+
echo "Setting up TD GPU env in Shifter"
76+
export TD_ENV="TD-GPU"
77+
export DESC_TD_INSTALL=/opt/desc/py
78+
source $DESC_TD_INSTALL/etc/profile.d/conda.sh
79+
# source $DESC_TD_INSTALL/bin/activate
80+
conda activate td-gpu
81+
export GSL_DIR=$CONDA_PREFIX
82+
export CFITSIO_DIR=$CONDA_PREFIX
83+
export YAML_DIR=$CONDA_PREFIX
84+
export ROOT_DIR=$ROOTSYS
85+
else
86+
export TD_ENV="TD-CPU-SCI-PIPE"
87+
unset LSST_HOME EUPS_PATH LSST_DEVEL EUPS_PKGROOT REPOSITORY_PATH PYTHONPATH
88+
# SHIFTER LSST Sci Pipelines env does not have the "-exact" suffice, while local NERSC builds do (mystery)
89+
export LSST_CONDA_ENV_NAME=lsst-scipipe-4.1.0
90+
source /opt/lsst/software/stack/loadLSST.bash
91+
setup lsst_distrib
92+
export GSL_DIR=$DESC_TD_INSTALL/conda/envs/$LSST_CONDA_ENV_NAME
93+
export CFITSIO_DIR=$DESC_TD_INSTALL/conda/envs/$LSST_CONDA_ENV_NAME
94+
export YAML_DIR=$DESC_TD_INSTALL/conda/envs/$LSST_CONDA_ENV_NAME
95+
export ROOT_DIR=$ROOTSYS
96+
97+
# For cosmosis and firecrown. Should try to find a better way to set these
98+
export CSL_DIR=$CONDA_PREFIX/lib/python3.10/site-packages/cosmosis/cosmosis-standard-library
99+
export FIRECROWN_SITE_PACKAGES=$CONDA_PREFIX/lib/python3.10/site-packages
100+
export FIRECROWN_DIR=/opt/lsst/software/stack/firecrown
101+
export FIRECROWN_EXAMPLES_DIR=$FIRECROWN_DIR/examples
102+
103+
# Fixes missing support in the Perlmutter libfabric:
104+
# https://docs.nersc.gov/development/languages/python/using-python-perlmutter/ #missing-support-for-matched-proberecv
105+
export MPI4PY_RC_RECV_MPROBE=0
106+
107+
# Tries to prevent cosmosis from launching any subprocesses, since that is
108+
# not allowed on Perlmutter.
109+
export COSMOSIS_NO_SUBPROCESS=1
110+
fi
111+
#
133112
elif [ $gpuenv ]
134113
then
135114
echo "Setting up TD GPU env"
136115
export TD_ENV="TD-GPU"
137116
# Making sure the absolutely necesary modules are loaded for GPU support
138117
module load gpu
139118
module load craype
140-
module load cray-mpich/8.1.25
141-
module load cudatoolkit/11.7
119+
module load cray-mpich
120+
module unload cudatoolkit
142121
module load evp-patch
143122

144-
export DESC_TD_INSTALL=/global/common/software/lsst/gitlab/td_env-dev/dev
123+
export DESC_TD_INSTALL=/global/common/software/lsst/gitlab/td_env-prod/stable
145124

146125
source $DESC_TD_INSTALL/conda/etc/profile.d/conda.sh
147126
conda activate td-gpu
148127

128+
export GSL_DIR=$CONDA_PREFIX
129+
export CFITSIO_DIR=$CONDA_PREFIX
130+
export YAML_DIR=$CONDA_PREFIX
131+
export ROOT_DIR=$ROOTSYS
132+
149133

150134
# Setup with LSST Science Pipelines
151135
elif [ -z "$nolsst" ]
@@ -175,6 +159,13 @@ then
175159

176160
fi
177161

162+
163+
# Set this after conda environment is setup
164+
python_ver_major=$(python -c 'import sys; print(sys.version_info.major)')
165+
python_ver_minor=$(python -c 'import sys; print(sys.version_info.minor)')
166+
export VERSION_LIBPYTHON="$python_ver_major.$python_ver_minor"
167+
168+
178169
# DIA Environment Variables
179170

180171

@@ -230,14 +221,24 @@ export PIPPIN_OUTPUT="$SNANA_SCRATCH/PIPPIN_OUTPUT"
230221
export PIPPIN_DIR="$TD_SOFTWARE/Pippin"
231222
export SBATCH_TEMPLATES="$SNANA_LSST_ROOT/SBATCH_TEMPLATES"
232223
export SNANA_DEBUG="$SNANA_LSST_USERS/kessler/debug"
233-
export SNANA_SETUP_COMMAND="source $TD/setup_td.sh"
224+
225+
if [[ "$gpuenv" ]]
226+
then
227+
export TD_GPU_ENV=1
228+
export SNANA_GPU_ENV=1
229+
export SNANA_SETUP_COMMAND="source $TD/setup_td.sh -g"
230+
export SNANA_IMAGE_DOCKER="lsstdesc/td-env-gpu:dev"
231+
else
232+
export SNANA_SETUP_COMMAND="source $TD/setup_td.sh"
233+
export SNANA_IMAGE_DOCKER="lsstdesc/td-env-cpu:stable"
234+
fi
234235
export TD_SETUP_COMMAND=$SNANA_SETUP_COMMAND
235-
export SNANA_IMAGE_DOCKER="lsstdesc/td-env:stable"
236+
236237

237238
# Add env var to point to bayeSN install
238-
export BAYESN_INSTALL=$DESC_TD_INSTALL/bayesn-public
239+
#export BAYESN_INSTALL=$DESC_TD_INSTALL/bayesn-public
239240

240-
export PATH=$PATH:${SNANA_DIR}/bin:${SNANA_DIR}/util:${PIPPIN_DIR}
241+
export PATH=$PATH:${SNANA_DIR}/bin:${SNANA_DIR}/util:${PIPPIN_DIR}:${SCONE_DIR}
241242

242243

243244
# For GCRCatalogs

0 commit comments

Comments
 (0)