feat: optionally download fieldmaps from xrootd or cvmfs (#689)

* feat: optionally download fieldmaps from xrootd

* fix: install xrootd client

* fix: uri

* test: build single-threaded

* do not download maps on --clean

* might as well support cvmfs too

* fix: cleanup PR diff

* fix: re-enable macOS in CI

* feat: how to rm fieldmaps

---------

Co-authored-by: Nathan Baltzell <baltzell@gmx.com>

Author: 2 people

.github/workflows/ci.yml

@@ -47,7 +47,17 @@ jobs:
         with:
           java-version: ${{ env.java_version }}
           distribution: ${{ env.java_distribution }}
-      - uses: cvmfs-contrib/github-action-cvmfs@v5
+      - name: install xrootd-client (linux)
+        if: ${{ matrix.runner == 'ubuntu-latest' }}
+        run: |
+          sudo apt -y update
+          sudo apt -y upgrade
+          sudo apt -y install xrootd-client
+      - name: install xrootd-client (macos)
+        if: ${{ matrix.runner == 'macos-latest' }}
+        run: brew install xrootd
+      - name: setup cvmfs
+        uses: cvmfs-contrib/github-action-cvmfs@v5
         with:
           cvmfs_repositories: 'oasis.opensciencegrid.org'
       - name: cvmfs
@@ -56,7 +66,7 @@ jobs:
         if: ${{ github.ref_type == 'tag' }}
         run: libexec/version-bump.sh ${{ github.ref_name }}
       - name: build
-        run: ./build-coatjava.sh --no-progress -T${{ env.nthreads }}
+        run: ./build-coatjava.sh --xrootd --no-progress -T${{ env.nthreads }}
       - name: tar # tarball to preserve permissions
         run: tar czvf coatjava.tar.gz coatjava
       - uses: actions/upload-artifact@v4
@@ -76,11 +86,16 @@ jobs:
         with:
           java-version: ${{ env.java_version }}
           distribution: ${{ env.java_distribution }}
+      - name: install xrootd-client
+        run: |
+          sudo apt -y update
+          sudo apt -y upgrade
+          sudo apt -y install xrootd-client
       - uses: cvmfs-contrib/github-action-cvmfs@v5
         with:
           cvmfs_repositories: 'oasis.opensciencegrid.org'
       - name: unit tests and spotbugs
-        run: ./build-coatjava.sh --spotbugs --unittests --no-progress -T${{ env.nthreads }}
+        run: ./build-coatjava.sh --xrootd --spotbugs --unittests --no-progress -T${{ env.nthreads }}
       - name: collect jacoco report
         run: validation/jacoco-aggregate.sh
       - name: publish jacoco report

build-coatjava.sh

@@ -19,6 +19,9 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]...
    --quiet           run more quietly
    --no-progress     no download progress printouts
 
+   --xrootd          use xrootd to download field maps
+   --cvmfs           use cvmfs to download field maps
+
    --help            show this message
 
   MAVEN_OPTIONS
@@ -30,6 +33,8 @@ cleanBuild="no"
 runSpotBugs="no"
 downloadMaps="yes"
 runUnitTests="no"
+useXrootd=false
+useCvmfs=false
 mvnArgs=()
 wgetArgs=()
 for xx in $@
@@ -48,6 +53,8 @@ do
       mvnArgs+=(--no-transfer-progress)
       wgetArgs+=(--no-verbose)
       ;;
+    --xrootd) useXrootd=true ;;
+    --cvmfs) useCvmfs=true ;;
     -h|--help)
       echo "$usage"
       exit 2
@@ -72,7 +79,13 @@ command_exists () {
 }
 download () {
     ret=0
-    if command_exists wget ; then
+    if $useXrootd; then
+        xrdcp $1 ./
+        ret=$?
+    elif $useCvmfs; then
+        cp $1 ./
+        ret=$?
+    elif command_exists wget ; then
         $wget $1
         ret=$?
     elif command_exists curl ; then
@@ -91,12 +104,14 @@ download () {
 # download the default field maps, as defined in libexec/env.sh:
 # (and duplicated in etc/services/reconstruction.yaml):
 source libexec/env.sh --no-classpath
-if [ $downloadMaps == "yes" ]; then
+magfield_dir=$src_dir/etc/data/magfield
+if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then
   echo 'Retrieving field maps ...'
   webDir=https://clasweb.jlab.org/clas12offline/magfield
-  locDir=etc/data/magfield
-  mkdir -p $locDir
-  cd $locDir
+  if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
+  if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
+  mkdir -p $magfield_dir
+  cd $magfield_dir
   for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
   do
     download $webDir/$map
@@ -119,8 +134,11 @@ if [ $cleanBuild == "yes" ]; then
   for target_dir in $(find $src_dir -type d -name target); do
     echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
   done
-  echo '''DONE CLEANING.
-  Now re-run without `--clean` to build.'''
+  echo """DONE CLEANING.
+  NOTE: if you want to remove locally downloaded magnetic field maps, run:
+    rm $magfield_dir/*.dat
+
+  Now re-run without \`--clean\` to build."""
   exit
 fi