r.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: R build

on:
  push:
    branches:
      - master
    paths:
      - 'common/**'
      - 'spark/**'
      - 'spark-shaded/**'
      - 'pom.xml'
      - 'R/**'
      - '.github/workflows/r.yml'
  pull_request:
    branches:
      - '*'
    paths:
      - 'common/**'
      - 'spark/**'
      - 'spark-shaded/**'
      - 'pom.xml'
      - 'R/**'
      - '.github/workflows/r.yml'

permissions:
  contents: read

env:
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=60
  JAI_CORE_VERSION: '1.1.3'
  JAI_CODEC_VERSION: '1.1.3'
  JAI_IMAGEIO_VERSION: '1.1'
  DO_NOT_TRACK: true

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

jobs:
  build:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        spark: [3.4.0, 3.5.0]
        hadoop: [3]
        scala: [2.12.15]
        r: [oldrel, release]
    env:
      SPARK_VERSION: ${{ matrix.spark }}
      HADOOP_VERSION: ${{ matrix.hadoop }}
      SCALA_VERSION: ${{ matrix.scala }}
      # Ensure the temporary auth token for this workflow, instead of the
      # bundled GitHub PAT from the `remotes` package is used for
      # `remotes::install_github()`
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Delete existing R binaries
        run: |
          for b in R Rscript
          do
            while [ -n "$(which "$b")" ]
            do
              sudo rm -v "$(which "$b")"
            done
          done
        shell: bash
      - name: Delete existing JDK installation(s)
        run: |
          sudo apt-get -y remove --purge default-jdk adoptopenjdk-11-hotspot || :
        shell: bash
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: r-lib/actions/setup-r@v2.11.4
        with:
          r-version: ${{ matrix.r }}
          use-public-rspm: true
      - name: Query R dependencies
        uses: r-lib/actions/setup-r-dependencies@v2.11.4
        with:
          cache: true
          extra-packages: |
            any::testthat
            any::rcmdcheck
          working-directory: './R'
      - name: Build and check R package
        uses: r-lib/actions/check-r-package@v2.11.4
        with:
          build_args: 'c("--no-build-vignettes", "--no-manual")'
          args: 'c("--no-build-vignettes", "--no-manual", "--no-tests")'
          error-on: '"error"'
          working-directory: './R'
        env:
          _R_CHECK_FORCE_SUGGESTS_: false
      - name: Install apache.sedona from source
        run: Rscript -e 'install.packages("./R/", repos = NULL, type = "source")'
      - uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '11'
          cache: 'maven'
      - uses: actions/setup-python@v6
        with:
          python-version: '3.11'
      - name: Install PySpark
        run: |
          pip3 install pyspark==${SPARK_VERSION}
      - name: Download JAI libraries
        run: |
          PYSPARK_PATH=$(python3 -c "import pyspark; print(pyspark.__path__[0])")
          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
          mv -v jai_core-${JAI_CORE_VERSION}.jar ${PYSPARK_PATH}/jars
          mv -v jai_codec-${JAI_CODEC_VERSION}.jar ${PYSPARK_PATH}/jars
          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar ${PYSPARK_PATH}/jars
          echo "PYSPARK_PATH=${PYSPARK_PATH}" >> $GITHUB_ENV
      - name: Build Sedona libraries
        run: |
          SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
          mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
      - name: Copy Sedona JARs to PySpark
        run: |
          find spark-shaded/target -name sedona-*.jar -exec cp {} ${PYSPARK_PATH}/jars/ \;
      - name: Run tests
        run: |
          # Set SPARK_HOME to PySpark path
          export SPARK_HOME=${PYSPARK_PATH}

          if [[ "${SPARK_VERSION:0:3}" < "3.3" ]]; then
            case "$HADOOP_VERSION" in
              3)
                export HADOOP_VERSION=3.2
                ;;
              2)
                export HADOOP_VERSION=2.7
                ;;
            esac
          fi
          export SPARKLYR_LOG_FILE='/tmp/sparklyr.log'
          source ./.github/workflows/scripts/prepare_sparklyr_sedona_test_env.sh
          echo "Apache Sedona jar files: ${SEDONA_JAR_FILES}"
          cd ./R/tests
          NOT_CRAN='true' Rscript testthat.R
        shell: bash
        timeout-minutes: 30
      - uses: actions/upload-artifact@v6
        if: failure()
        with:
          name: Worker logs
          path: /tmp/sparklyr.log
      - name: Dump worker logs on failure
        if: failure()
        run: cat /tmp/sparklyr.log