Benchmark PR #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark PR | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: "Dotty PR number" | |
| required: true | |
| type: string | |
| commits: | |
| # Space-separated SHAs. If empty, benchmarks merge base + PR head. | |
| description: "Commit SHAs to benchmark" | |
| required: false | |
| default: "" | |
| type: string | |
| runs: | |
| description: "Number of runs" | |
| required: false | |
| default: "1" | |
| type: string | |
| filter: | |
| description: "JMH benchmark filter (regex pattern)" | |
| required: false | |
| default: "" | |
| type: string | |
| workflow_call: | |
| inputs: | |
| pr: | |
| description: "Dotty PR number" | |
| required: true | |
| type: string | |
| commits: | |
| # Space-separated SHAs. If empty, benchmarks merge base + PR head. | |
| description: "Commit SHAs to benchmark" | |
| required: false | |
| default: "" | |
| type: string | |
| runs: | |
| description: "Number of runs" | |
| required: false | |
| default: "1" | |
| type: string | |
| filter: | |
| description: "JMH benchmark filter (regex pattern)" | |
| required: false | |
| default: "" | |
| type: string | |
| jobs: | |
| publish-pr: | |
| runs-on: [self-hosted, benchmarks] | |
| outputs: | |
| versions: ${{ steps.publish.outputs.versions }} | |
| versions_json: ${{ steps.publish.outputs.versions_json }} | |
| runs_array: ${{ steps.publish.outputs.runs_array }} | |
| steps: | |
| - name: Checkout dotty PR | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scala/scala3 | |
| ref: refs/pull/${{ inputs.pr }}/head | |
| fetch-depth: 0 | |
| path: dotty | |
| - name: Setup Coursier | |
| uses: coursier/setup-action@v1 | |
| - name: Setup sbt | |
| uses: sbt/setup-sbt@v1 | |
| - name: Determine commits to benchmark | |
| id: commits | |
| working-directory: dotty | |
| run: | | |
| if [ -n "${{ inputs.commits }}" ]; then | |
| COMMITS="${{ inputs.commits }}" | |
| else | |
| git fetch origin main | |
| MERGE_BASE=$(git merge-base origin/main HEAD) | |
| PR_HEAD=$(git rev-parse HEAD) | |
| COMMITS="$MERGE_BASE $PR_HEAD" | |
| fi | |
| echo "commits=$COMMITS" >> "$GITHUB_OUTPUT" | |
| echo "Commits to benchmark: $COMMITS" | |
| - name: Resolve or publish commits | |
| id: publish | |
| working-directory: dotty | |
| env: | |
| BENCHMARKBUILD: "yes" | |
| run: | | |
| NIGHTLIES_URL="https://repo.scala-lang.org/artifactory/maven-nightlies/org/scala-lang/scala3-compiler_3/maven-metadata.xml" | |
| METADATA=$(curl -sf "$NIGHTLIES_URL") || METADATA="" | |
| VERSIONS="" | |
| for COMMIT in ${{ steps.commits.outputs.commits }}; do | |
| echo "=== Processing commit $COMMIT ===" | |
| FULL_HASH=$(git rev-parse "$COMMIT") | |
| SHORT_HASH="${FULL_HASH:0:7}" | |
| # 1. Check if a nightly version exists for this commit | |
| NIGHTLY=$(echo "$METADATA" | grep -o "<version>[^<]*-${SHORT_HASH}-NIGHTLY</version>" | sed 's/<[^>]*>//g' | tail -1) | |
| if [ -n "$NIGHTLY" ]; then | |
| VERSION="$NIGHTLY" | |
| echo "Found nightly version: $VERSION" | |
| # 2. Check if already published locally | |
| elif EXISTING=$(find ~/.ivy2/local/org.scala-lang/scala3-compiler_3/ -maxdepth 1 -name "*-$FULL_HASH-*" -type d 2>/dev/null | head -1) && [ -n "$EXISTING" ]; then | |
| VERSION=$(basename "$EXISTING") | |
| echo "Already published locally: $VERSION, skipping" | |
| # 3. Publish locally as fallback | |
| else | |
| git checkout "$COMMIT" | |
| sbt --no-colors clean community-build/prepareCommunityBuild | |
| VERSION=$(cat community-build/scala3-bootstrapped.version) | |
| echo "Published version: $VERSION" | |
| fi | |
| VERSIONS="$VERSIONS $VERSION" | |
| done | |
| VERSIONS=$(echo "$VERSIONS" | xargs) | |
| echo "versions=$VERSIONS" >> "$GITHUB_OUTPUT" | |
| echo "All resolved versions: $VERSIONS" | |
| # Convert to JSON array for matrix | |
| VERSIONS_JSON=$(python3 -c "import sys, json; print(json.dumps(sys.argv[1].split()))" "$VERSIONS") | |
| echo "versions_json=$VERSIONS_JSON" >> "$GITHUB_OUTPUT" | |
| # Create runs array [1, 2, ..., runs] | |
| RUNS_ARRAY=$(python3 -c "import sys, json; print(json.dumps(list(range(1, int(sys.argv[1]) + 1))))" "${{ inputs.runs }}") | |
| echo "runs_array=$RUNS_ARRAY" >> "$GITHUB_OUTPUT" | |
| # Note: This relies on the PR compiler being published into the local machine | |
| # cache (Ivy/Coursier) and the subsequent benchmark jobs running on the same | |
| # self-hosted runner machine. GitHub does not guarantee runner affinity | |
| # between jobs. If/when multiple runners share the [self-hosted, benchmarks] | |
| # label, we'll need a different way to make the published artifacts available | |
| # (or merge publish+benchmark into one job / use a remote repository / use | |
| # concurrency + single runner). | |
| # | |
| # Note: Each job imports and pushes results independently. | |
| # | |
| # Note: the benchmark.yml workflow could be called with a list of versions to | |
| # benchmark in one job, but we run them separately with a matrix because a | |
| # single long-running job would time out after 6 hours. This is a bug in | |
| # GitHub actions. We should normally be able to run jobs on self-hosted | |
| # runners for up to 5 days, but this doesn't work with "workflow_call". See | |
| # https://github.com/orgs/community/discussions/50481 and | |
| # https://docs.github.com/en/actions/reference/limits. | |
| benchmark: | |
| needs: publish-pr | |
| strategy: | |
| matrix: | |
| version: ${{ fromJSON(needs.publish-pr.outputs.versions_json) }} | |
| run: ${{ fromJSON(needs.publish-pr.outputs.runs_array) }} | |
| uses: ./.github/workflows/benchmark.yml | |
| with: | |
| versions: ${{ matrix.version }} | |
| runs: "1" | |
| filter: ${{ inputs.filter }} | |
| secrets: inherit |