openvm/.github/workflows/benchmarks-execute.yml at 8ea50e3bf12876092d7d11c9b048a188e91e3934 · scroll-tech/openvm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
name: "Execution benchmarks"

on:
  workflow_call:
  push:
    branches: ["main"]
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    branches: ["**"]
    paths:
      - "benchmarks/execute/**"
      - "crates/circuits/**"
      - "crates/toolchain/**"
      - "crates/prof/**"
      - "crates/sdk/**"
      - "crates/vm/**"
      - "extensions/**"
      - "Cargo.toml"
      - ".github/workflows/benchmarks-execute.yml"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow_ref }}-benchmarks-execute-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  S3_FIXTURES_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/fixtures
  JEMALLOC_SYS_WITH_MALLOC_CONF: "retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true"
  TOOLCHAIN: "+nightly-2025-08-19"

jobs:
  codspeed-walltime-benchmarks:
    name: Run codspeed walltime benchmarks
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m5a.xlarge # 2.5Ghz clock speed
      - image=ubuntu24-full-x64
      - extras=s3-cache

    env:
      CODSPEED_RUNNER_MODE: walltime

    steps:
      - uses: runs-on/action@v2
      - uses: actions/checkout@v5
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install architecture specific tools
        run: |
          source ci/scripts/utils.sh
          install_s5cmd

      - name: Pull fixtures from S3
        run: |
          mkdir -p benchmarks/fixtures
          s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"

      - name: Install cargo-binstall
        uses: cargo-bins/cargo-binstall@main
      - name: Install codspeed
        run: cargo binstall --no-confirm --force cargo-codspeed

      - name: Build benchmarks
        working-directory: benchmarks/execute
        run: cargo $TOOLCHAIN codspeed build --profile maxperf --features tco
      - name: Run benchmarks
        uses: CodSpeedHQ/action@v3
        with:
          working-directory: benchmarks/execute
          run: cargo $TOOLCHAIN codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}

  codspeed-instrumentation-benchmarks:
    name: Run codspeed instrumentation benchmarks
    runs-on:
      - runs-on=${{ github.run_id }}/family=m5a.xlarge/image=ubuntu24-full-x64/extras=s3-cache
    if: github.event_name != 'pull_request'

    env:
      CODSPEED_RUNNER_MODE: instrumentation

    steps:
      - uses: runs-on/action@v2
      - uses: actions/checkout@v5
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install architecture specific tools
        run: |
          source ci/scripts/utils.sh
          install_s5cmd

      - name: Pull fixtures from S3
        run: |
          mkdir -p benchmarks/fixtures
          s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"

      - name: Install cargo-binstall
        uses: cargo-bins/cargo-binstall@main
      - name: Install codspeed
        run: cargo binstall --no-confirm --force cargo-codspeed

      - name: Build benchmarks
        working-directory: benchmarks/execute
        run: cargo $TOOLCHAIN codspeed build --features tco
      - name: Run benchmarks
        uses: CodSpeedHQ/action@v3
        with:
          working-directory: benchmarks/execute
          run: cargo $TOOLCHAIN codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}