Merge pull request #149 from ngrok/add-manual-publish-option

kmvachhani · web-flow · commit cb69b2d283ac · 2025-07-25T10:50:08.000-07:00
use manual workflow dispatch to publish artifacts
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,7 +14,8 @@ on:
         type: boolean
       publish:
         required: false
-        type: string
+        type: boolean
+        default: false
       setup:
         required: false
         type: string
@@ -26,6 +27,10 @@ jobs:
     if: ${{ inputs.if }}
     name: stable - ${{ inputs.target }} - python@3.12
     runs-on: ${{ inputs.host }}
+    env:
+      PYTHON_VERSION: "3.12"
+      MATURIN_VERSION: "v1.9.1"
+      XWIN_VERSION: "16"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -50,8 +55,22 @@ jobs:
         if: ${{ inputs.target == 'i686-pc-windows-msvc' }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ env.PYTHON_VERSION }}
           architecture: "x86"
+      - name: Download ARM64 Python SDK
+        if: ${{ inputs.target == 'aarch64-pc-windows-msvc' }}
+        run: |
+          curl -LO https://www.python.org/ftp/python/${{ env.PYTHON_VERSION }}.0/python-${{ env.PYTHON_VERSION }}.0-embed-arm64.zip
+          mkdir python-arm64
+          tar -xf python-${{ env.PYTHON_VERSION }}.0-embed-arm64.zip -C python-arm64
+      - name: Set ARM64 Python environment variables 
+        if: ${{ inputs.target == 'aarch64-pc-windows-msvc' }}
+        shell: pwsh
+        run: |
+          echo "PYTHONHOME=${{ github.workspace }}\python-arm64" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "PYTHONPATH=${{ github.workspace }}\python-arm64" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "PATH=${{ github.workspace }}\python-arm64;${env:PATH}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
       - name: Build and (Publish) musl for ${{ inputs.target }}
         uses: addnab/docker-run-action@v3
         if: ${{ inputs.target == 'x86_64-unknown-linux-musl' || inputs.target == 'aarch64-unknown-linux-musl' }}
@@ -72,8 +91,9 @@ jobs:
 
             if [ "${SHOULD_PUBLISH}" == 'true' ]; then
               echo "~~~~ maturin publishing"
-              . ./venv/bin/activate && maturin publish --no-sdist -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
+              . ./venv/bin/activate && maturin publish --no-sdist --skip-existing -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
             fi
+
       - name: Build and (Publish) aarch64-unknown-linux-gnu
         uses: addnab/docker-run-action@v3
         if: ${{ inputs.target == 'aarch64-unknown-linux-gnu' }}
@@ -101,7 +121,7 @@ jobs:
             if [ "${SHOULD_PUBLISH}" = 'true' ]; then
               echo "~~~~ maturin publishing"
               # pass '--debug' to avoid optimization, which breaks tls signature validation on this platform
-              . .env/bin/activate && maturin publish --target ${{ inputs.target }} --no-sdist --debug -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
+              . .env/bin/activate && maturin publish --target ${{ inputs.target }} --no-sdist --skip-existing --debug -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
             else
               echo "~~~~ maturin building"
               . .env/bin/activate && maturin build --target ${{ inputs.target }}
@@ -136,30 +156,59 @@ jobs:
             if [ "${SHOULD_PUBLISH}" = 'true' ]; then
               echo "~~~~ maturin publishing"
               # pass '--debug' to avoid optimization, which breaks tls signature validation on this platform
-              . .env/bin/activate && maturin publish --target ${{ inputs.target }} --no-sdist --debug -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
+              . .env/bin/activate && maturin publish --target ${{ inputs.target }} --no-sdist --skip-existing --debug -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
             else
               echo "~~~~ maturin building"
               RUST_BACKTRACE=1 . .env/bin/activate && maturin build --target ${{ inputs.target }} --verbose
             fi
+      - name: Build and (Publish) x86_64-unknown-linux-gnu
+        uses: addnab/docker-run-action@v3
+        if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }}
+        with:
+          image: ${{ inputs.docker }}
+          options: '-e SHOULD_PUBLISH=${{ inputs.publish }} --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build'
+          run: |
+            # Update and install required packages
+            sudo apt-get update
+            sudo apt-get install -y llvm-dev clang libclang-dev gcc-multilib
+
+            # Install Rust minimal toolchain
+            curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+            export PATH="$HOME/.cargo/bin:$PATH"
+
+            # Add necessary Rust targets and components
+            rustup component add llvm-tools-preview || true
+            rustup target add x86_64-unknown-linux-gnu
+
+            # Create and activate Python virtual environment
+            python3 -m venv .env
+            . .env/bin/activate && pip install -r requirements.txt
+            . .env/bin/activate && pip install patchelf
+
+            if [ "${SHOULD_PUBLISH}" = 'true' ]; then
+              echo "~~~~ maturin publishing"
+              . .env/bin/activate && maturin publish --target ${{ inputs.target }} --no-sdist --skip-existing -i python3.12 -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
+            else
+              echo "~~~~ maturin building"
+              . .env/bin/activate && maturin build --target ${{ inputs.target }}
+            fi
       - name: Build Python Extension Module (non-docker)
         if: ${{ inputs.publish != true && !inputs.docker && !contains(inputs.target,'android') }}
         uses: PyO3/maturin-action@v1
         with:
-          maturin-version: v1.6.0
+          maturin-version: ${{ env.MATURIN_VERSION }}
           target: ${{ inputs.target }}
-          # builds in release mode with the specified python version as the interpreter and the Cargo.toml file as the manifest
-          args: --release -i python3.12 --target ${{ inputs.target }} -m Cargo.toml
+          args: --release -i python${{ env.PYTHON_VERSION }} --target ${{ inputs.target }} -m Cargo.toml
         env:
-          # see https://github.com/PyO3/maturin/issues/2110
-          XWIN_VERSION: '16'
+          XWIN_VERSION: ${{ env.XWIN_VERSION }}
       - name: Publish to PyPI (non-docker)
         if: ${{ inputs.publish == true && !inputs.docker && !contains(inputs.target,'android') }}
         uses: PyO3/maturin-action@v1
         with:
           command: publish
-          maturin-version: v1.6.0
+          maturin-version: ${{ env.MATURIN_VERSION }}
           target: ${{ inputs.target }}
-          args: --no-sdist -i python3.12 -m Cargo.toml -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
+          args: --no-sdist --skip-existing -i python${{ env.PYTHON_VERSION }} -m Cargo.toml -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -4,7 +4,23 @@ env:
   APP_NAME: ngrok
   MACOSX_DEPLOYMENT_TARGET: '10.13'
 on:
+  workflow_call:
+    inputs:
+      publish:
+        required: false
+        default: false
+        type: boolean
+    secrets:
+      NGROK_AUTHTOKEN:
+        required: true
+      MATURIN_PASSWORD:
+        required: true
   push:
+    inputs:
+      publish:
+        required: false
+        default: false
+        type: boolean
     branches:
       - '**'
     tags-ignore:
@@ -15,11 +31,7 @@ on:
       - '**/*.gitignore'
       - .editorconfig
       - docs/**
-  workflow_dispatch:
-  pull_request:
-    types: [opened, reopened]
-    branches:
-      - main
+
 jobs:
   udeps:
     name: Udeps
@@ -61,42 +73,19 @@ jobs:
         run: |
           NGROK_AUTHTOKEN=${{ secrets.NGROK_AUTHTOKEN }} make mypy
           NGROK_AUTHTOKEN=${{ secrets.NGROK_AUTHTOKEN }} make testfast
-  decide:
-    name: Decide on Publishing
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Decide
-        id: decide
-        run: |
-          echo "github.ref: ${{ github.ref }}"
-          echo "github.repository: ${{ github.repository }}"
-          echo "git log:"
-          git log -1 --pretty=%B
-
-          if [ '${{ github.ref }}' == 'refs/heads/main' ] && [ '${{ github.repository }}' == 'ngrok/ngrok-python' ] && git log -1 --pretty=%B | grep "^Release [0-9]\+\.[0-9]\+\.[0-9]\+$"; then
-            echo "SHOULD_PUBLISH=true" >> $GITHUB_OUTPUT
-          else
-            echo "SHOULD_PUBLISH=false" >> $GITHUB_OUTPUT
-          fi
-          cat $GITHUB_OUTPUT
-    outputs:
-      publish: ${{ steps.decide.outputs.SHOULD_PUBLISH }}
 
   build:
     needs:
       - clippy
-      - decide
       - fmt
       - test
       - udeps
     uses: ./.github/workflows/build.yml
     secrets: inherit
     with:
+      publish: ${{ inputs.publish || false }}
       docker: ${{ matrix.settings.docker }}
       host: ${{ matrix.settings.host }}
-      if: true
-      publish: ${{ needs.decide.outputs.publish }}
       setup: ${{ matrix.settings.setup }}
       target: ${{ matrix.settings.target }}
     strategy:
@@ -123,6 +112,7 @@ jobs:
             docker: ghcr.io/rust-cross/manylinux2014-cross:armv7
           - host: ubuntu-latest
             target: x86_64-unknown-linux-gnu
+            docker: ghcr.io/rust-cross/manylinux2014-cross:x86_64
           - host: ubuntu-latest
             target: x86_64-unknown-linux-musl
             docker: alpine:3.21
@@ -136,7 +126,6 @@ jobs:
             target: x86_64-pc-windows-msvc 
   
   build-freebsd:
-    needs: decide
     runs-on: ubuntu-22.04
     name: Build FreeBSD
     timeout-minutes: 20
@@ -152,9 +141,8 @@ jobs:
           RUSTUP_HOME: /usr/local/rustup
           CARGO_HOME: /usr/local/cargo
           RUSTUP_IO_THREADS: 1
-          SHOULD_PUBLISH: ${{ needs.decide.outputs.publish }}
         with:
-          envs: RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS SHOULD_PUBLISH
+          envs: RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS
           usesh: true
           mem: 3000
           prepare: |
@@ -167,37 +155,12 @@ jobs:
             export PATH="/usr/local/cargo/bin:$PATH"
             python3 -m venv .env
             . .env/bin/activate && pip install -r requirements.txt
-            if [ "${SHOULD_PUBLISH}" == 'true' ]; then
-              echo "~~~~ maturin just building since pypi doesn't support BSD wheels"
-              # https://discuss.python.org/t/pypi-org-unsupported-platform-tag-openbsd-7-0-amd64/16302
-              # . .env/bin/activate && maturin publish --no-sdist -u __token__ -p ${{ secrets.MATURIN_PASSWORD }}
-              . .env/bin/activate && maturin build
-            else
-              echo "~~~~ maturin building"
-              . .env/bin/activate && maturin build
-            fi
-            # make the rsync back faster
+            echo "~~~~ maturin building"
+            . .env/bin/activate && maturin build
             rm -rf .env/ target/debug/ target/release/
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: bindings-freebsd
           path: target/wheels/*.whl
-          if-no-files-found: error
-  tag:
-    if: ${{ github.ref == 'refs/heads/main' && needs.decide.outputs.publish == true }}
-    needs:
-      - decide
-      - build
-    runs-on: ubuntu-latest
-    name: Tag Release
-    steps:
-      - uses: actions/checkout@v4
-      - uses: jrobsonchase/direnv-action@v0.7
-      - name: Tag
-        run: |
-          version="$(extract-crate-version ngrok-python)"
-          git config user.name "GitHub Action"
-          git config user.email noreply@ngrok.com
-          git tag -a -m "Version ${version}" v${version}
-          git push --tags
+          if-no-files-found: error
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,38 @@
+name: Publish All
+on:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    name: Run CI
+    uses: ./.github/workflows/ci.yml
+    secrets:
+      NGROK_AUTHTOKEN: ${{ secrets.NGROK_AUTHTOKEN }}
+      MATURIN_PASSWORD: ${{ secrets.MATURIN_PASSWORD }}
+    permissions:
+      contents: write
+    with:
+      publish: true
+  tag-release:
+    name: Tag Release
+    needs: ci
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jrobsonchase/direnv-action@v0.7
+      - name: Tag Release
+        run: |
+          version="$(grep '^version =' Cargo.toml | head -n1 | awk -F\" '{print $2}')"
+          tag="v${version}"
+          git config user.name "GitHub Action"
+          git config user.email noreply@ngrok.com
+          echo "Version: ${version} tag: $tag"
+          echo "Fetching all tags in the repository"
+          git fetch --tags
+          if git rev-parse "refs/tags/$tag" >/dev/null 2>&1; then
+            echo "Tag $tag already exists, skipping tag creation."
+          else
+            echo "Tag $tag does not exist, pushing tag."
+            git tag -a -m "Version ${version}" $tag
+            git push --tags
+          fi
diff --git a/pyproject.toml b/pyproject.toml
@@ -31,7 +31,7 @@ changelog = "https://github.com/ngrok/ngrok-python/blob/main/CHANGELOG.md"
 ngrok-asgi = "ngrok:__main__.asgi_cli"
 
 [build-system]
-requires = ["maturin>=1.5,<=1.6"]
+requires = ["maturin>=1.5,<=1.9.1"]
 build-backend = "maturin"
 
 [tool.maturin]
diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,7 @@
 aiohttp==3.9.5 # async HTTP client/server framework
 black==23.3.0 # code formatter
 furo==2024.8.6 # customisable Sphinx theme
-maturin==1.6.0 # build and publish Python packages
+maturin==1.9.1 # build and publish Python packages
 mypy==1.15.0 # static type checker
 myst-parser==4.0.1 # Sphinx extension for Markdown
 pytest-xdist==3.6.1 # run tests in parallel
