feat(sql_execution): Implement retry logic for userpod API #323
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| POETRY_VERSION: "2.2.0" | |
| POETRY_VIRTUALENVS_CREATE: true | |
| POETRY_VIRTUALENVS_IN_PROJECT: true | |
| POETRY_INSTALLER_PARALLEL: true | |
| jobs: | |
| spell-check: | |
| name: Spell Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Cache dependencies | |
| id: cache-deps | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-v2-${{ github.event.pull_request.head.repo.full_name || github.repository }}-py${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root --with dev | |
| - name: Run spell check | |
| run: poetry run codespell | |
| format: | |
| name: Format | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Cache dependencies | |
| id: cache-deps | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-v2-${{ github.event.pull_request.head.repo.full_name || github.repository }}-py${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root --with dev | |
| - name: Check formatting with black | |
| run: poetry run black . --check | |
| - name: Check linting with flake8 | |
| run: poetry run flake8 . | |
| license-check: | |
| name: License Check | |
| # This only checks production dependencies inclusive of the dev deps that are actually prod deps (see pyproject.toml). | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Install production dependencies | |
| run: | | |
| # Install production dependencies plus packages needed for license checking | |
| poetry install --no-interaction --no-root --only main --with license-check | |
| - name: Install pip-licenses | |
| run: poetry run python -m pip install pip-licenses | |
| - name: Check licenses | |
| run: | | |
| # Generate license reports | |
| poetry run pip-licenses --format=markdown --order=license > licenses.md | |
| poetry run pip-licenses --format=json > licenses.json | |
| # Display in logs and step summary | |
| cat licenses.md | |
| echo "## License Report" >> $GITHUB_STEP_SUMMARY | |
| cat licenses.md >> $GITHUB_STEP_SUMMARY | |
| # Only allow licenses compatible with Apache 2.0 (allowlist approach) | |
| # Ignored packages either have UNKNOWN licenses or not distributed | |
| poetry run pip-licenses --allow-only "Apache;MIT;BSD;ISC;Unlicense;CC0;Public Domain;Python Software Foundation;Mozilla Public License 2.0;GNU Library or Lesser General Public License (LGPL)" --partial-match --ignore-packages arro3-core click dependency-groups Flask jeepney jupyter_core matplotlib-inline MarkupSafe more-itertools pymssql PyMySQL SecretStorage sqlalchemy-spanner typing-extensions typing-inspection urllib3 | |
| echo "✅ All licenses are compatible with Apache 2.0" | |
| - name: Upload license report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: license-report | |
| path: | | |
| licenses.md | |
| licenses.json | |
| mypy: | |
| name: Typecheck - ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Cache dependencies | |
| id: cache-deps | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-v2-${{ github.event.pull_request.head.repo.full_name || github.repository }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root --with dev | |
| - name: Install project | |
| run: poetry install --no-interaction | |
| - name: Run mypy | |
| run: | | |
| poetry run mypy deepnote_toolkit/ --show-error-codes --no-error-summary | |
| - name: Run mypy on specific modules (if main check fails) | |
| if: failure() | |
| run: | | |
| echo "Main mypy check failed. Running on individual modules for detailed output:" | |
| for module in deepnote_toolkit/*.py; do | |
| if [ -f "$module" ]; then | |
| echo "Checking $module..." | |
| poetry run mypy "$module" --show-error-codes || true | |
| fi | |
| done | |
| gitleaks: | |
| name: Gitleaks check | |
| runs-on: ubuntu-latest | |
| # Only run for base repo, not forks | |
| if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Run Gitleaks | |
| uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} | |
| semgrep: | |
| name: Static Analysis | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| env: | |
| SEMGREP_BASELINE_REF: ${{ github.base_ref || 'main' }} | |
| SEMGREP_RULES: .semgrep/rules.yml | |
| SEMGREP_REPO_NAME: ${{ github.repository }} | |
| SEMGREP_REPO_URL: ${{ github.server_url }}/${{ github.repository }} | |
| container: | |
| image: returntocorp/semgrep | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Run Semgrep scan | |
| run: semgrep ci | |
| env: | |
| SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} | |
| qlty: | |
| name: Qlty Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Install qlty | |
| uses: qltysh/qlty-action/install@6bfd5815fe4171d7a566362f24d75e8454909d2a | |
| - name: Run qlty check | |
| run: qlty check | |
| - name: Run qlty code smells analysis | |
| run: qlty smells | |
| tests-unit: | |
| name: Test - Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Export version | |
| id: version | |
| uses: ./.github/actions/export-version | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Java 17 | |
| uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.local | |
| key: poetry-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ matrix.python-version }}-poetry${{ env.POETRY_VERSION }} | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Cache dependencies | |
| id: cache-deps | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-v2-${{ github.event.pull_request.head.repo.full_name || github.repository }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} | |
| - name: Add venv to PATH | |
| if: steps.cache-deps.outputs.cache-hit == 'true' | |
| run: echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH | |
| - name: Install dependencies | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root --with dev | |
| - name: Install project | |
| run: poetry install --no-interaction --only-root | |
| - name: Run unit tests | |
| env: | |
| TOOLKIT_VERSION: ${{ steps.version.outputs.VERSION }} | |
| run: | | |
| poetry run pytest tests/unit \ | |
| --cov=deepnote_toolkit \ | |
| --cov=installer \ | |
| --cov=deepnote_core \ | |
| --cov-branch \ | |
| --cov-report=term-missing:skip-covered \ | |
| --junitxml=junit.xml \ | |
| -o junit_family=legacy | |
| - name: Per-version coverage summary | |
| run: | | |
| echo "## Python ${{ matrix.python-version }} Coverage" >> $GITHUB_STEP_SUMMARY | |
| poetry run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: python-${{ matrix.python-version }} | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: coverage-${{ matrix.python-version }} | |
| path: .coverage | |
| retention-days: 1 | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| coverage-combine: | |
| name: Combine and Upload Coverage | |
| runs-on: ubuntu-latest | |
| needs: tests-unit | |
| if: always() | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install coverage | |
| run: pip install coverage[toml] | |
| - name: Download all coverage artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| pattern: coverage-* | |
| path: coverage-artifacts/ | |
| - name: Combine coverage files | |
| run: | | |
| shopt -s nullglob | |
| mkdir -p coverage-data | |
| i=0 | |
| for file in coverage-artifacts/*/.coverage; do | |
| cp "$file" "coverage-data/.coverage.$i" | |
| i=$((i + 1)) | |
| done | |
| coverage combine coverage-data/ | |
| coverage xml -o coverage-data/coverage.xml | |
| coverage report | |
| echo "## Combined Coverage Report" >> $GITHUB_STEP_SUMMARY | |
| coverage report --format=markdown >> $GITHUB_STEP_SUMMARY | |
| - name: Upload combined coverage to Codecov | |
| uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: ${{ github.repository }} | |
| files: ./coverage-data/coverage.xml | |
| flags: combined | |
| disable_search: true | |
| fail_ci_if_error: ${{ github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' }} | |
| - name: Upload combined coverage report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: coverage-combined-report | |
| path: coverage-data/coverage.xml | |
| retention-days: 30 | |
| audit-prod: | |
| name: Audit - Production | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Install poetry-plugin-export | |
| run: poetry self add poetry-plugin-export | |
| - name: Run audit for production dependencies | |
| run: | | |
| poetry export --without dev -f requirements.txt -o /tmp/requirements-prod.txt | |
| - uses: pypa/gh-action-pip-audit@f1c3022531130da3984dbdfb2786f6e1cba15556 # latest release (v1.1.0 doesn't contain `disable-pip`) | |
| with: | |
| inputs: /tmp/requirements-prod.txt | |
| require-hashes: true | |
| disable-pip: true | |
| # PYSEC-2023-121 (CVE-2022-4899 / GHSA-5c9c-6x87-f9vm) | |
| # * incorrectly flags zstd versions >= 1.5.4.0 as vulnerable to CVE-2022-4899 | |
| # * see https://github.com/pypa/advisory-database/blob/main/vulns/zstd/PYSEC-2023-121.yaml (our version ranges are outside of reported versions) | |
| # CVE-2026-0994 (protobuf DoS via nested Any messages in ParseDict) | |
| # * no fix version available yet | |
| # * low risk: only affects json_format.ParseDict() with untrusted input | |
| # PYSEC-2024-161 (pyarrow deserialization vulnerability) | |
| # * false positive: only affects Arrow R package, not PyArrow | |
| # * see CVE description: "This vulnerability only affects the arrow R package, not other Apache Arrow implementations" | |
| # * databricks-sqlalchemy 1.x caps pyarrow<17, but upgrading requires SQLAlchemy 2.x (which is not possible for some Python versions) | |
| # CVE-2026-32274 (black cache path injection via --python-cell-magics) | |
| # * dev-only dependency, not used with untrusted input | |
| # * fix requires black 26.x which changes formatting style; deferring upgrade | |
| # CVE-2026-27448, CVE-2026-27459 (pyopenssl callback vulnerabilities) | |
| # * transitive dep of snowflake-connector-python, not used directly | |
| # * blocked: snowflake-connector-python pins pyOpenSSL<26.0.0 (even in latest 4.3.0 as of 2026-03-17) | |
| # * upstream fix: https://github.com/snowflakedb/snowflake-connector-python/pull/2793 | |
| ignore-vulns: &ignore-vulns | | |
| PYSEC-2023-121 | |
| CVE-2026-0994 | |
| PYSEC-2024-161 | |
| CVE-2026-32274 | |
| CVE-2026-27448 | |
| CVE-2026-27459 | |
| audit-all: | |
| name: Audit - All | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| steps: | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: ${{ env.POETRY_VIRTUALENVS_CREATE }} | |
| virtualenvs-in-project: ${{ env.POETRY_VIRTUALENVS_IN_PROJECT }} | |
| installer-parallel: ${{ env.POETRY_INSTALLER_PARALLEL }} | |
| - name: Install poetry-plugin-export | |
| run: poetry self add poetry-plugin-export | |
| - name: Run audit for all dependencies | |
| run: | | |
| poetry export -f requirements.txt -o /tmp/requirements-all.txt | |
| - uses: pypa/gh-action-pip-audit@f1c3022531130da3984dbdfb2786f6e1cba15556 # latest release (v1.1.0 doesn't contain `disable-pip`) | |
| with: | |
| inputs: /tmp/requirements-all.txt | |
| require-hashes: true | |
| disable-pip: true | |
| ignore-vulns: *ignore-vulns |