test: improve dev environment for integration tests

.github/workflows/code-quality.yml

@@ -0,0 +1,67 @@
+name: Code quality
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: code-quality-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snok/install-poetry@v1
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: poetry
+      - run: poetry install --with dev
+      - name: isort (black profile)
+        run: poetry run isort --check --settings-path=pyproject.toml src/embit
+
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snok/install-poetry@v1
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: poetry
+      - run: poetry install --with dev
+      - name: Black
+        run: poetry run black --check src/embit
+
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snok/install-poetry@v1
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: poetry
+      - run: poetry install --with dev
+      - name: Pylint (errors-only)
+        run: poetry run pylint --errors-only --rcfile=pyproject.toml src/embit
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+          shellcheck tests/run.sh
+
+  typos:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Typos (spell check)
+        uses: crate-ci/typos@v1.45.0

.github/workflows/conventional-commits.yml

@@ -0,0 +1,57 @@
+name: Conventional commits
+
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: conventional-commits-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  commit-messages:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install checker
+        run: pip install "conventional-pre-commit==4.4.0"
+
+      - name: Verify pushed commits
+        if: github.event_name == 'push'
+        run: |
+          set -euo pipefail
+          before="${{ github.event.before }}"
+          after="${{ github.event.after }}"
+          check() {
+            git log -1 --format=%B "$1" > /tmp/commit-msg.txt
+            conventional-pre-commit /tmp/commit-msg.txt
+          }
+          if [ "$before" = "0000000000000000000000000000000000000000" ]; then
+            check "$after"
+            exit 0
+          fi
+          for rev in $(git rev-list --no-merges "${before}..${after}"); do
+            check "$rev"
+          done
+
+      - name: Fetch PR base (for range)
+        if: github.event_name == 'pull_request'
+        run: git fetch --no-tags origin ${{ github.event.pull_request.base.sha }}
+
+      - name: Verify PR commits
+        if: github.event_name == 'pull_request'
+        run: |
+          set -euo pipefail
+          base="${{ github.event.pull_request.base.sha }}"
+          for rev in $(git rev-list --no-merges "${base}..HEAD"); do
+            git log -1 --format=%B "$rev" > /tmp/commit-msg.txt
+            conventional-pre-commit /tmp/commit-msg.txt
+          done

.github/workflows/tests.yml

@@ -0,0 +1,142 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: tests-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install poetry
+      - run: poetry config virtualenvs.in-project true
+      - uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('poetry.lock') }}
+      - run: poetry install --with dev
+      - name: Unit tests
+        run: poetry run pytest tests/tests
+
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install poetry
+      - run: poetry config virtualenvs.in-project true
+      - uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('poetry.lock') }}
+      - run: poetry install --with dev
+
+      - name: Export daemon versions
+        run: grep -v '^#' tests/integration/daemon-versions.env | grep -v '^$' >> "$GITHUB_ENV"
+
+      - name: Restore daemon cache
+        id: daemon-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: .cache/embit-test-src-bin-cache
+          key: embit-daemons-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/integration/daemon-versions.env') }}
+          restore-keys: |
+            embit-daemons-v1-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Check if daemon binaries are present
+        id: need_toolchain
+        run: |
+          set -euo pipefail
+          HOST_ID="$(uname -s)-$(uname -m)"
+          B="${BITCOIN_REVISION:?}"; B="${B#v}"
+          E="${ELEMENTS_REVISION:?}"
+          ROOT="${GITHUB_WORKSPACE}/.cache/embit-test-src-bin-cache"
+          BC="${ROOT}/${HOST_ID}/bitcoin-v${B}/bitcoind"
+          EL="${ROOT}/${HOST_ID}/elements-${E}/elementsd"
+          if [[ -f "$BC" && -f "$EL" ]]; then
+            echo "Both daemons found in cache."
+            echo "install=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "install=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Install runtime libs (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libevent-dev \
+            libboost-system-dev \
+            libboost-filesystem-dev \
+            libboost-thread-dev \
+            libdb-dev \
+            libsqlite3-dev
+
+      - name: Install build toolchain (Linux)
+        if: steps.need_toolchain.outputs.install == 'true' && runner.os == 'Linux'
+        run: |
+          sudo apt-get install -y \
+            build-essential \
+            pkg-config \
+            libtool \
+            autotools-dev \
+            automake \
+            cmake \
+            git \
+            curl \
+            libssl-dev \
+            libboost-test-dev \
+            libboost-program-options-dev \
+            zlib1g-dev
+
+      - name: Install runtime libs (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install \
+            boost \
+            libevent \
+            openssl@3 \
+            berkeley-db@4 \
+            sqlite
+
+      - name: Install build toolchain (macOS)
+        if: steps.need_toolchain.outputs.install == 'true' && runner.os == 'macOS'
+        run: |
+          brew install \
+            autoconf \
+            automake \
+            libtool \
+            pkg-config \
+            cmake
+
+      - name: Integration tests
+        run: |
+          chmod +x tests/run.sh
+          poetry run bash tests/run.sh
+
+      - name: Save daemon cache
+        if: always() && steps.daemon-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .cache/embit-test-src-bin-cache
+          key: embit-daemons-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/integration/daemon-versions.env') }}

.gitignore

@@ -117,11 +117,6 @@ venv.bak/
 # mkdocs documentation
 /site
 
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
 # Pyre type checker
 .pyre/
 

.pre-commit-config.yaml

@@ -1,6 +1,89 @@
+# Two hooks (approach 1): unit-only is fast; integration still builds/restores
+# daemons.
+#
+# Skip when needed:
+#
+# SKIP=conventional-pre-commit,isort,black,pylint,shellcheck,typos,unit-test,integration-test git commit ...
+#
+# Typos uses --force-exclude so _typos.toml extend-exclude (tests, wordlists,
+# i18n fixtures) applies when hooks pass explicit paths.
+#
+# Mirrors .github/workflows/code-quality.yml
+# (isort → black → pylint → shellcheck → typos) and tests.yml.
+#
+# After changing this file: pre-commit install --install-hooks  (installs
+# pre-commit + commit-msg hooks).
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+
 repos:
--   repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.1
     hooks:
-    - id: black
-      language_version: python3
+      - id: isort
+        name: isort (black profile)
+        args: [--settings-path=pyproject.toml]
+        files: ^src/embit/
+
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        name: black
+        entry: black ./src/embit
+
+  - repo: https://github.com/pylint-dev/pylint
+    rev: v3.3.8
+    hooks:
+      - id: pylint
+        name: pylint (errors-only)
+        args: [--errors-only, --rcfile=pyproject.toml]
+        additional_dependencies: []
+        files: ^src/embit/
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.11.0.1
+    hooks:
+      - id: shellcheck
+        name: shellcheck (tests/run.sh)
+        files: ^tests/run\.sh$
+
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.45.0
+    hooks:
+      - id: typos
+        name: typos (spell check)
+        args: [--force-exclude]
+
+  - repo: local
+    hooks:
+      - id: test-clean
+        name: test-clean
+        entry: python -c 'import shutil, os; os.path.exists("htmlcov") and shutil.rmtree("htmlcov")'
+        language: system
+        pass_filenames: false
+        always_run: true
+        require_serial: true
+      - id: unit-test
+        name: unit-test
+        entry: poetry run pytest --cache-clear --cov src/embit --cov-report html --show-capture all --capture tee-sys -r A ./tests/tests
+        language: system
+        pass_filenames: false
+        always_run: true
+        require_serial: true
+      - id: integration-test
+        name: integration-test
+        entry: bash -c 'ulimit -n 10240 2>/dev/null; poetry run bash tests/run.sh'
+        language: system
+        pass_filenames: false
+        always_run: true
+        require_serial: true
+
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v4.4.0
+    hooks:
+      - id: conventional-pre-commit
+        name: conventional-commit-1.0
+        stages: [commit-msg]
+        args: []