train: 0.8.0 release candidate (alpha.16)

.github/workflows/ci.yml

@@ -27,6 +27,7 @@ env:
   CARGO_TERM_COLOR: always
   RUSTFLAGS: "-D warnings"
   # Reviewed ARM64 archive pin, shared by the producer and every CI consumer.
+  CI_LLAMA_AMD64_SHA256: 0dee60d855f864aa9e8a7200014baf9419e3a4c9ccb3f3609532aad0ed34f2c0
   CI_LLAMA_ARM64_SHA256: a92e404708feae91cbede43322e445971450de8a204f4ff51502a7f42f5df8b8
   # An explicit dispatch ref may select a tag independently of the workflow ref.
   CI_USE_CACHE: ${{ github.ref_type != 'tag' && !(github.event_name == 'workflow_dispatch' && inputs.ref != '') }}
@@ -95,8 +96,121 @@ jobs:
           path: ${{ runner.temp }}/browser-status-screenshots/
           if-no-files-found: ignore
 
-  # One input for every ARM consumer. PRs restore their develop base's cache;
+  # Every tracked Cargo.lock against the current RustSec database. Reviewed
+  # exceptions live only in elastos/.cargo/audit.toml, which cargo-audit
+  # reads from its working directory. No cache: the binary is a checksummed
+  # release and the database must be fresh.
+  cargo-audit:
+    name: cargo-audit
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
+          persist-credentials: false
+      - name: install checksummed cargo-audit
+        run: |
+          set -euo pipefail
+          archive=cargo-audit-x86_64-unknown-linux-musl-v0.22.2
+          directory="$RUNNER_TEMP/cargo-audit"
+          mkdir -p "$directory"
+          curl -fsSL --connect-timeout 30 --max-time 120 --retry 3 \
+            "https://github.com/rustsec/rustsec/releases/download/cargo-audit%2Fv0.22.2/$archive.tgz" \
+            -o "$directory/$archive.tgz"
+          echo "7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428  $directory/$archive.tgz" | sha256sum -c -
+          tar -xzf "$directory/$archive.tgz" -C "$directory" "$archive/cargo-audit"
+          echo "$directory/$archive" >> "$GITHUB_PATH"
+      - name: audit every Cargo.lock
+        working-directory: elastos
+        run: |
+          set -euo pipefail
+          # A failed listing must fail the job, not audit a partial list.
+          listing=$(git -C .. ls-files '*Cargo.lock')
+          locks=()
+          while IFS= read -r lock; do [[ -n "$lock" ]] && locks+=("$lock"); done <<< "$listing"
+          (( ${#locks[@]} > 0 ))
+          db="$RUNNER_TEMP/advisory-db"
+          fetch=()
+          failed=0
+          for lock in "${locks[@]}"; do
+            echo "::group::$lock"
+            cargo-audit audit --db "$db" ${fetch[@]+"${fetch[@]}"} --file "../$lock" || failed=1
+            echo "::endgroup::"
+            # One database fetch per run; every lockfile is checked against it.
+            fetch=(--no-fetch)
+          done
+          exit "$failed"
+
+  # One reviewed input per Linux architecture. PRs restore their develop base's cache;
   # only a develop push can save a verified recipe result in this namespace.
+  engine-llama-amd64:
+    name: engine-llama-amd64
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    needs: source-gate
+    container:
+      image: ubuntu@sha256:5ec03bb3441e8b0bf3b4f9cd4629a1ae763010dc3035bb8da3ae6cf026486401
+    defaults:
+      run:
+        shell: bash
+    env:
+      DEBIAN_FRONTEND: noninteractive
+      BUILD_CONTAINER_IMAGE: ubuntu@sha256:5ec03bb3441e8b0bf3b4f9cd4629a1ae763010dc3035bb8da3ae6cf026486401
+      ENGINE_BUILD_TOOLS: build-essential cmake git ca-certificates curl binutils file python3
+    steps:
+      - name: install engine build tools
+        run: |
+          apt-get update -qq
+          read -r -a tools <<< "$ENGINE_BUILD_TOOLS"
+          apt-get install -y -qq --no-install-recommends "${tools[@]}"
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
+          persist-credentials: false
+      - name: bind engine cache to build recipe
+        id: engine-recipe
+        run: |
+          recipe_hash=$( {
+            cat scripts/build/build-llama-server-bundle.sh
+            cat scripts/verify-linux-engine.py
+            printf '\n%s\n%s\n' "$BUILD_CONTAINER_IMAGE" "$ENGINE_BUILD_TOOLS"
+          } | sha256sum | cut -d ' ' -f 1)
+          echo "cache-key=develop-llama-amd64-${recipe_hash}" >> "$GITHUB_OUTPUT"
+      - name: restore pinned AMD64 engine
+        id: engine-cache
+        if: env.CI_USE_CACHE == 'true'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/llama-amd64-bundle
+          key: ${{ steps.engine-recipe.outputs.cache-key }}
+      - name: build and validate pinned AMD64 engine
+        id: engine-build
+        if: steps.engine-cache.outputs.cache-hit != 'true'
+        run: |
+          RECIPE_COMMIT="$(git -c safe.directory="$GITHUB_WORKSPACE" rev-parse HEAD)"
+          export RECIPE_COMMIT
+          bash scripts/build/build-llama-server-bundle.sh --platform linux-amd64 "$RUNNER_TEMP/llama-amd64-bundle"
+      - name: verify accepted AMD64 engine archive
+        run: bash scripts/build/build-llama-server-bundle.sh --platform linux-amd64 --verify-archive "$RUNNER_TEMP/llama-amd64-bundle"
+      - name: save pinned AMD64 engine
+        if: env.CI_USE_CACHE == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/develop' && steps.engine-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/llama-amd64-bundle
+          key: ${{ steps.engine-cache.outputs.cache-primary-key }}
+      - name: upload pinned AMD64 engine input
+        # Retain build receipts on a pin mismatch for review; dependent jobs
+        # require this job to succeed before they can consume the archive.
+        if: "!cancelled() && (steps.engine-cache.outputs.cache-hit == 'true' || steps.engine-build.outcome == 'success')"
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+        with:
+          name: llama-amd64-bundle
+          path: ${{ runner.temp }}/llama-amd64-bundle/
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 0
+
   engine-llama-arm64:
     name: engine-llama-arm64
     runs-on: ubuntu-24.04-arm
@@ -126,6 +240,7 @@ jobs:
         run: |
           recipe_hash=$( {
             cat scripts/build/build-llama-server-bundle.sh
+            cat scripts/verify-linux-engine.py
             printf '\n%s\n%s\n' "$BUILD_CONTAINER_IMAGE" "$ENGINE_BUILD_TOOLS"
           } | sha256sum | cut -d ' ' -f 1)
           echo "cache-key=develop-llama-arm64-${recipe_hash}" >> "$GITHUB_OUTPUT"
@@ -621,8 +736,8 @@ jobs:
   source-home-linux:
     name: source-home-linux (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
-    needs: source-gate
+    timeout-minutes: 120
+    needs: [source-gate, engine-llama-amd64]
     env:
       CI_APT_PACKAGES: coturn e2fsprogs ffmpeg musl-tools nasm pkg-config
     strategy:
@@ -723,6 +838,15 @@ jobs:
         with:
           path: ${{ steps.kubo-pin.outputs.paths }}
           key: ${{ steps.kubo-pin.outputs.key }}
+      - name: download pinned AMD64 engine input
+        if: matrix.os == 'ubuntu-24.04'
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
+        with:
+          name: llama-amd64-bundle
+          path: ${{ runner.temp }}/llama-amd64-bundle
+      - name: verify pinned AMD64 engine input
+        if: matrix.os == 'ubuntu-24.04'
+        run: bash scripts/build/build-llama-server-bundle.sh --platform linux-amd64 --verify-archive "$RUNNER_TEMP/llama-amd64-bundle"
       - name: download pinned ARM64 engine input
         if: matrix.os == 'ubuntu-22.04-arm'
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
@@ -737,6 +861,7 @@ jobs:
           ELASTOS_COLLABORATION_STARTUP_MODE: isolated
           SETUP_SOURCE_HOME_INSTALL_LLAMA_SERVER: '1'
           SETUP_SOURCE_HOME_INSTALL_KUBO: '1'
+          ELASTOS_LLAMA_AMD64_BUNDLE: ${{ runner.temp }}/llama-amd64-bundle/llama-b10516-bin-ubuntu22.04-amd64-cpu.tar.gz
           ELASTOS_LLAMA_ARM64_BUNDLE: ${{ runner.temp }}/llama-arm64-bundle/llama-b10516-bin-ubuntu22.04-arm64-cpu.tar.gz
         run: |
           set -euo pipefail
@@ -764,6 +889,7 @@ jobs:
             scripts/installed-provider-verify.sh 2>&1 | tee "${RUNNER_TEMP}/source-home-logs/installed-provider-verify.log"
       - name: local-carrier setup smoke
         env:
+          ELASTOS_LLAMA_AMD64_BUNDLE: ${{ runner.temp }}/llama-amd64-bundle/llama-b10516-bin-ubuntu22.04-amd64-cpu.tar.gz
           ELASTOS_LLAMA_ARM64_BUNDLE: ${{ runner.temp }}/llama-arm64-bundle/llama-b10516-bin-ubuntu22.04-arm64-cpu.tar.gz
         run: |
           set -euo pipefail
@@ -786,19 +912,30 @@ jobs:
         timeout-minutes: 20
         env:
           CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home
+          CI_MODEL_REPAIR_JOURNEY: ${{ matrix.os == 'ubuntu-24.04' }}
         run: scripts/ci-installed-journeys.sh prepare
       # Only the x86-64 job proves one reply Get from a separate holder Home over Carrier.
       - name: installed Marketplace Get and Assistant reply
-        timeout-minutes: 15
+        timeout-minutes: 30
         env:
           CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home
           CI_CARRIER_GET: ${{ matrix.os == 'ubuntu-24.04' }}
+          CI_FINITE_MEMORY: ${{ matrix.os == 'ubuntu-24.04' }}
         run: scripts/ci-installed-journeys.sh home
+      - name: installed model Stop, switch and interrupted Get recovery
+        if: matrix.os == 'ubuntu-24.04'
+        timeout-minutes: 30
+        env:
+          CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home
+          CI_MODEL_REPAIR_JOURNEY: 'true'
+        run: scripts/ci-installed-journeys.sh model-repair
       - name: summarize installed model journey
         if: always()
         env:
           CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home
           CI_CARRIER_GET: ${{ matrix.os == 'ubuntu-24.04' }}
+          CI_FINITE_MEMORY: ${{ matrix.os == 'ubuntu-24.04' }}
+          CI_MODEL_REPAIR_JOURNEY: ${{ matrix.os == 'ubuntu-24.04' }}
         run: scripts/ci-installed-journeys.sh summary
       - name: upload installed model journey
         if: always()
@@ -816,7 +953,7 @@ jobs:
         run: |
           set -euo pipefail
           # Incremental on top of setup-source-home's release builds.
-          cargo build --manifest-path elastos/Cargo.toml --workspace --release
+          cargo build --locked --manifest-path elastos/Cargo.toml --workspace --release
           scripts/package-release-binaries.sh
       - name: verify Jetson release compatibility
         if: matrix.os == 'ubuntu-22.04-arm'

.github/workflows/release-package.yml

@@ -206,6 +206,83 @@ jobs:
       - name: record the artifact
         run: echo "Mac inputs artifact ${{ steps.upload.outputs.artifact-id }}, zip sha256:${{ steps.upload.outputs.artifact-digest }}" >> "$GITHUB_STEP_SUMMARY"
 
+  engine-llama-amd64:
+    name: pinned AMD64 CPU engine
+    needs: admit
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    # Match ci.yml's Ubuntu 22.04 engine recipe and glibc 2.35 baseline.
+    container:
+      image: ubuntu@sha256:5ec03bb3441e8b0bf3b4f9cd4629a1ae763010dc3035bb8da3ae6cf026486401
+    defaults:
+      run:
+        shell: bash
+        working-directory: source
+    env:
+      SOURCE_TREE: ${{ needs.admit.outputs.source_tree }}
+      DEBIAN_FRONTEND: noninteractive
+      BUILD_CONTAINER_IMAGE: ubuntu@sha256:5ec03bb3441e8b0bf3b4f9cd4629a1ae763010dc3035bb8da3ae6cf026486401
+      ENGINE_BUILD_TOOLS: build-essential cmake git ca-certificates curl binutils file python3
+    steps:
+      - name: install engine build tools
+        working-directory: /tmp
+        run: |
+          apt-get update -qq
+          read -r -a tools <<< "$ENGINE_BUILD_TOOLS"
+          apt-get install -y -qq --no-install-recommends "${tools[@]}"
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+        with:
+          ref: ${{ inputs.source_commit }}
+          path: source
+          persist-credentials: false
+      - name: bind the source and approved AMD64 engine
+        id: engine-input
+        run: |
+          set -euo pipefail
+          [[ "$RUNNER_ENVIRONMENT/$RUNNER_OS/$RUNNER_ARCH" == github-hosted/Linux/X64 ]]
+          [[ "$(git rev-parse HEAD)/$(git rev-parse 'HEAD^{tree}')" == "$SOURCE_COMMIT/$SOURCE_TREE" ]]
+          [[ -z "$(git status --porcelain=v1 --untracked-files=all)" ]]
+          CI_LLAMA_AMD64_SHA256=$(python3 -I -S - <<'PYTHON'
+          import json
+          recipes = json.load(open("scripts/release-upstream-recipes.json"))["recipes"]
+          print(next(r["source"]["checksum"].removeprefix("sha256:") for r in recipes
+                     if r["component"] == "llama-server" and r["platform"] == "linux-amd64"))
+          PYTHON
+          )
+          echo "CI_LLAMA_AMD64_SHA256=$CI_LLAMA_AMD64_SHA256" >> "$GITHUB_ENV"
+      - name: bind engine cache to build recipe
+        id: engine-recipe
+        run: |
+          recipe_hash=$( {
+            cat scripts/build/build-llama-server-bundle.sh
+            cat scripts/verify-linux-engine.py
+            printf '\n%s\n%s\n' "$BUILD_CONTAINER_IMAGE" "$ENGINE_BUILD_TOOLS"
+          } | sha256sum | cut -d ' ' -f 1)
+          echo "cache-key=develop-llama-amd64-${recipe_hash}" >> "$GITHUB_OUTPUT"
+      - name: restore pinned AMD64 engine
+        id: engine-cache
+        # A miss takes the same verified cold build below.
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/llama-amd64-bundle
+          key: ${{ steps.engine-recipe.outputs.cache-key }}
+      - name: build and validate pinned AMD64 engine
+        id: engine-build
+        if: steps.engine-cache.outputs.cache-hit != 'true'
+        run: |
+          export RECIPE_COMMIT="$SOURCE_COMMIT"
+          bash scripts/build/build-llama-server-bundle.sh --platform linux-amd64 "$RUNNER_TEMP/llama-amd64-bundle"
+      - name: verify accepted AMD64 engine archive
+        run: |
+          bash scripts/build/build-llama-server-bundle.sh --platform linux-amd64 --verify-archive "$RUNNER_TEMP/llama-amd64-bundle"
+          [[ -z "$(git status --porcelain=v1 --untracked-files=all)" ]]
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
+        with:
+          name: release-llama-amd64-${{ inputs.source_commit }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/llama-amd64-bundle
+          if-no-files-found: error
+          retention-days: 14
+
   engine-llama-arm64:
     name: pinned Jetson CPU engine
     needs: admit
@@ -255,6 +332,7 @@ jobs:
         run: |
           recipe_hash=$( {
             cat scripts/build/build-llama-server-bundle.sh
+            cat scripts/verify-linux-engine.py
             printf '\n%s\n%s\n' "$BUILD_CONTAINER_IMAGE" "$ENGINE_BUILD_TOOLS"
           } | sha256sum | cut -d ' ' -f 1)
           echo "cache-key=develop-llama-arm64-${recipe_hash}" >> "$GITHUB_OUTPUT"
@@ -284,7 +362,7 @@ jobs:
 
   linux:
     name: ${{ matrix.platform }} N and N+1
-    needs: [admit, engine-llama-arm64]
+    needs: [admit, engine-llama-arm64, engine-llama-amd64]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 300
     permissions:
@@ -333,6 +411,12 @@ jobs:
       - name: install host prerequisites
         timeout-minutes: 15
         run: bash scripts/release-host-prerequisites.sh linux
+      - name: download pinned AMD64 engine input
+        if: matrix.platform == 'x86_64-linux'
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
+        with:
+          name: release-llama-amd64-${{ inputs.source_commit }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/llama-amd64-bundle
       - name: download pinned ARM64 engine input
         if: matrix.platform == 'aarch64-linux'
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
@@ -345,6 +429,7 @@ jobs:
           set -euo pipefail
           umask 077
           export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
+          export ELASTOS_LLAMA_AMD64_BUNDLE="$RUNNER_TEMP/llama-amd64-bundle/llama-b10516-bin-ubuntu22.04-amd64-cpu.tar.gz"
           export ELASTOS_LLAMA_ARM64_BUNDLE="$RUNNER_TEMP/llama-arm64-bundle/llama-b10516-bin-ubuntu22.04-arm64-cpu.tar.gz"
           inputs="$RELEASE_ROOT/inputs"
           # Fresh Cargo directories for each version; removed after each build.