Add index caching for use in development

.github/workflows/ci.yaml

@@ -818,3 +818,41 @@ jobs:
     secrets: inherit
     with:
       environment: "staging"
+
+  cache-index:
+    name: Cache realm index
+    needs: [change-check, test-web-assets]
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    concurrency:
+      group: cache-index-${{ github.run_id }}
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/init
+      - name: Download test web assets
+        uses: actions/download-artifact@b14cf4c92620c250e1c074ab0a5800e37df86765 # 4.2.0
+        with:
+          name: ${{ needs.test-web-assets.outputs.artifact_name }}
+          path: .test-web-assets-artifact
+      - name: Restore test web assets into workspace
+        shell: bash
+        run: |
+          shopt -s dotglob
+          cp -a .test-web-assets-artifact/. ./
+      - name: Index all realms and dump tables
+        run: mise run ci:cache-index 2>&1 | tee /tmp/cache-index.log
+        timeout-minutes: 45
+      - name: Upload service logs
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1
+        if: ${{ !cancelled() }}
+        with:
+          name: cache-index-log
+          path: /tmp/cache-index.log
+          retention-days: 7
+      - name: Upload index cache
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1
+        with:
+          name: boxel-index-cache
+          path: /tmp/boxel-index-cache.sql.gz
+          retention-days: 30

README.md

@@ -170,6 +170,24 @@ To use turbo mode but override a specific value:
 BOXEL_TURBO=true PRERENDER_COUNT=5 mise run dev
 ```
 
+##### Index cache
+
+When using `BOXEL_ENVIRONMENT` to work in an isolated worktree, the first startup normally indexes all realms from scratch, which is slow. Set `INDEX_CACHE=true` to download a pre-built index from CI instead:
+
+```bash
+INDEX_CACHE=true BOXEL_ENVIRONMENT=my-branch mise run dev-all
+```
+
+This downloads the latest `boxel_index` dump produced by CI on `main`, remaps URLs for your environment, and imports it into your local database. It also sets `REALM_SERVER_FULL_INDEX_ON_STARTUP=false` so the realm server skips the background reindex and relies on the file watcher for incremental updates. You can override this:
+
+```bash
+INDEX_CACHE=true REALM_SERVER_FULL_INDEX_ON_STARTUP=true BOXEL_ENVIRONMENT=my-branch mise run dev-all
+```
+
+Requires the `gh` CLI to be installed and authenticated.
+
+##### Services
+
 Here's what is spun up with `mise run dev`:
 
 | Port  | Description                                                                                   | `mise run dev` | `mise run services:realm-server-base` |

mise-tasks/ci/cache-index

@@ -0,0 +1,68 @@
+#!/bin/bash
+#MISE description="Index all realms and dump boxel_index tables for caching"
+#MISE dir="packages/realm-server"
+
+set -euo pipefail
+
+export PATH="./node_modules/.bin:$PATH"
+
+READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson"
+REALM_HOST="${REALM_BASE_URL#http://}"
+
+# Build readiness URLs for all enabled realms.
+# boxel-homepage is excluded: its source repo (cardstack/boxel-home) is private
+# and inaccessible to CI without extra credentials.
+READINESS_URLS=""
+for realm in base catalog external-catalog skills submissions experiments openrouter software-factory; do
+  READINESS_URLS="${READINESS_URLS:+${READINESS_URLS}|}http-get://${REALM_HOST}/${realm}/${READY_PATH}"
+done
+READINESS_URLS="${READINESS_URLS}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}|${HOST_URL}"
+
+echo "Starting services..."
+SKIP_BOXEL_HOMEPAGE=true \
+  NODE_NO_WARNINGS=1 \
+  run-p -ln start:icons start:host-dist start:pg start:prerender-dev start:prerender-manager-dev start:matrix start:smtp start:worker-development start:development &
+SERVICES_PID=$!
+
+ensure_services_running() {
+  if ! kill -0 "$SERVICES_PID" 2>/dev/null; then
+    wait "$SERVICES_PID"
+    echo "Services exited unexpectedly."
+    exit 1
+  fi
+}
+
+cleanup() {
+  echo "Stopping services..."
+  kill "$SERVICES_PID" 2>/dev/null || true
+  wait "$SERVICES_PID" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+# Register realm users with Matrix (waits for Synapse to be healthy).
+# Must happen while services are starting since the realm server needs
+# authenticated Matrix users.
+ensure_services_running
+echo "Registering realm users..."
+pnpm --dir=../matrix register-realm-users
+ensure_services_running
+
+echo "Waiting for realm readiness..."
+wait-on -t 1800000 $(echo "$READINESS_URLS" | tr '|' ' ') &
+WAIT_ON_PID=$!
+
+while kill -0 "$WAIT_ON_PID" 2>/dev/null; do
+  ensure_services_running
+  sleep 1
+done
+wait "$WAIT_ON_PID"
+
+echo "All realms ready. Dumping index tables..."
+docker exec boxel-pg pg_dump -U postgres --data-only \
+  -t boxel_index -t realm_versions -t realm_meta \
+  boxel > /tmp/boxel-index-cache.sql
+
+gzip /tmp/boxel-index-cache.sql
+
+echo "Index cache created at /tmp/boxel-index-cache.sql.gz"
+ls -lh /tmp/boxel-index-cache.sql.gz

mise-tasks/lib/dev-common.sh

@@ -3,6 +3,8 @@
 # Sourced (not executed) — sets variables and bootstraps infra.
 # Expects to run with MISE dir=packages/realm-server.
 
+export PATH="./node_modules/.bin:$PATH"
+
 READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson"
 
 # Phase 1 readiness URLs
@@ -35,5 +37,10 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then
   "$REPO_ROOT/scripts/ensure-branch-db.sh"
   echo "Running database migrations…"
   pnpm migrate
+  if [ "${INDEX_CACHE:-}" = "true" ]; then
+    if "$REPO_ROOT/scripts/import-cached-index.sh"; then
+      export REALM_SERVER_FULL_INDEX_ON_STARTUP="${REALM_SERVER_FULL_INDEX_ON_STARTUP:-false}"
+    fi
+  fi
   ./scripts/start-matrix.sh
 fi

packages/realm-server/package.json

@@ -86,6 +86,7 @@
     "undici": "catalog:",
     "uuid": "catalog:",
     "wait-for-localhost-cli": "catalog:",
+    "wait-on": "^9.0.4",
     "yaml": "catalog:",
     "yargs": "catalog:"
   },