Deploy to GPU server #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow to deploy the application to a GPU server | |
| # | |
| # Required GitHub repo secrets: | |
| # SSH_HOST, SSH_USER, SSH_PRIVATE_KEY - deploy target | |
| # DATABASE_URL - PostgreSQL connection string | |
| # BUNNY_PRIVATE_STORAGE - Bunny CDN storage URL | |
| # BUNNY_PRIVATE_STORAGE_KEY - Bunny storage AccessKey | |
| # Optional (have defaults or can be empty): | |
| # APP_PORT (default 47362), HF_TOKEN, BUNNY_PUBLIC_STORAGE, BUNNY_PUBLIC_STORAGE_KEY, | |
| # CUDA_VISIBLE_DEVICES, POLL_INTERVAL_SECONDS, STUCK_PROCESSING_MINUTES (default 5) | |
| # | |
| name: Deploy to GPU server | |
| # Concurrency control: ensures only one deployment runs at a time | |
| # If a new deployment starts, it will cancel any in-progress deployment | |
| concurrency: | |
| group: production | |
| cancel-in-progress: true | |
| # Trigger conditions: runs on push to main branch or manual trigger | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: # Allows manual triggering from GitHub Actions UI | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checkout the repository code to the GitHub Actions runner | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # Install rsync utility needed for efficient file synchronization | |
| - name: Install rsync | |
| run: sudo apt-get update && sudo apt-get install -y rsync | |
| # Configure SSH access to the deployment server | |
| - name: Setup SSH | |
| run: | | |
| # Create SSH directory if it doesn't exist | |
| mkdir -p ~/.ssh | |
| # Write the private key from GitHub secrets to SSH key file | |
| echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 | |
| # Set proper permissions for the private key (read/write for owner only) | |
| chmod 600 ~/.ssh/id_ed25519 | |
| # Add server's host key to known_hosts to avoid host verification prompts | |
| ssh-keyscan -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts | |
| # Synchronize code from GitHub Actions runner to the deployment server | |
| - name: Sync code to server | |
| run: | | |
| # Extract repository name from GITHUB_REPOSITORY (e.g., "owner/repo" -> "repo") | |
| REPO_NAME="${GITHUB_REPOSITORY#*/}" | |
| # Set destination path on the server | |
| DEST="~/deploy/$REPO_NAME" | |
| # rsync flags: | |
| # -a: archive mode (preserves permissions, timestamps, etc.) | |
| # -z: compress during transfer | |
| # --delete: delete files on destination that don't exist in source | |
| # --exclude: skip these directories (large files that shouldn't be synced) | |
| rsync -az --delete \ | |
| --exclude ".git/" \ | |
| --exclude "data/" \ | |
| --exclude "models/" \ | |
| --exclude "checkpoints/" \ | |
| --exclude "node_modules/" \ | |
| --exclude ".env" \ | |
| -e "ssh -i ~/.ssh/id_ed25519" \ | |
| ./ ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:"$DEST/" | |
| # Create .env on runner from GitHub Secrets (values are masked in logs) | |
| - name: Create .env for deployment | |
| run: | | |
| { | |
| echo "APP_PORT=${{ secrets.APP_PORT }}" | |
| echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" | |
| echo "BUNNY_PRIVATE_STORAGE=${{ secrets.BUNNY_PRIVATE_STORAGE }}" | |
| echo "BUNNY_PRIVATE_STORAGE_KEY=${{ secrets.BUNNY_PRIVATE_STORAGE_KEY }}" | |
| echo "BUNNY_PUBLIC_STORAGE=${{ secrets.BUNNY_PUBLIC_STORAGE }}" | |
| echo "BUNNY_PUBLIC_STORAGE_KEY=${{ secrets.BUNNY_PUBLIC_STORAGE_KEY }}" | |
| echo "HF_TOKEN=${{ secrets.HF_TOKEN }}" | |
| echo "CUDA_VISIBLE_DEVICES=${{ secrets.CUDA_VISIBLE_DEVICES }}" | |
| echo "POLL_INTERVAL_SECONDS=${{ secrets.POLL_INTERVAL_SECONDS }}" | |
| echo "STUCK_PROCESSING_MINUTES=${{ secrets.STUCK_PROCESSING_MINUTES }}" | |
| } > .env.deploy | |
| # Copy .env to server so docker-compose can substitute variables | |
| - name: Copy .env to server | |
| run: | | |
| REPO_NAME="${GITHUB_REPOSITORY#*/}" | |
| DEST="~/deploy/$REPO_NAME" | |
| scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=yes \ | |
| .env.deploy ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:"$DEST/.env" | |
| # Deploy the application using Docker Compose on the server | |
| - name: Compose up on server | |
| run: | | |
| REPO_NAME="${GITHUB_REPOSITORY#*/}" | |
| DEST="~/deploy/$REPO_NAME" | |
| # .env on the server (from previous step) is used by docker-compose for variable substitution | |
| ssh -i ~/.ssh/id_ed25519 ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << EOF | |
| set -euo pipefail | |
| cd $DEST | |
| docker version | |
| docker compose version | |
| docker compose -f docker-compose-prod.yml up -d --build --remove-orphans | |
| EOF | |
| # Health check: verify the application is running and healthy | |
| - name: Health check | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| REPO_NAME="${GITHUB_REPOSITORY#*/}" | |
| PORT="${{ secrets.APP_PORT }}" | |
| PORT="${PORT:-47362}" | |
| ssh -i ~/.ssh/id_ed25519 \ | |
| -o StrictHostKeyChecking=yes \ | |
| "${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}" \ | |
| "REPO_NAME='$REPO_NAME' PORT='$PORT' bash -s" <<'HEALTH_EOF' | |
| set -euo pipefail | |
| cd "$HOME/deploy/$REPO_NAME" | |
| echo "Waiting for container to start..." | |
| timeout=30 | |
| elapsed=0 | |
| while [ "$elapsed" -lt "$timeout" ]; do | |
| if docker compose -f docker-compose-prod.yml ps | grep -q "Up"; then | |
| echo "Container is running" | |
| break | |
| fi | |
| sleep 2 | |
| elapsed=$((elapsed + 2)) | |
| done | |
| echo "Performing health check on port $PORT..." | |
| max_attempts=10 | |
| attempt=1 | |
| wait_time=15 | |
| while [ "$attempt" -le "$max_attempts" ]; do | |
| echo "Attempt $attempt/$max_attempts..." | |
| response="$(curl -s -w $'\n%{http_code}' "http://localhost:$PORT/up" 2>/dev/null || true)" | |
| http_code="$(echo "$response" | tail -n1)" | |
| body="$(echo "$response" | head -n-1)" | |
| if [ "$http_code" = "200" ]; then | |
| echo "✅ Health check passed! Application is healthy." | |
| echo "Response: $body" | |
| exit 0 | |
| elif [ "$http_code" = "503" ]; then | |
| echo "⏳ Application is still booting (models loading)..." | |
| else | |
| echo "❌ Health check failed: HTTP $http_code" | |
| echo "Response: $body" | |
| fi | |
| attempt=$((attempt + 1)) | |
| sleep "$wait_time" | |
| done | |
| echo "❌ Health check failed after $max_attempts attempts" | |
| exit 1 | |
| HEALTH_EOF |