Compose updates & fixes

Author: welchr

.dockerignore

@@ -19,6 +19,8 @@
 !/sqlite3
 !/requirements
 !/bin/entrypoint-colocus.sh
+!/bin/entrypoint-migrate-and-load.sh
+!envs/env.test
 
 # Ignore unnecessary files inside allowed directories
 # This should go after the allowed directories

.github/workflows/ci.yml

@@ -30,24 +30,22 @@ jobs:
       - name: Run pre-commit
         run: uv run pre-commit run --all-files
 
-  # With no caching at all the entire ci process takes 4m 30s to complete!
   pytest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code Repository
         uses: actions/checkout@v4
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-
-      - name: Set up Python
-        run: uv python install 3.10
-
-      - name: Install dependencies
-        run: uv sync
-
-      - name: Run pytest
-        run: uv run pytest
+      - name: Run tests
         env:
-          DJANGO_SECRET_KEY: 1234
-          DJANGO_SETTINGS_MODULE: config.settings.test
+          COMPOSE_PROJECT_NAME: colocus_tests
+        run: |
+          docker compose --env-file envs/env.test -f docker-compose.tests.yml up \
+            --force-recreate \
+            --build \
+            --abort-on-container-exit \
+            --exit-code-from django
+
+      - name: Tear down
+        if: ${{ always() }}
+        run: docker compose --env-file envs/env.test -f docker-compose.tests.yml down --volumes --remove-orphans

.gitignore

@@ -337,3 +337,6 @@ database/**
 
 # Private settings
 workflow/config/config.yml
+
+# Docker compose
+docker-compose.override.yml

Dockerfile

@@ -59,8 +59,8 @@ RUN \
 COPY --chown=colocus:colocus . /opt/colocus/
 
 # Run test cases
-RUN env DJANGO_SECRET_KEY=1234 DJANGO_SETTINGS_MODULE=config.settings.test \
-  bash -c '. .venv/bin/activate && pytest'
+# RUN env DJANGO_SECRET_KEY=1234 DJANGO_SETTINGS_MODULE=config.settings.test \
+#  bash -c '. .venv/bin/activate && pytest'
 
 # Frequently changing metadata here to avoid cache misses
 ARG BUILD_DATE

README.md

@@ -48,7 +48,15 @@ docker compose up -d --build --force-recreate django
 docker compose exec db bash -c 'psql -U colocus -c "DROP DATABASE core WITH (FORCE)"'
 docker compose exec db bash /docker-entrypoint-initdb.d/init-db.sh
 docker compose exec django bash -c 'source .venv/bin/activate && python3 manage.py migrate --database=core'
-docker compose exec django bash -c 'source .venv/bin/activate && python3 scripts/load_dataset.py /data'
+docker compose exec django bash -c "
+  source .venv/bin/activate
+  for subdir in \$(find /data -mindepth 1 -maxdepth 1 -type d); do
+    subdir=\"\${subdir%/}\"
+    if [ -d \"\$subdir\" ]; then
+      uv run python3 scripts/load_dataset.py \"\${subdir}\"
+    fi
+  done
+"
 ```
 
 For debugging a new dataset load:

bin/compose-reload-data.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euxo pipefail
+
+docker compose exec db bash -c 'psql -U colocus -c "DROP DATABASE core WITH (FORCE)"'
+docker compose exec db bash /docker-entrypoint-initdb.d/init-db.sh
+docker compose exec django bash -c 'source .venv/bin/activate && python3 manage.py migrate --database=core'
+docker compose exec django bash -c "
+  source .venv/bin/activate
+  for subdir in \$(find /data -mindepth 1 -maxdepth 1 -type d); do
+    subdir=\"\${subdir%/}\"
+    if [ -d \"\$subdir\" ]; then
+      uv run python3 scripts/load_dataset.py \"\${subdir}\"
+    fi
+  done
+"

bin/compose-run-pytest.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euxo pipefail
+
+# This is to avoid conflict with how containers get named in the other compose stack
+export COMPOSE_PROJECT_NAME="colocus_tests"
+
+# Ensure cleanup on exit
+trap 'docker compose --env-file envs/env.test -f docker-compose.tests.yml down --volumes --remove-orphans' EXIT
+
+# Run the test suite
+docker compose --env-file envs/env.test -f docker-compose.tests.yml up --force-recreate --build --abort-on-container-exit --exit-code-from django
+
+# If you want to debug into the django container:
+# docker compose --env-file envs/env.test -f docker-compose.tests.yml run --entrypoint bash --rm django -l

bin/docker-build.sh

@@ -2,10 +2,13 @@
 set -euxo pipefail
 
 COLOCUS_VERSION=`git describe --tags --abbrev=11 | sed 's/^v//' | sed 's/-g/-/'`
+COLOCUS_BRANCH=`git describe --all --tags --abbrev=11 | sed 's#heads/##' | sed s'#/#-#' | sed 's/-g/-/'`
 GIT_SHA=`git rev-parse HEAD`
 BUILD_DATE=`date -u +'%Y-%m-%dT%H:%M:%SZ'`
 
-docker build --pull -t colocus:${COLOCUS_VERSION} \
+docker build --pull \
+  -t colocus:${COLOCUS_VERSION} \
+  -t colocus:${COLOCUS_BRANCH} \
   --build-arg MAKEFLAGS="-j 3" \
   --build-arg CMAKE_BUILD_PARALLEL_LEVEL=3 \
   --build-arg BUILD_DATE=${BUILD_DATE} \

bin/entrypoint-colocus.sh

@@ -1,53 +1,12 @@
 #!/bin/bash
 set -e
 
-source .venv/bin/activate
-
-# Function to check if migrations are needed
-check_migrations_needed() {
-  python3 manage.py showmigrations --plan | grep -q '\[ \]'
-}
-
-# Function to check if core app tables exist
-check_core_tables_exist() {
-    python3 -c "
-import os
-import django
-from django.db import connections
-
-# Check that django settings module is set
-if 'DJANGO_SETTINGS_MODULE' not in os.environ:
-  raise EnvironmentError('DJANGO_SETTINGS_MODULE environment variable is not set; set before running container')
-
-django.setup()
-
-try:
-  with connections['core'].cursor() as cursor:
-    cursor.execute(\"SELECT COUNT(*) FROM core_datasubmission\")
-    count = cursor.fetchone()[0]
-    exit(0 if count > 0 else 1)
-except Exception:
-  exit(1)
-"
-}
-
-# Check if core tables exist and migrations are up to date
-if check_core_tables_exist && ! check_migrations_needed; then
-  echo "Database already initialized and up to date"
-else
-  echo "Running database migrations..."
-  python3 manage.py migrate
-  python3 manage.py migrate --database=core
-
-  # Only load dataset if it's a fresh database
-  if ! check_core_tables_exist; then
-    echo "Loading initial dataset..."
-    python3 scripts/load_dataset.py /data
-  fi
-fi
+# First run migrations and load data if needed
+/opt/colocus/bin/entrypoint-migrate-and-load.sh
 
 # Now run uvicorn
 HOST=${UVICORN_HOST:-127.0.0.1}
 PORT=${UVICORN_PORT:-8000}
 WORKERS=${WEB_CONCURRENCY:-1}
-/opt/colocus/.venv/bin/uvicorn config.asgi:application --host $HOST --port $PORT --workers $WORKERS "$@"
+
+uv run uvicorn config.asgi:application --host $HOST --port $PORT --workers $WORKERS "$@"

bin/entrypoint-migrate-and-load.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+
+source .venv/bin/activate
+
+# Function to check if migrations are needed
+check_migrations_needed() {
+  uv run python3 manage.py showmigrations --plan | grep -q '\[ \]'
+}
+
+# Function to check if core app tables exist
+check_core_tables_exist() {
+    uv run python3 -c "
+import os
+import django
+from django.db import connections
+
+# Check that django settings module is set
+if 'DJANGO_SETTINGS_MODULE' not in os.environ:
+  raise EnvironmentError('DJANGO_SETTINGS_MODULE environment variable is not set; set before running container')
+
+django.setup()
+
+try:
+  with connections['core'].cursor() as cursor:
+    cursor.execute(\"SELECT COUNT(*) FROM core_datasubmission\")
+    count = cursor.fetchone()[0]
+    exit(0 if count > 0 else 1)
+except Exception:
+  exit(1)
+"
+}
+
+# Check if core tables exist and migrations are up to date
+if check_core_tables_exist && ! check_migrations_needed; then
+  echo "Database already initialized and up to date"
+else
+  echo "Running database migrations..."
+  uv run python3 manage.py migrate
+  uv run python3 manage.py migrate --database=core
+
+  # Only load datasets if it's a fresh database
+  if ! check_core_tables_exist; then
+    echo "Loading initial dataset(s)..."
+    for subdir in `find /data -type d -mindepth 1 -maxdepth 1`
+    do
+      # Remove the trailing slash
+      subdir="${subdir%/}"
+      if [ -d "$subdir" ]; then
+        # Load dataset
+        uv run python3 scripts/load_dataset.py "${subdir}"
+      fi
+    done
+  fi
+fi