docker-compose.yml

# main docker-compose file for local development.
#
# runs & configures by default:
# * `db` -> postgres db
# * `s3` -> minio
#
# optional profile: `web`:
# * `web` -> webserver
#
# optional profile: `builder`:
# * `builder-a` -> build-server 1
# * `builder-b` -> build-server 2
# ( two parallel build-servers, sharing nothing apart
# from the build queue they access)
#
# optional profile: `watcher`:
# * `registry-watcher` ->
#     - crates.io registry watcher
#     - repo-stats updater
#     - cdn invalidator
#     - release-rebuild-enqueuer
#
# optional profile: `metrics`:
# * `opentelemetry` -> a debug opentelemetry receiver
#
# optional profile: `full`: all of the above.
#
# Services purely for manual usage with `docker compose run` are:
# * `cli`: to run simple CLI commands that only need the database & S3
# * `builder-cli`: to run CLI commands that need the build environment.
# * `registry-watcher-cli`: to run CLI commands that need the crates.io index.
#
# CAVEATS:
# * the build-servers have to run on the `linux/amd64` platform,
#   while it doesn't matter for the rest of the services.
#   This means for example on a Mac, the layers will be
#   cached separately, once for `linux/amd64` and once for
#   `linux/arm64`, which increases build-time.
#
# * volumes: typically docker-native volumes are faster than mounts, but
#   sometimes annoying to inspect for debugging.
#   For now we choose:
#   * docker-native for DB, S3, rustwide workspace, crates.io index
#   * mounts for prefix
#
x-docker-cache: &docker-cache
    # shared configuration to cache docker layers across CI runs.
    # can just always be in in the `docker-compose.yml`, and will
    # just do nothing if we're not on GHA.
    # So feel free to add more techniques if needed.
    cache_from:
        - type=gha
    cache_to:
        - type=gha,mode=max

x-healthcheck: &healthcheck-interval
    interval: 10s
    timeout: 1s
    start_interval: 1s
    start_period: 10s

x-environment: &environment
    # needed default environment that can be overwritten in `.docker.env`
    # Example in `.docker.env.sample`
    RUST_BACKTRACE: ${RUST_BACKTRACE:-true}
    DOCSRS_LOG: ${DOCSRS_LOG:-docs_rs=trace,tower_http=debug,rustwide=info,aws_smithy_http=trace,axum=debug}

    # fixed environment to connect the different services in this file properly
    DOCSRS_PREFIX: /opt/docsrs/prefix
    DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
    DOCSRS_MIN_POOL_IDLE: 1
    DOCSRS_MIN_POOL_SIZE: 2
    DOCSRS_MAX_POOL_SIZE: 10
    DOCSRS_STORAGE_BACKEND: s3
    S3_ENDPOINT: http://s3:9000
    AWS_ACCESS_KEY_ID: cratesfyi
    AWS_SECRET_ACCESS_KEY: secret_key

x-builder-environment: &builder-environment
    <<: *environment
    # default builder environment that can be overwritten in `.docker.env`
    DOCSRS_INCLUDE_DEFAULT_TARGETS: ${DOCSRS_INCLUDE_DEFAULT_TARGETS:-false}
    DOCSRS_DOCKER_IMAGE: ${DOCSRS_DOCKER_IMAGE:-ghcr.io/rust-lang/crates-build-env/linux-micro}

    # fixed environment to connect the different services in this file properly
    DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
    DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics
    DOCSRS_DOCKER: true
    DOCSRS_BUILD_CPU_LIMIT: 2

x-build: &build
    context: .
    <<: *docker-cache
    dockerfile: ./dockerfiles/Dockerfile
    args:
        PROFILE: dev
        PROFILE_DIR: debug

x-builder: &builder
    build:
        <<: *build
        target: build-server
    # build servers only work with linux/amd64 for now.
    # This makes them much slower when working on a mac, but
    # it needs some work and digging into the builder to fix
    # this.
    platform: "linux/amd64"
    depends_on:
        - db
        - s3
    environment:
        <<: *builder-environment
    env_file:
        - .docker.env

x-registry-watcher: &registry-watcher
    build:
        <<: *build
        target: registry-watcher
    depends_on:
        - db
        - s3
    volumes:
        - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix"
        - crates-io-index:/opt/docsrs/crates.io-index
    environment:
        <<: *environment
        # the crates.io index can be shared between the registry watcher & its
        # CLI service.
        # So we configure it to be separate from the prefix.
        # Also on a native docker volume for performance
        REGISTRY_INDEX_PATH: /opt/docsrs/crates.io-index
        # configure the rebuild-queuer,
        DOCSRS_MAX_QUEUED_REBUILDS: ${DOCSRS_MAX_QUEUED_REBUILDS:-10}
    env_file:
        - .docker.env

services:
    web:
        build:
            <<: *build
            target: web-server
        depends_on:
            - db
            - s3
        ports:
            - "3000:3000"
        environment:
            <<: *environment
            DOCSRS_RENDER_THREADS: 2
            DOCSRS_S3_STATIC_ROOT_PATH: http://localhost:9000/rust-docs-rs
        env_file:
            - .docker.env
        healthcheck:
            <<: *healthcheck-interval
            test: curl --silent --fail localhost:3000/
        profiles:
            - web
            - full

    registry-watcher:
        <<: *registry-watcher
        profiles:
            - watcher
            - full

    registry-watcher-cli:
        <<: *registry-watcher
        profiles:
            # watcher-CLI should not be run as background daemon, just manually
            - manual

    builder-a:
        <<: *builder
        volumes:
            - "rustwide-builder-a:/opt/docsrs/rustwide"
            - "./ignored/docker-builder-a/prefix:/opt/docsrs/prefix"
            # this exposes the docker engine from the host machine
            # to the build-server inside the container.
            - "/var/run/docker.sock:/var/run/docker.sock"
        profiles:
            - builder
            - full

    builder-b:
        <<: *builder
        volumes:
            - "rustwide-builder-b:/opt/docsrs/rustwide"
            - "./ignored/docker-builder-b/prefix:/opt/docsrs/prefix"
            # this exposes the docker engine from the host machine
            # to the build-server inside the container.
            - "/var/run/docker.sock:/var/run/docker.sock"
        profiles:
            - builder
            - full

    builder-cli:
        <<: *builder
        volumes:
            - "rustwide-builder-cli:/opt/docsrs/rustwide"
            - "./ignored/docker-builder-cli/prefix:/opt/docsrs/prefix"
            # this exposes the docker engine from the host machine
            # to the build-server inside the container.
            - "/var/run/docker.sock:/var/run/docker.sock"
        profiles:
            # builder-CLI should not be run as background daemon, just manually
            - manual

    cli:
        build:
            <<: *build
            target: cli
        depends_on:
            # only for clarification.
            # When using "docker compose run", these dependencies are ignored,
            # we handle this in our `just` commands.
            - db
            - s3
        environment: *environment
        volumes:
            - "./ignored/docker-cli/prefix:/opt/docsrs/prefix"
        env_file:
            - .docker.env
        profiles:
            # CLI should not be run as background daemon, just manually
            - manual

    db:
        build:
            context: ./dockerfiles
            dockerfile: ./Dockerfile-postgres
            <<: *docker-cache
        volumes:
            - postgres-data:/var/lib/postgresql/data
        environment:
            POSTGRES_USER: cratesfyi
            POSTGRES_PASSWORD: password
        ports:
            # Use a non-standard port on the host to avoid conflicting
            # with existing postgres servers
            - "127.0.0.1:15432:5432"
        healthcheck:
            <<: *healthcheck-interval
            test: pg_isready --username cratesfyi

    s3:
        image: minio/minio
        entrypoint: >
            /bin/sh -c "
                mkdir -p /data/rust-docs-rs;
                minio server /data --console-address ":9001";
            "
        ports:
            - "127.0.0.1:9000:9000"
            - "127.0.0.1:9001:9001"
        volumes:
            - minio-data:/data
        environment:
            MINIO_ROOT_USER: cratesfyi
            MINIO_ROOT_PASSWORD: secret_key
        healthcheck:
            <<: *healthcheck-interval
            test: mc ready local

    opentelemetry:
        build:
            context: ./dockerfiles
            dockerfile: ./Dockerfile-opentelemetry
            <<: *docker-cache
        ports:
            - "127.0.0.1:4317:4317"
        healthcheck:
            <<: *healthcheck-interval
            test: curl --silent --fail http://localhost:13133/health

        profiles:
            # we rarely need to test with actual opentelemetry, so always
            # running it is a waste.
            - metrics
            - full

    gui_tests:
        platform: "linux/amd64"
        build:
            context: .
            dockerfile: ./dockerfiles/Dockerfile-gui-tests
            <<: *docker-cache
        network_mode: "host"
        extra_hosts:
            - "host.docker.internal:host-gateway"
        volumes:
            - "${PWD}:/build/out"
        profiles:
            # gui_tests should not be run as background daemon.
            # Run via `just run-gui-tests`.
            - manual

volumes:
    postgres-data: {}
    minio-data: {}
    crates-io-index: {}
    rustwide-builder-a: {}
    rustwide-builder-b: {}
    rustwide-builder-cli: {}