Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ Register HaRP as a deploy daemon in the AppAPI admin settings with the `HaRP Pro

## Connecting Docker Engines

On a Podman engine, HaRP requests the GPUs of ExApps on a CUDA daemon as the CDI device `nvidia.com/gpu=all`. This needs Podman 5.4 or later and the NVIDIA CDI specification on the host (`nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml`); without the specification the ExApp container does not start.

### Local Docker Engine

Mount the Docker socket into the HaRP container: `-v /var/run/docker.sock:/var/run/docker.sock`.
Expand Down
23 changes: 23 additions & 0 deletions haproxy_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1638,6 +1638,8 @@ async def docker_exapp_create(request: web.Request):
LOGGER.debug("Checking/Creating volume '%s' via Docker API", volume_name)
try:
async with session.get(inspect_volume_url) as resp_inspect:
# Podman, unlike Docker, sends its version with every answer.
podman_version = resp_inspect.headers.get("Libpod-Api-Version")
if resp_inspect.status == 200:
LOGGER.info("Volume '%s' already exists.", volume_name)
elif resp_inspect.status == 404:
Expand Down Expand Up @@ -1678,6 +1680,27 @@ async def docker_exapp_create(request: web.Request):
LOGGER.exception("Unexpected error during volume management for '%s'", volume_name)
raise web.HTTPInternalServerError(text="Unexpected error during volume management.") from e

# Podman ignores the "nvidia" driver; from version 5.4 it takes the GPUs as CDI devices.
if payload.compute_device == "cuda" and podman_version is not None:
version = re.match(r"(\d+)\.(\d+)", podman_version)
if version and (int(version[1]), int(version[2])) >= (5, 4):
LOGGER.info("Requesting the GPUs of '%s' from Podman as CDI devices.", container_name)
container_config["HostConfig"]["DeviceRequests"] = [
{"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=all"]}
]
# AppAPI's NVIDIA_* variables are for the NVIDIA runtime hook, which can conflict with the CDI devices.
container_config["Env"] = [
entry
for entry in container_config["Env"]
if not entry.startswith(("NVIDIA_VISIBLE_DEVICES=", "NVIDIA_DRIVER_CAPABILITIES="))
]
else:
LOGGER.warning(
"Podman %r ignores the GPU request for '%s': GPUs need Podman 5.4 or later.",
podman_version[:32],
container_name,
)

create_container_url = f"http://{DOCKER_API_HOST}:{docker_engine_port}/containers/create?name={container_name}"
LOGGER.debug(
"Attempting to create container '%s' with image '%s' via Docker API at %s",
Expand Down
93 changes: 93 additions & 0 deletions tests/unit/test_gpu_requests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
"""How `/docker/exapp/create` requests GPUs: the "nvidia" driver on Docker, CDI devices on Podman 5.4 or later."""

import asyncio
import logging

import pytest
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer

import haproxy_agent as agent

from .payloads import DOCKER_CREATE_PAYLOAD

AUTH = {"harp-shared-key": "unit-test-shared-key"}
NVIDIA_REQUEST = [{"Driver": "nvidia", "Count": -1, "Capabilities": [["compute", "utility"]]}]
CDI_REQUEST = [{"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=all"]}]
# AppAPI adds these for a "cuda" daemon.
NVIDIA_ENV = ["NVIDIA_VISIBLE_DEVICES=all", "NVIDIA_DRIVER_CAPABILITIES=compute,utility"]
OTHER_ENV = [agent.with_exapp_frp_token([entry])[0] for entry in DOCKER_CREATE_PAYLOAD["environment_variables"]]


def create_on_engine(monkeypatch, podman_version: str | None, compute_device: str) -> dict:
"""Create an ExApp container on a fake engine, Podman when a version is given; return the container config."""
configs = []

@web.middleware
async def engine_headers(request, handler):
response = await handler(request)
if podman_version is not None:
response.headers["Libpod-Api-Version"] = podman_version
return response

async def volume_missing(_request):
return web.Response(status=404)

async def volume_created(_request):
return web.json_response({}, status=201)

async def container_created(request):
configs.append(await request.json())
return web.json_response({"Id": "0123456789ab"}, status=201)

engine = web.Application(middlewares=[engine_headers])
engine.router.add_get("/volumes/{name}", volume_missing)
engine.router.add_post("/volumes/create", volume_created)
engine.router.add_post("/containers/create", container_created)
payload = {
**DOCKER_CREATE_PAYLOAD,
"compute_device": compute_device,
"environment_variables": [*DOCKER_CREATE_PAYLOAD["environment_variables"], *NVIDIA_ENV],
}

async def run():
async with TestServer(engine) as engine_server, TestClient(TestServer(agent.create_web_app())) as client:
monkeypatch.setattr(agent, "DOCKER_ENGINE_PORTS", [(engine_server.port, engine_server.port)])
headers = {**AUTH, "docker-engine-port": str(engine_server.port)}
response = await client.post("/docker/exapp/create", json=payload, headers=headers)
assert response.status == 201, await response.text()

asyncio.run(run())
return configs[0]


@pytest.mark.parametrize(
("podman_version", "device_requests", "env", "warning"),
[
(None, NVIDIA_REQUEST, OTHER_ENV + NVIDIA_ENV, False),
("6.1.3", CDI_REQUEST, OTHER_ENV, False),
("5.4.0-dev", CDI_REQUEST, OTHER_ENV, False),
("5.3.2", NVIDIA_REQUEST, OTHER_ENV + NVIDIA_ENV, True),
("unknown", NVIDIA_REQUEST, OTHER_ENV + NVIDIA_ENV, True),
],
)
def test_cuda_gets_cdi_devices_only_on_podman_5_4_or_later(
monkeypatch, caplog, podman_version, device_requests, env, warning
):
with caplog.at_level(logging.WARNING, logger="haproxy_agent"):
config = create_on_engine(monkeypatch, podman_version, "cuda")
assert config["HostConfig"]["DeviceRequests"] == device_requests
assert config["Env"] == env
assert ("GPUs need Podman 5.4 or later" in caplog.text) is warning


@pytest.mark.parametrize("podman_version", [None, "6.1.3"])
@pytest.mark.parametrize("compute_device", ["cpu", "rocm"])
def test_cpu_and_rocm_containers_are_the_same_on_docker_and_podman(monkeypatch, podman_version, compute_device):
config = create_on_engine(monkeypatch, podman_version, compute_device)
assert "DeviceRequests" not in config["HostConfig"]
assert config["Env"] == OTHER_ENV + NVIDIA_ENV
devices = [device["PathOnHost"] for device in config["HostConfig"].get("Devices", [])]
assert devices == (["/dev/kfd", "/dev/dri"] if compute_device == "rocm" else [])
2 changes: 1 addition & 1 deletion tests/unit/test_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ def test_create_permits_a_staged_mount_and_sends_the_normalized_mounts_to_the_en

class Response:
def __init__(self, status, body=None):
self.status, self.body = status, body or {}
self.status, self.body, self.headers = status, body or {}, {}

async def json(self):
return self.body
Expand Down