refactor: extract maintenance

app/http.php

@@ -8,6 +8,7 @@
 require_once __DIR__ . '/controllers.php';
 
 use OpenRuntimes\Executor\Runner\Docker;
+use OpenRuntimes\Executor\Runner\Maintenance;
 use OpenRuntimes\Executor\Runner\Repository\Runtimes;
 use OpenRuntimes\Executor\Runner\NetworkManager;
 use Swoole\Runtime;
@@ -45,6 +46,7 @@
         System::getEnv('OPR_EXECUTOR_DOCKER_HUB_USERNAME', ''),
         System::getEnv('OPR_EXECUTOR_DOCKER_HUB_PASSWORD', '')
     ));
+    $runtimes = new Runtimes();
 
     /* Create desired networks if they don't exist */
     $networks = explode(',', System::getEnv('OPR_EXECUTOR_NETWORK') ?: 'openruntimes-runtimes');
@@ -55,8 +57,14 @@
     $selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found');
     $networkManager->connectAll($selfContainer);
 
+    /* Start maintenance task */
+    $maintenance = new Maintenance($orchestration, $runtimes);
+    $maintenance->start(
+        (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'),
+        (int)System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60')
+    );
+
     /* Runner service, used to manage runtimes */
-    $runtimes = new Runtimes();
     $runner = new Docker($orchestration, $runtimes, $networkManager);
     Http::setResource('runner', fn () => $runner);
 

src/Executor/Runner/Docker.php

@@ -76,57 +76,6 @@ private function init(): void
 
         Console::success("Image pulling finished.");
 
-        /**
-         * Run a maintenance worker every X seconds to remove inactive runtimes
-         */
-        Console::info('Starting maintenance interval...');
-        $interval = (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'); // In seconds
-        Timer::tick($interval * 1000, function () {
-            Console::info("Running maintenance task ...");
-            // Stop idling runtimes
-            foreach ($this->runtimes as $runtimeName => $runtime) {
-                $inactiveThreshold = \time() - \intval(System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60'));
-                if ($runtime->updated < $inactiveThreshold) {
-                    go(function () use ($runtimeName, $runtime) {
-                        try {
-                            $this->orchestration->remove($runtime->name, true);
-                            Console::success("Successfully removed {$runtime->name}");
-                        } catch (Throwable $th) {
-                            Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
-                        } finally {
-                            $this->runtimes->remove($runtimeName);
-                        }
-                    });
-                }
-            }
-
-            // Clear leftover build folders
-            $localDevice = new Local();
-            $tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR;
-            $entries = $localDevice->getFiles($tmpPath);
-            $prefix = $tmpPath . System::getHostname() . '-';
-            foreach ($entries as $entry) {
-                if (\str_starts_with($entry, $prefix)) {
-                    $isActive = false;
-
-                    foreach ($this->runtimes as $runtimeName => $runtime) {
-                        if (\str_ends_with($entry, $runtimeName)) {
-                            $isActive = true;
-                            break;
-                        }
-                    }
-
-                    if (!$isActive) {
-                        $localDevice->deletePath($entry);
-                    }
-                }
-            }
-
-            Console::success("Maintanance task finished.");
-        });
-
-        Console::success('Maintenance interval started.');
-
         Process::signal(SIGINT, fn () => $this->cleanUp());
         Process::signal(SIGQUIT, fn () => $this->cleanUp());
         Process::signal(SIGKILL, fn () => $this->cleanUp());
@@ -599,7 +548,7 @@ public function createRuntime(
             // Silently try to kill container
             try {
                 $this->orchestration->remove($runtimeName, true);
-            } catch (Throwable $th) {
+            } catch (Throwable) {
             }
 
             $localDevice->deletePath($tmpFolder);
@@ -620,7 +569,7 @@ public function createRuntime(
             // Silently try to kill container
             try {
                 $this->orchestration->remove($runtimeName, true);
-            } catch (Throwable $th) {
+            } catch (Throwable) {
             }
 
             $localDevice->deletePath($tmpFolder);

src/Executor/Runner/Maintenance.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace OpenRuntimes\Executor\Runner;
+
+use OpenRuntimes\Executor\Runner\Repository\Runtimes;
+use Swoole\Timer;
+use Utopia\Console;
+use Utopia\Orchestration\Orchestration;
+use Utopia\Storage\Device\Local;
+use Utopia\System\System;
+
+use function Swoole\Coroutine\batch;
+
+/**
+ * Handles periodic cleanup of inactive runtimes and orphaned temp directories.
+ */
+class Maintenance
+{
+    private int|false $timerId = false;
+
+    public function __construct(
+        private Orchestration $orchestration,
+        private Runtimes $runtimes
+    ) {
+    }
+
+    /**
+     * Starts the maintenance loop. No-op if already running.
+     */
+    public function start(int $intervalSeconds, int $inactiveSeconds): void
+    {
+        if ($this->timerId !== false) {
+            return;
+        }
+
+        $intervalMs = $intervalSeconds * 1000;
+        $this->timerId = Timer::tick($intervalMs, fn () => $this->tick($inactiveSeconds));
+        Console::info("[Maintenance] Started task on interval $intervalSeconds seconds.");
+    }
+
+    /**
+     * Stops the maintenance loop. No-op if already stopped.
+     */
+    public function stop(): void
+    {
+        if ($this->timerId === false) {
+            return;
+        }
+
+        Timer::clear($this->timerId);
+        $this->timerId = false;
+        Console::info("[Maintenance] Stopped task.");
+    }
+
+    /**
+     * Removes runtimes inactive beyond the threshold and cleans up temporary files.
+     */
+    private function tick(int $inactiveSeconds): void
+    {
+        Console::info("[Maintenance] Running task with threshold $inactiveSeconds seconds.");
+
+        $threshold = \time() - $inactiveSeconds;
+        $candidates = array_filter(
+            $this->runtimes->list(),
+            fn ($runtime) => $runtime->updated < $threshold
+        );
+
+        // Remove from in-memory state before removing the container.
+        // Ensures availability, otherwise we would route requests to terminating runtimes.
+        $keys = array_keys($candidates);
+        foreach ($keys as $key) {
+            $this->runtimes->remove($key);
+        }
+        // Then, remove forcefully terminate the associated running container.
+        $jobs = array_map(
+            fn ($candidate) => fn () => $this->orchestration->remove($candidate->name, force: true),
+            $candidates
+        );
+        $results = batch($jobs);
+        $removed = \count(array_filter($results));
+
+        Console::info("[Maintenance] Removed {$removed}/" . \count($candidates) . " inactive runtimes.");
+
+        $this->cleanupTmp();
+    }
+
+    private function cleanupTmp(): void
+    {
+        $localDevice = new Local();
+        $tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR;
+        $prefix = $tmpPath . System::getHostname() . '-';
+
+        foreach ($localDevice->getFiles($tmpPath) as $entry) {
+            if (!\str_starts_with($entry, $prefix)) {
+                continue;
+            }
+
+            $runtimeName = substr($entry, \strlen($tmpPath));
+            if ($this->runtimes->exists($runtimeName)) {
+                continue;
+            }
+
+            if ($localDevice->deletePath($entry)) {
+                Console::info("[Maintenance] Removed {$entry}.");
+            }
+        }
+    }
+}

src/Executor/Runner/NetworkManager.php

@@ -67,31 +67,31 @@ public function removeAll(): void
     private function remove(string $network): void
     {
         if (!$this->orchestration->networkExists($network)) {
-            Console::error("Network {$network} does not exist");
+            Console::error("[NetworkManager] Network {$network} does not exist");
             return;
         }
 
         try {
             $this->orchestration->removeNetwork($network);
-            Console::success("Removed network: {$network}");
+            Console::success("[NetworkManager] Removed network: {$network}");
         } catch (\Throwable $e) {
-            Console::error("Failed to remove network {$network}: {$e->getMessage()}");
+            Console::error("[NetworkManager] Failed to remove network {$network}: {$e->getMessage()}");
         }
     }
 
     private function ensure(string $network): ?string
     {
         if ($this->orchestration->networkExists($network)) {
-            Console::info("Network {$network} already exists");
+            Console::info("[NetworkManager] Network {$network} already exists");
             return $network;
         }
 
         try {
             $this->orchestration->createNetwork($network, false);
-            Console::success("Created network: {$network}");
+            Console::success("[NetworkManager] Created network: {$network}");
             return $network;
         } catch (\Throwable $e) {
-            Console::error("Failed to create network {$network}: {$e->getMessage()}");
+            Console::error("[NetworkManager] Failed to create network {$network}: {$e->getMessage()}");
             return null;
         }
     }

src/Executor/Runner/Repository/Runtimes.php

@@ -86,6 +86,20 @@ public function remove(string $id): bool
         return $this->runtimes->del($id);
     }
 
+    /**
+     * List all runtimes.
+     *
+     * @return array<Runtime> An array of Runtime objects.
+     */
+    public function list(): array
+    {
+        $runtimes = [];
+        foreach ($this->runtimes as $runtimeKey => $runtime) {
+            $runtimes[$runtimeKey] = Runtime::fromArray($runtime);
+        }
+        return $runtimes;
+    }
+
     // Iterator traits
     public function current(): Runtime
     {