From 3ddd696288c3205b82df31e795f282c2b26f7a6c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 30 Dec 2025 12:22:51 +0000 Subject: [PATCH] refactor: extract maintenance --- app/http.php | 10 +- src/Executor/Runner/Docker.php | 55 +--------- src/Executor/Runner/Maintenance.php | 108 ++++++++++++++++++++ src/Executor/Runner/NetworkManager.php | 12 +-- src/Executor/Runner/Repository/Runtimes.php | 14 +++ 5 files changed, 139 insertions(+), 60 deletions(-) create mode 100644 src/Executor/Runner/Maintenance.php diff --git a/app/http.php b/app/http.php index 969d461..465a1cb 100644 --- a/app/http.php +++ b/app/http.php @@ -8,6 +8,7 @@ require_once __DIR__ . '/controllers.php'; use OpenRuntimes\Executor\Runner\Docker; +use OpenRuntimes\Executor\Runner\Maintenance; use OpenRuntimes\Executor\Runner\Repository\Runtimes; use OpenRuntimes\Executor\Runner\NetworkManager; use Swoole\Runtime; @@ -45,6 +46,7 @@ System::getEnv('OPR_EXECUTOR_DOCKER_HUB_USERNAME', ''), System::getEnv('OPR_EXECUTOR_DOCKER_HUB_PASSWORD', '') )); + $runtimes = new Runtimes(); /* Create desired networks if they don't exist */ $networks = explode(',', System::getEnv('OPR_EXECUTOR_NETWORK') ?: 'openruntimes-runtimes'); @@ -55,8 +57,14 @@ $selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found'); $networkManager->connectAll($selfContainer); + /* Start maintenance task */ + $maintenance = new Maintenance($orchestration, $runtimes); + $maintenance->start( + (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'), + (int)System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60') + ); + /* Runner service, used to manage runtimes */ - $runtimes = new Runtimes(); $runner = new Docker($orchestration, $runtimes, $networkManager); Http::setResource('runner', fn () => $runner); diff --git a/src/Executor/Runner/Docker.php b/src/Executor/Runner/Docker.php index dc13ae4..e78604d 100644 --- a/src/Executor/Runner/Docker.php +++ b/src/Executor/Runner/Docker.php @@ -76,57 +76,6 @@ private function init(): void Console::success("Image pulling finished."); - /** - * Run a maintenance worker every X seconds to remove inactive runtimes - */ - Console::info('Starting maintenance interval...'); - $interval = (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'); // In seconds - Timer::tick($interval * 1000, function () { - Console::info("Running maintenance task ..."); - // Stop idling runtimes - foreach ($this->runtimes as $runtimeName => $runtime) { - $inactiveThreshold = \time() - \intval(System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60')); - if ($runtime->updated < $inactiveThreshold) { - go(function () use ($runtimeName, $runtime) { - try { - $this->orchestration->remove($runtime->name, true); - Console::success("Successfully removed {$runtime->name}"); - } catch (Throwable $th) { - Console::error('Inactive Runtime deletion failed: ' . $th->getMessage()); - } finally { - $this->runtimes->remove($runtimeName); - } - }); - } - } - - // Clear leftover build folders - $localDevice = new Local(); - $tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR; - $entries = $localDevice->getFiles($tmpPath); - $prefix = $tmpPath . System::getHostname() . '-'; - foreach ($entries as $entry) { - if (\str_starts_with($entry, $prefix)) { - $isActive = false; - - foreach ($this->runtimes as $runtimeName => $runtime) { - if (\str_ends_with($entry, $runtimeName)) { - $isActive = true; - break; - } - } - - if (!$isActive) { - $localDevice->deletePath($entry); - } - } - } - - Console::success("Maintanance task finished."); - }); - - Console::success('Maintenance interval started.'); - Process::signal(SIGINT, fn () => $this->cleanUp()); Process::signal(SIGQUIT, fn () => $this->cleanUp()); Process::signal(SIGKILL, fn () => $this->cleanUp()); @@ -599,7 +548,7 @@ public function createRuntime( // Silently try to kill container try { $this->orchestration->remove($runtimeName, true); - } catch (Throwable $th) { + } catch (Throwable) { } $localDevice->deletePath($tmpFolder); @@ -620,7 +569,7 @@ public function createRuntime( // Silently try to kill container try { $this->orchestration->remove($runtimeName, true); - } catch (Throwable $th) { + } catch (Throwable) { } $localDevice->deletePath($tmpFolder); diff --git a/src/Executor/Runner/Maintenance.php b/src/Executor/Runner/Maintenance.php new file mode 100644 index 0000000..2b935de --- /dev/null +++ b/src/Executor/Runner/Maintenance.php @@ -0,0 +1,108 @@ +timerId !== false) { + return; + } + + $intervalMs = $intervalSeconds * 1000; + $this->timerId = Timer::tick($intervalMs, fn () => $this->tick($inactiveSeconds)); + Console::info("[Maintenance] Started task on interval $intervalSeconds seconds."); + } + + /** + * Stops the maintenance loop. No-op if already stopped. + */ + public function stop(): void + { + if ($this->timerId === false) { + return; + } + + Timer::clear($this->timerId); + $this->timerId = false; + Console::info("[Maintenance] Stopped task."); + } + + /** + * Removes runtimes inactive beyond the threshold and cleans up temporary files. + */ + private function tick(int $inactiveSeconds): void + { + Console::info("[Maintenance] Running task with threshold $inactiveSeconds seconds."); + + $threshold = \time() - $inactiveSeconds; + $candidates = array_filter( + $this->runtimes->list(), + fn ($runtime) => $runtime->updated < $threshold + ); + + // Remove from in-memory state before removing the container. + // Ensures availability, otherwise we would route requests to terminating runtimes. + $keys = array_keys($candidates); + foreach ($keys as $key) { + $this->runtimes->remove($key); + } + // Then, remove forcefully terminate the associated running container. + $jobs = array_map( + fn ($candidate) => fn () => $this->orchestration->remove($candidate->name, force: true), + $candidates + ); + $results = batch($jobs); + $removed = \count(array_filter($results)); + + Console::info("[Maintenance] Removed {$removed}/" . \count($candidates) . " inactive runtimes."); + + $this->cleanupTmp(); + } + + private function cleanupTmp(): void + { + $localDevice = new Local(); + $tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR; + $prefix = $tmpPath . System::getHostname() . '-'; + + foreach ($localDevice->getFiles($tmpPath) as $entry) { + if (!\str_starts_with($entry, $prefix)) { + continue; + } + + $runtimeName = substr($entry, \strlen($tmpPath)); + if ($this->runtimes->exists($runtimeName)) { + continue; + } + + if ($localDevice->deletePath($entry)) { + Console::info("[Maintenance] Removed {$entry}."); + } + } + } +} diff --git a/src/Executor/Runner/NetworkManager.php b/src/Executor/Runner/NetworkManager.php index 4ecc339..35b7120 100644 --- a/src/Executor/Runner/NetworkManager.php +++ b/src/Executor/Runner/NetworkManager.php @@ -67,31 +67,31 @@ public function removeAll(): void private function remove(string $network): void { if (!$this->orchestration->networkExists($network)) { - Console::error("Network {$network} does not exist"); + Console::error("[NetworkManager] Network {$network} does not exist"); return; } try { $this->orchestration->removeNetwork($network); - Console::success("Removed network: {$network}"); + Console::success("[NetworkManager] Removed network: {$network}"); } catch (\Throwable $e) { - Console::error("Failed to remove network {$network}: {$e->getMessage()}"); + Console::error("[NetworkManager] Failed to remove network {$network}: {$e->getMessage()}"); } } private function ensure(string $network): ?string { if ($this->orchestration->networkExists($network)) { - Console::info("Network {$network} already exists"); + Console::info("[NetworkManager] Network {$network} already exists"); return $network; } try { $this->orchestration->createNetwork($network, false); - Console::success("Created network: {$network}"); + Console::success("[NetworkManager] Created network: {$network}"); return $network; } catch (\Throwable $e) { - Console::error("Failed to create network {$network}: {$e->getMessage()}"); + Console::error("[NetworkManager] Failed to create network {$network}: {$e->getMessage()}"); return null; } } diff --git a/src/Executor/Runner/Repository/Runtimes.php b/src/Executor/Runner/Repository/Runtimes.php index 6cbf0ba..d1a46c3 100644 --- a/src/Executor/Runner/Repository/Runtimes.php +++ b/src/Executor/Runner/Repository/Runtimes.php @@ -86,6 +86,20 @@ public function remove(string $id): bool return $this->runtimes->del($id); } + /** + * List all runtimes. + * + * @return array An array of Runtime objects. + */ + public function list(): array + { + $runtimes = []; + foreach ($this->runtimes as $runtimeKey => $runtime) { + $runtimes[$runtimeKey] = Runtime::fromArray($runtime); + } + return $runtimes; + } + // Iterator traits public function current(): Runtime {