diff --git a/app/Enums/RolePermissionModels.php b/app/Enums/RolePermissionModels.php index 0ecfc69989..3237b6e865 100644 --- a/app/Enums/RolePermissionModels.php +++ b/app/Enums/RolePermissionModels.php @@ -6,6 +6,7 @@ enum RolePermissionModels: string { case ApiKey = 'apiKey'; case Allocation = 'allocation'; + case BackupHost = 'backupHost'; case DatabaseHost = 'databaseHost'; case Database = 'database'; case Egg = 'egg'; diff --git a/app/Extensions/BackupAdapter/BackupAdapterSchemaInterface.php b/app/Extensions/BackupAdapter/BackupAdapterSchemaInterface.php new file mode 100644 index 0000000000..54e2ccbe61 --- /dev/null +++ b/app/Extensions/BackupAdapter/BackupAdapterSchemaInterface.php @@ -0,0 +1,23 @@ + */ + private array $schemas = []; + + /** @return BackupAdapterSchemaInterface[] */ + public function getAll(): array + { + return $this->schemas; + } + + public function get(string $id): ?BackupAdapterSchemaInterface + { + return array_get($this->schemas, $id); + } + + public function register(BackupAdapterSchemaInterface $schema): void + { + if (array_key_exists($schema->getId(), $this->schemas)) { + return; + } + + $this->schemas[$schema->getId()] = $schema; + } + + /** @return array */ + public function getMappings(): array + { + return collect($this->schemas)->mapWithKeys(fn ($schema) => [$schema->getId() => $schema->getName()])->all(); + } +} diff --git a/app/Extensions/BackupAdapter/Schemas/BackupAdapterSchema.php b/app/Extensions/BackupAdapter/Schemas/BackupAdapterSchema.php new file mode 100644 index 0000000000..10bf40eca8 --- /dev/null +++ b/app/Extensions/BackupAdapter/Schemas/BackupAdapterSchema.php @@ -0,0 +1,14 @@ +getId()); + } +} diff --git a/app/Extensions/BackupAdapter/Schemas/S3BackupSchema.php b/app/Extensions/BackupAdapter/Schemas/S3BackupSchema.php new file mode 100644 index 0000000000..95705423ff --- /dev/null +++ b/app/Extensions/BackupAdapter/Schemas/S3BackupSchema.php @@ -0,0 +1,207 @@ +configuration; + $config['version'] = 'latest'; + + if (!empty($config['key']) && !empty($config['secret'])) { + $config['credentials'] = Arr::only($config, ['key', 'secret', 'token']); + } + + return new S3Client($config); + } + + public function getId(): string + { + return 's3'; + } + + public function createBackup(Backup $backup): void + { + $this->repository->setServer($backup->server)->create($backup); + } + + public function deleteBackup(Backup $backup): void + { + $client = $this->createClient($backup->backupHost); + + $client->deleteObject([ + 'Bucket' => $backup->backupHost->configuration['bucket'], + 'Key' => "{$backup->server->uuid}/$backup->uuid.tar.gz", + ]); + } + + public function getDownloadLink(Backup $backup, User $user): string + { + $client = $this->createClient($backup->backupHost); + + $request = $client->createPresignedRequest( + $client->getCommand('GetObject', [ + 'Bucket' => $backup->backupHost->configuration['bucket'], + 'Key' => "{$backup->server->uuid}/$backup->uuid.tar.gz", + 'ContentType' => 'application/x-gzip', + ]), + CarbonImmutable::now()->addMinutes(5) + ); + + return $request->getUri()->__toString(); + } + + /** @return Component[] */ + public function getConfigurationForm(): array + { + return [ + TextInput::make('configuration.region') + ->label(trans('admin/setting.backup.s3.default_region')) + ->required(), + TextInput::make('configuration.key') + ->label(trans('admin/setting.backup.s3.access_key')) + ->required(), + TextInput::make('configuration.secret') + ->label(trans('admin/setting.backup.s3.secret_key')) + ->required(), + TextInput::make('configuration.bucket') + ->label(trans('admin/setting.backup.s3.bucket')) + ->required(), + TextInput::make('configuration.endpoint') + ->label(trans('admin/setting.backup.s3.endpoint')) + ->required(), + Toggle::make('configuration.use_path_style_endpoint') + ->label(trans('admin/setting.backup.s3.use_path_style_endpoint')) + ->inline(false) + ->onIcon(TablerIcon::Check) + ->offIcon(TablerIcon::X) + ->onColor('success') + ->offColor('danger') + ->live() + ->stateCast(new BooleanStateCast(false)), + ]; + } + + /** @return array{parts: string[], part_size: int} */ + public function getUploadParts(Backup $backup, int $size): array + { + $expires = CarbonImmutable::now()->addMinutes(config('backups.presigned_url_lifespan', 60)); + + // Params for generating the presigned urls + $params = [ + 'Bucket' => $backup->backupHost->configuration['bucket'], + 'Key' => "{$backup->server->uuid}/$backup->uuid.tar.gz", + 'ContentType' => 'application/x-gzip', + ]; + + $storageClass = $backup->backupHost->configuration['storage_class']; + if (!is_null($storageClass)) { + $params['StorageClass'] = $storageClass; + } + + $client = $this->createClient($backup->backupHost); + + // Execute the CreateMultipartUpload request + $result = $client->execute($client->getCommand('CreateMultipartUpload', $params)); + + // Get the UploadId from the CreateMultipartUpload request, this is needed to create + // the other presigned urls. + $params['UploadId'] = $result->get('UploadId'); + + // Retrieve configured part size + $maxPartSize = config('backups.max_part_size', BackupRemoteUploadController::DEFAULT_MAX_PART_SIZE); + if ($maxPartSize <= 0) { + $maxPartSize = BackupRemoteUploadController::DEFAULT_MAX_PART_SIZE; + } + + // Create as many UploadPart presigned urls as needed + $parts = []; + for ($i = 0; $i < ($size / $maxPartSize); $i++) { + $parts[] = $client->createPresignedRequest( + $client->getCommand('UploadPart', array_merge($params, ['PartNumber' => $i + 1])), + $expires + )->getUri()->__toString(); + } + + // Set the upload_id on the backup in the database. + $backup->update(['upload_id' => $params['UploadId']]); + + return [ + 'parts' => $parts, + 'part_size' => $maxPartSize, + ]; + } + + /** + * Marks a multipart upload in a given S3-compatible instance as failed or successful for the given backup. + * + * @param ?array $parts + * + * @throws Exception + */ + public function completeMultipartUpload(Backup $backup, bool $successful, ?array $parts): void + { + // This should never really happen, but if it does don't let us fall victim to Amazon's + // wildly fun error messaging. Just stop the process right here. + if (empty($backup->upload_id)) { + // A failed backup doesn't need to error here, this can happen if the backup encounters + // an error before we even start the upload. AWS gives you tooling to clear these failed + // multipart uploads as needed too. + if (!$successful) { + return; + } + + throw new Exception('Cannot complete backup request: no upload_id present on model.'); + } + + $params = [ + 'Bucket' => $backup->backupHost->configuration['bucket'], + 'Key' => "{$backup->server->uuid}/$backup->uuid.tar.gz", + 'UploadId' => $backup->upload_id, + ]; + + $client = $this->createClient($backup->backupHost); + + if (!$successful) { + $client->execute($client->getCommand('AbortMultipartUpload', $params)); + + return; + } + + // Otherwise send a CompleteMultipartUpload request. + $params['MultipartUpload'] = [ + 'Parts' => [], + ]; + + if (is_null($parts)) { + $params['MultipartUpload']['Parts'] = $client->execute($client->getCommand('ListParts', $params))['Parts']; + } else { + foreach ($parts as $part) { + $params['MultipartUpload']['Parts'][] = [ + 'ETag' => $part['etag'], + 'PartNumber' => $part['part_number'], + ]; + } + } + + $client->execute($client->getCommand('CompleteMultipartUpload', $params)); + } +} diff --git a/app/Extensions/BackupAdapter/Schemas/WingsBackupSchema.php b/app/Extensions/BackupAdapter/Schemas/WingsBackupSchema.php new file mode 100644 index 0000000000..06a1b31138 --- /dev/null +++ b/app/Extensions/BackupAdapter/Schemas/WingsBackupSchema.php @@ -0,0 +1,64 @@ +repository->setServer($backup->server)->create($backup); + } + + /** @throws Exception */ + public function deleteBackup(Backup $backup): void + { + try { + $this->repository->setServer($backup->server)->delete($backup); + } catch (Exception $exception) { + // Don't fail the request if the Daemon responds with a 404, just assume the backup + // doesn't actually exist and remove its reference from the Panel as well. + if ($exception->getCode() !== Response::HTTP_NOT_FOUND) { + throw $exception; + } + } + } + + public function getDownloadLink(Backup $backup, User $user): string + { + $token = $this->jwtService + ->setExpiresAt(CarbonImmutable::now()->addMinutes(15)) + ->setUser($user) + ->setClaims([ + 'backup_uuid' => $backup->uuid, + 'server_uuid' => $backup->server->uuid, + ]) + ->handle($backup->server->node, $user->id . $backup->server->uuid); + + return $backup->server->node->getConnectionAddress() . '/download/backup?token=' . $token->toString(); + } + + /** @return Component[] */ + public function getConfigurationForm(): array + { + return [ + TextEntry::make(trans('admin/backuphost.no_configuration')), + ]; + } +} diff --git a/app/Extensions/Backups/BackupManager.php b/app/Extensions/Backups/BackupManager.php deleted file mode 100644 index 1f79500932..0000000000 --- a/app/Extensions/Backups/BackupManager.php +++ /dev/null @@ -1,177 +0,0 @@ - - */ - protected array $adapters = []; - - /** - * The registered custom driver creators. - * - * @var array - */ - protected array $customCreators; - - public function __construct(protected Application $app) {} - - /** - * Returns a backup adapter instance. - */ - public function adapter(?string $name = null): FilesystemAdapter - { - return $this->get($name ?: $this->getDefaultAdapter()); - } - - /** - * Set the given backup adapter instance. - */ - public function set(string $name, FilesystemAdapter $disk): self - { - $this->adapters[$name] = $disk; - - return $this; - } - - /** - * Gets a backup adapter. - */ - protected function get(string $name): FilesystemAdapter - { - return $this->adapters[$name] = $this->resolve($name); - } - - /** - * Resolve the given backup disk. - */ - protected function resolve(string $name): FilesystemAdapter - { - $config = $this->getConfig($name); - - if (empty($config['adapter'])) { - throw new InvalidArgumentException("Backup disk [$name] does not have a configured adapter."); - } - - $adapter = $config['adapter']; - - if (isset($this->customCreators[$name])) { - return $this->callCustomCreator($config); - } - - $adapterMethod = 'create' . Str::studly($adapter) . 'Adapter'; - if (method_exists($this, $adapterMethod)) { - $instance = $this->{$adapterMethod}($config); - - Assert::isInstanceOf($instance, FilesystemAdapter::class); - - return $instance; - } - - throw new InvalidArgumentException("Adapter [$adapter] is not supported."); - } - - /** - * Calls a custom creator for a given adapter type. - * - * @param array{adapter: string} $config - */ - protected function callCustomCreator(array $config): mixed - { - return $this->customCreators[$config['adapter']]($this->app, $config); - } - - /** - * Creates a new daemon adapter. - * - * @param array $config - */ - public function createWingsAdapter(array $config): FilesystemAdapter - { - return new InMemoryFilesystemAdapter(); - } - - /** - * Creates a new S3 adapter. - * - * @param array $config - */ - public function createS3Adapter(array $config): FilesystemAdapter - { - $config['version'] = 'latest'; - - if (!empty($config['key']) && !empty($config['secret'])) { - $config['credentials'] = Arr::only($config, ['key', 'secret', 'token']); - } - - $client = new S3Client($config); - - return new S3Filesystem($client, $config['bucket'], $config['prefix'] ?? '', $config['options'] ?? []); - } - - /** - * Returns the configuration associated with a given backup type. - * - * @return array - */ - protected function getConfig(string $name): array - { - return config("backups.disks.$name") ?: []; - } - - /** - * Get the default backup driver name. - */ - public function getDefaultAdapter(): string - { - return config('backups.default'); - } - - /** - * Set the default session driver name. - */ - public function setDefaultAdapter(string $name): void - { - config()->set('backups.default', $name); - } - - /** - * Unset the given adapter instances. - * - * @param string|string[] $adapter - */ - public function forget(array|string $adapter): self - { - $adapters = &$this->adapters; - foreach ((array) $adapter as $adapterName) { - unset($adapters[$adapterName]); - } - - return $this; - } - - /** - * Register a custom adapter creator closure. - */ - public function extend(string $adapter, Closure $callback): self - { - $this->customCreators[$adapter] = $callback; - - return $this; - } -} diff --git a/app/Extensions/Filesystem/S3Filesystem.php b/app/Extensions/Filesystem/S3Filesystem.php deleted file mode 100644 index 64aa95bb48..0000000000 --- a/app/Extensions/Filesystem/S3Filesystem.php +++ /dev/null @@ -1,38 +0,0 @@ - $options - */ - public function __construct( - private S3ClientInterface $client, - private string $bucket, - string $prefix = '', - array $options = [], - ) { - parent::__construct( - $client, - $bucket, - $prefix, - null, - null, - $options, - ); - } - - public function getClient(): S3ClientInterface - { - return $this->client; - } - - public function getBucket(): string - { - return $this->bucket; - } -} diff --git a/app/Extensions/Tasks/Schemas/CreateBackupSchema.php b/app/Extensions/Tasks/Schemas/CreateBackupSchema.php index 98927ab08c..411c5ade3e 100644 --- a/app/Extensions/Tasks/Schemas/CreateBackupSchema.php +++ b/app/Extensions/Tasks/Schemas/CreateBackupSchema.php @@ -8,7 +8,7 @@ final class CreateBackupSchema extends TaskSchema { - public function __construct(private InitiateBackupService $backupService) {} + public function __construct(private InitiateBackupService $initiateService) {} public function getId(): string { @@ -17,7 +17,7 @@ public function getId(): string public function runTask(Task $task): void { - $this->backupService->setIgnoredFiles(explode(PHP_EOL, $task->payload))->handle($task->server, null, true); + $this->initiateService->setIgnoredFiles(explode(PHP_EOL, $task->payload))->handle($task->server, null, true); } public function canCreate(Schedule $schedule): bool diff --git a/app/Filament/Admin/Pages/Settings.php b/app/Filament/Admin/Pages/Settings.php index fea6c11c0d..f29c73f711 100644 --- a/app/Filament/Admin/Pages/Settings.php +++ b/app/Filament/Admin/Pages/Settings.php @@ -6,7 +6,6 @@ use App\Extensions\Avatar\AvatarService; use App\Extensions\Captcha\CaptchaService; use App\Extensions\OAuth\OAuthService; -use App\Models\Backup; use App\Notifications\MailTested; use App\Traits\EnvironmentWriterTrait; use App\Traits\Filament\CanCustomizeHeaderActions; @@ -490,16 +489,6 @@ private function mailSettings(): array private function backupSettings(): array { return [ - ToggleButtons::make('APP_BACKUP_DRIVER') - ->label(trans('admin/setting.backup.backup_driver')) - ->columnSpanFull() - ->inline() - ->options([ - Backup::ADAPTER_DAEMON => 'Wings', - Backup::ADAPTER_AWS_S3 => 'S3', - ]) - ->live() - ->default(env('APP_BACKUP_DRIVER', config('backups.default'))), Section::make(trans('admin/setting.backup.throttle')) ->description(trans('admin/setting.backup.throttle_help')) ->columns() @@ -519,41 +508,6 @@ private function backupSettings(): array ->suffix('Seconds') ->default(config('backups.throttles.period')), ]), - Section::make(trans('admin/setting.backup.s3.s3_title')) - ->columns() - ->visible(fn (Get $get) => $get('APP_BACKUP_DRIVER') === Backup::ADAPTER_AWS_S3) - ->schema([ - TextInput::make('AWS_DEFAULT_REGION') - ->label(trans('admin/setting.backup.s3.default_region')) - ->required() - ->default(config('backups.disks.s3.region')), - TextInput::make('AWS_ACCESS_KEY_ID') - ->label(trans('admin/setting.backup.s3.access_key')) - ->required() - ->default(config('backups.disks.s3.key')), - TextInput::make('AWS_SECRET_ACCESS_KEY') - ->label(trans('admin/setting.backup.s3.secret_key')) - ->required() - ->default(config('backups.disks.s3.secret')), - TextInput::make('AWS_BACKUPS_BUCKET') - ->label(trans('admin/setting.backup.s3.bucket')) - ->required() - ->default(config('backups.disks.s3.bucket')), - TextInput::make('AWS_ENDPOINT') - ->label(trans('admin/setting.backup.s3.endpoint')) - ->required() - ->default(config('backups.disks.s3.endpoint')), - Toggle::make('AWS_USE_PATH_STYLE_ENDPOINT') - ->label(trans('admin/setting.backup.s3.use_path_style_endpoint')) - ->inline(false) - ->onIcon(TablerIcon::Check) - ->offIcon(TablerIcon::X) - ->onColor('success') - ->offColor('danger') - ->live() - ->stateCast(new BooleanStateCast(false)) - ->default(env('AWS_USE_PATH_STYLE_ENDPOINT', config('backups.disks.s3.use_path_style_endpoint'))), - ]), ]; } diff --git a/app/Filament/Admin/Resources/BackupHosts/BackupHostResource.php b/app/Filament/Admin/Resources/BackupHosts/BackupHostResource.php new file mode 100644 index 0000000000..22f8a6a5b3 --- /dev/null +++ b/app/Filament/Admin/Resources/BackupHosts/BackupHostResource.php @@ -0,0 +1,162 @@ +count() ?: null; + } + + public static function getNavigationLabel(): string + { + return static::getPluralModelLabel(); + } + + public static function getModelLabel(): string + { + return trans_choice('admin/backuphost.model_label', 1); + } + + public static function getPluralModelLabel(): string + { + return trans_choice('admin/backuphost.model_label', 2); + } + + public static function getNavigationGroup(): ?string + { + return trans('admin/dashboard.advanced'); + } + + /** @throws Exception */ + public static function defaultTable(Table $table): Table + { + return $table + ->columns([ + TextColumn::make('name') + ->label(trans('admin/backuphost.name')), + TextColumn::make('schema') + ->label(trans('admin/backuphost.schema')) + ->badge(), + TextColumn::make('backups_count') + ->counts('backups') + ->label(trans('admin/backuphost.backups')), + TextColumn::make('nodes.name') + ->badge() + ->placeholder(trans('admin/backuphost.all_nodes')), + ]) + ->recordActions([ + ViewAction::make() + ->hidden(fn ($record) => static::getEditAuthorizationResponse($record)->allowed()), + EditAction::make(), + ]) + ->toolbarActions([ + CreateAction::make(), + ]) + ->emptyStateIcon(TablerIcon::FileZip) + ->emptyStateDescription(trans('admin/backuphost.local_backups_only')) + ->emptyStateHeading(trans('admin/backuphost.no_backup_hosts')); + } + + /** @throws Exception */ + public static function defaultForm(Schema $schema): Schema + { + return $schema + ->components([ + TextInput::make('name') + ->label(trans('admin/backuphost.name')) + ->required(), + Select::make('schema') + ->label(trans('admin/backuphost.schema')) + ->required() + ->selectablePlaceholder(false) + ->searchable() + ->options(fn (BackupAdapterService $service) => $service->getMappings()) + ->live(onBlur: true), + Select::make('node_ids') + ->label(trans('admin/backuphost.linked_nodes')) + ->multiple() + ->searchable() + ->preload() + ->relationship('nodes', 'name', fn (Builder $query) => $query->whereIn('nodes.id', user()?->accessibleNodes()->pluck('id'))), + Section::make(trans('admin/backuphost.configuration')) + ->columnSpanFull() + ->columns() + ->schema(function (?BackupHost $backupHost, Get $get, BackupAdapterService $service) { + $schema = $get('schema') ?? $backupHost?->schema; + + if (!$schema) { + return []; + } + + $schema = $service->get($schema); + + if ($schema) { + return $schema->getConfigurationForm(); + } + + return []; + }), + ]); + } + + /** @return class-string[] */ + public static function getDefaultRelations(): array + { + return [ + BackupsRelationManager::class, + ]; + } + + /** @return array */ + public static function getDefaultPages(): array + { + return [ + 'index' => ListBackupHosts::route('/'), + 'create' => CreateBackupHost::route('/create'), + 'view' => ViewBackupHost::route('/{record}'), + 'edit' => EditBackupHost::route('/{record}/edit'), + ]; + } +} diff --git a/app/Filament/Admin/Resources/BackupHosts/Pages/CreateBackupHost.php b/app/Filament/Admin/Resources/BackupHosts/Pages/CreateBackupHost.php new file mode 100644 index 0000000000..995f535d93 --- /dev/null +++ b/app/Filament/Admin/Resources/BackupHosts/Pages/CreateBackupHost.php @@ -0,0 +1,18 @@ + */ + protected function getDefaultHeaderActions(): array + { + return [ + DeleteAction::make() + ->label(fn (BackupHost $backupHost) => $backupHost->backups()->count() > 0 ? trans('admin/backuphost.delete_help') : trans('filament-actions::delete.single.modal.actions.delete.label')) + ->disabled(fn (BackupHost $backupHost) => $backupHost->backups()->count() > 0) + ->hidden(fn () => BackupHost::count() === 1), + Action::make('save') + ->hiddenLabel() + ->action('save') + ->keyBindings(['mod+s']) + ->tooltip(trans('filament-panels::resources/pages/edit-record.form.actions.save.label')) + ->icon(TablerIcon::DeviceFloppy), + ]; + } + + protected function getFormActions(): array + { + return []; + } +} diff --git a/app/Filament/Admin/Resources/BackupHosts/Pages/ListBackupHosts.php b/app/Filament/Admin/Resources/BackupHosts/Pages/ListBackupHosts.php new file mode 100644 index 0000000000..a18a185083 --- /dev/null +++ b/app/Filament/Admin/Resources/BackupHosts/Pages/ListBackupHosts.php @@ -0,0 +1,16 @@ + */ + protected function getDefaultHeaderActions(): array + { + return [ + EditAction::make(), + ]; + } +} diff --git a/app/Filament/Admin/Resources/BackupHosts/RelationManagers/BackupsRelationManager.php b/app/Filament/Admin/Resources/BackupHosts/RelationManagers/BackupsRelationManager.php new file mode 100644 index 0000000000..e2ce0a5528 --- /dev/null +++ b/app/Filament/Admin/Resources/BackupHosts/RelationManagers/BackupsRelationManager.php @@ -0,0 +1,42 @@ +recordTitleAttribute('name') + ->heading(null) + ->columns([ + TextColumn::make('name') + ->label(trans('server/backup.actions.create.name')) + ->searchable(), + BytesColumn::make('bytes') + ->label(trans('server/backup.size')), + DateTimeColumn::make('created_at') + ->label(trans('server/backup.created_at')) + ->since() + ->sortable(), + TextColumn::make('status') + ->label(trans('server/backup.status')) + ->badge(), + IconColumn::make('is_locked') + ->label(trans('server/backup.is_locked')) + ->visibleFrom('md') + ->trueIcon(TablerIcon::Lock) + ->falseIcon(TablerIcon::LockOpen), + ]); + } +} diff --git a/app/Filament/Admin/Resources/Servers/Pages/EditServer.php b/app/Filament/Admin/Resources/Servers/Pages/EditServer.php index d171287f1d..0e6fc9e957 100644 --- a/app/Filament/Admin/Resources/Servers/Pages/EditServer.php +++ b/app/Filament/Admin/Resources/Servers/Pages/EditServer.php @@ -4,6 +4,8 @@ use App\Enums\SuspendAction; use App\Enums\TablerIcon; +use App\Extensions\BackupAdapter\BackupAdapterService; +use App\Extensions\BackupAdapter\Schemas\WingsBackupSchema; use App\Filament\Admin\Resources\Servers\ServerResource; use App\Filament\Components\Actions\DeleteServerIcon; use App\Filament\Components\Actions\PreviewStartupAction; @@ -12,7 +14,6 @@ use App\Filament\Components\StateCasts\ServerConditionStateCast; use App\Filament\Server\Pages\Console; use App\Models\Allocation; -use App\Models\Backup; use App\Models\Egg; use App\Models\Server; use App\Models\User; @@ -976,16 +977,20 @@ protected function getDefaultTabs(): array ->disabled(fn (Server $server) => user()?->accessibleNodes()->count() <= 1 || $server->isInConflictState()) ->modalHeading(trans('admin/server.transfer')) ->schema($this->transferServer()) - ->action(function (TransferServerService $transfer, Server $server, $data) { + ->action(function (TransferServerService $transfer, BackupAdapterService $backupService, Server $server, $data) { try { $selectedBackupUuids = Arr::get($data, 'backups', []); $transfer->handle($server, Arr::get($data, 'node_id'), Arr::get($data, 'allocation_id'), Arr::get($data, 'allocation_additional', []), $selectedBackupUuids); $server->backups ->whereNotIn('uuid', $selectedBackupUuids) - ->where('disk', Backup::ADAPTER_DAEMON) - ->each(function ($backup) { - $backup->delete(); + ->each(function ($backup) use ($backupService) { + $schema = $backupService->get($backup->backupHost->schema); + + // Wings backups that aren't transferred only need to be delete on the panel, wings will cleanup the backup files automatically + if ($schema instanceof WingsBackupSchema) { + $backup->delete(); + } }); Notification::make() @@ -1077,17 +1082,17 @@ protected function transferServer(): array ->placeholder(trans('admin/server.select_additional')), Grid::make() ->columnSpanFull() - ->schema([ + ->schema(fn (BackupAdapterService $backupService) => [ CheckboxList::make('backups') ->label(trans('admin/server.backups')) ->bulkToggleable() - ->options(fn (Server $server) => $server->backups->where('disk', Backup::ADAPTER_DAEMON)->mapWithKeys(fn ($backup) => [$backup->uuid => $backup->name])) - ->columns(fn (Server $record) => (int) ceil($record->backups->where('disk', Backup::ADAPTER_DAEMON)->count() / 4)), + ->options(fn (Server $server) => $server->backups->filter(fn ($backup) => $backupService->get($backup->backupHost->schema) instanceof WingsBackupSchema)->mapWithKeys(fn ($backup) => [$backup->uuid => $backup->name])) + ->columns(fn (Server $record) => (int) ceil($record->backups->filter(fn ($backup) => $backupService->get($backup->backupHost->schema) instanceof WingsBackupSchema)->count() / 4)), Text::make('backup_helper') ->columnSpanFull() ->content(trans('admin/server.warning_backups')), ]) - ->hidden(fn (Server $server) => $server->backups->where('disk', Backup::ADAPTER_DAEMON)->count() === 0), + ->hidden(fn (Server $server, BackupAdapterService $backupService) => $server->backups->filter(fn ($backup) => $backupService->get($backup->backupHost->schema) instanceof WingsBackupSchema)->count() === 0), ]; } diff --git a/app/Filament/Server/Resources/Backups/BackupResource.php b/app/Filament/Server/Resources/Backups/BackupResource.php index f12536a5c8..e82e8b4eb2 100644 --- a/app/Filament/Server/Resources/Backups/BackupResource.php +++ b/app/Filament/Server/Resources/Backups/BackupResource.php @@ -170,7 +170,7 @@ public static function defaultTable(Table $table): Table ->color('primary') ->icon(TablerIcon::Download) ->authorize(fn () => user()?->can(SubuserPermission::BackupDownload, $server)) - ->url(fn (DownloadLinkService $downloadLinkService, Backup $backup, Request $request) => $downloadLinkService->handle($backup, $request->user()), true) + ->url(fn (DownloadLinkService $downloadLinkService, Backup $backup) => $downloadLinkService->handle($backup, user()), true) ->visible(fn (Backup $backup) => $backup->status === BackupStatus::Successful), Action::make('exclude_restore') ->label(trans('server/backup.actions.restore.title')) @@ -207,17 +207,13 @@ public static function defaultTable(Table $table): Table ->property(['name' => $backup->name, 'truncate' => $data['truncate']]); $log->transaction(function () use ($downloadLinkService, $daemonRepository, $backup, $server, $data) { - // If the backup is for an S3 file we need to generate a unique Download link for - // it that will allow daemon to actually access the file. - if ($backup->disk === Backup::ADAPTER_AWS_S3) { - $url = $downloadLinkService->handle($backup, user()); - } + $url = $downloadLinkService->handle($backup, user()); // Update the status right away for the server so that we know not to allow certain // actions against it via the Panel API. $server->update(['status' => ServerState::RestoringBackup]); - $daemonRepository->setServer($server)->restore($backup, $url ?? null, $data['truncate']); + $daemonRepository->setServer($server)->restore($backup, $url, $data['truncate']); }); return Notification::make() diff --git a/app/Http/Controllers/Api/Client/Servers/BackupController.php b/app/Http/Controllers/Api/Client/Servers/BackupController.php index d5492cbcd8..284b07e2ba 100644 --- a/app/Http/Controllers/Api/Client/Servers/BackupController.php +++ b/app/Http/Controllers/Api/Client/Servers/BackupController.php @@ -4,6 +4,7 @@ use App\Enums\ServerState; use App\Enums\SubuserPermission; +use App\Extensions\BackupAdapter\BackupAdapterService; use App\Facades\Activity; use App\Http\Controllers\Api\Client\ClientApiController; use App\Http\Requests\Api\Client\Servers\Backups\RenameBackupRequest; @@ -33,6 +34,7 @@ public function __construct( private readonly DeleteBackupService $deleteBackupService, private readonly InitiateBackupService $initiateBackupService, private readonly DownloadLinkService $downloadLinkService, + private readonly BackupAdapterService $backupService ) { parent::__construct(); } @@ -191,7 +193,8 @@ public function download(Request $request, Server $server, Backup $backup): Json throw new AuthorizationException(); } - if ($backup->disk !== Backup::ADAPTER_AWS_S3 && $backup->disk !== Backup::ADAPTER_DAEMON) { + $schema = $this->backupService->get($backup->backupHost->schema); + if (!$schema) { throw new BadRequestHttpException('The backup requested references an unknown disk driver type and cannot be downloaded.'); } @@ -264,17 +267,13 @@ public function restore(RestoreBackupRequest $request, Server $server, Backup $b ->property(['name' => $backup->name, 'truncate' => $request->input('truncate')]); $log->transaction(function () use ($backup, $server, $request) { - // If the backup is for an S3 file we need to generate a unique Download link for - // it that will allow daemon to actually access the file. - if ($backup->disk === Backup::ADAPTER_AWS_S3) { - $url = $this->downloadLinkService->handle($backup, $request->user()); - } + $url = $this->downloadLinkService->handle($backup, $request->user()); // Update the status right away for the server so that we know not to allow certain // actions against it via the Panel API. $server->update(['status' => ServerState::RestoringBackup]); - $this->daemonRepository->setServer($server)->restore($backup, $url ?? null, $request->input('truncate')); + $this->daemonRepository->setServer($server)->restore($backup, $url, $request->input('truncate')); }); return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php index fa149c31e6..cb9727773e 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php @@ -3,20 +3,16 @@ namespace App\Http\Controllers\Api\Remote\Backups; use App\Exceptions\Http\HttpForbiddenException; -use App\Extensions\Backups\BackupManager; -use App\Extensions\Filesystem\S3Filesystem; +use App\Extensions\BackupAdapter\BackupAdapterService; +use App\Extensions\BackupAdapter\Schemas\S3BackupSchema; use App\Http\Controllers\Controller; use App\Models\Backup; use App\Models\Node; -use App\Models\Server; -use Carbon\CarbonImmutable; -use Exception; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Symfony\Component\HttpKernel\Exception\ConflictHttpException; -use Throwable; class BackupRemoteUploadController extends Controller { @@ -25,117 +21,45 @@ class BackupRemoteUploadController extends Controller /** * BackupRemoteUploadController constructor. */ - public function __construct(private BackupManager $backupManager) {} + public function __construct(private BackupAdapterService $backupService) {} /** * Returns the required presigned urls to upload a backup to S3 cloud storage. * - * @throws Exception - * @throws Throwable + * @throws BadRequestHttpException * @throws ModelNotFoundException + * @throws HttpForbiddenException + * @throws ConflictHttpException */ public function __invoke(Request $request, string $backup): JsonResponse { - // Get the node associated with the request. /** @var Node $node */ $node = $request->attributes->get('node'); - // Get the size query parameter. $size = (int) $request->query('size'); if (empty($size)) { throw new BadRequestHttpException('A non-empty "size" query parameter must be provided.'); } - /** @var Backup $model */ - $model = Backup::query() - ->where('uuid', $backup) - ->firstOrFail(); + $backup = Backup::where('uuid', $backup)->firstOrFail(); // Check that the backup is "owned" by the node making the request. This avoids other nodes // from messing with backups that they don't own. - /** @var Server $server */ - $server = $model->server; - if ($server->node_id !== $node->id) { + if ($backup->server->node_id !== $node->id) { throw new HttpForbiddenException('You do not have permission to access that backup.'); } - // Prevent backups that have already been completed from trying to - // be uploaded again. - if (!is_null($model->completed_at)) { + // Prevent backups that have already been completed from trying to be uploaded again. + if (!is_null($backup->completed_at)) { throw new ConflictHttpException('This backup is already in a completed state.'); } - // Ensure we are using the S3 adapter. - $adapter = $this->backupManager->adapter(); - if (!$adapter instanceof S3Filesystem) { - throw new BadRequestHttpException('The configured backup adapter is not an S3 compatible adapter.'); + // Ensure we are using the S3 schema. + $schema = $this->backupService->get($backup->backupHost->schema); + if (!$schema instanceof S3BackupSchema) { + throw new BadRequestHttpException('The configured backup schema is not an S3 compatible.'); } - // The path where backup will be uploaded to - $path = sprintf('%s/%s.tar.gz', $model->server->uuid, $model->uuid); - - // Get the S3 client - $client = $adapter->getClient(); - $expires = CarbonImmutable::now()->addMinutes(config('backups.presigned_url_lifespan', 60)); - - // Params for generating the presigned urls - $params = [ - 'Bucket' => $adapter->getBucket(), - 'Key' => $path, - 'ContentType' => 'application/x-gzip', - ]; - - $storageClass = config('backups.disks.s3.storage_class'); - if (!is_null($storageClass)) { - $params['StorageClass'] = $storageClass; - } - - // Execute the CreateMultipartUpload request - $result = $client->execute($client->getCommand('CreateMultipartUpload', $params)); - - // Get the UploadId from the CreateMultipartUpload request, this is needed to create - // the other presigned urls. - $params['UploadId'] = $result->get('UploadId'); - - // Retrieve configured part size - $maxPartSize = $this->getConfiguredMaxPartSize(); - - // Create as many UploadPart presigned urls as needed - $parts = []; - for ($i = 0; $i < ($size / $maxPartSize); $i++) { - $parts[] = $client->createPresignedRequest( - $client->getCommand('UploadPart', array_merge($params, ['PartNumber' => $i + 1])), - $expires - )->getUri()->__toString(); - } - - // Set the upload_id on the backup in the database. - $model->update(['upload_id' => $params['UploadId']]); - - return new JsonResponse([ - 'parts' => $parts, - 'part_size' => $maxPartSize, - ]); - } - - /** - * Get the configured maximum size of a single part in the multipart upload. - * - * The function tries to retrieve a configured value from the configuration. - * If no value is specified, a fallback value will be used. - * - * Note if the received config cannot be converted to int (0), is zero or is negative, - * the fallback value will be used too. - * - * The fallback value is {@see BackupRemoteUploadController::DEFAULT_MAX_PART_SIZE}. - */ - private function getConfiguredMaxPartSize(): int - { - $maxPartSize = config('backups.max_part_size', self::DEFAULT_MAX_PART_SIZE); - if ($maxPartSize <= 0) { - $maxPartSize = self::DEFAULT_MAX_PART_SIZE; - } - - return $maxPartSize; + return new JsonResponse($schema->getUploadParts($backup, $size)); } } diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php index ba69fd0047..66ed949324 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php @@ -2,10 +2,9 @@ namespace App\Http\Controllers\Api\Remote\Backups; -use App\Exceptions\DisplayException; use App\Exceptions\Http\HttpForbiddenException; -use App\Extensions\Backups\BackupManager; -use App\Extensions\Filesystem\S3Filesystem; +use App\Extensions\BackupAdapter\BackupAdapterService; +use App\Extensions\BackupAdapter\Schemas\S3BackupSchema; use App\Facades\Activity; use App\Http\Controllers\Controller; use App\Http\Requests\Api\Remote\ReportBackupCompleteRequest; @@ -13,7 +12,6 @@ use App\Models\Node; use App\Models\Server; use Carbon\CarbonImmutable; -use Exception; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; @@ -24,7 +22,7 @@ class BackupStatusController extends Controller /** * BackupStatusController constructor. */ - public function __construct(private BackupManager $backupManager) {} + public function __construct(private BackupAdapterService $backupService) {} /** * Handles updating the state of a backup. @@ -73,9 +71,9 @@ public function index(ReportBackupCompleteRequest $request, string $backup): Jso // Check if we are using the s3 backup adapter. If so, make sure we mark the backup as // being completed in S3 correctly. - $adapter = $this->backupManager->adapter(); - if ($adapter instanceof S3Filesystem) { - $this->completeMultipartUpload($model, $adapter, $successful, $request->input('parts')); + $schema = $this->backupService->get($model->backupHost->schema); + if ($schema instanceof S3BackupSchema) { + $schema->completeMultipartUpload($model, $successful, $request->input('parts')); } }); @@ -106,59 +104,4 @@ public function restore(Request $request, string $backup): JsonResponse return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); } - - /** - * Marks a multipart upload in a given S3-compatible instance as failed or successful for the given backup. - * - * @param ?array $parts - * - * @throws Exception - * @throws DisplayException - */ - protected function completeMultipartUpload(Backup $backup, S3Filesystem $adapter, bool $successful, ?array $parts): void - { - // This should never really happen, but if it does don't let us fall victim to Amazon's - // wildly fun error messaging. Just stop the process right here. - if (empty($backup->upload_id)) { - // A failed backup doesn't need to error here, this can happen if the backup encounters - // an error before we even start the upload. AWS gives you tooling to clear these failed - // multipart uploads as needed too. - if (!$successful) { - return; - } - - throw new DisplayException('Cannot complete backup request: no upload_id present on model.'); - } - - $params = [ - 'Bucket' => $adapter->getBucket(), - 'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid), - 'UploadId' => $backup->upload_id, - ]; - - $client = $adapter->getClient(); - if (!$successful) { - $client->execute($client->getCommand('AbortMultipartUpload', $params)); - - return; - } - - // Otherwise send a CompleteMultipartUpload request. - $params['MultipartUpload'] = [ - 'Parts' => [], - ]; - - if (is_null($parts)) { - $params['MultipartUpload']['Parts'] = $client->execute($client->getCommand('ListParts', $params))['Parts']; - } else { - foreach ($parts as $part) { - $params['MultipartUpload']['Parts'][] = [ - 'ETag' => $part['etag'], - 'PartNumber' => $part['part_number'], - ]; - } - } - - $client->execute($client->getCommand('CompleteMultipartUpload', $params)); - } } diff --git a/app/Models/Backup.php b/app/Models/Backup.php index 9687f79ab5..bdb14b24c8 100644 --- a/app/Models/Backup.php +++ b/app/Models/Backup.php @@ -20,7 +20,8 @@ * @property string $uuid * @property string $name * @property string[] $ignored_files - * @property string $disk + * @property int $backup_host_id + * @property BackupHost $backupHost * @property string|null $checksum * @property int $bytes * @property CarbonImmutable|null $completed_at @@ -65,10 +66,6 @@ class Backup extends Model implements Validatable public const RESOURCE_NAME = 'backup'; - public const ADAPTER_DAEMON = 'wings'; - - public const ADAPTER_AWS_S3 = 's3'; - protected $attributes = [ 'is_successful' => false, 'is_locked' => false, @@ -87,7 +84,7 @@ class Backup extends Model implements Validatable 'is_locked' => ['boolean'], 'name' => ['required', 'string'], 'ignored_files' => ['array'], - 'disk' => ['required', 'string'], + 'backup_host_id' => ['required', 'numeric', 'exists:backup_hosts,id'], 'checksum' => ['nullable', 'string'], 'bytes' => ['numeric'], 'upload_id' => ['nullable', 'string'], @@ -120,6 +117,11 @@ public function server(): BelongsTo return $this->belongsTo(Server::class); } + public function backupHost(): BelongsTo + { + return $this->belongsTo(BackupHost::class); + } + /** * @param Builder $query * @return BackupQueryBuilder diff --git a/app/Models/BackupHost.php b/app/Models/BackupHost.php new file mode 100644 index 0000000000..773e228c57 --- /dev/null +++ b/app/Models/BackupHost.php @@ -0,0 +1,62 @@ + $configuration + * @property CarbonImmutable $created_at + * @property CarbonImmutable $updated_at + * @property Collection|Node[] $nodes + * @property int|null $nodes_count + * @property Collection|Backup[] $backups + * @property int|null $backups_count + */ +class BackupHost extends Model implements Validatable +{ + use HasFactory; + use HasValidation; + + public const RESOURCE_NAME = 'backup_host'; + + protected $fillable = [ + 'name', + 'schema', + 'configuration', + ]; + + /** @var array */ + public static array $validationRules = [ + 'name' => ['required', 'string', 'max:255'], + 'schema' => ['required', 'string', 'max:255'], + 'configuration' => ['nullable', 'array'], + ]; + + protected function casts(): array + { + return [ + 'configuration' => 'array', + ]; + } + + public function nodes(): BelongsToMany + { + return $this->belongsToMany(Node::class); + } + + public function backups(): HasMany + { + return $this->hasMany(Backup::class); + } +} diff --git a/app/Models/Node.php b/app/Models/Node.php index 3edf89d9d1..889639b1da 100644 --- a/app/Models/Node.php +++ b/app/Models/Node.php @@ -61,6 +61,8 @@ * @property-read int|null $roles_count * @property-read Collection $servers * @property-read int|null $servers_count + * @property-read Collection $backupHosts + * @property-read int|null $backup_hosts_count * * @method static \Database\Factories\NodeFactory factory($count = null, $state = []) * @method static \Illuminate\Database\Eloquent\Builder|Node newModelQuery() @@ -308,14 +310,17 @@ public function allocations(): HasMany return $this->hasMany(Allocation::class); } - /** - * @return BelongsToMany - */ + /** @return BelongsToMany */ public function databaseHosts(): BelongsToMany { return $this->belongsToMany(DatabaseHost::class); } + public function backupHosts(): BelongsToMany + { + return $this->belongsToMany(BackupHost::class); + } + public function roles(): HasManyThrough { return $this->hasManyThrough(Role::class, NodeRole::class, 'node_id', 'id', 'id', 'role_id'); diff --git a/app/Policies/BackupHostPolicy.php b/app/Policies/BackupHostPolicy.php new file mode 100644 index 0000000000..dde248a681 --- /dev/null +++ b/app/Policies/BackupHostPolicy.php @@ -0,0 +1,29 @@ +nodes as $node) { + if (!$user->canTarget($node)) { + return false; + } + } + + return null; + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 6d3f89cf3e..aa2d77c49c 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -117,8 +117,6 @@ public function boot( 'Up-to-Date' => $versionService->isLatestPanel() ? 'Yes' : 'No', ]); - AboutCommand::add('Drivers', 'Backups', config('backups.default')); - AboutCommand::add('Environment', 'Installation Directory', base_path()); } diff --git a/app/Providers/BackupsServiceProvider.php b/app/Providers/BackupsServiceProvider.php deleted file mode 100644 index 0d970afe3f..0000000000 --- a/app/Providers/BackupsServiceProvider.php +++ /dev/null @@ -1,28 +0,0 @@ -app->singleton(BackupManager::class, function ($app) { - return new BackupManager($app); - }); - } - - /** - * @return class-string[] - */ - public function provides(): array - { - return [BackupManager::class]; - } -} diff --git a/app/Providers/Extensions/BackupAdapterServiceProvider.php b/app/Providers/Extensions/BackupAdapterServiceProvider.php new file mode 100644 index 0000000000..795668bebb --- /dev/null +++ b/app/Providers/Extensions/BackupAdapterServiceProvider.php @@ -0,0 +1,26 @@ +app->singleton(BackupAdapterService::class, function ($app) { + $service = new BackupAdapterService(); + + // Default Backup adapter providers + $service->register(new WingsBackupSchema($app->make(DaemonBackupRepository::class), $app->make(NodeJWTService::class))); + $service->register(new S3BackupSchema($app->make(DaemonBackupRepository::class))); + + return $service; + }); + } +} diff --git a/app/Repositories/Daemon/DaemonBackupRepository.php b/app/Repositories/Daemon/DaemonBackupRepository.php index 4ec41d9b00..ecbec2bc08 100644 --- a/app/Repositories/Daemon/DaemonBackupRepository.php +++ b/app/Repositories/Daemon/DaemonBackupRepository.php @@ -8,28 +8,16 @@ class DaemonBackupRepository extends DaemonRepository { - protected ?string $adapter; - - /** - * Sets the backup adapter for this execution instance. - */ - public function setBackupAdapter(string $adapter): self - { - $this->adapter = $adapter; - - return $this; - } - /** * Tells the remote Daemon to begin generating a backup for the server. * * @throws ConnectionException */ - public function backup(Backup $backup): Response + public function create(Backup $backup): Response { return $this->getHttpClient()->post("/api/servers/{$this->server->uuid}/backup", [ - 'adapter' => $this->adapter ?? config('backups.default'), + 'adapter' => $backup->backupHost->schema, 'uuid' => $backup->uuid, 'ignore' => implode("\n", $backup->ignored_files), ] @@ -45,7 +33,7 @@ public function restore(Backup $backup, ?string $url = null, bool $truncate = fa { return $this->getHttpClient()->post("/api/servers/{$this->server->uuid}/backup/$backup->uuid/restore", [ - 'adapter' => $backup->disk, + 'adapter' => $backup->backupHost->schema, 'truncate_directory' => $truncate, 'download_url' => $url ?? '', ] diff --git a/app/Services/Backups/DeleteBackupService.php b/app/Services/Backups/DeleteBackupService.php index c5064a2d09..49090e81cc 100644 --- a/app/Services/Backups/DeleteBackupService.php +++ b/app/Services/Backups/DeleteBackupService.php @@ -3,23 +3,15 @@ namespace App\Services\Backups; use App\Exceptions\Service\Backup\BackupLockedException; -use App\Extensions\Backups\BackupManager; -use App\Extensions\Filesystem\S3Filesystem; +use App\Extensions\BackupAdapter\BackupAdapterService; use App\Models\Backup; -use App\Repositories\Daemon\DaemonBackupRepository; -use Aws\S3\S3Client; use Exception; use Illuminate\Database\ConnectionInterface; -use Illuminate\Http\Response; use Throwable; class DeleteBackupService { - public function __construct( - private ConnectionInterface $connection, - private BackupManager $manager, - private DaemonBackupRepository $daemonBackupRepository - ) {} + public function __construct(private readonly ConnectionInterface $connection, private readonly BackupAdapterService $backupService) {} /** * Deletes a backup from the system. If the backup is stored in S3 a request @@ -39,47 +31,15 @@ public function handle(Backup $backup): void throw new BackupLockedException(); } - if ($backup->disk === Backup::ADAPTER_AWS_S3) { - $this->deleteFromS3($backup); - - return; + $schema = $this->backupService->get($backup->backupHost->schema); + if (!$schema) { + throw new Exception('Backup has unknown backup adapter.'); } - $this->connection->transaction(function () use ($backup) { - try { - $this->daemonBackupRepository->setServer($backup->server)->delete($backup); - } catch (Exception $exception) { - // Don't fail the request if the Daemon responds with a 404, just assume the backup - // doesn't actually exist and remove its reference from the Panel as well. - if ($exception->getCode() !== Response::HTTP_NOT_FOUND) { - throw $exception; - } - } + $this->connection->transaction(function () use ($schema, $backup) { + $schema->deleteBackup($backup); $backup->delete(); }); } - - /** - * Deletes a backup from an S3 disk. - * - * @throws Throwable - */ - protected function deleteFromS3(Backup $backup): void - { - $this->connection->transaction(function () use ($backup) { - $backup->delete(); - - /** @var S3Filesystem $adapter */ - $adapter = $this->manager->adapter(Backup::ADAPTER_AWS_S3); - - /** @var S3Client $client */ - $client = $adapter->getClient(); - - $client->deleteObject([ - 'Bucket' => $adapter->getBucket(), - 'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid), - ]); - }); - } } diff --git a/app/Services/Backups/DownloadLinkService.php b/app/Services/Backups/DownloadLinkService.php index 0b3eaba15a..d9f1159cd4 100644 --- a/app/Services/Backups/DownloadLinkService.php +++ b/app/Services/Backups/DownloadLinkService.php @@ -2,60 +2,28 @@ namespace App\Services\Backups; -use App\Extensions\Backups\BackupManager; -use App\Extensions\Filesystem\S3Filesystem; +use App\Extensions\BackupAdapter\BackupAdapterService; use App\Models\Backup; use App\Models\User; -use App\Services\Nodes\NodeJWTService; -use Carbon\CarbonImmutable; +use Exception; class DownloadLinkService { - /** - * DownloadLinkService constructor. - */ - public function __construct(private BackupManager $backupManager, private NodeJWTService $jwtService) {} + public function __construct(private readonly BackupAdapterService $backupService) {} /** * Returns the URL that allows for a backup to be downloaded by an individual * user, or by the daemon control software. + * + * @throws Exception */ public function handle(Backup $backup, User $user): string { - if ($backup->disk === Backup::ADAPTER_AWS_S3) { - return $this->getS3BackupUrl($backup); + $schema = $this->backupService->get($backup->backupHost->schema); + if (!$schema) { + throw new Exception('Backup has unknown backup adapter.'); } - $token = $this->jwtService - ->setExpiresAt(CarbonImmutable::now()->addMinutes(15)) - ->setUser($user) - ->setClaims([ - 'backup_uuid' => $backup->uuid, - 'server_uuid' => $backup->server->uuid, - ]) - ->handle($backup->server->node, $user->id . $backup->server->uuid); - - return sprintf('%s/download/backup?token=%s', $backup->server->node->getConnectionAddress(), $token->toString()); - } - - /** - * Returns a signed URL that allows us to download a file directly out of a non-public - * S3 bucket by using a signed URL. - */ - protected function getS3BackupUrl(Backup $backup): string - { - /** @var S3Filesystem $adapter */ - $adapter = $this->backupManager->adapter(Backup::ADAPTER_AWS_S3); - - $request = $adapter->getClient()->createPresignedRequest( - $adapter->getClient()->getCommand('GetObject', [ - 'Bucket' => $adapter->getBucket(), - 'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid), - 'ContentType' => 'application/x-gzip', - ]), - CarbonImmutable::now()->addMinutes(5) - ); - - return $request->getUri()->__toString(); + return $schema->getDownloadLink($backup, $user); } } diff --git a/app/Services/Backups/InitiateBackupService.php b/app/Services/Backups/InitiateBackupService.php index 67aff86463..ba57c241e1 100644 --- a/app/Services/Backups/InitiateBackupService.php +++ b/app/Services/Backups/InitiateBackupService.php @@ -3,10 +3,11 @@ namespace App\Services\Backups; use App\Exceptions\Service\Backup\TooManyBackupsException; -use App\Extensions\Backups\BackupManager; +use App\Extensions\BackupAdapter\BackupAdapterService; use App\Models\Backup; +use App\Models\BackupHost; use App\Models\Server; -use App\Repositories\Daemon\DaemonBackupRepository; +use Exception; use Illuminate\Database\ConnectionInterface; use Ramsey\Uuid\Uuid; use Symfony\Component\HttpKernel\Exception\TooManyRequestsHttpException; @@ -25,9 +26,8 @@ class InitiateBackupService */ public function __construct( private readonly ConnectionInterface $connection, - private readonly DaemonBackupRepository $daemonBackupRepository, private readonly DeleteBackupService $deleteBackupService, - private readonly BackupManager $backupManager + private readonly BackupAdapterService $backupService ) {} /** @@ -110,20 +110,28 @@ public function handle(Server $server, ?string $name = null, bool $override = fa $this->deleteBackupService->handle($oldest); } - return $this->connection->transaction(function () use ($server, $name) { - /** @var Backup $backup */ - $backup = Backup::query()->create([ + // TODO: select backup host via frontend + $backupHost = $server->node->backupHosts()->first(); + if (!$backupHost) { + $backupHost = BackupHost::doesntHave('nodes')->firstOrFail(); + } + + $schema = $this->backupService->get($backupHost->schema); + if (!$schema) { + throw new Exception('Backup host has unknown backup adapter.'); + } + + return $this->connection->transaction(function () use ($backupHost, $schema, $server, $name) { + $backup = Backup::create([ 'server_id' => $server->id, 'uuid' => Uuid::uuid4()->toString(), 'name' => trim($name) ?: sprintf('Backup at %s', now()->toDateTimeString()), 'ignored_files' => array_values($this->ignoredFiles ?? []), - 'disk' => $this->backupManager->getDefaultAdapter(), + 'backup_host_id' => $backupHost->id, 'is_locked' => $this->isLocked, ]); - $this->daemonBackupRepository->setServer($server) - ->setBackupAdapter($this->backupManager->getDefaultAdapter()) - ->backup($backup); + $schema->createBackup($backup); return $backup; }); diff --git a/app/Services/Servers/TransferServerService.php b/app/Services/Servers/TransferServerService.php index 05a0fcc254..abaa788901 100644 --- a/app/Services/Servers/TransferServerService.php +++ b/app/Services/Servers/TransferServerService.php @@ -2,6 +2,8 @@ namespace App\Services\Servers; +use App\Extensions\BackupAdapter\BackupAdapterService; +use App\Extensions\BackupAdapter\Schemas\WingsBackupSchema; use App\Models\Allocation; use App\Models\Backup; use App\Models\Node; @@ -20,19 +22,23 @@ class TransferServerService * TransferService constructor. */ public function __construct( - private ConnectionInterface $connection, - private NodeJWTService $nodeJWTService, + private readonly ConnectionInterface $connection, + private readonly NodeJWTService $nodeJWTService, + private readonly BackupAdapterService $backupService ) {} - /** - * @param string[] $backup_uuids - */ + /** @param string[] $backup_uuids */ // TODO: add backup uuids to ServerTransfer model private function notify(ServerTransfer $transfer, UnencryptedToken $token, array $backup_uuids = []): void { - $backups = []; - if (config('backups.default') === Backup::ADAPTER_DAEMON) { - $backups = $backup_uuids; - } + // Make sure only wings backups of the current server are forwarded in the wings request + $backups = Backup::where('server_id', $transfer->server_id) + ->whereIn('uuid', $backup_uuids) + ->with('backupHost') + ->get() + ->filter(fn (Backup $backup) => $this->backupService->get($backup->backupHost->schema) instanceof WingsBackupSchema) + ->pluck('uuid') + ->all(); + Http::daemon($transfer->oldNode)->post("/api/servers/{$transfer->server->uuid}/transfer", [ 'url' => $transfer->newNode->getConnectionAddress() . '/api/transfers', 'token' => 'Bearer ' . $token->toString(), @@ -48,11 +54,11 @@ private function notify(ServerTransfer $transfer, UnencryptedToken $token, array * Starts a transfer of a server to a new node. * * @param int[] $additional_allocations - * @param string[] $backup_uuid + * @param string[] $backup_uuids * * @throws Throwable */ - public function handle(Server $server, int $node_id, ?int $allocation_id = null, ?array $additional_allocations = [], ?array $backup_uuid = []): bool + public function handle(Server $server, int $node_id, ?int $allocation_id = null, ?array $additional_allocations = [], array $backup_uuids = []): bool { $additional_allocations = array_map(intval(...), $additional_allocations); @@ -103,7 +109,7 @@ public function handle(Server $server, int $node_id, ?int $allocation_id = null, ->handle($transfer->newNode, $server->uuid, 'sha256'); // Notify the source node of the pending outgoing transfer. - $this->notify($transfer, $token, $backup_uuid); + $this->notify($transfer, $token, $backup_uuids); return true; } diff --git a/bootstrap/providers.php b/bootstrap/providers.php index 66735abce0..e7133617cd 100644 --- a/bootstrap/providers.php +++ b/bootstrap/providers.php @@ -2,9 +2,9 @@ use App\Providers\ActivityLogServiceProvider; use App\Providers\AppServiceProvider; -use App\Providers\BackupsServiceProvider; use App\Providers\EventServiceProvider; use App\Providers\Extensions\AvatarServiceProvider; +use App\Providers\Extensions\BackupAdapterServiceProvider; use App\Providers\Extensions\CaptchaServiceProvider; use App\Providers\Extensions\FeatureServiceProvider; use App\Providers\Extensions\OAuthServiceProvider; @@ -19,9 +19,9 @@ return [ ActivityLogServiceProvider::class, AppServiceProvider::class, - BackupsServiceProvider::class, EventServiceProvider::class, AvatarServiceProvider::class, + BackupAdapterServiceProvider::class, CaptchaServiceProvider::class, FeatureServiceProvider::class, OAuthServiceProvider::class, diff --git a/config/backups.php b/config/backups.php index c3bf16cf71..d765bb2f79 100644 --- a/config/backups.php +++ b/config/backups.php @@ -4,11 +4,6 @@ use App\Models\Backup; return [ - // The backup driver to use for this Panel instance. All client generated server backups - // will be stored in this location by default. It is possible to change this once backups - // have been made, without losing data. - 'default' => env('APP_BACKUP_DRIVER', Backup::ADAPTER_DAEMON), - // This value is used to determine the lifespan of UploadPart presigned urls that daemon // uses to upload backups to S3 storage. Value is in minutes, so this would default to an hour. 'presigned_url_lifespan' => (int) env('BACKUP_PRESIGNED_URL_LIFESPAN', 60), @@ -31,37 +26,4 @@ 'limit' => (int) env('BACKUP_THROTTLE_LIMIT', 2), 'period' => (int) env('BACKUP_THROTTLE_PERIOD', 600), ], - - 'disks' => [ - // There is no configuration for the local disk for Daemon. That configuration - // is determined by the Daemon configuration, and not the Panel. - 'wings' => [ - 'adapter' => Backup::ADAPTER_DAEMON, - ], - - // Configuration for storing backups in Amazon S3. This uses the same credentials - // specified in filesystems.php but does include some more specific settings for - // backups, notably bucket, location, and use_accelerate_endpoint. - 's3' => [ - 'adapter' => Backup::ADAPTER_AWS_S3, - - 'region' => env('AWS_DEFAULT_REGION'), - 'key' => env('AWS_ACCESS_KEY_ID'), - 'secret' => env('AWS_SECRET_ACCESS_KEY'), - - // The S3 bucket to use for backups. - 'bucket' => env('AWS_BACKUPS_BUCKET'), - - // The location within the S3 bucket where backups will be stored. Backups - // are stored within a folder using the server's UUID as the name. Each - // backup for that server lives within that folder. - 'prefix' => env('AWS_BACKUPS_BUCKET') ?? '', - - 'endpoint' => env('AWS_ENDPOINT'), - 'use_path_style_endpoint' => env('AWS_USE_PATH_STYLE_ENDPOINT', false), - 'use_accelerate_endpoint' => env('AWS_BACKUPS_USE_ACCELERATE', false), - - 'storage_class' => env('AWS_BACKUPS_STORAGE_CLASS'), - ], - ], ]; diff --git a/database/Factories/BackupFactory.php b/database/Factories/BackupFactory.php index ff37b60fa8..dbe5b918db 100644 --- a/database/Factories/BackupFactory.php +++ b/database/Factories/BackupFactory.php @@ -24,7 +24,6 @@ public function definition(): array return [ 'uuid' => Uuid::uuid4()->toString(), 'name' => $this->faker->sentence(), - 'disk' => Backup::ADAPTER_DAEMON, 'is_successful' => true, 'created_at' => CarbonImmutable::now(), 'completed_at' => CarbonImmutable::now(), diff --git a/database/Factories/BackupHostFactory.php b/database/Factories/BackupHostFactory.php new file mode 100644 index 0000000000..c1bcfde642 --- /dev/null +++ b/database/Factories/BackupHostFactory.php @@ -0,0 +1,28 @@ + $this->faker->colorName(), + 'schema' => 'wings', + 'configuration' => null, + ]; + } +} diff --git a/database/migrations/2026_01_16_081858_create_backup_hosts_table.php b/database/migrations/2026_01_16_081858_create_backup_hosts_table.php new file mode 100644 index 0000000000..058f8f1dd2 --- /dev/null +++ b/database/migrations/2026_01_16_081858_create_backup_hosts_table.php @@ -0,0 +1,85 @@ +increments('id'); + $table->string('name'); + $table->string('schema'); + $table->json('configuration')->nullable(); + $table->timestamps(); + }); + + Schema::create('backup_host_node', function (Blueprint $table) { + $table->unsignedInteger('node_id'); + $table->foreign('node_id')->references('id')->on('nodes')->cascadeOnDelete(); + + $table->unsignedInteger('backup_host_id'); + $table->foreign('backup_host_id')->references('id')->on('backup_hosts')->cascadeOnDelete(); + + $table->timestamps(); + + $table->unique(['node_id']); + }); + + Schema::table('backups', function (Blueprint $table) { + $table->unsignedInteger('backup_host_id')->after('disk'); + $table->foreign('backup_host_id')->references('id')->on('backup_hosts'); + + $table->dropColumn('disk'); + }); + + $oldDriver = env('APP_BACKUP_DRIVER', 'wings'); + + $oldConfiguration = null; + if ($oldDriver === 's3') { + $oldConfiguration = [ + 'region' => env('AWS_DEFAULT_REGION'), + 'key' => env('AWS_ACCESS_KEY_ID'), + 'secret' => env('AWS_SECRET_ACCESS_KEY'), + 'bucket' => env('AWS_BACKUPS_BUCKET'), + 'prefix' => env('AWS_BACKUPS_BUCKET', ''), + 'endpoint' => env('AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('AWS_USE_PATH_STYLE_ENDPOINT', false), + 'use_accelerate_endpoint' => env('AWS_BACKUPS_USE_ACCELERATE', false), + 'storage_class' => env('AWS_BACKUPS_STORAGE_CLASS'), + ]; + } + + $backupHost = BackupHost::create([ + 'name' => $oldDriver === 's3' ? 'Remote' : 'Local', + 'schema' => $oldDriver, + 'configuration' => $oldConfiguration, + ]); + + DB::table('backups')->update(['backup_host_id' => $backupHost->id]); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('backups', function (Blueprint $table) { + $table->string('disk')->after('backup_host_id'); + + $table->dropForeign(['backup_host_id']); + $table->dropColumn('backup_host_id'); + }); + + Schema::dropIfExists('backup_hosts'); + + Schema::dropIfExists('backup_host_node'); + } +}; diff --git a/lang/en/admin/backuphost.php b/lang/en/admin/backuphost.php new file mode 100644 index 0000000000..631080f760 --- /dev/null +++ b/lang/en/admin/backuphost.php @@ -0,0 +1,14 @@ + 'Backup Host|Backup Hosts', + 'name' => 'Name', + 'schema' => 'Schema', + 'backups' => 'Backups', + 'linked_nodes' => 'Linked Nodes', + 'all_nodes' => 'All Nodes', + 'configuration' => 'Configuration', + 'no_configuration' => 'No additional configuration required', + 'no_backup_hosts' => 'No Backup Hosts', + 'local_backups_only' => 'All backups will be created locally on the respective node', +]; diff --git a/tests/Integration/Api/Client/Server/Backup/BackupAuthorizationTest.php b/tests/Integration/Api/Client/Server/Backup/BackupAuthorizationTest.php index bf6ff79a76..1ec6a16a83 100644 --- a/tests/Integration/Api/Client/Server/Backup/BackupAuthorizationTest.php +++ b/tests/Integration/Api/Client/Server/Backup/BackupAuthorizationTest.php @@ -3,6 +3,7 @@ namespace App\Tests\Integration\Api\Client\Server\Backup; use App\Models\Backup; +use App\Models\BackupHost; use App\Models\Subuser; use App\Services\Backups\DeleteBackupService; use App\Tests\Integration\Api\Client\ClientApiIntegrationTestCase; @@ -25,9 +26,11 @@ public function test_access_to_a_servers_backup_is_restricted_properly(string $m // to do anything with the backups for that server. Subuser::factory()->create(['server_id' => $server2->id, 'user_id' => $user->id]); - $backup1 = Backup::factory()->create(['server_id' => $server1->id, 'completed_at' => CarbonImmutable::now()]); - $backup2 = Backup::factory()->create(['server_id' => $server2->id, 'completed_at' => CarbonImmutable::now()]); - $backup3 = Backup::factory()->create(['server_id' => $server3->id, 'completed_at' => CarbonImmutable::now()]); + $backupHost = BackupHost::factory()->create(); + + $backup1 = Backup::factory()->create(['server_id' => $server1->id, 'backup_host_id' => $backupHost->id, 'completed_at' => CarbonImmutable::now()]); + $backup2 = Backup::factory()->create(['server_id' => $server2->id, 'backup_host_id' => $backupHost->id, 'completed_at' => CarbonImmutable::now()]); + $backup3 = Backup::factory()->create(['server_id' => $server3->id, 'backup_host_id' => $backupHost->id, 'completed_at' => CarbonImmutable::now()]); $this->instance(DeleteBackupService::class, $mock = \Mockery::mock(DeleteBackupService::class)); diff --git a/tests/Integration/Api/Client/Server/Backup/DeleteBackupTest.php b/tests/Integration/Api/Client/Server/Backup/DeleteBackupTest.php index 8a8f805a5a..e2038269cb 100644 --- a/tests/Integration/Api/Client/Server/Backup/DeleteBackupTest.php +++ b/tests/Integration/Api/Client/Server/Backup/DeleteBackupTest.php @@ -5,6 +5,7 @@ use App\Enums\SubuserPermission; use App\Events\ActivityLogged; use App\Models\Backup; +use App\Models\BackupHost; use App\Repositories\Daemon\DaemonBackupRepository; use App\Tests\Integration\Api\Client\ClientApiIntegrationTestCase; use Illuminate\Http\Response; @@ -26,7 +27,8 @@ public function test_user_without_permission_cannot_delete_backup(): void { [$user, $server] = $this->generateTestAccount([SubuserPermission::BackupCreate]); - $backup = Backup::factory()->create(['server_id' => $server->id]); + $backupHost = BackupHost::factory()->create(); + $backup = Backup::factory()->create(['server_id' => $server->id, 'backup_host_id' => $backupHost->id]); $this->actingAs($user)->deleteJson($this->link($backup)) ->assertStatus(Response::HTTP_FORBIDDEN); @@ -43,8 +45,9 @@ public function test_backup_can_be_deleted(): void [$user, $server] = $this->generateTestAccount([SubuserPermission::BackupDelete]); + $backupHost = BackupHost::factory()->create(); /** @var Backup $backup */ - $backup = Backup::factory()->create(['server_id' => $server->id]); + $backup = Backup::factory()->create(['server_id' => $server->id, 'backup_host_id' => $backupHost->id]); $this->repository->expects('setServer->delete')->with( \Mockery::on(function ($value) use ($backup) { diff --git a/tests/Integration/Services/Backups/DeleteBackupServiceTest.php b/tests/Integration/Services/Backups/DeleteBackupServiceTest.php index b8a796c2c3..6e2d378808 100644 --- a/tests/Integration/Services/Backups/DeleteBackupServiceTest.php +++ b/tests/Integration/Services/Backups/DeleteBackupServiceTest.php @@ -3,9 +3,8 @@ namespace App\Tests\Integration\Services\Backups; use App\Exceptions\Service\Backup\BackupLockedException; -use App\Extensions\Backups\BackupManager; -use App\Extensions\Filesystem\S3Filesystem; use App\Models\Backup; +use App\Models\BackupHost; use App\Repositories\Daemon\DaemonBackupRepository; use App\Services\Backups\DeleteBackupService; use App\Tests\Integration\IntegrationTestCase; @@ -17,8 +16,11 @@ class DeleteBackupServiceTest extends IntegrationTestCase public function test_locked_backup_cannot_be_deleted(): void { $server = $this->createServerModel(); + + $backupHost = BackupHost::factory()->create(); $backup = Backup::factory()->create([ 'server_id' => $server->id, + 'backup_host_id' => $backupHost->id, 'is_locked' => true, ]); @@ -30,8 +32,11 @@ public function test_locked_backup_cannot_be_deleted(): void public function test_failed_backup_that_is_locked_can_be_deleted(): void { $server = $this->createServerModel(); + + $backupHost = BackupHost::factory()->create(); $backup = Backup::factory()->create([ 'server_id' => $server->id, + 'backup_host_id' => $backupHost->id, 'is_locked' => true, 'is_successful' => false, ]); @@ -49,7 +54,9 @@ public function test_failed_backup_that_is_locked_can_be_deleted(): void public function test_exception_thrown_due_to_missing_backup_is_ignored(): void { $server = $this->createServerModel(); - $backup = Backup::factory()->create(['server_id' => $server->id]); + + $backupHost = BackupHost::factory()->create(); + $backup = Backup::factory()->create(['server_id' => $server->id, 'backup_host_id' => $backupHost->id]); $mock = $this->mock(DaemonBackupRepository::class); $mock->expects('setServer->delete')->with($backup)->andThrow(new ConnectionException(code: 404)); @@ -64,7 +71,9 @@ public function test_exception_thrown_due_to_missing_backup_is_ignored(): void public function test_exception_is_thrown_if_not404(): void { $server = $this->createServerModel(); - $backup = Backup::factory()->create(['server_id' => $server->id]); + + $backupHost = BackupHost::factory()->create(); + $backup = Backup::factory()->create(['server_id' => $server->id, 'backup_host_id' => $backupHost->id]); $mock = $this->mock(DaemonBackupRepository::class); $mock->expects('setServer->delete')->with($backup)->andThrow(new ConnectionException(code: 500)); @@ -77,28 +86,4 @@ public function test_exception_is_thrown_if_not404(): void $this->assertNull($backup->deleted_at); } - - public function test_s3_object_can_be_deleted(): void - { - $server = $this->createServerModel(); - $backup = Backup::factory()->create([ - 'disk' => Backup::ADAPTER_AWS_S3, - 'server_id' => $server->id, - ]); - - $manager = $this->mock(BackupManager::class); - $adapter = $this->mock(S3Filesystem::class); - - $manager->expects('adapter')->with(Backup::ADAPTER_AWS_S3)->andReturn($adapter); - - $adapter->expects('getBucket')->andReturn('foobar'); - $adapter->expects('getClient->deleteObject')->with([ - 'Bucket' => 'foobar', - 'Key' => sprintf('%s/%s.tar.gz', $server->uuid, $backup->uuid), - ]); - - $this->app->make(DeleteBackupService::class)->handle($backup); - - $this->assertSoftDeleted($backup); - } }