diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index de1ccfefcf64..6c9d26461cb2 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -8,7 +8,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; -import type * as Path from "effect/Path"; +import * as Path from "effect/Path"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts"; @@ -656,6 +656,72 @@ it.live("removes clean submodules recursively, including deinitialized repositor ), ); +it.live("removes submodules whose tags match the remote's tags off every remote branch", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, repo, worktree, submodule, command } = yield* submoduleCleanupFixture(); + const head = (yield* command(submodule, ["rev-parse", "HEAD"])).stdout.trim(); + const tree = (yield* command(submodule, ["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const release = (yield* command(repo, [ + "commit-tree", + tree, + "-p", + head, + "-m", + "release", + ])).stdout.trim(); + yield* command(repo, ["tag", "v0.9.1", release]); + yield* command(submodule, ["fetch", "origin", "tag", "v0.9.1"]); + // Pinning the submodule at the release puts the tag's commit in HEAD and the reflog. + yield* command(submodule, ["checkout", "--detach", "v0.9.1"]); + yield* command(worktree, ["commit", "-am", "pin module release"]); + const { relative } = yield* Path.Path; + yield* command(submodule, ["config", "remote.origin.url", relative(submodule, repo)]); + const report = yield* service.runNow; + expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ + outcome: "removed", + reason: expect.not.stringContaining("remote-tracking"), + }); + expect(yield* fs.exists(worktree)).toBe(false); + }), + ), +); + +it.live("keeps submodules whose tags the remote lacks or has at another commit", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, repo, worktree, submodule, command } = yield* submoduleCleanupFixture(); + const commitOff = (cwd: string, message: string, parent = "HEAD") => + Effect.gen(function* () { + const tree = (yield* command(cwd, ["rev-parse", "HEAD^{tree}"])).stdout.trim(); + return (yield* command(cwd, [ + "commit-tree", + tree, + "-p", + parent, + "-m", + message, + ])).stdout.trim(); + }); + yield* command(repo, ["tag", "v0.9.1", yield* commitOff(repo, "release")]); + const local = yield* commitOff(submodule, "local release"); + for (const tag of ["local-only", "v0.9.1"]) { + yield* command(submodule, ["tag", tag, local]); + expect( + (yield* service.runNow).entries.find((entry) => entry.path === worktree), + ).toMatchObject({ outcome: "kept", reason: expect.stringContaining("remote-tracking") }); + yield* command(submodule, ["tag", "-d", tag]); + } + yield* command(submodule, ["fetch", "origin", "tag", "v0.9.1"]); + yield* command(submodule, ["branch", "work", yield* commitOff(submodule, "work", "v0.9.1")]); + expect( + (yield* service.runNow).entries.find((entry) => entry.path === worktree), + ).toMatchObject({ outcome: "kept", reason: expect.stringContaining("remote-tracking") }); + expect(yield* fs.exists(worktree)).toBe(true); + }), + ), +); + it.live("keeps nested submodule files protected by the current keep rule", () => runCleanupTest( Effect.gen(function* () { diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index a5f9cd7eefa1..9d6b65c45e16 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -47,6 +47,7 @@ import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import { NON_INTERACTIVE_GIT_ENV } from "./vcs/GitVcsDriverCore.ts"; import { removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -677,8 +678,99 @@ const make = Effect.gen(function* () { return file === undefined ? null : `has ignored files (${file})`; }); + const unpushedHistoryReason = Effect.fn("StorageCleanup.unpushedHistoryReason")(function* ( + cwd: string, + directory: string, + remoteTags: Map | null>, + ) { + const inspect = (args: string[], options?: Partial) => + git.execute({ + operation: "StorageCleanup.submodules", + cwd, + args: ["--git-dir", directory, "--work-tree", cwd, ...args], + maxOutputBytes: 64 * 1024, + ...options, + }); + const exceeded = "submodule inspection exceeded the limit"; + const unpushed = "submodule has commits not on a remote-tracking ref"; + const reachable = Effect.fnUntraced(function* (stdin?: string) { + const result = yield* inspect( + [ + "rev-list", + "--max-count=1", + ...(stdin === undefined ? [] : ["--stdin"]), + "HEAD", + "--all", + "--reflog", + "--not", + "--remotes", + ], + stdin === undefined ? {} : { stdin }, + ); + if (result.stdoutTruncated || result.stderrTruncated) return exceeded; + return result.stdout.trim() === "" ? null : unpushed; + }); + const all = yield* reachable(); + if (all !== unpushed) return all; + // Clones fetch every upstream tag, so a tag a remote has at the same object is pushed. + const tags = yield* inspect( + ["for-each-ref", "--format=%(objectname)%09%(refname)", "refs/tags"], + { + maxOutputBytes: 16 * 1024 * 1024, + }, + ); + const remotes = yield* inspect(["remote"]); + const worktree = yield* inspect(["config", "core.worktree"], { allowNonZeroExit: true }); + if (tags.stdoutTruncated || tags.stderrTruncated || remotes.stdoutTruncated) return exceeded; + // Relative remote paths resolve from the submodule's work tree. Deinit unsets it. + const base = path.resolve(directory, worktree.stdout.trim() || "."); + const local = tags.stdout.split("\n").filter(Boolean); + const pushed = new Set(); + for (const remote of remotes.stdout.split("\n").filter(Boolean)) { + if (pushed.size === local.length) break; + const url = yield* inspect(["ls-remote", "--get-url", "--", remote]); + if (url.stdoutTruncated) return exceeded; + const location = url.stdout.trim(); + const key = + path.isAbsolute(location) || /^[^/]*:/.test(location) + ? location + : path.resolve(base, location); + let published = remoteTags.get(key); + if (published === undefined) { + published = yield* git + .execute({ + operation: "StorageCleanup.remoteTags", + cwd: base, + args: ["--git-dir", directory, "ls-remote", "--tags", "--refs", "--", remote], + env: NON_INTERACTIVE_GIT_ENV, + timeoutMs: 20_000, + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe( + Effect.map((result) => + result.stdoutTruncated ? null : new Set(result.stdout.split("\n")), + ), + // An unreachable remote vouches for no tags. + Effect.orElseSucceed(() => new Set()), + ); + remoteTags.set(key, published); + } + if (published === null) return exceeded; + for (const line of local) if (published.has(line)) pushed.add(line); + } + if (pushed.size === 0) return unpushed; + return yield* reachable( + // Exclude the verified objects, not the names: a tag can move after the listing. + [...pushed].map((line) => `^${line.slice(0, line.indexOf("\t"))}\n`).join(""), + ); + }); + const submoduleRemoval = Effect.fn("StorageCleanup.submoduleRemoval")( - function* (cwd: string, rules: WorktreeCleanupRules) { + function* ( + cwd: string, + rules: WorktreeCleanupRules, + remoteTags: Map | null>, + ) { const inspect = (args: string[], maxOutputBytes = 64 * 1024) => git.execute({ operation: "StorageCleanup.submodules", @@ -722,23 +814,8 @@ const make = Effect.gen(function* () { if ((yield* fs.stat(directory)).type !== "Directory") return { force: false, reason: "submodule repository could not be inspected" }; if (yield* fs.exists(path.join(directory, "HEAD"))) { - const unpushed = yield* inspect([ - "--git-dir", - directory, - "--work-tree", - cwd, - "rev-list", - "--max-count=1", - "HEAD", - "--all", - "--reflog", - "--not", - "--remotes", - ]); - if (unpushed.stdoutTruncated || unpushed.stderrTruncated) - return { force: false, reason: "submodule inspection exceeded the limit" }; - if (unpushed.stdout.trim() !== "") - return { force: false, reason: "submodule has commits not on a remote-tracking ref" }; + const unpushed = yield* unpushedHistoryReason(cwd, directory, remoteTags); + if (unpushed !== null) return { force: false, reason: unpushed }; pending.push(path.join(directory, "modules")); } else { for (const name of yield* fs.readDirectory(directory)) @@ -865,6 +942,7 @@ const make = Effect.gen(function* () { const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread])); const refreshedDefaultRefs = new Map>(); const defaultBranches = new Map(); + const remoteTags = new Map | null>(); const groups = Map.groupBy( snapshot.threads.filter((thread) => thread.worktreePath !== null), (thread) => path.resolve(thread.worktreePath!), @@ -1209,7 +1287,7 @@ const make = Effect.gen(function* () { // Clean only untracked files. Ordinary worktrees retain Git's final tracked-edit check. if (settings.worktreeKeepWhen === "tracked-changes") yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); - const submodules = yield* submoduleRemoval(worktreePath, settings); + const submodules = yield* submoduleRemoval(worktreePath, settings, remoteTags); if (submodules.reason !== null) return keep(submodules.reason); if (submodules.force) { const changes = yield* localChangesReason(worktreePath, settings); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 4dabf3b493eb..b6ae2aa9fadb 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -111,7 +111,7 @@ const LIST_REFS_REFRESH_COALESCE_TTL = Duration.seconds(5); const LIST_REFS_REFRESH_FAILURE_COOLDOWN = Duration.seconds(30); const STATUS_DEFAULT_BRANCH_CACHE_TTL = Duration.minutes(5); const STATUS_ORIGIN_EXISTS_CACHE_TTL = Duration.minutes(5); -const STATUS_UPSTREAM_REFRESH_ENV = Object.freeze({ +export const NON_INTERACTIVE_GIT_ENV = Object.freeze({ GCM_INTERACTIVE: "never", GIT_ASKPASS: "", GIT_TERMINAL_PROMPT: "0", @@ -1377,7 +1377,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // healthy writer. A separate safety limit prevents a hung transport from // occupying its worker slot indefinitely. const started = yield* executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { - env: STATUS_UPSTREAM_REFRESH_ENV, + env: NON_INTERACTIVE_GIT_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), forceKillAfterMs: 1_000, @@ -4582,7 +4582,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const fetchRemote = Effect.fn("fetchRemote")(function* (input: GitVcsDriver.GitFetchRemoteInput) { const args = ["fetch", "--quiet", "--", input.remoteName]; const options = { - env: STATUS_UPSTREAM_REFRESH_ENV, + env: NON_INTERACTIVE_GIT_ENV, fallbackErrorDetail: `git fetch ${input.remoteName} failed`, }; // Another fetch in the same repository, such as the background status