diff --git a/evals/triage-finding/pnpm-lock.yaml b/evals/triage-finding/pnpm-lock.yaml index 691230e275..14e6ded965 100644 --- a/evals/triage-finding/pnpm-lock.yaml +++ b/evals/triage-finding/pnpm-lock.yaml @@ -2339,8 +2339,8 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - kareem@3.3.0: - resolution: {integrity: sha512-kpSuLD3/7RenBnjnJdOHXCKC8dTd1JzeOiJhN0necWWci6cC+qX+VuwPnMVgb+a4+KNJSfgqahpnfWaeDXCimw==} + kareem@3.4.1: + resolution: {integrity: sha512-UlgCAFNAdk/HLSsbfkmbH8pJqpEN52Ek8RG53Ys1i3Q3gbCQruSrWpplZlgIGyDHX/LNoLW97ei3OG43s7UhUA==} engines: {node: '>=18.0.0'} keyv-file@5.3.5: @@ -2464,8 +2464,8 @@ packages: resolution: {integrity: sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==} engines: {node: '>=20.19.0'} - mongodb@7.5.0: - resolution: {integrity: sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==} + mongodb@7.7.0: + resolution: {integrity: sha512-XG1xXEqyouAV8GCb3PuRaujofeXKpAJAPvXYWZgHl6Dz4QHSJ1nRLl0+GpqaLTwKvC0RFE9truOur34RNFY1nw==} engines: {node: '>=20.19.0'} peerDependencies: '@aws-sdk/credential-providers': ^3.806.0 @@ -2491,16 +2491,16 @@ packages: socks: optional: true - mongoose@9.9.4: - resolution: {integrity: sha512-Gc7buf0ExrOZ3t8MD8tVzTek7Qr5d+tEwj4GRFmHCtTZvpaDb38EVsXgCkYacPL9ICAftgS+FGe+dQHLRLHhcw==} + mongoose@9.11.4: + resolution: {integrity: sha512-+xME0fduXl1sWF+NZpJHZaNcwQSt+X2vbJ/h7GRSZ/ltLycAkGZSEABhzic0+q3IMF66KqF0viEE5+frchKCAQ==} engines: {node: '>=20.19.0'} mpath@0.9.0: resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} engines: {node: '>=4.0.0'} - mquery@6.0.0: - resolution: {integrity: sha512-b2KQNsmgtkscfeDgkYMcWGn9vZI9YoXh802VDEwE6qc50zxBFQ0Oo8ROkawbPAsXCY1/Z1yp0MagqsZStPWJjw==} + mquery@6.0.1: + resolution: {integrity: sha512-/Nijmq3JcpHwDM+5CwBp5yEl6fUa6jLsIhPsoQywlGXU/ZrKIvSpBzfaGnW9BLqcne/CoUwd1ch7MRqptDgFTA==} engines: {node: '>=20.19.0'} ms@2.0.0: @@ -5903,7 +5903,7 @@ snapshots: safe-buffer: 5.2.1 optional: true - kareem@3.3.0: + kareem@3.4.1: optional: true keyv-file@5.3.5: @@ -6039,7 +6039,7 @@ snapshots: whatwg-url: 14.2.0 optional: true - mongodb@7.5.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): + mongodb@7.7.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): dependencies: '@mongodb-js/saslprep': 1.5.0 bson: 7.3.2 @@ -6050,13 +6050,13 @@ snapshots: socks: 2.8.10 optional: true - mongoose@9.9.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): + mongoose@9.11.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): dependencies: '@standard-schema/spec': 1.1.0 - kareem: 3.3.0 - mongodb: 7.5.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) + kareem: 3.4.1 + mongodb: 7.7.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) mpath: 0.9.0 - mquery: 6.0.0 + mquery: 6.0.1 ms: 2.1.3 sift: 17.1.3 transitivePeerDependencies: @@ -6072,7 +6072,7 @@ snapshots: mpath@0.9.0: optional: true - mquery@6.0.0: + mquery@6.0.1: optional: true ms@2.0.0: {} @@ -6094,7 +6094,7 @@ snapshots: apparatus: 0.0.10 dotenv: 17.4.2 memjs: 1.3.2 - mongoose: 9.9.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) + mongoose: 9.11.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) pg: 8.23.0 redis: 5.12.1(@opentelemetry/api@1.9.1) safe-stable-stringify: 2.5.0 diff --git a/github-action/scripts/setup-ubuntu-sandbox.sh b/github-action/scripts/setup-ubuntu-sandbox.sh index 17db9d0ce6..24ed38009e 100644 --- a/github-action/scripts/setup-ubuntu-sandbox.sh +++ b/github-action/scripts/setup-ubuntu-sandbox.sh @@ -7,4 +7,9 @@ $nrconf{override_rc}{qr(^php8\.3-fpm\.service$)} = 0; EOF sudo apt-get update sudo apt-get install --yes bubblewrap apparmor-profiles -sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict +bwrap_profile=/usr/share/apparmor/extra-profiles/bwrap-userns-restrict +# Ubuntu 26.04 installs this profile under /etc/apparmor.d. +if [[ ! -f "$bwrap_profile" ]]; then + bwrap_profile=/etc/apparmor.d/bwrap-userns-restrict +fi +sudo apparmor_parser -r "$bwrap_profile" diff --git a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts index 4fdd1d00c6..830b9ebf23 100644 --- a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts +++ b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts @@ -10,7 +10,6 @@ import { dirname, isAbsolute, parse, relative, sep } from "node:path"; import { parseArgs } from "node:util"; import { mkdir, readFile, writeFile } from "./helper-files"; import { expandHome, windowsFiles, windowsJoin } from "./resolve-security-md"; -import { decodePosixBytes, encodePosixPath } from "./posix-path"; import { compareCodePoints, stringifyJson } from "./json"; import { print, reportCommandError, ArgumentError } from "./rank-worklists"; import { decodeUtf8 } from "./utf8"; @@ -33,6 +32,7 @@ import { blobSamples, changedPaths, directoryPaths, + decodeGitPath, encodeGitPath, ignoredTrackedPaths, runRipgrep, @@ -128,12 +128,12 @@ async function scopeCandidates(repo: string, scope: string): Promise { "!**/{.git,.svn}", "--glob", "!**/{.git,.svn}/**", - "--", - relative(repo, scope) || ".", ], repo, + relative(repo, scope) || ".", ); - } catch { + } catch (error) { + if (windows && environmentValue("RIPGREP_CONFIG_PATH")) throw error; const ignoreNames = [".gitignore", ".ignore", ".rgignore"]; const isIgnoreFile = (path: string) => { try { @@ -175,7 +175,7 @@ async function scopeCandidates(repo: string, scope: string): Promise { throw new Error( `Could not enumerate scoped repository files${result.signal ? ` (${result.signal})` : ""}: ${result.stderr.trim()}`, ); - return decodePosixBytes(result.stdout) + return decodeGitPath(result.stdout) .split("\0") .filter(Boolean) .map((path) => append(repo, path)); @@ -281,10 +281,9 @@ export async function generateInventory( "!**/{.git,.svn}", "--glob", "!**/{.git,.svn}/**", - "--", - requested, ], repo, + requested, ).catch((error: unknown) => { throw new Error( `could not run ripgrep: ${error instanceof Error ? error.message : String(error)}`, @@ -316,13 +315,13 @@ export async function generateInventory( encodeGitPath(paths.map((path) => prefix + path + "\0").join("")), ]); } - for (const path of decodePosixBytes(bytes).split("\0").filter(Boolean)) { + for (const path of decodeGitPath(bytes).split("\0").filter(Boolean)) { if (path.split("/").some((part) => sourceMetadata.has(part))) continue; if (/[\r\n]/u.test(path)) throw new Error( "Repository contains a path that cannot fit in the file inventory", ); - inventoryRows.push(encodePosixPath(path + "\n")); + inventoryRows.push(encodeGitPath(path + "\n")); } } else if (diff) { if ( diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts index 6a3403a33a..4925c709aa 100644 --- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts +++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts @@ -10,11 +10,12 @@ import { } from "node:child_process"; import { randomUUID } from "node:crypto"; import type { Writable } from "node:stream"; -import { +import pathSyntax, { basename, dirname, delimiter, isAbsolute, + parse, resolve, sep, } from "node:path"; @@ -83,6 +84,23 @@ function selectedBundledRipgrep(candidate: string): boolean { } } +// Filesystem resolution can return different Win32 spellings for the same root. +export function toolPathInRepository( + root: string, + candidate: string, + paths = pathSyntax, +): boolean { + const name = paths.relative( + paths.toNamespacedPath(root), + paths.toNamespacedPath(candidate), + ); + return ( + !paths.isAbsolute(name) && + name !== ".." && + !name.startsWith(`..${paths.sep}`) + ); +} + function trustedTool( name: "git" | "rg", target: string, @@ -149,14 +167,7 @@ function trustedTool( ) continue; const inRepository = [resolve(candidate), invocation, resolved].some( - (path) => { - try { - inside(protectedRoot, path); - return true; - } catch { - return false; - } - }, + (path) => toolPathInRepository(protectedRoot, path), ); if (inRepository && !(name === "rg" && selectedBundledRipgrep(resolved))) { if (configured !== undefined) @@ -353,19 +364,64 @@ function gitProcess(repo: string, args: string[]) { export async function runRipgrep( args: string[], repo: string, + scope: string, ): Promise { - if (!windows) return runTool("rg", args, repo); + const scopedArgs = [...args, "--", scope]; + if (!windows) return runTool("rg", scopedArgs, repo); const inherited = inheritedToolPaths([ "RIPGREP_CONFIG_PATH", "CODEX_SECURITY_GIT", ]); - const rawWindowsLookup = [repo, ...args, ...Object.values(inherited)].some( - rawToolValue, - ); + const rawWindowsLookup = [ + repo, + ...scopedArgs, + ...Object.values(inherited), + ].some(rawToolValue); const command = trustedTool("rg", repo, rawWindowsLookup); if (!command) throw Object.assign(new Error("spawn rg ENOENT"), { code: "ENOENT" }); - return runTool(command, args, repo); + if (!repo.startsWith("\\\\?\\") && repo.length < 260) + return runTool(command, scopedArgs, repo); + + const portable = (path: string) => { + const value = path.replaceAll("/", "\\"); + return /^\\\\\\?\\UNC\\/iu.test(value) + ? "\\\\" + value.slice(8) + : value.replace(/^\\\\\\?\\(?=[a-z]:\\)/iu, ""); + }; + const root = portable(repo); + if (root.length < 260 && sameFile(repo, root)) + return runTool(command, scopedArgs, root); + if (environmentValue("RIPGREP_CONFIG_PATH")) { + // Operator globs and ignore files can be relative to the working directory. + try { + return await runTool(command, scopedArgs, repo); + } catch (cause) { + throw new Error( + "Could not start ripgrep in the Windows repository directory required by RIPGREP_CONFIG_PATH.", + { cause }, + ); + } + } + // CreateProcessW can't use a long or extended current directory. Supply the + // scope as an operand and return the same relative names as a run from repo. + const result = await runTool( + command, + [...args, "--path-separator", "/", "--", resolve(repo, scope)], + parse(root).root, + ); + if (result.status !== 0) return result; + const prefix = /^\.([/\\]|$)/u.test(scope) ? "./" : ""; + const rows = decodeGitPath(result.stdout).split("\0").filter(Boolean); + const output = rows.map((path) => { + const absolute = portable(path); + if (!isAbsolute(absolute)) + throw new Error( + "Ripgrep returned a relative path for an absolute scope.", + ); + return prefix + inside(root, absolute) + "\0"; + }); + return { ...result, stdout: encodeGitPath(output.join("")) }; } export async function runTool( diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts index cecc2c27af..4f795d540c 100644 --- a/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts +++ b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts @@ -20,8 +20,9 @@ import { isMissingPathError } from "./helper-files"; export const windows = process.platform === "win32"; export const fsPath = (path: string) => windows ? toNamespacedPath(path) : encodePosixPath(path); +// Git and ripgrep return POSIX separators. Win32 verbatim paths require backslashes. export const append = (parent: string, child: string) => - `${parent || "."}${parent.endsWith(sep) ? "" : sep}${child}`; + `${parent || "."}${parent.endsWith(sep) ? "" : sep}${windows ? child.replaceAll("/", sep) : child}`; export const stat = (path: string) => windows ? windowsFiles().stat(Buffer.from(path, "utf16le")) diff --git a/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts index 2dbe96d704..97f193d141 100644 --- a/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts +++ b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import childProcess from "node:child_process"; import { createRequire, syncBuiltinESMExports } from "node:module"; import { constants, tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, posix, win32 } from "node:path"; import { importSource } from "./import-module.ts"; import type { WindowsBinding } from "../../native/windows-binding.mjs"; @@ -12,8 +12,37 @@ const helperUrl = new URL( "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs", import.meta.url, ); -const { git, runTool } = await importSource("src/helpers/inventory-git.ts", { - define: { "import.meta.url": JSON.stringify(helperUrl.href) }, +const { git, runTool, toolPathInRepository } = await importSource( + "src/helpers/inventory-git.ts", + { + define: { "import.meta.url": JSON.stringify(helperUrl.href) }, + }, +); + +test("tool containment recognizes equivalent Windows repository paths", () => { + const contained = (root: string, path: string) => + toolPathInRepository(root, path, win32); + for (const root of [String.raw`C:\repo`, String.raw`\\server\share\repo`]) { + const extended = win32.toNamespacedPath(root); + const child = win32.join(root, "tools", "placeholder"); + assert.equal(contained(extended, child), true); + assert.equal(contained(root, win32.toNamespacedPath(child)), true); + assert.equal(contained(extended, root), true); + assert.equal(contained(root, `${root}-other`), false); + } + assert.equal(contained("C:\\repo", "D:\\repo\\tools"), false); + assert.equal( + contained(String.raw`\\server\share\repo`, String.raw`\\server\other\repo`), + false, + ); + const volume = String.raw`\\?\Volume{00000000-0000-0000-0000-000000000001}\repo`; + assert.equal(contained(volume, win32.join(volume, "tools")), true); + assert.equal(contained(volume, "C:\\repo\\tools"), false); + assert.equal(toolPathInRepository("/repo", "/repo/..notes", posix), true); + assert.equal( + toolPathInRepository("/repo", "/repo-other/tools", posix), + false, + ); }); for (const [name, details] of [ diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json index 1329685ef6..c6429a208c 100644 --- a/plugins/codex-security/plugin-files.json +++ b/plugins/codex-security/plugin-files.json @@ -83,6 +83,7 @@ "scripts/snapshot_sqlite.py", "scripts/threat_model_projection.py", "scripts/validate_scan_contract.py", + "scripts/windows_paths.py", "scripts/windows_scan_local_files.py", "scripts/workbench/__init__.py", "scripts/workbench/handoff.py", diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index a206d66218..29185af08d 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_saved_results as saved_results from deep_scan_config import resolve_deep_scan_setup +from windows_paths import extended_path, filesystem_path, portable_path, portable_relative_path from workbench.handoff import require_current_continuation from workbench.storage import ( archived_deep_path, @@ -193,6 +194,10 @@ def require_owned_scan( return scan, workspace +def scan_directory_path(scan: sqlite3.Row, *parts: str) -> Path: + return filesystem_path(Path(scan["scan_dir"]).joinpath(*parts)) + + def deep_scan_path( scan: sqlite3.Row, value: str, @@ -205,18 +210,19 @@ def deep_scan_path( if not supplied.is_absolute(): raise SystemExit(f"{label} must be an absolute path inside the scan directory.") try: - resolved = supplied.resolve(strict=True) + resolved = filesystem_path(supplied).resolve(strict=True) + portable_resolved = portable_path(resolved) scan_dir = dependencies().require_canonical_scan_directory(Path(scan["scan_dir"])) - resolved.relative_to(scan_dir) + portable_relative_path(resolved, scan_dir) except (OSError, RuntimeError, ValueError) as exc: raise SystemExit(f"{label} must be an existing path inside the scan directory.") from exc - if os.path.normcase(resolved) != os.path.normcase(supplied.absolute()): + if os.path.normcase(portable_resolved) != os.path.normcase(portable_path(supplied.absolute())): raise SystemExit(f"{label} must be a canonical non-symlink path.") if kind == "file" and not resolved.is_file(): raise SystemExit(f"{label} must be a regular file.") if kind == "directory" and not resolved.is_dir(): raise SystemExit(f"{label} must be a directory.") - return str(resolved) + return str(portable_resolved) def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str: @@ -224,22 +230,26 @@ def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str: supplied = Path(value).expanduser() if not supplied.is_absolute(): raise SystemExit(f"{label} must be an absolute path inside the scan directory.") - if supplied.exists(): + if filesystem_path(supplied).exists(): return deep_scan_path(scan, str(supplied), label, kind="file") parent = Path(deep_scan_path(scan, str(supplied.parent), label, kind="directory")) output = parent / supplied.name - if os.path.normcase(output) != os.path.normcase(supplied.absolute()): + if os.path.normcase(output) != os.path.normcase(portable_path(supplied.absolute())): raise SystemExit(f"{label} must be a canonical non-symlink path.") return str(output) +def temporary_artifact_path(path: Path, suffix: str) -> Path: + return filesystem_path(path.with_name(f".{path.name}.{uuid.uuid4()}.{suffix}")) + + def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]: """Publish the staged file and return the paths needed to finalize or undo the move.""" - staged = Path(staged_path) - output = Path(output_path) + staged = filesystem_path(Path(staged_path)) + output = filesystem_path(Path(output_path)) if staged == output: raise SystemExit("A staged Deep Scan artifact must not be its published output path.") - backup = output.with_name(f".{output.name}.{uuid.uuid4()}.backup") if output.exists() else None + backup = temporary_artifact_path(output, "backup") if output.exists() else None if backup is not None: os.replace(output, backup) try: @@ -287,9 +297,13 @@ def publication_matches_snapshot(publication: Path, snapshot: Path) -> bool: def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: - discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" return { - name: deep_scan_path(scan, str(discovery_dir / filename), label, kind="file") + name: deep_scan_path( + scan, + str(scan_directory_path(scan, "artifacts", "02_discovery", filename)), + label, + kind="file", + ) for name, filename, label in ( ("inScopeFilesPath", "in_scope_files.txt", "Canonical in-scope inventory path"), ("candidateLedgerPath", "candidate_ledger.jsonl", "Canonical candidate ledger path"), @@ -321,15 +335,17 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A run["canonical_inventory_path"] is None and run["status"] == "succeeded" and manifest_path is not None - and manifest_path != str(Path(scan["scan_dir"]) / "scan-manifest.json") - and (Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "in_scope_files.txt").exists() + and portable_path(Path(manifest_path)) + != portable_path(scan_directory_path(scan, "scan-manifest.json")) + and scan_directory_path(scan, "artifacts", "02_discovery", "in_scope_files.txt").exists() ): canonical_artifacts = canonical_discovery_artifacts(scan) if ( run["terminal_reason"] == "capped" and run["completion_sequence"] == 0 and deep_scan_deadline_reached(run) - and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size != 0 + and filesystem_path(Path(canonical_artifacts["candidateLedgerPath"])).stat().st_size + != 0 ): raise SystemExit( "A capped Deep Scan without completed discoveries requires an empty " @@ -351,7 +367,7 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A else scan["user_context"] ), "scanDir": scan["scan_dir"], - "workerUsageDirectory": str(worker_usage_directory(run["scan_id"])), + "workerUsageDirectory": str(portable_path(worker_usage_directory(run["scan_id"]))), "schemaVersion": run["schema_version"], "workflowVersion": run["workflow_version"], "coordinatorGeneration": run["coordinator_generation"], @@ -701,7 +717,7 @@ def begin_deep_scan_for_target( target = dependencies().require_target(args.target_path) dependencies().require_scannable_target(target) scope = dependencies().require_scope(args.scope, "deep", target) - target_path = str(target) + target_path = str(portable_path(target)) existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: return begin_deep_scan_for_scan(connection, existing["id"], thread_id, args) @@ -782,10 +798,7 @@ def begin_deep_scan_for_target( workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: raise SystemExit("workflow-version is required.") - root = dependencies().resolve_scan_root(args.scan_root) - target_root = (root / dependencies().safe_segment(target.name)).resolve() - if target_root == target or target in target_root.parents: - raise SystemExit("The scan artifact directory must be outside the selected target.") + target_root = dependencies().scan_target_root(args.scan_root, target) create_private_directory(target_root) model = optional_text(args.model, maximum=200) reasoning_effort = optional_text(args.reasoning_effort, maximum=32) @@ -906,11 +919,11 @@ def coordinator_lease_is_live( seconds=DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS ) heartbeat_time = _parse_timestamp(str(run["updated_at"])) - heartbeat_path = ( - Path(scan["scan_dir"]) - / "artifacts" - / "deep_discovery" - / f"coordinator-heartbeat-{run['coordinator_generation']}.json" + heartbeat_path = scan_directory_path( + scan, + "artifacts", + "deep_discovery", + f"coordinator-heartbeat-{run['coordinator_generation']}.json", ) try: heartbeat = json.loads(heartbeat_path.read_text(encoding="utf-8")) @@ -1067,9 +1080,9 @@ def recover_expired_coordinator( def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None: """Reconcile ledger backups with reducer snapshots after an interrupted publication.""" scan = dependencies().require_scan(connection, scan_id) - ledger = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl" + ledger = scan_directory_path(scan, "artifacts", "02_discovery", "candidate_ledger.jsonl") backups = sorted( - ledger.parent.glob(f".{ledger.name}.*.backup"), + extended_path(ledger.parent).glob(f".{ledger.name}.*.backup"), key=lambda backup: backup.stat().st_mtime_ns, reverse=True, ) @@ -1086,7 +1099,7 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id (scan_id,), ) for reducer in reducers: - snapshot = Path(reducer["artifact_dir"]) / "canonical" / ledger.name + snapshot = filesystem_path(Path(reducer["artifact_dir"]) / "canonical" / ledger.name) if not snapshot.exists(): continue published = publication_matches_snapshot(ledger, snapshot) @@ -1468,7 +1481,9 @@ def finish_deep_scan_locked( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) ) - standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json") + standard_scan_manifest = manifest_path == str( + portable_path(scan_directory_path(scan, "scan-manifest.json")) + ) buffered_worker_ids = { row["id"] @@ -1522,7 +1537,7 @@ def finish_deep_scan_locked( for artifact_name in ("scan-manifest.json", "findings.json"): deep_scan_path( scan, - str(Path(scan["scan_dir"]) / artifact_name), + str(scan_directory_path(scan, artifact_name)), f"Canonical parent {artifact_name}", kind="file", ) @@ -1548,7 +1563,8 @@ def finish_deep_scan_locked( and ( standard_scan_manifest or canonical_artifacts is not None - and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0 + and filesystem_path(Path(canonical_artifacts["candidateLedgerPath"])).stat().st_size + == 0 ) ) if saved_iteration is None and not zero_discovery_deadline: @@ -1644,7 +1660,9 @@ def fail_deep_scan_locked( kind="file", ) ) - elif recorded_manifest == str(Path(scan["scan_dir"]) / "scan-manifest.json"): + elif recorded_manifest is not None and portable_path( + Path(recorded_manifest) + ) == portable_path(scan_directory_path(scan, "scan-manifest.json")): manifest_path = recorded_manifest if run["status"] in {"failed", "interrupted"} or scan["status"] == "failed": if ( diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 37633caa43..603d2e3fd3 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -38,6 +38,7 @@ ) from posix_scan_local_files import managed_fd, stage_write from review_state import checkpoint_review_state, review_state, review_summary +from windows_paths import filesystem_path, portable_path, portable_relative_path SCHEMA_VERSION = "1.0" PRODUCER_NAME = "codex-security-plugin" @@ -135,7 +136,7 @@ def _loads_json(value: str | bytes) -> Any: def _read_json(path: Path) -> dict[str, Any]: try: - payload = _loads_json(path.read_text(encoding="utf-8")) + payload = _loads_json(filesystem_path(path).read_text(encoding="utf-8")) _require_safe_json_value(payload, str(path)) except FileNotFoundError as exc: raise ContractError(f"missing required contract artifact: {path}") from exc @@ -471,7 +472,7 @@ def _scan_local_directory_error(message: str, error: OSError) -> ContractError: def _require_scan_directory(scan_dir: Path, context: str = "scan directory") -> Path: - scan_dir = scan_dir.absolute() + scan_dir = filesystem_path(scan_dir.absolute()) try: metadata = scan_dir.lstat() except OSError as exc: @@ -486,7 +487,7 @@ def _require_scan_directory(scan_dir: Path, context: str = "scan directory") -> raise _scan_local_directory_error( f"{context} {scan_dir}: expected an existing non-symlink directory: {exc}", exc ) from exc - if os.path.normcase(resolved) != os.path.normcase(scan_dir): + if os.path.normcase(portable_path(resolved)) != os.path.normcase(portable_path(scan_dir)): raise ContractError(f"{context}: expected a canonical non-symlink directory") return resolved @@ -494,10 +495,14 @@ def _require_scan_directory(scan_dir: Path, context: str = "scan directory") -> def _validate_scan_local_output_path(scan_dir: Path, path: Path, relative_path: str) -> None: try: resolved_parent = path.parent.resolve(strict=True) - resolved_parent.relative_to(scan_dir) + portable_relative_path(resolved_parent, scan_dir) except (OSError, RuntimeError, ValueError) as exc: raise ContractError(f"{relative_path}: expected a path inside the scan directory") from exc - if os.path.normcase(resolved_parent) != os.path.normcase(path.parent) or path.is_symlink(): + if ( + os.path.normcase(portable_path(resolved_parent)) + != os.path.normcase(portable_path(path.parent)) + or path.is_symlink() + ): raise ContractError( f"{relative_path}: expected a non-symlink path inside the scan directory" ) @@ -2552,7 +2557,7 @@ def _github_line_hash_cache( if source_root is None: return {} try: - source_root = source_root.resolve(strict=True) + source_root = filesystem_path(source_root).resolve(strict=True) except (OSError, RuntimeError): return {} requested_lines_by_path: dict[str, set[int]] = {} @@ -2792,7 +2797,7 @@ def build_sarif_projection( """Build SARIF for a sealed scan, using the optional source directory to resolve locations.""" if source_root is not None: try: - source_root = source_root.resolve(strict=True) + source_root = filesystem_path(source_root).resolve(strict=True) source_root_is_directory = source_root.is_dir() except (OSError, RuntimeError): source_root_is_directory = False @@ -3001,9 +3006,9 @@ def write_export_output(scan_dir: Path, output: Path, export_format: str, conten if export_format not in output_paths: raise ContractError(f"unsupported export format: {export_format}") scan_dir = _require_scan_directory(scan_dir) - output = Path(os.path.abspath(output)) + output = filesystem_path(Path(os.path.abspath(output))) try: - relative_output = output.relative_to(scan_dir).as_posix() + relative_output = portable_relative_path(output, scan_dir).as_posix() except ValueError: for ancestor in output.parents: try: diff --git a/plugins/codex-security/scripts/windows_paths.py b/plugins/codex-security/scripts/windows_paths.py new file mode 100644 index 0000000000..f66699a372 --- /dev/null +++ b/plugins/codex-security/scripts/windows_paths.py @@ -0,0 +1,106 @@ +"""Portable filesystem path spelling for Windows long-path support.""" + +from __future__ import annotations + +import argparse +import os +import re +import string +from pathlib import Path + +WINDOWS_DIRECTORY_PATH_LIMIT = 248 +# The default Win32 VFS allows MAX_PATH * 4 UTF-8 bytes; the pager needs eight for "-journal". +_WIN32_SQLITE_FILENAME_BYTES = 260 * 4 - 8 +_NAMESPACE_PREFIXES = ("\\\\?\\", "\\\\.\\") +_DOS_DEVICE = re.compile(r"(?:AUX|CON|CONIN\$|CONOUT\$|NUL|PRN|(?:COM|LPT)[1-9¹²³])", re.ASCII) + + +def _ordinary_component(name: str) -> bool: + # Verbatim paths bypass the DOS parsing of names, trailing dots/spaces and streams. + return bool( + name + and not name.endswith((" ", ".")) + and not any(ord(c) < 32 or c in '<>:"|?*' for c in name) + and not _DOS_DEVICE.fullmatch(name.partition(".")[0].rstrip(" ").upper()) + ) + + +def extended_path(path: Path) -> Path: + """Use absolute Win32 extended-length spelling for an equivalent filesystem path.""" + if os.name != "nt" or str(path).startswith(_NAMESPACE_PREFIXES): + return path + value = os.path.abspath(path) + candidate = Path("\\\\?\\UNC\\" + value[2:] if value.startswith("\\\\") else "\\\\?\\" + value) + return candidate if portable_path(candidate) != candidate else path + + +def filesystem_path(path: Path) -> Path: + """Use Win32 extended-length spelling when a path is near legacy limits.""" + + if os.name != "nt" or str(path).startswith(_NAMESPACE_PREFIXES): + return path + value = os.path.abspath(path) + path_length = len(value.encode("utf-16-le", errors="surrogatepass")) // 2 + return extended_path(Path(value)) if path_length >= WINDOWS_DIRECTORY_PATH_LIMIT else path + + +def portable_path(path: Path) -> Path: + """Remove an extended-length prefix only when DOS addresses the same path.""" + + if os.name != "nt": + return path + value = str(path) + if value[:8].lower() == "\\\\?\\unc\\": + tail = value[8:].rstrip("\\") + components = tail.split("\\") + if len(components) >= 2 and all(_ordinary_component(part) for part in components): + return Path("\\\\" + value[8:]) + elif value.startswith("\\\\?\\") and value[4:5] in string.ascii_letters and value[5:7] == ":\\": + tail = value[7:] + if not tail or all(_ordinary_component(part) for part in tail.split("\\")): + return Path(value[4:]) + return path + + +def sqlite_filename(path: Path) -> Path: + """Keep a usable ordinary filename when its equivalent prefix alone exceeds SQLite's limit.""" + if os.name == "nt" and len(str(path).encode("utf-8", errors="surrogatepass")) > ( + _WIN32_SQLITE_FILENAME_BYTES + ): + ordinary = portable_path(path) + if ( + len(str(ordinary).encode("utf-8", errors="surrogatepass")) + <= _WIN32_SQLITE_FILENAME_BYTES + ): + return ordinary + return path + + +def uses_windows_namespace(path: Path) -> bool: + """Whether ordinary drive/UNC spelling cannot represent the selected object.""" + return os.name == "nt" and str(portable_path(path)).startswith(_NAMESPACE_PREFIXES) + + +def _comparable_paths(path: Path, root: Path) -> tuple[Path, Path]: + if os.name == "nt" and path.anchor != root.anchor and path.is_absolute() and root.is_absolute(): + # An extended child can contain a verbatim-only name below an ordinary root. + return extended_path(path), extended_path(root) + return path, root + + +def portable_is_relative_to(path: Path, root: Path) -> bool: + path, root = _comparable_paths(path, root) + return path.is_relative_to(root) + + +def portable_relative_path(path: Path, root: Path) -> Path: + path, root = _comparable_paths(path, root) + return path.relative_to(root) + + +def main() -> None: + argparse.ArgumentParser(description=__doc__).parse_args() + + +if __name__ == "__main__": + main() diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py index 7329c0e0db..eff12a08cb 100644 --- a/plugins/codex-security/scripts/workbench/retained_coverage.py +++ b/plugins/codex-security/scripts/workbench/retained_coverage.py @@ -21,6 +21,7 @@ write_scan_local_bytes, ) from review_state import uses_legacy_review +from windows_paths import portable_relative_path from workbench.retained_records import ( _append_previous_finding, _competing_rejection_receipts, @@ -266,7 +267,9 @@ def capture_head(relative: str, canonical: Path | None = None) -> dict[str, tupl latest_reducer: str | None = None if reducer is not None: try: - latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix() + latest_reducer = portable_relative_path( + Path(reducer["result_manifest_path"]), scan_dir + ).as_posix() paths[latest_reducer] = None reducer_paths.add(latest_reducer) except ValueError: @@ -312,7 +315,9 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None checkpoints(f"{output}/checkpoints", worker["id"], attempt) if worker["result_manifest_path"]: try: - current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() + current_path = portable_relative_path( + Path(worker["result_manifest_path"]), scan_dir + ).as_posix() paths[current_path] = worker["id"] current_results.add(current_path) except ValueError: diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py index d8c018a1e7..1c11793f2f 100644 --- a/plugins/codex-security/scripts/workbench/saved_sources.py +++ b/plugins/codex-security/scripts/workbench/saved_sources.py @@ -21,6 +21,7 @@ write_scan_local_bytes, ) from review_state import review_state, uses_legacy_review +from windows_paths import portable_relative_path from workbench_coverage_receipts import capture_receipt_checkpoints @@ -691,7 +692,7 @@ def _latest_iteration(workers: Iterable[Any]) -> Any | None: def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]: - output = Path(worker["artifact_dir"]).relative_to(scan_dir) + output = portable_relative_path(Path(worker["artifact_dir"]), scan_dir) attempts = (output.parent if output.name == "output" else output) / "attempts" archived = [ ((attempts / name).as_posix(), int(name.split("-")[1])) @@ -733,7 +734,9 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st ): try: yield ( - Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(), + portable_relative_path( + Path(worker["result_manifest_path"]), scan_dir + ).as_posix(), worker["kind"], ) except ValueError: diff --git a/plugins/codex-security/scripts/workbench/storage.py b/plugins/codex-security/scripts/workbench/storage.py index 119e5b1633..542a9bef09 100644 --- a/plugins/codex-security/scripts/workbench/storage.py +++ b/plugins/codex-security/scripts/workbench/storage.py @@ -4,9 +4,21 @@ import os import sqlite3 +import sys from pathlib import Path from typing import Any +# SDK hosts also load this file directly with runpy, outside the workbench package. +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from windows_paths import ( + extended_path, + filesystem_path, + portable_path, + portable_relative_path, + uses_windows_namespace, +) + def state_dir(*, canonical: bool = True) -> Path: """Return the workbench state path, resolving symlinks unless canonical is false.""" @@ -18,12 +30,16 @@ def state_dir(*, canonical: bool = True) -> Path: os.environ["CODEX_HOME"] if os.environ.get("CODEX_HOME", "").strip() else "~/.codex" ).expanduser() path = codex_home / "state" / "plugins" / "codex-security" - return path.resolve() if canonical else path.absolute() + return filesystem_path(path).resolve() if canonical else path.absolute() def resolve_scan_root(scan_root: str | None) -> Path: """Resolve an explicit scan output root, defaulting to the scans directory under workbench state.""" - return Path(scan_root).expanduser().resolve() if scan_root else state_dir() / "scans" + return ( + filesystem_path(Path(scan_root).expanduser()).resolve() + if scan_root + else filesystem_path(state_dir() / "scans") + ) def worker_usage_directory(scan_id: str) -> Path: @@ -31,18 +47,32 @@ def worker_usage_directory(scan_id: str) -> Path: return state_dir() / "codex-home" / "worker-usage" / scan_id +def relocate_scan_path(scan_dir: Path, archive_dir: Path, value: str | None) -> str | None: + """Move a stored scan-local path to the archive, preserving verbatim-only names.""" + if value is None: + return None + path = Path(value) + try: + relative = portable_relative_path(path, portable_path(scan_dir)) + except ValueError: + return value + if uses_windows_namespace(path): + return str(portable_path(extended_path(archive_dir) / relative)) + return str(portable_path(archive_dir) / relative) + + def archived_deep_path(scan_dir: Path, value: str | None) -> str | None: """Resolve old archive rows that still name the original sibling output.""" if value is None or ".previous-" not in scan_dir.name: return value - path = Path(value) - if not path.is_relative_to(scan_dir.parent): + try: + parts = portable_relative_path(Path(value), portable_path(scan_dir.parent)).parts + except ValueError: return value - parts = path.relative_to(scan_dir.parent).parts if not parts or ".." in parts or not scan_dir.name.startswith(f"{parts[0]}.previous-"): return value # Keep missing files inside the archive too. Never follow the replacement scan's output. - return str(scan_dir.joinpath(*parts[1:])) + return relocate_scan_path(scan_dir.parent / parts[0], scan_dir, value) def saved_deep_workers(connection: sqlite3.Connection, scan: sqlite3.Row) -> list[Any]: diff --git a/plugins/codex-security/scripts/workbench_coverage_receipts.py b/plugins/codex-security/scripts/workbench_coverage_receipts.py index 4b27b39977..cf1a144366 100644 --- a/plugins/codex-security/scripts/workbench_coverage_receipts.py +++ b/plugins/codex-security/scripts/workbench_coverage_receipts.py @@ -16,6 +16,7 @@ open_scan_local_file_descriptor, path_within_scope, ) +from windows_paths import portable_relative_path from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id @@ -115,7 +116,7 @@ def resolve_coverage_receipts( directory = Path(relative).parent if directory.name == "checkpoints": directory = directory.parent - output = Path(worker["artifact_dir"]).relative_to(scan_dir) + output = portable_relative_path(Path(worker["artifact_dir"]), scan_dir) worker_prefix = (output.parent if output.name == "output" else output).as_posix() + "/" active_prefix = output.as_posix() + "/" if directory != output: diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 88ac47f22a..1e28d4ca4a 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -54,6 +54,14 @@ open_scan_local_file_descriptor, ) from finding_preview import bounded_finding_details +from windows_paths import ( + extended_path, + filesystem_path, + portable_is_relative_to, + portable_path, + portable_relative_path, + sqlite_filename, +) from workbench import handoff from workbench.storage import ( create_private_directory, @@ -169,15 +177,15 @@ def stale_claim_before(seconds: int = CLAIM_LEASE_SECONDS) -> str: def database_path() -> Path: - return state_dir() / "workbench.sqlite3" + return filesystem_path(state_dir() / "workbench.sqlite3") @contextmanager def scan_completion_lock(scan_id: str) -> Any: """Hold the cross-process lock for a scan ID until the context exits.""" - lock_dir = state_dir() / "completion-locks" + lock_dir = filesystem_path(state_dir() / "completion-locks") create_private_directory(lock_dir) - lock_path = lock_dir / f"{require_uuid(scan_id, 'scan-id')}.lock" + lock_path = filesystem_path(lock_dir / f"{require_uuid(scan_id, 'scan-id')}.lock") descriptor = os.open( lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), @@ -265,7 +273,7 @@ def connect(*, deferred: bool = False) -> sqlite3.Connection: raise for attempt in range(SQLITE_RETRY_ATTEMPTS): try: - connection = sqlite3.connect(path, timeout=5) + connection = sqlite3.connect(sqlite_filename(path), timeout=5) except sqlite3.OperationalError: print(guidance, file=sys.stderr) raise @@ -639,8 +647,8 @@ def save_workspace(connection: sqlite3.Connection, args: argparse.Namespace) -> def scan_target_root(scan_root: str | None, target: Path) -> Path: root = resolve_scan_root(scan_root) - target_root = (root / safe_segment(target.name)).resolve() - if target_root == target or target in target_root.parents: + target_root = filesystem_path(root / safe_segment(target.name)).resolve() + if portable_is_relative_to(target_root, target): raise SystemExit("The scan artifact directory must be outside the selected target.") return target_root @@ -738,7 +746,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict return workspace_state(connection, workspace["id"], result_scan_id=active["id"]) if workspace["updated_at"] != workspace_version: raise SystemExit("Codex Security setup changed while the scan was starting. Try again.") - current_target = require_remediation_target(str(target)) + current_target = require_remediation_target(workspace["target_path"]) current_target_metadata = current_target.stat() if (current_target_metadata.st_dev, current_target_metadata.st_ino) != ( target_metadata.st_dev, @@ -751,7 +759,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict owned_active_scan = deep_scan.existing_deep_scan_for_target( connection, workspace["thread_id"], - str(target), + workspace["target_path"], scope, ) if owned_active_scan is not None: @@ -799,8 +807,8 @@ def _start_prompt_driven_scan( args.diff_head_revision, args.diff_content_digest, ) - target = Path(inspected["target"]["targetPath"]) - target_path = str(target) + target_path = inspected["target"]["targetPath"] + target = require_target(target_path) scope = inspected["scope"] diff_target = inspected["diffTarget"] user_context = user_context_argument(args) @@ -1126,7 +1134,8 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + if portable_path(Path(run["manifest_path"])) + == portable_path(scan_dir / "scan-manifest.json") else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message) @@ -1153,7 +1162,7 @@ def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[ inventory = Path(artifacts["inScopeFilesPath"]) inventory_descriptor = open_scan_local_file_descriptor( scan_dir, - inventory.relative_to(scan_dir).as_posix(), + portable_relative_path(inventory, scan_dir).as_posix(), "Canonical Deep Scan in-scope inventory", ) with os.fdopen(inventory_descriptor, "rb") as source: @@ -1161,7 +1170,7 @@ def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[ in_scope = {re.sub(r"^(?:\./)+", "", line) for line in lines if line} descriptor = open_scan_local_file_descriptor( scan_dir, - ledger.relative_to(scan_dir).as_posix(), + portable_relative_path(ledger, scan_dir).as_posix(), "Canonical Deep Scan candidate ledger", ) with os.fdopen(descriptor, "r", encoding="utf-8") as source: @@ -1428,7 +1437,7 @@ def add_warning() -> None: if path is not None: connection.execute( "INSERT INTO scan_artifacts (scan_id, kind, path, created_at) VALUES (?, ?, ?, ?)", - (scan["id"], kind, str(path), timestamp), + (scan["id"], kind, str(portable_path(path)), timestamp), ) connection.execute("DELETE FROM finding_occurrences WHERE scan_id = ?", (scan["id"],)) index_findings(connection, scan["id"], findings, timestamp) @@ -1468,7 +1477,9 @@ def add_warning() -> None: def require_cli_scan_directory(repository: Path, args: argparse.Namespace) -> Path: scan_dir = require_canonical_scan_directory(Path(args.scan_dir).expanduser()) - if scan_dir == repository or repository in scan_dir.parents or scan_dir in repository.parents: + if portable_is_relative_to(scan_dir, repository) or portable_is_relative_to( + repository, scan_dir + ): raise SystemExit( "The scan artifact directory must be outside and not contain the selected target." ) @@ -1598,13 +1609,16 @@ def register_cli_scan( scan_dir = None # Revalidate the directory and its owner under the next lock. time.sleep(min(0.05, remaining)) connection.execute("BEGIN IMMEDIATE") - target_id = ensure_security_target(connection, str(repository)) - if parent_scan_id is not None: - parent = require_scan(connection, parent_scan_id) - if parent["target_id"] != target_id: - raise SystemExit( - "A rerun must belong to the same repository as its parent scan." - ) + parent = ( + require_scan(connection, parent_scan_id) if parent_scan_id is not None else None + ) + target_id = ensure_security_target( + connection, + str(portable_path(repository)), + preferred_target_id=parent["target_id"] if parent is not None else None, + ) + if parent is not None and parent["target_id"] != target_id: + raise SystemExit("A rerun must belong to the same repository as its parent scan.") connection.execute( """ @@ -1617,7 +1631,7 @@ def register_cli_scan( ( workspace_id, target_id, - str(repository), + str(portable_path(repository)), repository.name, scope, mode, @@ -1656,7 +1670,9 @@ def register_cli_scan( ), ) if workflow_id is not None: - register_workflow_scan(connection, workflow_id, scan_id, str(scan_dir), timestamp) + register_workflow_scan( + connection, workflow_id, scan_id, str(portable_path(scan_dir)), timestamp + ) scan = connection.execute( "SELECT scans.*, snapshot AS lifecycle_json FROM scans " "JOIN scan_lifecycles ON scan_id = scans.id WHERE scans.id = ?", @@ -1667,10 +1683,14 @@ def register_cli_scan( connection.rollback() raise return { - **({"archivedScanDir": str(archived_scan_dir)} if archived_scan_dir is not None else {}), + **( + {"archivedScanDir": str(portable_path(archived_scan_dir))} + if archived_scan_dir is not None + else {} + ), "contract": scan_contract(scan), "lifecycle": json.loads(scan["lifecycle_json"]), - "scanDir": str(scan_dir), + "scanDir": str(portable_path(scan_dir)), "scanId": scan_id, "scopeFileCount": scope_file_count, "targetId": target_id, @@ -2650,12 +2670,12 @@ def scan_result( continue path = available_artifact_path(Path(scan["scan_dir"]), Path(row["path"])) if path is not None: - artifacts[row["kind"]] = str(path) + artifacts[row["kind"]] = str(portable_path(path)) sarif_path = available_artifact_path( Path(scan["scan_dir"]), Path(scan["scan_dir"]) / "exports" / "results.sarif" ) if sarif_path is not None: - artifacts["sarifReport"] = str(sarif_path) + artifacts["sarifReport"] = str(portable_path(sarif_path)) model_fields = saved_results.threat_model_fields(_WORKBENCH_DB_CONTEXT, scan) if path := model_fields.get("threatModelPath"): artifacts["threatModel"] = path @@ -2949,7 +2969,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] artifacts.append(report_path) poc_relative = report_relative.parent / "poc" - poc_root = scan_dir.joinpath(*poc_relative.parts) + poc_root = filesystem_path(scan_dir.joinpath(*poc_relative.parts)) try: if not stat.S_ISDIR(poc_root.stat(follow_symlinks=False).st_mode): return artifacts @@ -2957,7 +2977,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] return artifacts for directories_seen, (current_directory, directory_names, file_names) in enumerate( - os.walk(poc_root, topdown=True, followlinks=False), start=1 + os.walk(extended_path(poc_root), topdown=True, followlinks=False), start=1 ): if directories_seen > FINDING_ARTIFACT_DIRECTORIES_LIMIT: break @@ -2968,7 +2988,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] for file_name in sorted(file_names): candidate = current_path / file_name try: - relative_path = candidate.relative_to(scan_dir).as_posix() + relative_path = portable_relative_path(candidate, scan_dir).as_posix() except ValueError: continue if not scan_local_regular_file(scan_dir, relative_path): @@ -3133,34 +3153,41 @@ def patch_artifact_preview( def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None: try: resolved_scan_dir = require_canonical_scan_directory(scan_dir) - resolved = candidate.resolve(strict=True) - resolved.relative_to(resolved_scan_dir) + resolved = filesystem_path(candidate).resolve(strict=True) + portable_relative_path(resolved, resolved_scan_dir) except (FileNotFoundError, RuntimeError, SystemExit, ValueError): return None - if os.path.normcase(resolved) != os.path.normcase(candidate) or not candidate.is_file(): + if ( + os.path.normcase(portable_path(resolved)) != os.path.normcase(portable_path(candidate)) + or not resolved.is_file() + ): return None return resolved def artifact_path(scan_dir: Path, file_name: str, *, required: bool) -> Path | None: scan_dir = require_canonical_scan_directory(scan_dir) - candidate = scan_dir / file_name + candidate = filesystem_path(scan_dir / file_name) try: resolved = candidate.resolve(strict=True) - resolved.relative_to(scan_dir.resolve()) + portable_relative_path(resolved, scan_dir) except (FileNotFoundError, RuntimeError, ValueError) as exc: if not required and isinstance(exc, FileNotFoundError): return None raise SystemExit( f"{file_name}: expected a regular file inside the scan directory." ) from exc - if os.path.normcase(resolved) != os.path.normcase(candidate) or not candidate.is_file(): + if ( + os.path.normcase(portable_path(resolved)) != os.path.normcase(portable_path(candidate)) + or not candidate.is_file() + ): raise SystemExit(f"{file_name}: expected a regular non-symlink file.") return resolved def require_canonical_scan_directory(scan_dir: Path) -> Path: - scan_dir = scan_dir.absolute() + portable_scan_dir = portable_path(scan_dir.absolute()) + scan_dir = filesystem_path(portable_scan_dir) try: metadata = scan_dir.lstat() resolved = scan_dir.resolve(strict=True) @@ -3168,9 +3195,9 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path: raise SystemExit( "Scan directory must be an existing canonical non-symlink directory." ) from exc - if not stat.S_ISDIR(metadata.st_mode) or os.path.normcase(resolved) != os.path.normcase( - scan_dir - ): + if not stat.S_ISDIR(metadata.st_mode) or os.path.normcase( + portable_path(resolved) + ) != os.path.normcase(portable_scan_dir): raise SystemExit("Scan directory must be an existing canonical non-symlink directory.") # Re-check ownership and parents so the scan directory cannot be substituted. if os.name != "nt": @@ -3213,7 +3240,7 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None: ) deep_scan.configure(command_context) if args.command == "resolve-scan-root": - print(json.dumps({"scanRoot": str(resolve_scan_root(args.scan_root))})) + print(json.dumps({"scanRoot": str(portable_path(resolve_scan_root(args.scan_root)))})) return if args.command in {"inspect-target", "inspect-setup"}: print(json.dumps(inspect_command(args), allow_nan=False, sort_keys=True)) @@ -3361,7 +3388,7 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None: remediation_state.cancel_finding_remediation_request(connection, args), ) elif args.command == "database-info": - result = {"databasePath": str(database_path())} + result = {"databasePath": str(portable_path(database_path()))} elif args.command == "finding-workflow": result = finding_workflow(connection, json.load(sys.stdin), now()) else: diff --git a/plugins/codex-security/scripts/workbench_native_indexes.py b/plugins/codex-security/scripts/workbench_native_indexes.py index 98c7a75011..57f471e888 100644 --- a/plugins/codex-security/scripts/workbench_native_indexes.py +++ b/plugins/codex-security/scripts/workbench_native_indexes.py @@ -12,6 +12,7 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_scan_history as scan_history +from windows_paths import filesystem_path from workbench_constants import FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, FINDINGS_PAGE_MAX from workbench_validation import bounded_output_text, timestamp_key @@ -267,7 +268,7 @@ def list_repositories( path = os.path.expanduser(repository["targetPath"]) try: repository["checkoutAvailable"] = Path(path).is_absolute() and os.path.isdir( - os.path.realpath(path) + os.path.realpath(filesystem_path(Path(path))) ) except (OSError, RecursionError): repository["checkoutAvailable"] = False diff --git a/plugins/codex-security/scripts/workbench_publication.py b/plugins/codex-security/scripts/workbench_publication.py index b87630493f..672867bb0a 100644 --- a/plugins/codex-security/scripts/workbench_publication.py +++ b/plugins/codex-security/scripts/workbench_publication.py @@ -28,6 +28,7 @@ write_sarif_projection, ) from validate_scan_contract import validate_contract +from windows_paths import portable_path, sqlite_filename def destination_key(destination: Any) -> str: @@ -225,7 +226,7 @@ def finding_issues(db: Any, payload: Any) -> dict[str, Any]: except PermissionError: pass # Let SQLite report the open failure used by the host's state fallback. if payload["action"] == "inspect": - database_uri = f"file:{quote(str(path), safe='')}?mode=ro" + database_uri = f"file:{quote(str(sqlite_filename(path)), safe='')}?mode=ro" try: connection = sqlite3.connect(database_uri, uri=True, timeout=5) except sqlite3.OperationalError as exc: @@ -364,7 +365,7 @@ def export_findings( if path is None: raise SystemExit(f"Could not export Codex Security findings as {args.format.upper()}.") return { - "export": {"format": args.format, "path": str(path)}, + "export": {"format": args.format, "path": str(portable_path(path))}, "scan": db.scan_result(connection, scan), "workspace": db.workspace_state(connection, scan["workspace_id"]), } diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index af4d506d56..9a407f0951 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -39,6 +39,7 @@ ) from finalize_scan_contract import finding_candidate_id as finding_candidate_id from review_state import checkpoint_review_state, review_state, review_summary, uses_legacy_review +from windows_paths import portable_relative_path from workbench.retained_coverage import merge_saved_results as merge_saved_results from workbench.saved_coverage import ( _saved_coverage_id as _saved_coverage_id, @@ -170,7 +171,7 @@ def has_saved_source() -> bool: return has_saved_source() manifest = _read_scan_local_json( scan_dir, - manifest_path.relative_to(scan_dir).as_posix(), + portable_relative_path(manifest_path, scan_dir).as_posix(), "Saved scan manifest", ) manifest_scan = manifest.get("scan") @@ -259,7 +260,7 @@ def _recovery_source_digests( if manifest_path is not None: manifest = _read_scan_local_json( scan_dir, - manifest_path.relative_to(scan_dir).as_posix(), + portable_relative_path(manifest_path, scan_dir).as_posix(), "Saved scan manifest", ) manifest_scan = manifest.get("scan") @@ -1026,7 +1027,7 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) try: - relative = Path(args.draft_path).relative_to(scan_dir).as_posix() + relative = portable_relative_path(Path(args.draft_path), scan_dir).as_posix() except ValueError as exc: raise SystemExit( "Scan draft must be inside the registered scan drafts directory." @@ -1056,7 +1057,9 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: checkpoint_contents = None if args.checkpoint_path is not None: try: - checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix() + checkpoint_relative = portable_relative_path( + Path(args.checkpoint_path), scan_dir + ).as_posix() except ValueError as exc: raise SystemExit( "Scan checkpoint must be inside the registered scan drafts directory." diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py index 3315ca0d88..41059bf318 100644 --- a/plugins/codex-security/scripts/workbench_scan_history.py +++ b/plugins/codex-security/scripts/workbench_scan_history.py @@ -25,6 +25,7 @@ ) from report_projection import SEVERITY_ORDER from review_state import review_state +from windows_paths import extended_path, filesystem_path, portable_path from workbench_constants import ARTIFACTS, FINDINGS_PAGE_MAX from workbench_scan_start import scan_target_identity from workbench_scan_usage import stored_scan_cost_fields @@ -211,7 +212,7 @@ def coverage_summary_for_history( def _windows_path_key(value: str) -> str: - return os.path.normcase(os.path.realpath(value)) + return os.path.normcase(portable_path(Path(os.path.realpath(filesystem_path(Path(value)))))) def _same_repository( @@ -222,8 +223,10 @@ def _same_repository( ) -> bool: if before["target_id"] is not None and before["target_id"] == after["target_id"]: return True - before_target = Path(before["target_path"]) - after_target = Path(after["target_path"]) + before_target = filesystem_path(Path(before["target_path"])) + after_target = filesystem_path(Path(after["target_path"])) + if portable_path(before_target) == portable_path(after_target): + return True if before_target.resolve() == after_target.resolve(): return True common_dir = ("rev-parse", "--path-format=absolute", "--git-common-dir") @@ -232,7 +235,8 @@ def _same_repository( if ( before_git_dir is not None and (after_git_dir := after_git(after_target, *common_dir)) is not None - and Path(before_git_dir).resolve() == Path(after_git_dir).resolve() + and portable_path(filesystem_path(Path(before_git_dir)).resolve()) + == portable_path(filesystem_path(Path(after_git_dir)).resolve()) ): return True origin = ("remote", "get-url", "origin") @@ -333,12 +337,12 @@ def scan_summary(row: sqlite3.Row) -> dict[str, Any]: }, "recipeAvailable": row["recipe_json"] is not None, "reasoningEffort": row["reasoning_effort"], - "scanDir": row["scan_dir"], + "scanDir": str(portable_path(Path(row["scan_dir"]))), "scanId": row["id"], "scope": row["scope"], "startedAt": row["started_at"], "targetId": row["target_id"], - "targetPath": row["target_path"], + "targetPath": str(portable_path(Path(row["target_path"]))), "targetRevision": row["target_revision"], "targetSummary": row["target_summary"], "updatedAt": max( @@ -354,6 +358,17 @@ def scan_summary(row: sqlite3.Row) -> dict[str, Any]: } +def _requested_repository(connection: sqlite3.Connection, repository: Path) -> sqlite3.Row: + repository_path = str(portable_path(repository)) + return connection.execute( + """ + SELECT COALESCE((SELECT id FROM security_targets WHERE current_path IN (?, ?)), '') + AS target_id, ? AS target_path + """, + (repository_path, str(extended_path(Path(repository_path))), repository_path), + ).fetchone() + + def list_scans( connection: sqlite3.Connection, args: argparse.Namespace | None = None ) -> dict[str, Any]: @@ -363,14 +378,9 @@ def list_scans( clauses: list[str] = [] values: list[Any] = [] if args is not None and args.repository: - repository = Path(args.repository).expanduser().resolve() - requested_repository = connection.execute( - """ - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - """, - (str(repository), str(repository)), - ).fetchone() + repository = filesystem_path(Path(args.repository).expanduser()).resolve() + repository_path = str(portable_path(repository)) + requested_repository = _requested_repository(connection, repository) requested_git = cache(git_output) related_target_ids = [ target["target_id"] @@ -379,15 +389,15 @@ def list_scans( ) if _same_repository(target, requested_repository, after_git=requested_git) ] - repository_clauses = ["scans.target_path = ?"] - values.append(str(repository)) + repository_clauses = ["scans.target_path IN (?, ?)"] + values.extend((repository_path, str(extended_path(Path(repository_path))))) if related_target_ids: placeholders = ", ".join("?" for _ in related_target_ids) repository_clauses.append(f"scans.target_id IN ({placeholders})") values.extend(related_target_ids) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: - scan_root = str(Path(args.scan_root).expanduser().resolve()) + scan_root = str(portable_path(filesystem_path(Path(args.scan_root).expanduser()).resolve())) prefix = scan_root.rstrip(os.sep) + os.sep if os.name == "nt": scan_root = _windows_path_key(scan_root) @@ -515,14 +525,9 @@ def list_unmatched_scan_pairs( backfill_finding_details: Callable[[sqlite3.Connection, sqlite3.Row], None], read_coverage: Callable[[sqlite3.Row], dict[str, Any]], ) -> dict[str, Any]: - repository = Path(args.repository).expanduser().resolve() - requested = connection.execute( - """ - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - """, - (str(repository), str(repository)), - ).fetchone() + repository = filesystem_path(Path(args.repository).expanduser()).resolve() + repository_path = str(portable_path(repository)) + requested = _requested_repository(connection, repository) selected = [] requested_git = cache(git_output) membership: dict[tuple[str | None, str], bool] = {} @@ -594,7 +599,7 @@ def list_unmatched_scan_pairs( ) return { "batches": batches, - "repository": str(repository), + "repository": repository_path, "scanCount": len(selected), "skippedPairs": skipped, "unavailableScans": len(selected) - len(available), diff --git a/plugins/codex-security/scripts/workbench_scan_start.py b/plugins/codex-security/scripts/workbench_scan_start.py index 8a93b69cf7..70009ffdbd 100644 --- a/plugins/codex-security/scripts/workbench_scan_start.py +++ b/plugins/codex-security/scripts/workbench_scan_start.py @@ -19,7 +19,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from filesystem_identity import serialize_filesystem_identity from finalize_scan_contract import write_scan_local_bytes -from workbench.storage import state_dir +from windows_paths import ( + extended_path, + portable_is_relative_to, + portable_path, +) +from workbench.storage import relocate_scan_path, state_dir from workbench_feedback import get_scan_feedback from workbench_target import ( directory_content_digest, @@ -121,6 +126,8 @@ def archive_scan( before_archive: Callable[[], None] | None = None, ) -> Iterator[Path | None]: """Yield the archive directory while preparing the original scan path for reuse.""" + portable_scan_dir = portable_path(scan_dir) + scan_dir_spellings = (str(portable_scan_dir), str(extended_path(portable_scan_dir))) archived_scan_dir = ( canonical_directory(Path(args.archived_scan_dir).expanduser()) if args.archived_scan_dir is not None @@ -128,14 +135,17 @@ def archive_scan( ) if archived_scan_dir is not None and ( not args.archive_existing - or archived_scan_dir.parent != scan_dir.parent + or portable_path(archived_scan_dir).parent != portable_scan_dir.parent or not archived_scan_dir.name.startswith(f"{scan_dir.name}.previous-") ): raise SystemExit("The archived scan must be a previous sibling of the scan directory.") - previous_scan = connection.execute( - "SELECT id, status FROM scans WHERE scan_dir = ?", (str(scan_dir),) - ).fetchone() + owners = connection.execute( + "SELECT id, status FROM scans WHERE scan_dir IN (?, ?)", scan_dir_spellings + ).fetchall() + if len(owners) > 1: + raise SystemExit("Cannot archive an output directory claimed by multiple scans.") + previous_scan = owners[0] if owners else None if previous_scan is not None: if not args.archive_existing: raise SystemExit( @@ -169,7 +179,9 @@ def archive_scan( if row[2] ), ] - if any(path == scan_dir or scan_dir in path.parents for path in protected_directories): + if any( + portable_is_relative_to(path, portable_scan_dir) for path in protected_directories + ): raise SystemExit( "Cannot archive output containing the workbench state or active database." ) @@ -180,7 +192,9 @@ def archive_scan( if before_archive is not None: before_archive() archived_scan_dir = Path( - tempfile.mkdtemp(prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent) + tempfile.mkdtemp( + prefix=f"{scan_dir.name}.previous-", dir=extended_path(scan_dir.parent) + ) ).resolve() archived_scan_dir.rmdir() scan_dir.rename(archived_scan_dir) @@ -193,28 +207,29 @@ def archive_scan( scan_dir.mkdir(mode=0o700) scan_dir.chmod(0o700) if previous_scan is not None: + archive_dir = portable_path(archived_scan_dir) + old_sarif = portable_scan_dir / "exports" / "results.sarif" connection.execute( "UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?", - (str(archived_scan_dir), timestamp, previous_scan["id"]), + (str(archive_dir), timestamp, previous_scan["id"]), ) connection.execute( """UPDATE finding_workflows SET scan_dir = ?, - results_json = CASE WHEN json_extract(results_json, '$.scan.sarifPath') = ? + results_json = CASE WHEN json_extract(results_json, '$.scan.sarifPath') IN (?, ?) THEN json_set(results_json, '$.scan.sarifPath', ?) ELSE results_json END - WHERE scan_id = ? AND scan_dir = ?""", + WHERE scan_id = ? AND scan_dir IN (?, ?)""", ( - str(archived_scan_dir), - str(scan_dir / "exports" / "results.sarif"), - str(archived_scan_dir / "exports" / "results.sarif"), + str(archive_dir), + str(old_sarif), + str(extended_path(old_sarif)), + str(archive_dir / "exports" / "results.sarif"), previous_scan["id"], - str(scan_dir), + *scan_dir_spellings, ), ) def archive_path(value: str | None) -> str | None: - if value is None or not Path(value).is_relative_to(scan_dir): - return value - return str(archived_scan_dir / Path(value).relative_to(scan_dir)) + return relocate_scan_path(scan_dir, archive_dir, value) connection.create_function("archive_path", 1, archive_path) try: @@ -253,7 +268,7 @@ def archive_path(value: str | None) -> str | None: if moved: if not connection.in_transaction: current_owner = connection.execute( - "SELECT id FROM scans WHERE scan_dir = ?", (str(scan_dir),) + "SELECT id FROM scans WHERE scan_dir IN (?, ?)", scan_dir_spellings ).fetchone() if current_owner is not None and ( previous_scan is None or current_owner["id"] != previous_scan["id"] @@ -303,7 +318,7 @@ def insert_running_scan( scan_dir = Path( tempfile.mkdtemp( prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", - dir=target_root, + dir=extended_path(target_root), ) ).resolve() connection.execute( @@ -322,7 +337,7 @@ def insert_running_scan( scan_id, workspace["id"], workspace["target_id"], - str(target), + workspace["target_path"], *target_identity, json.dumps(source_scopes, allow_nan=False, separators=(",", ":"), sort_keys=True), scope, @@ -331,7 +346,7 @@ def insert_running_scan( workspace["thread_id"] if workspace["default_mode"] == "deep" else None, *scan_diff_identity(diff_target or None), target_summary, - str(scan_dir), + str(portable_path(scan_dir)), optional_text(model, maximum=200), optional_text(reasoning_effort, maximum=32), handoff_status, diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index dc4b7a845e..2ea0cd30e6 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -20,6 +20,7 @@ from urllib.parse import quote sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import extended_path, filesystem_path, portable_path from workbench.storage import worker_usage_directory TOKEN_FIELDS = { @@ -460,9 +461,9 @@ def _worker_usage_receipts( ) -> tuple[dict[str, RolloutUsage], set[str], set[str]]: if scan["mode"] != "deep": return {}, set(), set() - directory = worker_usage_directory(scan["id"]) + directory = filesystem_path(worker_usage_directory(scan["id"])) try: - if directory.resolve(strict=True) != directory: + if portable_path(directory.resolve(strict=True)) != portable_path(directory): raise ValueError("Worker usage directory is not a canonical protected directory.") except FileNotFoundError: return {}, set(), set() @@ -473,7 +474,7 @@ def _worker_usage_receipts( missing: set[str] = set() attempts: set[str] = set() scope: set[str] = set() - for path in sorted(directory.glob("*.json")): + for path in sorted(extended_path(directory).glob("*.json")): try: if _rollout_path(str(path)) is None: raise ValueError("Worker usage receipt path is unavailable.") @@ -838,17 +839,17 @@ def _rollout_path(value: object) -> Path | None: return None try: try: - candidate.lstat() + filesystem_path(candidate).lstat() except FileNotFoundError: # Native cold storage may compress an indexed JSONL file in place. if candidate.suffix != ".jsonl": return None candidate = candidate.with_suffix(".jsonl.zst") - resolved = candidate.resolve(strict=True) + resolved = filesystem_path(candidate).resolve(strict=True) if not resolved.is_file(): return None - if resolved == candidate: + if portable_path(resolved) == portable_path(candidate.absolute()): return resolved if sys.platform == "darwin" and candidate.parts[1] in {"var", "tmp"}: diff --git a/plugins/codex-security/scripts/workbench_setup.py b/plugins/codex-security/scripts/workbench_setup.py index fcfeafedea..e8e67fbd74 100644 --- a/plugins/codex-security/scripts/workbench_setup.py +++ b/plugins/codex-security/scripts/workbench_setup.py @@ -10,6 +10,12 @@ from typing import Any from deep_scan_config import resolve_deep_scan_setup +from windows_paths import ( + filesystem_path, + portable_path, + portable_relative_path, + uses_windows_namespace, +) from workbench_cli import parse_args from workbench_constants import DIFF_TARGET_KINDS, EMPTY_GIT_TREE, EMPTY_GIT_TREES, optional_text from workbench_target import ( @@ -26,7 +32,11 @@ def require_target(value: str) -> Path: expanded = Path(value).expanduser() if not expanded.is_absolute(): raise SystemExit("Scan target must be an absolute local directory path.") - target = expanded.resolve() + if uses_windows_namespace(expanded): + raise SystemExit("Scan target must have an ordinary Windows drive or UNC path.") + target = filesystem_path(expanded).resolve() + if uses_windows_namespace(target): + raise SystemExit("Scan target must have an ordinary Windows drive or UNC path.") if not target.is_dir(): raise SystemExit(f"Scan target is not a readable local directory: {target}") return target @@ -37,7 +47,7 @@ def inspect_target(target: Path) -> dict[str, Any]: return { "displayName": target.name, "targetMetadata": git_target_metadata(target), - "targetPath": str(target), + "targetPath": str(portable_path(target)), } @@ -177,7 +187,7 @@ def require_review_changes_target(target: Path) -> str: """Return the checked-out revision or raise SystemExit if target is not the worktree root.""" revision = require_git_worktree_head(target) repository_root = git_output(target, "rev-parse", "--show-toplevel") - if repository_root is None or not Path(repository_root).samefile(target): + if repository_root is None or not filesystem_path(Path(repository_root)).samefile(target): raise SystemExit( "Review changes requires the checked-out Git repository root as the target." ) @@ -204,10 +214,10 @@ def require_scope(scope: str, mode: str, target: Path) -> str: if ".." in requested_scope.parts: raise SystemExit("Scan scope must stay inside the scanned target.") try: - resolved_scope = ( + resolved_scope = filesystem_path( requested_scope if requested_scope.is_absolute() else target / requested_scope ).resolve() - relative_scope = resolved_scope.relative_to(target) + relative_scope = portable_relative_path(resolved_scope, target) except (RuntimeError, ValueError) as exc: raise SystemExit("Scan scope must stay inside the scanned target.") from exc normalized = relative_scope.as_posix() or "." diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py index 15cfc67913..51ad99368a 100644 --- a/plugins/codex-security/scripts/workbench_target.py +++ b/plugins/codex-security/scripts/workbench_target.py @@ -21,6 +21,13 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) from filesystem_identity import stored_filesystem_identity_matches +from windows_paths import ( + extended_path, + filesystem_path, + portable_is_relative_to, + portable_path, + portable_relative_path, +) from windows_scan_local_files import copy_directory_junction from workbench_constants import GIT_REPOSITORY_ENVIRONMENT @@ -236,7 +243,7 @@ def trusted_git_executable(protected_root: Path) -> str | None: candidates = iter((candidate,)) try: - repository = _protected_repository_root(protected_root) + repository = portable_path(_protected_repository_root(protected_root)) except (OSError, RuntimeError): return None @@ -260,8 +267,7 @@ def trusted_git_executable(protected_root: Path) -> str | None: ): continue if any( - path == repository or repository in path.parents - for path in (lexical, invocation, canonical) + portable_is_relative_to(path, repository) for path in (lexical, invocation, canonical) ): if configured is not None: raise SystemExit("CODEX_SECURITY_GIT must stay outside the protected repository.") @@ -307,10 +313,12 @@ def git_command( "-c", "i18n.logOutputEncoding=UTF-8", "-C", - str(target), + str(portable_path(target)), ] if git_dir is not None and work_tree is not None: - command.extend(["--git-dir", str(git_dir), "--work-tree", str(work_tree)]) + command.extend( + ["--git-dir", str(portable_path(git_dir)), "--work-tree", str(portable_path(work_tree))] + ) full_command = [*command, *args] if executable is None: empty_output = "" if text else b"" @@ -402,7 +410,7 @@ def local_status(name: str, status: str) -> str | None: return None if status == "D": return status - path = repo / name + path = filesystem_path(repo / name) try: if ( status in {"M", "R"} @@ -438,13 +446,13 @@ def diff_scope_file_count(target: Path, diff_target: dict[str, str]) -> int | No # A partial clone can lack a tree needed even to enumerate the changed paths. return None for path, status in changes: - name = path.relative_to(target).as_posix() + name = portable_relative_path(path, target).as_posix() if not local or status == "D": committed[name] = f"{base if status == 'D' else head}:{name}" else: try: - resolved = path.resolve(strict=True) - if not resolved.is_relative_to(target): + resolved = filesystem_path(path).resolve(strict=True) + if not portable_is_relative_to(resolved, target): continue with resolved.open("rb") as stream: binary = _stream_is_binary(stream, os.fstat(stream.fileno()).st_size) @@ -475,7 +483,7 @@ def candidate_source_lines( base, head = diff_target["baseRevision"], diff_target["headRevision"] local = diff_target["kind"] == "working_tree" changed = { - path.relative_to(target).as_posix(): status + portable_relative_path(path, target).as_posix(): status for path, status in git_changed_paths( target, base, head, "local-patch" if local else "revisions" ) @@ -722,7 +730,7 @@ def worktree_content_digest_for_context( raise SystemExit("Could not snapshot the selected working-tree changes.") for raw_path in sorted(path for path in untracked.split(b"\0") if path): relative_path = os.fsdecode(raw_path) - path = (work_tree or repository) / relative_path + path = filesystem_path((work_tree or repository) / relative_path) try: metadata = path.lstat() except OSError as exc: @@ -761,14 +769,14 @@ def git_worktree_context(target: Path) -> tuple[Path, str]: root = git_output(target, "rev-parse", "--show-toplevel") if root is None: raise SystemExit("Could not inspect the selected Git working tree.") - repository = Path(root).resolve() + repository = filesystem_path(Path(root)).resolve() prefix = git_bytes(target, "rev-parse", "--show-prefix") if prefix is None: raise SystemExit("Could not inspect the selected Git working tree.") pathspec = os.fsdecode(prefix.removesuffix(b"\n")).removesuffix("/") or "." - scoped = (repository / pathspec).resolve() + scoped = filesystem_path(repository / pathspec).resolve() try: - if not scoped.is_relative_to(repository) or not scoped.samefile(target): + if not portable_is_relative_to(scoped, repository) or not scoped.samefile(target): raise ValueError("Git working tree does not contain the selected target") except (OSError, ValueError) as exc: raise SystemExit("Scan target must stay inside its Git working tree.") from exc @@ -792,13 +800,13 @@ def git_submodule_entries(target: Path) -> tuple[tuple[Path, str], ...]: if mode != b"160000": continue relative_path = Path(os.fsdecode(raw_path)).relative_to(pathspec) - entries.append((target / relative_path, object_id.decode("ascii"))) + entries.append((filesystem_path(target / relative_path), object_id.decode("ascii"))) return tuple(entries) def require_clean_submodule_worktrees(target: Path) -> None: for submodule, expected_revision in git_submodule_entries(target): - relative_path = str(submodule.relative_to(target)) + relative_path = str(portable_relative_path(submodule, target)) if not submodule.exists(): continue try: @@ -847,8 +855,8 @@ def _directory_link_boundary(target: Path, path: Path, linked_prefixes: dict[str # The index can retain descendants of a directory replaced by a link. # Keep the link's identity in snapshots without following its contents. prefix = target - for component in path.relative_to(target).parts[:-1]: - prefix /= component + for component in portable_relative_path(path, target).parts[:-1]: + prefix = filesystem_path(prefix / component) key = str(prefix) if key not in linked_prefixes: metadata = prefix.lstat() @@ -908,7 +916,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: continue # Git's index spelling can differ after a case-only directory rename. # Compare each scope-depth prefix once, without following symlink leaves. - prefix = repository.joinpath(*relative.parts[:scope_depth]) + prefix = filesystem_path(repository.joinpath(*relative.parts[:scope_depth])) key = str(prefix) if key not in matching_prefixes: try: @@ -919,7 +927,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: # POSIX volumes; WindowsPath equality also folds distinct names. if not matching_prefixes[key]: continue - path = target.joinpath(*relative.parts[scope_depth:]) + path = filesystem_path(target.joinpath(*relative.parts[scope_depth:])) try: path = _directory_link_boundary(target, path, linked_prefixes) metadata = path.lstat() @@ -940,7 +948,9 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: ): continue nested_repository_root = git_output(path, "rev-parse", "--show-toplevel") - if nested_repository_root is not None and Path(nested_repository_root).samefile(path): + if nested_repository_root is not None and filesystem_path( + Path(nested_repository_root) + ).samefile(path): nested_paths = git_directory_snapshot_paths(path) if nested_paths is not None: nested_worktrees.add((metadata.st_dev, metadata.st_ino)) @@ -970,11 +980,15 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: ) if directories is None: raise SystemExit("Could not inspect directories in the selected Git working tree.") - pending = [target / os.fsdecode(path) for path in directories.split(b"\0") if path] + pending = [ + filesystem_path(target / os.fsdecode(path)) for path in directories.split(b"\0") if path + ] while pending: children: list[Path] = [] for path in pending: - if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(target).parts): + if not SOURCE_METADATA_DIRECTORIES.isdisjoint( + portable_relative_path(path, target).parts + ): continue try: path = _directory_link_boundary(target, path, linked_prefixes) @@ -990,7 +1004,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: # Its own inventory already applied its repository exclusions. continue try: - children.extend(path.iterdir()) + children.extend(filesystem_path(child) for child in path.iterdir()) except PermissionError: # Match Git's treatment of unreadable descendants. continue @@ -1004,7 +1018,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: continue if not (stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode)): continue - relative = (Path(pathspec) / path.relative_to(target)).as_posix() + relative = (Path(pathspec) / portable_relative_path(path, target)).as_posix() candidates[os.fsencode(relative)] = path if not candidates: break @@ -1028,7 +1042,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: excluded = set(ignored.stdout.split(b"\0")) pending = [path for raw, path in candidates.items() if raw not in excluded] paths.extend(junctions.values()) - return sorted({str(path): path for path in paths}.values(), key=str) + entries = {str(portable_relative_path(path, target)): path for path in paths} + return [entries[name] for name in sorted(entries)] def source_directory_snapshot_paths( @@ -1039,15 +1054,16 @@ def source_directory_snapshot_paths( onerror: Callable[[OSError], None] | None = None, ) -> list[Path]: paths: list[Path] = [] + excluded_portable = {portable_path(path) for path in excluded} for directory, directories, files in os.walk( - target, topdown=True, followlinks=False, onerror=onerror + extended_path(target), topdown=True, followlinks=False, onerror=onerror ): parent = Path(directory) for name in directories[:]: path = parent / name - if ( - name in SOURCE_METADATA_DIRECTORIES and not include_git_metadata - ) or path in excluded: + if (name in SOURCE_METADATA_DIRECTORIES and not include_git_metadata) or portable_path( + path + ) in excluded_portable: directories.remove(name) continue paths.append(path) @@ -1058,7 +1074,7 @@ def source_directory_snapshot_paths( parent / name for name in files if (name not in SOURCE_METADATA_DIRECTORIES or include_git_metadata) - and parent / name not in excluded + and portable_path(parent / name) not in excluded_portable ) return sorted(paths) @@ -1074,7 +1090,9 @@ def raise_walk_error(error: OSError) -> None: raise error excluded_relative = [ - path.relative_to(target) for path in excluded if path.is_relative_to(target) + portable_relative_path(path, target) + for path in excluded + if portable_is_relative_to(path, target) ] paths = ( source_directory_snapshot_paths( @@ -1088,7 +1106,7 @@ def raise_walk_error(error: OSError) -> None: digest = hashlib.sha256() update_digest_field(digest, b"format", b"codex-security-directory/v1") for path in paths: - relative_path = path.relative_to(target) + relative_path = portable_relative_path(path, target) if any(relative_path.is_relative_to(path) for path in excluded_relative): continue try: @@ -1128,7 +1146,9 @@ def directory_snapshot_regular_file_count(target: Path) -> int: try: metadata = path.lstat() except OSError as exc: - raise SystemExit(f"Could not inspect local file: {path.relative_to(target)}") from exc + raise SystemExit( + f"Could not inspect local file: {portable_relative_path(path, target)}" + ) from exc if stat.S_ISREG(metadata.st_mode): count += 1 return count @@ -1138,13 +1158,15 @@ def copy_directory_excluding( source: Path, destination: Path, excluded: tuple[Path, ...] ) -> list[Path]: excluded_relative = [ - path.relative_to(source) for path in excluded if path.is_relative_to(source) + portable_relative_path(path, source) + for path in excluded + if portable_is_relative_to(path, source) ] junctions: list[Path] = [] def ignored(directory: str, names: list[str]) -> list[str]: - relative = Path(directory).relative_to(source) + relative = portable_relative_path(Path(directory), source) skipped = [ path.name for path in excluded_relative @@ -1163,11 +1185,13 @@ def ignored(directory: str, names: list[str]) -> list[str]: skipped.append(name) return skipped - shutil.copytree(source, destination, symlinks=True, ignore=ignored) + shutil.copytree( + extended_path(source), extended_path(destination), symlinks=True, ignore=ignored + ) # Empty placeholders retain parent directories while a patch is reversed. for path in junctions: - (destination / path).mkdir() + filesystem_path(destination / path).mkdir() return junctions @@ -1176,8 +1200,9 @@ def restore_directory_junctions(source: Path, destination: Path, junctions: list destination = destination.resolve() for path in junctions: placeholder = require_remediation_target(str(destination / path)) - os.chmod(placeholder, stat.S_IMODE((source / path).lstat().st_mode)) - copy_directory_junction(source / path, placeholder) + source_junction = filesystem_path(source / path) + os.chmod(placeholder, stat.S_IMODE(source_junction.lstat().st_mode)) + copy_directory_junction(source_junction, placeholder) def copy_git_worktree_files( @@ -1199,10 +1224,10 @@ def copy_git_worktree_files( excluded_relative = [] for path in excluded: try: - relative_path = path.relative_to(repository) + relative_path = portable_relative_path(path, repository) except ValueError: try: - relative_path = Path(pathspec) / path.relative_to(source) + relative_path = Path(pathspec) / portable_relative_path(path, source) except ValueError: continue excluded_relative.append(relative_path) @@ -1215,10 +1240,10 @@ def copy_git_worktree_files( relative = Path(os.fsdecode(raw_path)) if any(relative.is_relative_to(path) for path in excluded_relative): continue - source_path = repository / relative + source_path = filesystem_path(repository / relative) try: source_path = _directory_link_boundary(repository, source_path, linked_prefixes) - relative = source_path.relative_to(repository) + relative = portable_relative_path(source_path, repository) metadata = source_path.lstat() except (FileNotFoundError, NotADirectoryError): continue @@ -1228,8 +1253,8 @@ def copy_git_worktree_files( if stat.S_ISDIR(metadata.st_mode) and getattr(metadata, "st_reparse_tag", 0) & 0x20000000: junctions.add(relative) continue - destination_path = destination / relative - destination_path.parent.mkdir(parents=True, exist_ok=True) + destination_path = filesystem_path(destination / relative) + filesystem_path(destination_path.parent).mkdir(parents=True, exist_ok=True) if stat.S_ISLNK(metadata.st_mode): destination_path.symlink_to(os.readlink(source_path)) elif stat.S_ISREG(metadata.st_mode): @@ -1246,7 +1271,7 @@ def copy_git_worktree_files( raise SystemExit(f"Unsupported Git working-tree file type: {relative}") if _WINDOWS: for path in git_directory_snapshot_paths(source) or []: - relative = Path(pathspec) / path.relative_to(source) + relative = Path(pathspec) / portable_relative_path(path, source) if any( relative.is_relative_to(root) for root in [*excluded_relative, *nested_worktrees] ): @@ -1258,8 +1283,8 @@ def copy_git_worktree_files( ): junctions.add(relative) for path in junctions: - (destination / path).mkdir(parents=True, exist_ok=True) - copied_target = destination if pathspec == "." else destination / pathspec + filesystem_path(destination / path).mkdir(parents=True, exist_ok=True) + copied_target = filesystem_path(destination if pathspec == "." else destination / pathspec) copied_target.mkdir(parents=True, exist_ok=True) return copied_target, sorted(junctions) @@ -1316,16 +1341,17 @@ def require_remediation_target(value: str) -> Path: if not stored.is_absolute(): raise SystemExit("Remediation target must be an absolute local directory path.") try: - resolved = stored.resolve(strict=True) + filesystem_target = filesystem_path(stored) + resolved = filesystem_target.resolve(strict=True) except (FileNotFoundError, OSError) as exc: raise SystemExit( "Remediation is unavailable because the selected checkout is no longer accessible." ) from exc - if resolved != stored or not stored.is_dir(): + if portable_path(resolved) != portable_path(filesystem_target) or not resolved.is_dir(): raise SystemExit( "Remediation is unavailable because the selected checkout path was replaced. Start a new scan." ) - return stored + return resolved def require_scan_target_identity(scan: sqlite3.Row) -> Path: diff --git a/plugins/codex-security/scripts/workbench_target_state.py b/plugins/codex-security/scripts/workbench_target_state.py index 23daebb954..9ebc983aea 100644 --- a/plugins/codex-security/scripts/workbench_target_state.py +++ b/plugins/codex-security/scripts/workbench_target_state.py @@ -5,9 +5,14 @@ import argparse import hashlib import sqlite3 +import sys from datetime import datetime, timezone from pathlib import Path +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import extended_path, portable_path + def stable_target_id(target: Path) -> str: """Hash the supplied path spelling into the workbench ID for a local workspace.""" @@ -37,15 +42,40 @@ def backfill_security_targets(connection: sqlite3.Connection) -> None: ) -def ensure_security_target(connection: sqlite3.Connection, target_path: str) -> str: +def ensure_security_target( + connection: sqlite3.Connection, target_path: str, *, preferred_target_id: str | None = None +) -> str: """Return the ID already associated with the path or insert a deterministic ID for it.""" + target = portable_path(Path(target_path)) + target_path = str(target) + extended_target_path = str(extended_path(target)) existing = connection.execute( - "SELECT id FROM security_targets WHERE current_path = ?", - (target_path,), - ).fetchone() - if existing is not None: - return str(existing["id"]) - target_id = stable_target_id(Path(target_path)) + "SELECT id, current_path FROM security_targets WHERE current_path IN (?, ?)", + (target_path, extended_target_path), + ).fetchall() + if existing: + current = next((row for row in existing if row["current_path"] == target_path), existing[0]) + target_id = str(current["id"]) + if len(existing) > 1: + # Keep sealed IDs. A rerun can reuse its parent's matching legacy record. + return next( + (str(row["id"]) for row in existing if row["id"] == preferred_target_id), + target_id, + ) + if current["current_path"] != target_path: + timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + connection.execute( + "UPDATE security_targets SET current_path = ?, display_name = ?, updated_at = ? " + "WHERE id = ?", + (target_path, target.name, timestamp, target_id), + ) + for table, condition in (("workspaces", ""), ("scans", " AND status = 'running'")): + connection.execute( + f"UPDATE {table} SET target_path = ? WHERE target_id = ? AND target_path = ?{condition}", + (target_path, target_id, extended_target_path), + ) + return target_id + target_id = stable_target_id(target) timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") connection.execute( """ diff --git a/plugins/codex-security/tests/test_finding_issue_receipts.py b/plugins/codex-security/tests/test_finding_issue_receipts.py index 81244f719c..26119c94f0 100644 --- a/plugins/codex-security/tests/test_finding_issue_receipts.py +++ b/plugins/codex-security/tests/test_finding_issue_receipts.py @@ -86,6 +86,43 @@ def legacy_database(path: Path, validated: dict, destination: dict) -> None: connection.commit() +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows SQLite") +def test_record_and_inspect_at_the_windows_sqlite_filename_limit(tmp_path: Path) -> None: + scan_dir, validated = source(tmp_path) + name = "workbench.sqlite3" + state = tmp_path + # Keep each directory component short while crossing SQLite's limit with the prefix. + while 1030 - len(str(state / name).encode("utf-8")) > 180: + state /= "界" * 40 + padding = 1030 - len(str(state / name).encode("utf-8")) - 1 + state /= "s" * padding + path = state / name + assert len(str(path).encode("utf-8")) == 1030 + native_path = load_script("windows_paths").extended_path(path) + native_path.parent.mkdir(parents=True) + + # Establish that this SQLite build can write the ordinary filename with WAL. + try: + with closing(sqlite3.connect(path)) as control: + assert control.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + except sqlite3.OperationalError as exc: + if str(exc) == "unable to open database file": + pytest.skip("native SQLite cannot write the ordinary 1,030-byte control filename") + raise + + receipt = receipt_for(validated) + expected = [{**receipt, "scanId": validated["manifest"]["scan"]["id"]}] + assert issues(state, scan_dir, "record", receipts=[receipt])["receipts"] == expected + with closing(sqlite3.connect(path)) as control: + assert control.execute("SELECT external_id FROM finding_issue_receipts").fetchall() == [ + ("SEC-101",) + ] + written = native_path.read_bytes() + assert issues(state, scan_dir, "inspect")["receipts"] == expected + assert native_path.read_bytes() == written + + def test_inspection_of_portable_source_does_not_create_state(tmp_path: Path) -> None: scan_dir, validated = source(tmp_path) state = tmp_path / "state" diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py new file mode 100644 index 0000000000..4b42b13093 --- /dev/null +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import ntpath +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path, PureWindowsPath +from types import SimpleNamespace + +import pytest +from test_workbench_scan_archive import register_scan +from workbench_test_support import fail_scan, load_script, run_workbench + + +@pytest.fixture +def windows_paths(monkeypatch): + module = load_script("windows_paths") + monkeypatch.setattr(module, "Path", PureWindowsPath) + monkeypatch.setattr(module, "os", SimpleNamespace(name="nt", path=ntpath)) + return module + + +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +@pytest.mark.parametrize("character", ["a", "\U0001f600", "\ud800"]) +def test_filesystem_path_counts_utf16_units_and_preserves_spelling(windows_paths, root, character): + prefix = str(PureWindowsPath(root)) + units = len(character.encode("utf-16-le", errors="surrogatepass")) // 2 + for length in (247, 248): + count, remainder = divmod(length - len(prefix), units) + path = PureWindowsPath(prefix + character * count + "a" * remainder) + expected = windows_paths.extended_path(path) if length == 248 else path + assert windows_paths.filesystem_path(path) == expected + assert windows_paths.portable_path(windows_paths.extended_path(path)) == path + assert windows_paths.extended_path( + windows_paths.extended_path(path) + ) == windows_paths.extended_path(path) + assert windows_paths.portable_relative_path( + windows_paths.extended_path(path / "child"), path + ) == PureWindowsPath("child") + + +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_sqlite_uses_an_equivalent_filename_within_the_win32_pager_budget(windows_paths, root): + directory = PureWindowsPath(root, *(["界" * 80] * 4)) + filename = "workbench.sqlite3" + base_bytes = len(str(directory / filename).encode("utf-8")) + 1 + ordinary = directory / ("s" * (1030 - base_bytes)) / filename + extended = windows_paths.filesystem_path(ordinary) + assert len(str(ordinary).encode("utf-8")) == 1030 < len(str(extended).encode("utf-8")) + assert windows_paths.sqlite_filename(extended) == ordinary + smaller = windows_paths.filesystem_path(directory / filename) + assert windows_paths.sqlite_filename(smaller) == smaller + literal = extended.with_name("workbench.sqlite3.") + assert windows_paths.sqlite_filename(literal) == literal + + +@pytest.mark.parametrize( + "value", + [ + "//?/C:/checkout./source", + "//?/C:/checkout /source", + "//?/C:/AUX.txt/source", + "//?/C:/LPT¹ .log", + "//?/C:/dir/file:stream", + "//?/UNC/server/share/checkout.", + "//?/UNC/./pipe/name", + "//?/Volume{123}/source", + "//./pipe/name", + ], +) +def test_verbatim_only_names_preserve_their_namespace(windows_paths, value): + path = PureWindowsPath(value) + assert windows_paths.uses_windows_namespace(path) + assert windows_paths.portable_path(path) == path + assert windows_paths.filesystem_path(path) == path + assert windows_paths.extended_path(path) == path + + +def test_windows_path_conversion_keeps_relative_spelling_and_literal_leaf(windows_paths): + name = PureWindowsPath("sub/source.py") + assert windows_paths.filesystem_path(name) == name + checkout = PureWindowsPath("C:/checkout") + literal = PureWindowsPath("//?/C:/checkout/reports.") + assert windows_paths.portable_relative_path(literal, checkout) == PureWindowsPath("reports.") + assert windows_paths.portable_is_relative_to(literal / "tool.exe", checkout) + assert not windows_paths.portable_is_relative_to(literal, checkout / "reports") + assert windows_paths.extended_path(checkout / "reports.") == checkout / "reports." + assert not windows_paths.uses_windows_namespace(windows_paths.extended_path(checkout)) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires Windows namespace I/O") +def test_target_rejects_verbatim_only_sibling_without_changing_its_identity(tmp_path): + paths = load_script("windows_paths") + setup = load_script("workbench_setup") + ordinary = tmp_path / "reports" + ordinary.mkdir() + literal = paths.extended_path(tmp_path) / "reports." + literal.mkdir() + try: + assert not literal.samefile(ordinary) + assert paths.portable_path(literal).samefile(literal) + assert setup.require_target(str(ordinary)).samefile(ordinary) + with pytest.raises(SystemExit, match="ordinary Windows drive or UNC"): + setup.require_target(str(literal)) + finally: + literal.rmdir() + + +def test_target_lookup_preserves_both_legacy_ids_after_a_path_spelling_collision( + windows_paths, monkeypatch, workbench_db +): + targets = load_script("workbench_target_state") + monkeypatch.setattr(targets, "Path", PureWindowsPath) + monkeypatch.setattr(targets, "portable_path", windows_paths.portable_path) + monkeypatch.setattr(targets, "extended_path", windows_paths.extended_path) + path = PureWindowsPath("C:/repository") + alias = windows_paths.extended_path(path) + for target_id, stored_path in (("original", alias), ("current", path)): + workbench_db.execute( + "INSERT INTO security_targets (id, current_path, display_name, created_at, updated_at) " + "VALUES (?, ?, 'Repository', '2026-01-01', '2026-01-01')", + (target_id, str(stored_path)), + ) + assert targets.ensure_security_target(workbench_db, str(alias)) == "current" + assert ( + targets.ensure_security_target(workbench_db, str(path), preferred_target_id="original") + == "original" + ) + assert { + tuple(row) for row in workbench_db.execute("SELECT id, current_path FROM security_targets") + } == {("original", str(alias)), ("current", str(path))} + + +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_archive_relocation_preserves_verbatim_artifact_names(windows_paths, monkeypatch, root): + monkeypatch.setitem(sys.modules, "windows_paths", windows_paths) + storage = load_script("workbench/storage") + monkeypatch.setattr(storage, "Path", PureWindowsPath) + + scan_dir = PureWindowsPath(root) / "scans" / "run" + archive = scan_dir.with_name("run.previous-test") + for name, archived_root in ( + ("result.json.", windows_paths.extended_path(archive)), + ("result.json", archive), + ): + stored = windows_paths.extended_path(scan_dir) / "worker" / name + expected = archived_root / "worker" / name + assert storage.archived_deep_path(archive, str(stored)) == str(expected) + assert storage.relocate_scan_path(scan_dir, archive, str(stored)) == str(expected) + + +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_archive_relocation_respects_ordinary_and_literal_parent_spellings( + windows_paths, monkeypatch, root +): + monkeypatch.setitem(sys.modules, "windows_paths", windows_paths) + storage = load_script("workbench/storage") + monkeypatch.setattr(storage, "Path", PureWindowsPath) + + parent = PureWindowsPath(root) / "scans" + source = windows_paths.extended_path(parent) / "run" + archive = source.with_name("run.previous-old") + stored = parent / "run" / "worker" / "result.json." + expected = parent / archive.name / "worker" / stored.name + assert storage.relocate_scan_path(source, archive, str(stored)) == str(expected) + assert storage.archived_deep_path(archive, str(stored)) == str(expected) + + literal_parent = windows_paths.extended_path(parent.parent) / "scans." + source = literal_parent / "run" + archive = source.with_name("run.previous-old") + ordinary = parent.with_name("scans.") / "run" / "worker" / "result.json" + literal = source / "worker" / "result.json" + for path, expected in ((ordinary, ordinary), (literal, archive / "worker" / literal.name)): + assert storage.relocate_scan_path(source, archive, str(path)) == str(expected) + assert storage.archived_deep_path(archive, str(path)) == str(expected) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="exercise native Win32 files and SQLite paths") +def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Path, request): + paths = load_script("windows_paths") + root = tmp_path + while len(str(root)) < 270: + root = root / ("long-directory-" * 3) + repository, state_dir, output = (root / part for part in ("repository", "state", "output")) + paths.filesystem_path(repository).mkdir(parents=True) + paths.filesystem_path(output).mkdir() + first = register_scan(state_dir, repository, output) + scan_id = first["scanId"] + fail_scan(state_dir, scan_id, "Archive the previous run") + paths.filesystem_path(output / "report.md").write_text("# Previous review\n") + archived = root / "output.previous-test" + literal = paths.extended_path(output) / "coverage.json." + archived_literal = paths.extended_path(archived) / literal.name + request.addfinalizer(lambda: literal.unlink(missing_ok=True)) + request.addfinalizer(lambda: archived_literal.unlink(missing_ok=True)) + paths.filesystem_path(output / "coverage.json").write_text("ordinary") + literal.write_text("verbatim") + database = paths.filesystem_path(state_dir / "workbench.sqlite3") + with sqlite3.connect(database) as connection: + target_id = connection.execute( + "SELECT target_id FROM scans WHERE id = ?", (scan_id,) + ).fetchone()[0] + repository_alias = str(paths.extended_path(repository)) + connection.execute( + "UPDATE security_targets SET current_path = ? WHERE id = ?", + (repository_alias, target_id), + ) + connection.execute( + "UPDATE scans SET target_path = ?, scan_dir = ? WHERE id = ?", + (repository_alias, str(paths.extended_path(output)), scan_id), + ) + connection.execute( + "INSERT INTO scan_artifacts (scan_id, kind, path, created_at) VALUES (?, ?, ?, ?)", + ( + scan_id, + "markdownReport", + str(paths.extended_path(output / "report.md")), + "2026-08-13T00:00:00Z", + ), + ) + connection.execute( + "INSERT INTO scan_artifacts (scan_id, kind, path, created_at) VALUES (?, ?, ?, ?)", + ( + scan_id, + "coverage", + str(literal), + "2026-08-13T00:00:00Z", + ), + ) + for spelling in [repository, paths.extended_path(repository)]: + listed = run_workbench(state_dir, "list-scans", "--repository", str(spelling))["scans"] + assert [scan["scanId"] for scan in listed] == [scan_id] + + paths.filesystem_path(output).rename(paths.filesystem_path(archived)) + paths.filesystem_path(output).mkdir() + current = register_scan( + state_dir, repository, output, "--archive-existing", "--archived-scan-dir", str(archived) + ) + assert current["scanDir"] == str(output) + assert current["archivedScanDir"] == str(archived) + assert paths.filesystem_path(archived / "report.md").read_text() == "# Previous review\n" + with sqlite3.connect(database) as connection: + assert connection.execute( + "SELECT scan_dir FROM scans WHERE id = ?", (scan_id,) + ).fetchone() == (str(archived),) + artifacts = dict( + connection.execute( + "SELECT kind, path FROM scan_artifacts WHERE scan_id = ?", (scan_id,) + ) + ) + assert artifacts["markdownReport"] == str(archived / "report.md") + assert Path(artifacts["coverage"]).read_text() == "verbatim" + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows path spelling") +def test_git_snapshot_digest_is_independent_of_root_spelling(tmp_path: Path): + paths = load_script("windows_paths") + target = load_script("workbench_target") + subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True) + (tmp_path / "z.txt").write_text("short") + nested = tmp_path + while len(str(nested / "a.txt")) < paths.WINDOWS_DIRECTORY_PATH_LIMIT: + nested /= "alpha-" * 6 + paths.filesystem_path(nested).mkdir(parents=True, exist_ok=True) + paths.filesystem_path(nested / "a.txt").write_text("long") + assert target.directory_content_digest(tmp_path) == target.directory_content_digest( + paths.extended_path(tmp_path) + ) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows path spelling") +def test_git_discovery_rejects_a_tool_under_a_verbatim_only_directory(tmp_path, monkeypatch): + paths = load_script("windows_paths") + target = load_script("workbench_target") + literal = paths.extended_path(tmp_path) / "tools." + literal.mkdir() + candidate = literal / "git.exe" + try: + candidate.touch() + monkeypatch.setenv("CODEX_SECURITY_GIT", str(candidate)) + with pytest.raises(SystemExit, match="outside the protected repository"): + target.trusted_git_executable(tmp_path) + finally: + candidate.unlink(missing_ok=True) + literal.rmdir() + + +@pytest.mark.cross_platform +@pytest.mark.parametrize("extended_worker", [False, True]) +def test_saved_worker_results_accept_mixed_scan_root_spellings(tmp_path, extended_worker): + paths = load_script("windows_paths") + sources = load_script("workbench/saved_sources") + ordinary = paths.portable_path(tmp_path) + extended = paths.extended_path(ordinary) + root, stored = (ordinary, extended) if extended_worker else (extended, ordinary) + worker = { + "kind": "discovery", + "artifact_dir": str(stored / "worker" / "output"), + "result_manifest_path": str(stored / "worker" / "last-result.json"), + "attempt": 2, + } + receipts = load_script("workbench_coverage_receipts") + assert ( + receipts.resolve_coverage_receipts( + root, {"receiptRefs": []}, worker, "worker/output/result.json" + ) + == [] + ) + assert sources._worker_outputs(root, worker) == [("worker/output", 2)] + assert ("worker/last-result.json", "discovery") in list( + sources._saved_result_paths(root, [worker]) + ) + with pytest.raises(ValueError): + sources._worker_outputs(root, {**worker, "artifact_dir": str(stored.parent / "sibling")}) diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 536378f70a..c5df116def 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -40,6 +40,7 @@ "dist", "schemas", "_bundled_plugin", + "!_bundled_plugin/**/__pycache__/**", "LICENSE", "README.md", "docs/cli.md", diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.ts index 89fb8b80cd..49a96e1a1e 100644 --- a/sdk/typescript/tests-ts/inventory-input.test.ts +++ b/sdk/typescript/tests-ts/inventory-input.test.ts @@ -12,10 +12,22 @@ import { statSync, writeFileSync, } from "node:fs"; -import { delimiter, dirname, join, parse, relative, sep } from "node:path"; +import { + delimiter, + dirname, + join, + parse, + relative, + sep, + win32, +} from "node:path"; import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js"; import { nodeCommand } from "./support/shell.js"; import { git } from "./git-fixture.js"; +import { + hasWindowsLoopbackShare, + windowsLoopbackPath, +} from "./windows-helper-location.js"; import { PLUGIN_ROOT } from "./plugin-root.js"; import { resolveCodexCommand } from "../src/runtime.js"; import { bundledCodexSdkEnvironment } from "../src/codex-sdk-environment.js"; @@ -27,6 +39,88 @@ const pathNameBytes = process.platform === "darwin" ? Buffer.from("雪") : Buffer.from([0xff]); afterEach(temporary.cleanup); +for (const spelling of ["ordinary", "extended", "unc"] as const) { + test.skipIf( + process.platform !== "win32" || + (spelling === "unc" && !hasWindowsLoopbackShare), + )( + `long Windows ${spelling} inventory preserves ignores and scope-relative output`, + () => { + const f = fixture(); + rmSync(join(f.repo, ".git"), { recursive: true, force: true }); + let repo = f.repo; + while (repo.length < 270) repo = join(repo, "nested-🌲🌲🌲🌲🌲🌲"); + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, ".ignore"), "/src/ignored.py\n"); + writeFileSync(join(repo, "src", "visible.py"), "visible"); + writeFileSync(join(repo, "src", "ignored.py"), "ignored"); + const input = + spelling === "unc" + ? windowsLoopbackPath(repo) + : spelling === "extended" + ? win32.toNamespacedPath(repo) + : repo; + const scopes = join(f.root, "scopes.json"); + writeFileSync(scopes, '["src"]'); + for (const command of [ + "generate-in-scope-files", + "make-repo-scope-input", + ]) { + const result = spawnSync( + node, + [ + join(PLUGIN_ROOT, "mcp", "helpers.mjs"), + command, + "--repo", + input, + "--out", + f.out, + ...(command === "generate-in-scope-files" + ? ["--scope", "./src"] + : ["--scopes-file", scopes]), + ], + { env: f.toolEnvironment, encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + const output = readFileSync(f.out, "utf8").trim(); + if (command === "generate-in-scope-files") + expect(output).toBe("./src/visible.py"); + else expect(JSON.parse(output).path).toBe("src/visible.py"); + } + }, + ); +} + +test.skipIf(process.platform !== "win32")( + "an extended Windows repository keeps operator globs anchored to the checkout", + () => { + const f = fixture(); + f.write("src/visible.py"); + f.write("src/hidden.py"); + const config = join(f.root, "rg.conf"); + writeFileSync(config, "--glob\n!/src/hidden.py\n"); + const result = spawnSync( + node, + [ + join(PLUGIN_ROOT, "mcp", "helpers.mjs"), + "generate-in-scope-files", + "--repo", + win32.toNamespacedPath(f.repo), + "--scope", + "src", + "--out", + f.out, + ], + { + env: { ...f.toolEnvironment, RIPGREP_CONFIG_PATH: config }, + encoding: "utf8", + }, + ); + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(f.out, "utf8").trim()).toBe("src/visible.py"); + }, +); + function fixture() { const root = temporary.create("codex-security-inventory-"), repo = join(root, "repository"), diff --git a/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts b/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts index a263dfcf8c..57890dd9af 100644 --- a/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts +++ b/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts @@ -40,6 +40,7 @@ async function fixture() { const plugin = join(root, "plugin"); const paths = [ "scripts/workbench_scan_usage.py", + "scripts/windows_paths.py", "scripts/windows_scan_local_files.py", "scripts/read_codex_session.mjs", "scripts/workbench/__init__.py", diff --git a/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts b/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts index 6d5dd2ea90..4a85e875d3 100644 --- a/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts +++ b/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts @@ -24,15 +24,16 @@ function probe(program: string, ...args: string[]): void { test("normalizes absolute Windows scopes without accepting escapes", () => { probe( [ - "import sys", + "import ntpath, sys", "from pathlib import PureWindowsPath", "from types import SimpleNamespace", "sys.path.insert(0, sys.argv[1])", - "import workbench_setup as workbench", + "import windows_paths, workbench_setup as workbench", "class WindowsPath(PureWindowsPath):", " def resolve(self): return self", " def is_dir(self): return True", - "workbench.Path = WindowsPath", + "workbench.Path = windows_paths.Path = WindowsPath", + "windows_paths.os = SimpleNamespace(name='nt', path=ntpath)", "workbench.os = SimpleNamespace(name='nt')", "target = WindowsPath('C:/repository')", "assert workbench.require_scope(r'C:\\repository\\src', 'standard', target) == 'src'",