From 37def7c0cd01a24df0240a62500f75e01ac3a95a Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:21:07 -0700 Subject: [PATCH 01/32] fix(windows): support long paths in scan tooling --- .../scripts/config_preflight.py | 12 +- .../scripts/generate_in_scope_files.py | 59 ++-- .../scripts/generate_rank_input.py | 88 +++--- .../scripts/normalize_candidates.py | 29 +- .../scripts/resolve_security_md.py | 31 +- .../_bundled_plugin/scripts/windows_paths.py | 44 +++ .../_bundled_plugin/scripts/workbench_db.py | 44 ++- .../scripts/workbench_scan_start.py | 14 +- .../scripts/workbench_target.py | 77 +++-- sdk/typescript/plugin-files.json | 1 + sdk/typescript/tests-ts/api.test.ts | 35 ++- sdk/typescript/tests-ts/runtime.test.ts | 278 +++++++++++++++++- 12 files changed, 582 insertions(+), 130 deletions(-) create mode 100644 sdk/typescript/_bundled_plugin/scripts/windows_paths.py diff --git a/sdk/typescript/_bundled_plugin/scripts/config_preflight.py b/sdk/typescript/_bundled_plugin/scripts/config_preflight.py index ee4f1ab018..e96d77380f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/config_preflight.py +++ b/sdk/typescript/_bundled_plugin/scripts/config_preflight.py @@ -12,6 +12,10 @@ from pathlib import Path from typing import Any +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import filesystem_path, portable_path + try: import tomllib except ModuleNotFoundError: # pragma: no cover - Python 3.10 only @@ -243,7 +247,7 @@ def project_trust_level( projects = config.get("projects") if not isinstance(projects, dict): continue - project = projects.get(str(project_root)) + project = projects.get(str(portable_path(project_root))) if not isinstance(project, dict): continue trust_level = project.get("trust_level") @@ -265,7 +269,7 @@ def project_config_paths(project_root: Path, cwd: Path) -> list[Path]: def discover_config_paths( *, cwd: Path, profile_layer_path: Path | None ) -> tuple[list[Path], dict[str, Any]]: - resolved_cwd = cwd.expanduser().resolve() + resolved_cwd = filesystem_path(cwd.expanduser()).resolve() if not resolved_cwd.is_dir(): raise ValueError(f"cwd must be a directory, got {str(resolved_cwd)!r}") @@ -279,8 +283,8 @@ def discover_config_paths( if trust_level == "trusted": paths.extend(project_config_paths(project_root, resolved_cwd)) return paths, { - "cwd": str(resolved_cwd), - "project_root": str(project_root), + "cwd": str(portable_path(resolved_cwd)), + "project_root": str(portable_path(project_root)), "project_trust_level": trust_level, "project_layers_loaded": trust_level == "trusted", } diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py index d32397117b..2471ccf1ca 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py @@ -4,11 +4,16 @@ from __future__ import annotations import argparse +import os import subprocess import sys import tempfile from pathlib import Path +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import filesystem_path, portable_path + class InventoryError(ValueError): """Raised when the repository, scope, or inventory cannot be used safely.""" @@ -17,7 +22,7 @@ class InventoryError(ValueError): def resolve_repository(value: str) -> Path: """Resolve the repository once so every scope is bound to its real root.""" try: - repository = Path(value).expanduser().resolve(strict=True) + repository = filesystem_path(Path(value).expanduser()).resolve(strict=True) except (OSError, ValueError) as error: raise InventoryError(f"--repo: cannot resolve repository: {value}") from error if not repository.is_dir(): @@ -33,12 +38,12 @@ def resolve_scope(repository: Path, value: str) -> str: requested = Path(value).expanduser() scope = requested if requested.is_absolute() else repository / requested try: - resolved = scope.resolve(strict=True) + resolved = filesystem_path(scope).resolve(strict=True) except (OSError, ValueError) as error: raise InventoryError(f"--scope: path does not exist: {value}") from error try: - relative = resolved.relative_to(repository) + relative = portable_path(resolved).relative_to(portable_path(repository)) except ValueError as error: raise InventoryError(f"--scope: path must remain inside --repo: {value}") from error @@ -54,7 +59,7 @@ def resolve_output(value: str) -> Path: """Reject direct symlink outputs without constraining the artifact root.""" if not value or "\0" in value: raise InventoryError("--out: expected an inventory file path") - requested = Path(value).expanduser() + requested = filesystem_path(Path(value).expanduser()) if requested.is_symlink(): raise InventoryError("--out: refusing to replace a symbolic link") try: @@ -68,6 +73,8 @@ def resolve_output(value: str) -> Path: def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int: """Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``.""" + absolute_search = os.name == "nt" and str(repository).startswith("\\\\?\\") + search_path = str(repository / scope) if absolute_search else scope command = [ "rg", "--files", @@ -78,13 +85,13 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int: "--glob", "!.git/**", "--", - scope, + search_path, ] with tempfile.TemporaryFile(mode="w+b") as inventory: try: result = subprocess.run( command, - cwd=repository, + cwd=None if absolute_search else repository, stdout=inventory, stderr=subprocess.PIPE, check=False, @@ -100,7 +107,22 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int: raise InventoryError(message) inventory.seek(0) - rows = sorted(inventory) + if absolute_search: + rows = [] + for raw_line in inventory: + candidate = Path(os.fsdecode(raw_line.rstrip(b"\r\n"))) + try: + relative = candidate.relative_to(repository).as_posix() + except ValueError as error: + raise InventoryError( + f"ripgrep returned a path outside --repo: {candidate}" + ) from error + if scope in (".", "./"): + relative = "./" + relative + rows.append(os.fsencode(relative) + b"\n") + rows.sort() + else: + rows = sorted(inventory) return write_inventory(output, rows) @@ -113,7 +135,6 @@ def generate_diff_in_scope_files( output: Path, ) -> int: """Reuse the existing diff selection without generating previews or duplicate worklists.""" - sys.path.insert(0, str(Path(__file__).resolve().parent)) from generate_rank_input import git_changed_paths, path_is_excluded from rank_preview import ( DEFAULT_PREVIEW_BYTES, @@ -129,11 +150,13 @@ def generate_diff_in_scope_files( eligible = [ (path, status) for path, status in changed - if not path_is_excluded(path.relative_to(repository)) + if not path_is_excluded( + portable_path(path).relative_to(portable_path(repository)) + ) and path.suffix.lower() in TEXT_CODE_EXTENSIONS ] revision_paths = [ - path.relative_to(repository) + portable_path(path).relative_to(portable_path(repository)) for path, status in eligible if mode == "revisions" and status != "D" ] @@ -148,7 +171,7 @@ def generate_diff_in_scope_files( ) for path, status in eligible: - relative = path.relative_to(repository) + relative = portable_path(path).relative_to(portable_path(repository)) if status != "D": if mode == "revisions": contents = revision_blobs[relative] @@ -158,12 +181,14 @@ def generate_diff_in_scope_files( ) if is_binary_sample(contents): continue - elif ( - path.is_symlink() - or not path.is_file() - or preview_for(path, DEFAULT_PREVIEW_BYTES)[1] - ): - continue + else: + path = filesystem_path(path) + if ( + path.is_symlink() + or not path.is_file() + or preview_for(path, DEFAULT_PREVIEW_BYTES)[1] + ): + continue relative_path = relative.as_posix() if "\n" in relative_path or "\r" in relative_path: raise InventoryError( diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py index 92018d74bd..a371fcf7aa 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py @@ -44,6 +44,7 @@ preview_for_bytes, ) from workbench_target import git_blob_bytes, git_directory_snapshot_paths +from windows_paths import extended_path, filesystem_path, portable_path EXCLUDED_DIRS = { ".cache", @@ -303,10 +304,12 @@ def resolve_scope( scope_path = Path(scope).expanduser() if expand_user else Path(scope) if not scope_path.is_absolute(): scope_path = repo / scope_path + scope_path = filesystem_path(scope_path) + repository = filesystem_path(repo).resolve() + portable_repository = portable_path(repository) if reject_symlinks: - repository = repo.resolve() try: - relative = scope_path.relative_to(repository) + relative = portable_path(scope_path).relative_to(portable_repository) except ValueError as exc: raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc ancestor = repository @@ -316,7 +319,7 @@ def resolve_scope( raise SystemExit(f"Scope must be inside repo: {scope_path}") ancestor = ancestor.parent continue - ancestor /= part + ancestor = filesystem_path(ancestor / part) try: metadata = ancestor.stat(follow_symlinks=False) except OSError as exc: @@ -324,9 +327,8 @@ def resolve_scope( if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000: raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}") scope_path = scope_path.resolve() - repo_resolved = repo.resolve() try: - scope_path.relative_to(repo_resolved) + portable_path(scope_path).relative_to(portable_repository) except ValueError as exc: raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc if not scope_path.is_dir() and not scope_path.is_file(): @@ -441,7 +443,7 @@ def require_unique_paths(rows: list[JsonRow], label: str) -> None: def make_repo_rank_input(args: argparse.Namespace) -> None: - repo = Path(args.repo).expanduser().resolve() + repo = filesystem_path(Path(args.repo).expanduser()).resolve() if not repo.is_dir(): raise SystemExit(f"Repo path not found: {repo}") scopes = [args.scope] @@ -457,20 +459,25 @@ def make_repo_rank_input(args: argparse.Namespace) -> None: } rows_by_path: dict[str, JsonRow] = {} for scope_abs in resolved_scopes: - scope_rel = scope_abs.relative_to(repo) + scope_rel = portable_path(scope_abs).relative_to(portable_path(repo)) area = args.area or scope_rel.as_posix() - candidates = (scope_abs,) if scope_abs.is_file() else scope_abs.rglob("*") + candidates = ( + (scope_abs,) if scope_abs.is_file() else extended_path(scope_abs).rglob("*") + ) for path in candidates: + path = filesystem_path(path) try: if path.is_symlink() or not path.is_file(): continue - path.resolve(strict=True).relative_to(repo) + portable_path(path.resolve(strict=True)).relative_to(portable_path(repo)) except (OSError, ValueError): continue - rel = path.relative_to(repo) + rel = portable_path(path).relative_to(portable_path(repo)) directly_requested = path in directly_requested_files excluded_path = ( - path.relative_to(scope_abs if scope_abs.is_dir() else scope_abs.parent) + portable_path(path).relative_to( + portable_path(scope_abs if scope_abs.is_dir() else scope_abs.parent) + ) if explicit_scopes else rel ) @@ -499,17 +506,19 @@ def make_repo_rank_input(args: argparse.Namespace) -> None: ) rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"])) - output = Path(args.out).expanduser() + output = filesystem_path(Path(args.out).expanduser()) write_jsonl(output, rows) print(f"Wrote {len(rows)} rows to {output}") def make_repo_scope_input(args: argparse.Namespace) -> None: - repo = Path(args.repo).expanduser().resolve() + repo = filesystem_path(Path(args.repo).expanduser()).resolve() if not repo.is_dir(): raise SystemExit(f"Repo path not found: {repo}") - scopes = load_scopes_file(Path(args.scopes_file).expanduser()) + scopes = load_scopes_file( + filesystem_path(Path(args.scopes_file).expanduser()) + ) rows_by_path: dict[str, JsonRow] = {} for scope in scopes: scope_path = resolve_scope(repo, scope, expand_user=False, reject_symlinks=True) @@ -529,7 +538,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: "--glob", "!.git/**", "--", - str(scope_path.relative_to(repo)), + str(portable_path(scope_path).relative_to(portable_path(repo))), ] try: result = subprocess.run(command, cwd=repo, capture_output=True, check=False) @@ -546,7 +555,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: ) or any( path.name in ignore_names - for path in scope_path.rglob("*") + for path in extended_path(scope_path).rglob("*") if path.is_file() ) ) @@ -554,19 +563,24 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: raise SystemExit( "Could not safely enumerate ignored scoped files without Git or ripgrep." ) from exc - candidates = scope_path.rglob("*") + candidates = extended_path(scope_path).rglob("*") else: if result.returncode not in (0, 1): detail = result.stderr.decode("utf-8", errors="replace").strip() raise SystemExit(f"Could not enumerate scoped repository files: {detail}") candidates = ( - repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path + filesystem_path(repo / os.fsdecode(path)) + for path in result.stdout.split(b"\0") + if path ) for path in candidates: + path = filesystem_path(path) try: if path.is_symlink() or not path.is_file(): continue - relative = path.resolve(strict=True).relative_to(repo) + relative = portable_path(path.resolve(strict=True)).relative_to( + portable_path(repo) + ) except (OSError, ValueError): continue if ".git" in relative.parts: @@ -574,7 +588,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()}) rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"])) - output = Path(args.out).expanduser() + output = filesystem_path(Path(args.out).expanduser()) write_jsonl(output, rows) print(f"Wrote {len(rows)} scoped paths to {output}") @@ -664,18 +678,18 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple def make_diff_rank_input(args: argparse.Namespace) -> None: - repo = Path(args.repo).expanduser().resolve() + repo = filesystem_path(Path(args.repo).expanduser()).resolve() if not repo.is_dir(): raise SystemExit(f"Repo path not found: {repo}") changed = [ (path, status) for path, status in git_changed_paths(repo, args.base, args.head, args.mode) - if not path_is_excluded(path.relative_to(repo)) + if not path_is_excluded(portable_path(path).relative_to(portable_path(repo))) and path.suffix.lower() in TEXT_CODE_EXTENSIONS ] revision_paths = [ - path.relative_to(repo) + portable_path(path).relative_to(portable_path(repo)) for path, status in changed if args.mode == "revisions" and status != "D" ] @@ -691,7 +705,7 @@ def make_diff_rank_input(args: argparse.Namespace) -> None: rows: list[JsonRow] = [] for path, status in changed: - rel = path.relative_to(repo) + rel = portable_path(path).relative_to(portable_path(repo)) if status == "D": preview = "" @@ -704,23 +718,25 @@ def make_diff_rank_input(args: argparse.Namespace) -> None: preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes) if is_binary: continue - elif path.is_symlink(): - preview = "" - elif path.is_file(): - try: - path.resolve(strict=True).relative_to(repo) - except (OSError, ValueError): + else: + path = filesystem_path(path) + if path.is_symlink(): preview = "" + elif path.is_file(): + try: + portable_path(path.resolve(strict=True)).relative_to(portable_path(repo)) + except (OSError, ValueError): + preview = "" + else: + preview, is_binary = preview_for(path, args.preview_bytes) + if is_binary: + continue else: - preview, is_binary = preview_for(path, args.preview_bytes) - if is_binary: - continue - else: - preview = "" + preview = "" rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview}) rows.sort(key=lambda row: str(row["path"])) - output = Path(args.out).expanduser() + output = filesystem_path(Path(args.out).expanduser()) write_jsonl(output, rows) print(f"Wrote {len(rows)} rows to {output}") diff --git a/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py b/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py index a445acd704..56e50b0c5f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py +++ b/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py @@ -12,6 +12,10 @@ from pathlib import Path, PurePosixPath from typing import Any +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import filesystem_path, portable_path + CWE = re.compile(r"(?i)CWE-(\d+)") ROLES = { "entrypoint": 0, @@ -70,9 +74,9 @@ def relative_file(value: Any, repo_root: Path) -> tuple[str, Path]: or (sys.platform == "win32" and re.match(r"^[A-Za-z]:", raw)) ): raise ValueError("path: expected a repository-relative path without traversal") - resolved = (repo_root / raw).resolve(strict=True) + resolved = filesystem_path(repo_root / raw).resolve(strict=True) try: - relative = resolved.relative_to(repo_root).as_posix() + relative = portable_path(resolved).relative_to(portable_path(repo_root)).as_posix() except ValueError as error: raise ValueError("path: must resolve inside --repo-root") from error if not resolved.is_file(): @@ -187,9 +191,13 @@ def is_scope_file(value: str) -> bool: candidate = PurePosixPath(line) if candidate.is_absolute() or ".." in candidate.parts or "\0" in line: raise ValueError(f"in-scope file row {number}: unsafe deleted path") from error - resolved = (repo_root / line).resolve(strict=False) + resolved = filesystem_path(repo_root / line).resolve(strict=False) try: - relative = resolved.relative_to(repo_root).as_posix() + relative = ( + portable_path(resolved) + .relative_to(portable_path(repo_root)) + .as_posix() + ) except ValueError as escaped: raise ValueError( f"in-scope file row {number}: path escapes repository" @@ -281,12 +289,17 @@ def main() -> None: ) args = parser.parse_args() try: - repo_root = Path(args.repo_root).expanduser().resolve(strict=True) + repo_root = filesystem_path(Path(args.repo_root).expanduser()).resolve(strict=True) if not repo_root.is_dir(): raise ValueError("--repo-root: expected a directory") - output = Path(args.out).expanduser().resolve(strict=False) - scope_path = Path(args.in_scope_files).expanduser().resolve(strict=True) - inputs = sorted({Path(value).expanduser().resolve(strict=True) for value in args.input}) + output = filesystem_path(Path(args.out).expanduser()).resolve(strict=False) + scope_path = filesystem_path(Path(args.in_scope_files).expanduser()).resolve(strict=True) + inputs = sorted( + { + filesystem_path(Path(value).expanduser()).resolve(strict=True) + for value in args.input + } + ) if output in inputs: raise ValueError("--out: must not also be an input") if output == scope_path: diff --git a/sdk/typescript/_bundled_plugin/scripts/resolve_security_md.py b/sdk/typescript/_bundled_plugin/scripts/resolve_security_md.py index 59b12539f4..40d1ba0fdd 100644 --- a/sdk/typescript/_bundled_plugin/scripts/resolve_security_md.py +++ b/sdk/typescript/_bundled_plugin/scripts/resolve_security_md.py @@ -10,6 +10,10 @@ import sys from pathlib import Path +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import extended_path, filesystem_path, portable_path + MAX_SECURITY_MD_BYTES = 1024 * 1024 @@ -19,14 +23,14 @@ class ResolutionError(ValueError): def _inside(path: Path, root: Path, label: str) -> Path: try: - return path.relative_to(root) + return portable_path(path).relative_to(portable_path(root)) except ValueError as exc: raise ResolutionError(f"{label} is outside the scan root: {path}") from exc def _resolve_root(repo: Path) -> Path: try: - root = repo.expanduser().resolve(strict=True) + root = filesystem_path(repo.expanduser()).resolve(strict=True) except OSError as exc: raise ResolutionError(f"scan root does not exist: {repo}") from exc if not root.is_dir(): @@ -36,7 +40,7 @@ def _resolve_root(repo: Path) -> Path: def list_security_md(repo: Path) -> list[str]: """Return a stable, safely framed inventory without traversing Git metadata.""" - root = _resolve_root(repo) + root = extended_path(_resolve_root(repo)) def raise_walk_error(error: OSError) -> None: raise error @@ -49,7 +53,7 @@ def raise_walk_error(error: OSError) -> None: for name in sorted(subdirectories): if name == ".git": continue - directory_stat = (Path(directory) / name).stat(follow_symlinks=False) + directory_stat = filesystem_path(Path(directory) / name).stat(follow_symlinks=False) if not stat.S_ISDIR(directory_stat.st_mode): continue reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) @@ -59,9 +63,9 @@ def raise_walk_error(error: OSError) -> None: subdirectories[:] = safe_subdirectories if "SECURITY.md" not in filenames: continue - policy = Path(directory) / "SECURITY.md" + policy = filesystem_path(Path(directory) / "SECURITY.md") if policy.is_file() or policy.is_symlink(): - policies.append(policy.relative_to(root).as_posix()) + policies.append(_inside(policy, root, "SECURITY.md").as_posix()) return sorted(policies) @@ -73,7 +77,7 @@ def resolve_security_md(repo: Path, scope: Path) -> str: if not requested_scope.is_absolute(): requested_scope = root / requested_scope try: - resolved_scope = requested_scope.resolve(strict=True) + resolved_scope = filesystem_path(requested_scope).resolve(strict=True) except OSError as exc: raise ResolutionError(f"scan scope does not exist: {requested_scope}") from exc _inside(resolved_scope, root, "scan scope") @@ -83,15 +87,15 @@ def resolve_security_md(repo: Path, scope: Path) -> str: directories = [root] current = root for part in relative_directory.parts: - current /= part + current = filesystem_path(current / part) directories.append(current) sections: list[str] = [] for directory in directories: - policy = directory / "SECURITY.md" + policy = filesystem_path(directory / "SECURITY.md") if not policy.is_file(): continue - resolved_policy = policy.resolve(strict=True) + resolved_policy = filesystem_path(policy).resolve(strict=True) _inside(resolved_policy, root, "SECURITY.md") try: with resolved_policy.open("rb") as policy_file: @@ -104,7 +108,7 @@ def resolve_security_md(repo: Path, scope: Path) -> str: if not content.strip(): continue - source = policy.relative_to(root).as_posix() + source = _inside(policy, root, "SECURITY.md").as_posix() section = f"## SECURITY.md source: {json.dumps(source)}\n\n{content}" if not section.endswith("\n"): section += "\n" @@ -146,8 +150,9 @@ def main() -> int: if args.out == Path("-"): sys.stdout.buffer.write(guidance.encode("utf-8")) else: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(guidance, encoding="utf-8") + output = filesystem_path(args.out.expanduser()) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(guidance, encoding="utf-8") except (OSError, ResolutionError) as exc: print(f"resolve_security_md.py: error: {exc}", file=sys.stderr) return 2 diff --git a/sdk/typescript/_bundled_plugin/scripts/windows_paths.py b/sdk/typescript/_bundled_plugin/scripts/windows_paths.py new file mode 100644 index 0000000000..168874ce66 --- /dev/null +++ b/sdk/typescript/_bundled_plugin/scripts/windows_paths.py @@ -0,0 +1,44 @@ +"""Portable filesystem path spelling for Windows long-path support.""" + +from __future__ import annotations + +import os +from pathlib import Path + +WINDOWS_DIRECTORY_PATH_LIMIT = 248 + + +def extended_path(path: Path) -> Path: + """Use absolute Win32 extended-length spelling for a filesystem path.""" + if os.name != "nt": + return path + value = os.path.abspath(path) + if value.startswith("\\\\?\\"): + return Path(value) + if value.startswith("\\\\"): + return Path("\\\\?\\UNC\\" + value[2:]) + return Path("\\\\?\\" + value) + + +def filesystem_path(path: Path) -> Path: + """Use Win32 extended-length spelling when a path is near legacy limits.""" + + if os.name != "nt": + return path + value = os.path.abspath(path) + if value.startswith("\\\\?\\") or len(value) >= WINDOWS_DIRECTORY_PATH_LIMIT: + return extended_path(Path(value)) + return Path(value) + + +def portable_path(path: Path) -> Path: + """Remove Win32 extended-length spelling before persisting a path.""" + + value = str(path) + if os.name != "nt": + return path + if value.startswith("\\\\?\\UNC\\"): + return Path("\\\\" + value[8:]) + if value.startswith("\\\\?\\"): + return Path(value[4:]) + return path diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index f6e803b4bf..d5ce5a142e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -129,6 +129,7 @@ sqlite_busy, user_text, ) +from windows_paths import filesystem_path, portable_path FINDING_ARTIFACT_DIRECTORIES_LIMIT = 80 FINDING_ARTIFACTS_LIMIT = 40 @@ -251,7 +252,7 @@ def require_target(value: str) -> Path: expanded = Path(value).expanduser() if not expanded.is_absolute(): raise SystemExit("Scan target must be an absolute local directory path.") - target = expanded.resolve() + target = filesystem_path(expanded).resolve() if not target.is_dir(): raise SystemExit(f"Scan target is not a readable local directory: {target}") return target @@ -262,7 +263,7 @@ def inspect_target(target_path: str) -> dict[str, Any]: return { "displayName": target.name, "targetMetadata": git_target_metadata(target), - "targetPath": str(target), + "targetPath": str(portable_path(target)), } @@ -402,7 +403,10 @@ def inspect_setup(args: argparse.Namespace) -> dict[str, Any]: def require_review_changes_target(target: Path) -> str: revision = require_git_worktree_head(target) repository_root = git_output(target, "rev-parse", "--show-toplevel") - if repository_root is None or Path(repository_root).resolve() != target: + if ( + repository_root is None + or filesystem_path(Path(repository_root)).resolve() != target + ): raise SystemExit( "Review changes requires the checked-out Git repository root as the target." ) @@ -609,11 +613,11 @@ def require_scope(scope: str, mode: str, target: Path) -> str: raise SystemExit("Scan scope must stay inside the scanned target.") try: resolved_scope = ( - Path(parsed.as_posix()).resolve() + filesystem_path(Path(parsed.as_posix())).resolve() if parsed.is_absolute() - else (target / parsed.as_posix()).resolve() + else filesystem_path(target / parsed.as_posix()).resolve() ) - relative_scope = resolved_scope.relative_to(target) + relative_scope = portable_path(resolved_scope).relative_to(portable_path(target)) except (RuntimeError, ValueError) as exc: raise SystemExit("Scan scope must stay inside the scanned target.") from exc normalized = relative_scope.as_posix() or "." @@ -792,9 +796,18 @@ def save_workspace(connection: sqlite3.Connection, args: argparse.Namespace) -> def scan_target_root(scan_root: str | None, target: Path) -> Path: - root = Path(scan_root).expanduser().resolve() if scan_root else state_dir() / "scans" - target_root = (root / safe_segment(target.name)).resolve() - if target_root == target or target in target_root.parents: + root = ( + filesystem_path(Path(scan_root).expanduser()).resolve() + if scan_root + else state_dir() / "scans" + ) + target_root = filesystem_path(root / safe_segment(target.name)).resolve() + portable_target = portable_path(target) + portable_target_root = portable_path(target_root) + if ( + portable_target_root == portable_target + or portable_target in portable_target_root.parents + ): raise SystemExit("The scan artifact directory must be outside the selected target.") return target_root @@ -864,7 +877,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict return workspace_state(connection, workspace["id"]) if workspace["updated_at"] != workspace_version: raise SystemExit("Codex Security setup changed while the scan was starting. Try again.") - current_target = require_remediation_target(str(target)) + current_target = require_remediation_target(workspace["target_path"]) current_target_metadata = current_target.stat() if (current_target_metadata.st_dev, current_target_metadata.st_ino) != ( target_metadata.st_dev, @@ -877,7 +890,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict owned_active_scan = deep_scan.existing_deep_scan_for_target( connection, workspace["thread_id"], - str(target), + workspace["target_path"], scope, ) if owned_active_scan is not None: @@ -936,8 +949,8 @@ def _start_prompt_driven_scan( args.diff_head_revision, args.diff_content_digest, ) - target = Path(inspected["target"]["targetPath"]) - target_path = str(target) + target_path = inspected["target"]["targetPath"] + target = require_target(target_path) scope = inspected["scope"] diff_target = inspected["diffTarget"] user_context = user_text(args.user_context) @@ -1644,7 +1657,8 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) connection.execute("BEGIN IMMEDIATE") try: archive_scan(connection, args, scan_dir, timestamp, require_canonical_scan_directory) - target_id = ensure_security_target(connection, str(repository)) + repository_path = str(portable_path(repository)) + target_id = ensure_security_target(connection, repository_path) if parent_scan_id is not None: parent = require_scan(connection, parent_scan_id) if parent["target_id"] != target_id: @@ -1661,7 +1675,7 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) ( workspace_id, target_id, - str(repository), + repository_path, repository.name, scope, mode, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index 7cb1f62174..2101e505d9 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -23,6 +23,7 @@ worktree_content_digest, ) from workbench_validation import optional_text, user_text +from windows_paths import portable_path def safe_segment(value: str) -> str: @@ -93,6 +94,7 @@ def archive_scan( timestamp: str, canonical_directory: Callable[[Path], Path], ) -> None: + portable_scan_dir = portable_path(scan_dir) archived_scan_dir = ( canonical_directory(Path(args.archived_scan_dir).expanduser()) if args.archived_scan_dir is not None @@ -106,7 +108,7 @@ def archive_scan( raise SystemExit("The archived scan must be a previous sibling of the scan directory.") previous_scan = connection.execute( - "SELECT id, status FROM scans WHERE scan_dir = ?", (str(scan_dir),) + "SELECT id, status FROM scans WHERE scan_dir = ?", (str(portable_scan_dir),) ).fetchone() if previous_scan is None: return @@ -131,17 +133,17 @@ def archive_scan( ).resolve() connection.execute( "UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?", - (str(archived_scan_dir), timestamp, previous_scan["id"]), + (str(portable_path(archived_scan_dir)), timestamp, previous_scan["id"]), ) for artifact in artifacts: try: - relative_path = Path(artifact["path"]).relative_to(scan_dir) + relative_path = Path(artifact["path"]).relative_to(portable_scan_dir) except ValueError: continue connection.execute( "UPDATE scan_artifacts SET path = ? WHERE scan_id = ? AND kind = ?", ( - str(archived_scan_dir / relative_path), + str(portable_path(archived_scan_dir / relative_path)), previous_scan["id"], artifact["kind"], ), @@ -191,7 +193,7 @@ def insert_running_scan( scan_id, workspace["id"], workspace["target_id"], - str(target), + workspace["target_path"], *target_identity, scope, workspace["default_mode"], @@ -202,7 +204,7 @@ def insert_running_scan( diff_target["headRevision"] if diff_target else None, diff_target.get("contentDigest") if diff_target else None, target_summary, - str(scan_dir), + str(portable_path(scan_dir)), optional_text(model, maximum=200), optional_text(reasoning_effort, maximum=32), handoff_status, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target.py index 05faf6e036..bff7ede31a 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target.py @@ -17,6 +17,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from filesystem_identity import stored_filesystem_identity_matches from workbench_constants import GIT_REPOSITORY_ENVIRONMENT +from windows_paths import extended_path, filesystem_path, portable_path + + +def portable_relative_path(path: Path, root: Path) -> Path: + return portable_path(path).relative_to(portable_path(root)) def git_output( @@ -207,7 +212,7 @@ def worktree_content_digest_for_context( update_digest_field(digest, b"tracked-diff", tracked) for raw_path in sorted(path for path in untracked.split(b"\0") if path): relative_path = os.fsdecode(raw_path) - path = (work_tree or repository) / relative_path + path = filesystem_path((work_tree or repository) / relative_path) try: metadata = path.lstat() except OSError as exc: @@ -262,9 +267,9 @@ def git_worktree_context(target: Path) -> tuple[Path, str]: root = git_output(target, "rev-parse", "--show-toplevel") if root is None: raise SystemExit("Could not inspect the selected Git working tree.") - repository = Path(root).resolve() + repository = filesystem_path(Path(root)).resolve() try: - relative = target.resolve().relative_to(repository) + relative = portable_relative_path(target.resolve(), repository) except ValueError as exc: raise SystemExit("Scan target must stay inside its Git working tree.") from exc return repository, relative.as_posix() or "." @@ -286,7 +291,9 @@ def git_submodule_entries(target: Path) -> tuple[tuple[Path, str], ...]: ) from exc if mode != b"160000": continue - entries.append((repository / os.fsdecode(raw_path), object_id.decode("ascii"))) + entries.append( + (filesystem_path(repository / os.fsdecode(raw_path)), object_id.decode("ascii")) + ) return tuple(entries) @@ -296,7 +303,7 @@ def git_submodule_paths(target: Path) -> tuple[Path, ...]: def require_clean_submodule_worktrees(target: Path) -> None: for submodule, expected_revision in git_submodule_entries(target): - relative_path = str(submodule.relative_to(target)) + relative_path = str(portable_relative_path(submodule, target)) if not submodule.exists(): continue try: @@ -305,7 +312,10 @@ def require_clean_submodule_worktrees(target: Path) -> None: continue root = git_output(submodule, "rev-parse", "--show-toplevel") try: - is_initialized = root is not None and Path(root).resolve() == submodule.resolve() + is_initialized = ( + root is not None + and filesystem_path(Path(root)).resolve() == submodule.resolve() + ) except OSError: is_initialized = False if not is_initialized: @@ -360,7 +370,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: raise SystemExit("Could not inspect files in the selected Git working tree.") paths: list[Path] = [] for raw_path in (raw_path for raw_path in listed.split(b"\0") if raw_path): - path = repository / os.fsdecode(raw_path) + path = filesystem_path(repository / os.fsdecode(raw_path)) try: metadata = path.lstat() except FileNotFoundError: @@ -372,16 +382,17 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: nested_repository_root = git_output(path, "rev-parse", "--show-toplevel") if ( nested_repository_root is not None - and Path(nested_repository_root).resolve() == path.resolve() + and portable_path(filesystem_path(Path(nested_repository_root)).resolve()) + == portable_path(path.resolve()) ): nested_paths = git_directory_snapshot_paths(path) if nested_paths is not None: paths.extend(nested_paths) continue paths.extend( - nested_path - for nested_path in path.rglob("*") - if ".git" not in nested_path.relative_to(path).parts + filesystem_path(nested_path) + for nested_path in extended_path(path).rglob("*") + if ".git" not in portable_relative_path(nested_path, path).parts ) return sorted(set(paths)) @@ -390,16 +401,16 @@ def directory_content_digest(target: Path, *, excluded: tuple[Path, ...] = ()) - excluded_relative = [] for path in excluded: try: - excluded_relative.append(path.relative_to(target)) + excluded_relative.append(portable_relative_path(path, target)) except ValueError: continue paths = git_directory_snapshot_paths(target) if paths is None: - paths = sorted(target.rglob("*")) + paths = sorted(extended_path(target).rglob("*")) digest = hashlib.sha256() update_digest_field(digest, b"format", b"codex-security-directory/v1") for path in paths: - relative_path = path.relative_to(target) + relative_path = portable_relative_path(path, target) if any( relative_path == excluded_path or excluded_path in relative_path.parents for excluded_path in excluded_relative @@ -438,13 +449,15 @@ def directory_content_digest(target: Path, *, excluded: tuple[Path, ...] = ()) - def directory_snapshot_regular_file_count(target: Path) -> int: paths = git_directory_snapshot_paths(target) if paths is None: - paths = sorted(target.rglob("*")) + paths = sorted(extended_path(target).rglob("*")) count = 0 for path in paths: try: metadata = path.lstat() except OSError as exc: - raise SystemExit(f"Could not inspect local file: {path.relative_to(target)}") from exc + raise SystemExit( + f"Could not inspect local file: {portable_relative_path(path, target)}" + ) from exc if stat.S_ISREG(metadata.st_mode): count += 1 return count @@ -454,19 +467,21 @@ def copy_directory_excluding(source: Path, destination: Path, excluded: tuple[Pa excluded_relative = [] for path in excluded: try: - excluded_relative.append(path.relative_to(source)) + excluded_relative.append(portable_relative_path(path, source)) except ValueError: continue def ignored(directory: str, names: list[str]) -> list[str]: - relative = Path(directory).relative_to(source) + relative = portable_relative_path(Path(directory), source) return [ path.name for path in excluded_relative if path.parent == relative and path.name in names ] - shutil.copytree(source, destination, symlinks=True, ignore=ignored) + shutil.copytree( + extended_path(source), extended_path(destination), symlinks=True, ignore=ignored + ) def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Path, ...]) -> Path: @@ -486,7 +501,7 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat excluded_relative = [] for path in excluded: try: - excluded_relative.append(path.relative_to(repository)) + excluded_relative.append(portable_relative_path(path, repository)) except ValueError: continue destination.mkdir() @@ -497,13 +512,13 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat for excluded_path in excluded_relative ): continue - source_path = repository / relative + source_path = filesystem_path(repository / relative) try: metadata = source_path.lstat() except FileNotFoundError: continue - destination_path = destination / relative - destination_path.parent.mkdir(parents=True, exist_ok=True) + destination_path = filesystem_path(destination / relative) + filesystem_path(destination_path.parent).mkdir(parents=True, exist_ok=True) if stat.S_ISLNK(metadata.st_mode): destination_path.symlink_to(os.readlink(source_path)) elif stat.S_ISREG(metadata.st_mode): @@ -516,7 +531,9 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat (destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n") else: raise SystemExit(f"Unsupported Git working-tree file type: {relative}") - copied_target = destination if pathspec == "." else destination / pathspec + copied_target = filesystem_path( + destination if pathspec == "." else destination / pathspec + ) copied_target.mkdir(parents=True, exist_ok=True) return copied_target @@ -535,7 +552,7 @@ def git_target_metadata(target: Path) -> dict[str, Any]: and is_worktree and revision is not None and repository_root is not None - and Path(repository_root).resolve() == target + and filesystem_path(Path(repository_root)).resolve() == target ) metadata: dict[str, Any] = { "hasHead": revision is not None, @@ -563,16 +580,20 @@ def require_remediation_target(value: str) -> Path: if not stored.is_absolute(): raise SystemExit("Remediation target must be an absolute local directory path.") try: - resolved = stored.resolve(strict=True) + filesystem_target = filesystem_path(stored) + resolved = filesystem_target.resolve(strict=True) except (FileNotFoundError, OSError) as exc: raise SystemExit( "Remediation is unavailable because the selected checkout is no longer accessible." ) from exc - if resolved != stored or not stored.is_dir(): + if ( + portable_path(resolved) != portable_path(filesystem_target) + or not resolved.is_dir() + ): raise SystemExit( "Remediation is unavailable because the selected checkout path was replaced. Start a new scan." ) - return stored + return resolved def require_scan_target_identity(scan: sqlite3.Row) -> Path: diff --git a/sdk/typescript/plugin-files.json b/sdk/typescript/plugin-files.json index b8d7c2be27..448f0517ce 100644 --- a/sdk/typescript/plugin-files.json +++ b/sdk/typescript/plugin-files.json @@ -52,6 +52,7 @@ "scripts/validate_report_format.py", "scripts/validate_scan_contract.py", "scripts/validate_tracking_source.py", + "scripts/windows_paths.py", "scripts/windows_scan_local_files.py", "scripts/workbench/__init__.py", "scripts/workbench/handoff.py", diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 049680b98e..604fb84778 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -4538,7 +4538,7 @@ describe("CodexSecurity orchestration", () => { ...Array.from( { length: 1024 }, (_, index) => - `scope-${String(index).padStart(4, "0")}-${"a".repeat(115)}.ts`, + `scope-${String(index).padStart(4, "0")}-${"a".repeat(180)}.ts`, ), ); await mkdir(repository); @@ -4760,6 +4760,39 @@ describe("CodexSecurity orchestration", () => { ).toEqual([...paths].sort()); for (const separator of ["\u0085", "\u2028", "\u2029"]) expect(scopedSourceInputContents).not.toContain(separator); + const candidateInput = join(scanDir, "long-path-candidate.jsonl"); + const candidateScope = join(scanDir, "long-path-scope.txt"); + const normalizedCandidates = join(scanDir, "long-path-normalized.jsonl"); + const candidatePath = paths.at(-1)!; + await writeFile(candidateScope, `${candidatePath}\n`); + await writeFile( + candidateInput, + `${JSON.stringify({ + cwe_ids: ["CWE-20"], + locations: [{ path: candidatePath, start_line: 1, role: "sink" }], + summary: "summary", + evidence: "evidence", + })}\n`, + ); + execFileSync( + interpreter!, + [ + "-B", + join(PLUGIN_ROOT, "scripts", "normalize_candidates.py"), + "--input", + candidateInput, + "--out", + normalizedCandidates, + "--repo-root", + repository, + "--in-scope-files", + candidateScope, + ], + { stdio: "pipe" }, + ); + expect( + JSON.parse((await readFile(normalizedCandidates, "utf8")).trim()), + ).toMatchObject({ locations: [{ path: candidatePath }] }); const manifest = join(scanDir, "scan-manifest.json"); const coverage = join(scanDir, "coverage.json"); await writeFile( diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 65006ff9f1..d2e23e3452 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -247,6 +247,13 @@ describe("plugin runtime preparation", () => { join(repository, "nested", "tracked-secret.py"), "secret = True\n", ); + const mixedLongPath = + process.platform === "win32" ? `long-file-${"c".repeat(180)}.ts` : null; + if (mixedLongPath !== null) { + const mixedLongFile = join(repository, mixedLongPath); + expect(mixedLongFile.length).toBeGreaterThan(260); + await writeFile(mixedLongFile, "export {};\n"); + } for (const args of [ ["init", "--quiet", repository], ["-C", repository, "add", "--force", "--", "nested/tracked-secret.py"], @@ -259,13 +266,16 @@ describe("plugin runtime preparation", () => { expect(python).not.toBeNull(); const output = join(root, "inventory.txt"); const repeatedOutput = join(root, "inventory-repeated.txt"); - const generatorArguments = (destination: string) => + const generatorArguments = ( + destination: string, + sourceRepository = repository, + ) => [ "-I", "-B", join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), "--repo", - repository, + sourceRepository, "--scope", ".", "--out", @@ -330,6 +340,270 @@ describe("plugin runtime preparation", () => { .split(/\r?\n/u); expect(posixRows).toContain(String.raw`./literal\backslash.txt`); expect(posixRows).toContain("./literal:colon.txt"); + } else { + expect(rows).toContain(`./${mixedLongPath}`); + const committed = spawnSync( + "git", + [ + "-C", + repository, + "-c", + "user.name=Codex Security Test", + "-c", + "user.email=codex-security-test@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "-m", + "base", + ], + { encoding: "utf8" }, + ); + expect(committed.status, committed.stderr).toBe(0); + const mixedScopeOutput = join(root, "inventory-mixed-long-scope.txt"); + const mixedScopeInventory = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + mixedLongPath!, + "--out", + mixedScopeOutput, + ], + { encoding: "utf8" }, + ); + expect(mixedScopeInventory.status, mixedScopeInventory.stderr).toBe(0); + expect(await readFile(mixedScopeOutput, "utf8")).toBe( + `${mixedLongPath}\n`, + ); + const mixedDiffOutput = join(root, "inventory-mixed-long-diff.txt"); + const mixedDiffInventory = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + ".", + "--out", + mixedDiffOutput, + "--diff-base", + "HEAD", + "--diff-mode", + "local-patch", + ], + { encoding: "utf8" }, + ); + expect(mixedDiffInventory.status, mixedDiffInventory.stderr).toBe(0); + expect(await readFile(mixedDiffOutput, "utf8")).toBe( + `${mixedLongPath}\n`, + ); + await writeFile(join(repository, "SECURITY.md"), "# mixed policy\n"); + const mixedPolicy = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "resolve_security_md.py"), + "--repo", + repository, + "--scope", + mixedLongPath!, + "--out", + "-", + ], + { encoding: "utf8" }, + ); + expect(mixedPolicy.status, mixedPolicy.stderr).toBe(0); + expect(mixedPolicy.stdout).toBe( + '## SECURITY.md source: "SECURITY.md"\n\n# mixed policy\n', + ); + const longRepository = join( + root, + `long-${"a".repeat(100)}`, + `long-${"b".repeat(100)}`, + "repository", + ); + await mkdir(longRepository, { recursive: true }); + await writeFile(join(longRepository, "SECURITY.md"), "# policy\n"); + await writeFile(join(longRepository, "app.ts"), "export {};\n"); + expect(longRepository.length).toBeGreaterThan(260); + const longOutput = join(root, "inventory-long-path.txt"); + const inventory = spawnSync( + python!, + generatorArguments(longOutput, longRepository), + { encoding: "utf8" }, + ); + expect(inventory.status, inventory.stderr).toBe(0); + expect(await readFile(longOutput, "utf8")).toBe( + "./SECURITY.md\n./app.ts\n", + ); + + const policy = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "resolve_security_md.py"), + "--repo", + longRepository, + "--scope", + "app.ts", + "--out", + "-", + ], + { encoding: "utf8" }, + ); + expect(policy.status, policy.stderr).toBe(0); + expect(policy.stdout).toBe( + '## SECURITY.md source: "SECURITY.md"\n\n# policy\n', + ); + + const candidates = join(root, "long-path-candidates.jsonl"); + const normalized = join(root, "long-path-normalized.jsonl"); + await writeFile( + candidates, + `${JSON.stringify({ + cwe_ids: ["CWE-20"], + locations: [{ path: "app.ts", start_line: 1, role: "sink" }], + summary: "summary", + evidence: "evidence", + })}\n`, + ); + const normalization = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "normalize_candidates.py"), + "--input", + candidates, + "--out", + normalized, + "--repo-root", + longRepository, + "--in-scope-files", + longOutput, + ], + { encoding: "utf8" }, + ); + expect(normalization.status, normalization.stderr).toBe(0); + expect( + JSON.parse((await readFile(normalized, "utf8")).trim()), + ).toMatchObject({ + cwe_ids: ["CWE-20"], + locations: [{ path: "app.ts", start_line: 1, role: "sink" }], + }); + + const stateDirectory = join(root, "long-path-state"); + const scanRoot = join(root, "long-path-scans"); + const workbench = join(PLUGIN_ROOT, "scripts", "workbench_db.py"); + const mixedStarted = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "start-headless-standard-scan", + "--thread-id", + "mixed-long-path-test", + "--target-path", + repository, + "--scope", + ".", + "--scan-root", + scanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(mixedStarted.status, mixedStarted.stderr).toBe(0); + expect(JSON.parse(mixedStarted.stdout)).toMatchObject({ + scan: { targetPath: repository }, + workspace: { targetPath: repository }, + }); + const inspected = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "inspect-target", + "--target-path", + longRepository, + ], + { encoding: "utf8" }, + ); + expect(inspected.status, inspected.stderr).toBe(0); + expect(JSON.parse(inspected.stdout)).toMatchObject({ + displayName: "repository", + targetPath: longRepository, + }); + + const preflight = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, pathlib, runpy, sys", + "module = runpy.run_path(sys.argv[1])", + "_, metadata = module['discover_config_paths'](cwd=pathlib.Path(sys.argv[2]), profile_layer_path=None)", + "print(json.dumps(metadata))", + ].join("\n"), + join(PLUGIN_ROOT, "scripts", "config_preflight.py"), + longRepository, + ], + { encoding: "utf8" }, + ); + expect(preflight.status, preflight.stderr).toBe(0); + expect(JSON.parse(preflight.stdout)).toMatchObject({ + cwd: longRepository, + project_root: longRepository, + }); + + const started = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "start-headless-standard-scan", + "--thread-id", + "long-path-test", + "--target-path", + longRepository, + "--scope", + ".", + "--scan-root", + scanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(started.status, started.stderr).toBe(0); + expect(JSON.parse(started.stdout)).toMatchObject({ + scan: { targetPath: longRepository }, + workspace: { targetPath: longRepository }, + }); } }); From f3ccf52bfd209fc829c50c36e8abdee1ca7857e5 Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:10:38 -0700 Subject: [PATCH 02/32] fix(windows): finalize scans through long paths --- .../scripts/finalize_scan_contract.py | 16 +++-- .../scripts/windows_scan_local_files.py | 4 +- sdk/typescript/tests-ts/runtime.test.ts | 71 +++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index 4939cfae15..8545110b20 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -23,6 +23,10 @@ from typing import Any, TextIO from urllib.parse import quote, urlsplit +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import filesystem_path + SCHEMA_VERSION = "1.0" PRODUCER_NAME = "codex-security-plugin" FINGERPRINT_ALGORITHM = "codex-security/v1" @@ -81,7 +85,7 @@ def _loads_json(value: str | bytes) -> Any: def _read_json(path: Path) -> dict[str, Any]: try: - payload = _loads_json(path.read_text(encoding="utf-8")) + payload = _loads_json(filesystem_path(path).read_text(encoding="utf-8")) _require_safe_json_value(payload, str(path)) except FileNotFoundError as exc: raise ContractError(f"missing required contract artifact: {path}") from exc @@ -214,7 +218,7 @@ def _require_safe_relative_path(value: str, context: str, *, allow_dot: bool = F def _require_scan_directory(scan_dir: Path) -> Path: - scan_dir = scan_dir.absolute() + scan_dir = filesystem_path(scan_dir.absolute()) try: metadata = scan_dir.lstat() except OSError as exc: @@ -1618,7 +1622,7 @@ def _github_primary_location_line_hash( return None primary_location = _sarif_primary_location(finding) try: - source_root = source_root.resolve(strict=True) + source_root = filesystem_path(source_root).resolve(strict=True) except (OSError, RuntimeError): return None relative_path = _require_safe_relative_path(primary_location["path"], "SARIF source location") @@ -1640,7 +1644,7 @@ def _github_line_hash_cache( if source_root is None: return {} try: - source_root = source_root.resolve(strict=True) + source_root = filesystem_path(source_root).resolve(strict=True) except (OSError, RuntimeError): return {} requested_lines_by_path: dict[str, set[int]] = {} @@ -1884,7 +1888,7 @@ def build_sarif_projection( ) -> dict[str, Any]: if source_root is not None: try: - source_root = source_root.resolve(strict=True) + source_root = filesystem_path(source_root).resolve(strict=True) source_root_is_directory = source_root.is_dir() except (OSError, RuntimeError): source_root_is_directory = False @@ -2027,7 +2031,7 @@ def write_export_output(scan_dir: Path, output: Path, export_format: str, conten if export_format not in EXPORT_PATHS: raise ContractError(f"unsupported export format: {export_format}") scan_dir = _require_scan_directory(scan_dir) - output = Path(os.path.abspath(output)) + output = filesystem_path(Path(os.path.abspath(output))) try: relative_output = output.relative_to(scan_dir).as_posix() except ValueError: diff --git a/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py b/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py index baef9508f9..25a1010332 100644 --- a/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py +++ b/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py @@ -29,6 +29,8 @@ from ctypes import wintypes from pathlib import Path, PurePosixPath +from windows_paths import filesystem_path + _msvcrt = importlib.import_module("msvcrt") if os.name == "nt" else None @@ -354,7 +356,7 @@ def _verify_regular_file(handle: int, expected_path: Path) -> None: def _canonical_scan_directory(scan_dir: Path) -> tuple[Path, tuple[int, int]]: - absolute = Path(scan_dir).absolute() + absolute = filesystem_path(Path(scan_dir).absolute()) try: expected = absolute.lstat() canonical = absolute.resolve(strict=True) diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index d2e23e3452..072249feff 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -466,6 +466,77 @@ describe("plugin runtime preparation", () => { '## SECURITY.md source: "SECURITY.md"\n\n# policy\n', ); + await mkdir(join(longRepository, "src")); + await writeFile( + join(longRepository, "src", "extract.py"), + Array.from({ length: 50 }, (_, index) => `line_${index + 1}`).join( + "\n", + ) + "\n", + ); + const longScanDirectory = join(longRepository, "scan-output"); + await mkdir(longScanDirectory); + await Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + (filename) => + copyFile( + join(PLUGIN_ROOT, "examples", "completed-scan", filename), + join(longScanDirectory, filename), + ), + ), + ); + const longSarifOutput = join( + longScanDirectory, + "exports", + "results.sarif", + ); + const longSchemaDirectory = join(longRepository, "schemas"); + await mkdir(longSchemaDirectory); + await Promise.all( + [ + "scan-manifest.schema.json", + "findings.schema.json", + "coverage.schema.json", + ].map((filename) => + copyFile( + join(PLUGIN_ROOT, "schemas", filename), + join(longSchemaDirectory, filename), + ), + ), + ); + const finalization = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "finalize_scan_contract.py"), + "--scan-dir", + longScanDirectory, + "--source-root", + longRepository, + "--schema-dir", + longSchemaDirectory, + ], + { encoding: "utf8" }, + ); + expect(finalization.status, finalization.stderr).toBe(0); + expect( + await readFile(join(longScanDirectory, "report.md"), "utf8"), + ).toContain("# Security Review: example/repo"); + expect(JSON.parse(await readFile(longSarifOutput, "utf8"))).toMatchObject({ + version: "2.1.0", + runs: [ + { + results: [ + { + partialFingerprints: { + primaryLocationLineHash: expect.any(String), + }, + }, + ], + }, + ], + }); + const candidates = join(root, "long-path-candidates.jsonl"); const normalized = join(root, "long-path-normalized.jsonl"); await writeFile( From 0d34df1626c51c0a2f28a63ba3b6b8363b2868b0 Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:24:49 -0700 Subject: [PATCH 03/32] fix(windows): support long deep scan paths --- .../scripts/deep_scan_workbench.py | 95 ++++++++++----- .../_bundled_plugin/scripts/workbench_db.py | 24 ++-- sdk/typescript/tests-ts/runtime.test.ts | 111 ++++++++++++++++-- 3 files changed, 174 insertions(+), 56 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index 113f7c156e..e14e9b80f2 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -26,6 +26,7 @@ worktree_content_digest, ) from workbench_validation import optional_text, require_uuid, user_text +from windows_paths import filesystem_path, portable_path DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled") @@ -196,6 +197,10 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path: return dependencies().require_canonical_scan_directory(scan_dir) +def scan_directory_path(scan: sqlite3.Row) -> Path: + return filesystem_path(Path(scan["scan_dir"])) + + def safe_segment(value: str) -> str: return dependencies().safe_segment(value) @@ -264,25 +269,28 @@ def deep_scan_path( if not supplied.is_absolute(): raise SystemExit(f"{label} must be an absolute path inside the scan directory.") try: - resolved = supplied.resolve(strict=True) - scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) - resolved.relative_to(scan_dir) + resolved = type(supplied)(filesystem_path(supplied)).resolve(strict=True) + scan_dir = require_canonical_scan_directory(scan_directory_path(scan)) + portable_resolved = portable_path(resolved) + portable_resolved.relative_to(portable_path(scan_dir)) except (OSError, RuntimeError, ValueError) as exc: raise SystemExit(f"{label} must be an existing path inside the scan directory.") from exc - if os.path.normcase(resolved) != os.path.normcase(supplied.absolute()): + if os.path.normcase(portable_resolved) != os.path.normcase( + portable_path(supplied.absolute()) + ): raise SystemExit(f"{label} must be a canonical non-symlink path.") if kind == "file" and not resolved.is_file(): raise SystemExit(f"{label} must be a regular file.") if kind == "directory" and not resolved.is_dir(): raise SystemExit(f"{label} must be a directory.") - return str(resolved) + return str(portable_resolved) def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str: supplied = Path(value).expanduser() if not supplied.is_absolute(): raise SystemExit(f"{label} must be an absolute path inside the scan directory.") - if supplied.exists(): + if filesystem_path(supplied).exists(): return deep_scan_path(scan, str(supplied), label, kind="file") parent = Path(deep_scan_path(scan, str(supplied.parent), label, kind="directory")) output = parent / supplied.name @@ -292,8 +300,8 @@ def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str: def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]: - staged = Path(staged_path) - output = Path(output_path) + staged = filesystem_path(Path(staged_path)) + output = filesystem_path(Path(output_path)) if staged == output: raise SystemExit("A staged Deep Scan artifact must not be its published output path.") backup = output.with_name(f".{output.name}.{uuid.uuid4()}.backup") if output.exists() else None @@ -323,7 +331,7 @@ def finish_staged_file(promotion: tuple[Path, Path, Path | None]) -> None: def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: - discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" + discovery_dir = scan_directory_path(scan) / "artifacts" / "02_discovery" artifacts = { "inScopeFilesPath": discovery_dir / "in_scope_files.txt", "candidateLedgerPath": discovery_dir / "candidate_ledger.jsonl", @@ -364,15 +372,24 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A run["canonical_inventory_path"] is None and run["status"] == "succeeded" and run["manifest_path"] is not None - and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json") - and (Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "in_scope_files.txt").exists() + and run["manifest_path"] + != str(portable_path(scan_directory_path(scan) / "scan-manifest.json")) + and ( + scan_directory_path(scan) + / "artifacts" + / "02_discovery" + / "in_scope_files.txt" + ).exists() ): canonical_artifacts = canonical_discovery_artifacts(scan) if ( run["terminal_reason"] == "capped" and run["completion_sequence"] == 0 and deep_scan_deadline_reached(run) - and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size != 0 + and filesystem_path(Path(canonical_artifacts["candidateLedgerPath"])) + .stat() + .st_size + != 0 ): raise SystemExit( "A capped Deep Scan without completed discoveries requires an empty " @@ -691,7 +708,7 @@ def begin_deep_scan_for_target( target = require_target(args.target_path) require_scannable_target(target) scope = require_scope(args.scope, "deep", target) - target_path = str(target) + target_path = str(portable_path(target)) existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: return begin_deep_scan_for_scan(connection, existing["id"], thread_id, args) @@ -757,10 +774,17 @@ def begin_deep_scan_for_target( if workflow_version is None: raise SystemExit("workflow-version is required.") root = ( - Path(args.scan_root).expanduser().resolve() if args.scan_root else state_dir() / "scans" + filesystem_path(Path(args.scan_root).expanduser()).resolve() + if args.scan_root + else state_dir() / "scans" ) - target_root = (root / safe_segment(target.name)).resolve() - if target_root == target or target in target_root.parents: + target_root = filesystem_path(root / safe_segment(target.name)).resolve() + portable_target = portable_path(target) + portable_target_root = portable_path(target_root) + if ( + portable_target_root == portable_target + or portable_target in portable_target_root.parents + ): raise SystemExit("The scan artifact directory must be outside the selected target.") target_root.mkdir(parents=True, exist_ok=True) user_context = user_text(args.user_context) @@ -770,10 +794,12 @@ def begin_deep_scan_for_target( scan_id = str(uuid.uuid4()) timestamp = now() target_id = ensure_security_target(connection, target_path) - scan_dir = Path( - tempfile.mkdtemp( - prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", - dir=target_root, + scan_dir = filesystem_path( + Path( + tempfile.mkdtemp( + prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", + dir=target_root, + ) ) ).resolve() connection.execute( @@ -817,7 +843,7 @@ def begin_deep_scan_for_target( scope, user_context, thread_id, - str(scan_dir), + str(portable_path(scan_dir)), model, reasoning_effort, timestamp, @@ -887,7 +913,7 @@ def coordinator_lease_is_live( ) heartbeat_time = datetime.fromisoformat(str(run["updated_at"])) heartbeat_path = ( - Path(scan["scan_dir"]) + scan_directory_path(scan) / "artifacts" / "deep_discovery" / f"coordinator-heartbeat-{run['coordinator_generation']}.json" @@ -1047,7 +1073,7 @@ def recover_expired_coordinator( def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None: scan = require_scan(connection, scan_id) - ledger = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl" + ledger = scan_directory_path(scan) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl" backups = sorted( ledger.parent.glob(f".{ledger.name}.*.backup"), key=lambda backup: backup.stat().st_mtime_ns, @@ -1066,7 +1092,7 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id (scan_id,), ) for reducer in reducers: - snapshot = Path(reducer["artifact_dir"]) / "canonical" / ledger.name + snapshot = filesystem_path(Path(reducer["artifact_dir"])) / "canonical" / ledger.name if not snapshot.exists(): continue published = ledger.exists() and ledger.samefile(snapshot) @@ -1531,7 +1557,7 @@ def commit_deep_scan_dedup_locked( "Staged candidate ledger path", kind="file", ) - discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" + discovery_dir = scan_directory_path(scan) / "artifacts" / "02_discovery" deep_scan_path( scan, str(discovery_dir / "in_scope_files.txt"), @@ -1567,11 +1593,11 @@ def commit_deep_scan_dedup_locked( if not inputs or any(row["merge_state"] != "merging" for row in inputs): raise SystemExit("Dedup inputs are not in the claimed merging state.") if candidate_ledger_path and canonical_candidate_ledger_path: - canonical_path = Path(canonical_candidate_ledger_path) + canonical_path = filesystem_path(Path(canonical_candidate_ledger_path)) publication_copy = canonical_path.with_name( f".{canonical_path.name}.{uuid.uuid4()}.publish" ) - os.link(candidate_ledger_path, publication_copy) + os.link(filesystem_path(Path(candidate_ledger_path)), publication_copy) promotion = promote_staged_file( str(publication_copy), canonical_candidate_ledger_path, @@ -1651,7 +1677,9 @@ def finish_deep_scan_locked( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) ) - standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json") + standard_scan_manifest = manifest_path == str( + portable_path(scan_directory_path(scan) / "scan-manifest.json") + ) failure_capped = False if ( @@ -1662,12 +1690,12 @@ def finish_deep_scan_locked( for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, - str(Path(scan["scan_dir"]) / artifact_name), + str(scan_directory_path(scan) / artifact_name), f"Canonical parent {artifact_name}", kind="file", ) coverage = _read_scan_local_json( - Path(scan["scan_dir"]), "coverage.json", "Canonical parent coverage.json" + scan_directory_path(scan), "coverage.json", "Canonical parent coverage.json" ) deferred = coverage.get("deferred") failure_capped = ( @@ -1766,7 +1794,7 @@ def finish_deep_scan_locked( for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, - str(Path(scan["scan_dir"]) / artifact_name), + str(scan_directory_path(scan) / artifact_name), f"Canonical parent {artifact_name}", kind="file", ) @@ -1792,7 +1820,10 @@ def finish_deep_scan_locked( and ( standard_scan_manifest or canonical_artifacts is not None - and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0 + and filesystem_path(Path(canonical_artifacts["candidateLedgerPath"])) + .stat() + .st_size + == 0 ) ) if successful_reducer is None and not zero_discovery_deadline: diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index d795853374..d95911c2cc 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -1190,7 +1190,8 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + if run["manifest_path"] + == str(portable_path(scan_dir / "scan-manifest.json")) else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message, maximum=2400) @@ -1566,7 +1567,7 @@ def add_warning() -> None: if path is not None: connection.execute( "INSERT INTO scan_artifacts (scan_id, kind, path, created_at) VALUES (?, ?, ?, ?)", - (scan["id"], kind, str(path), timestamp), + (scan["id"], kind, str(portable_path(path)), timestamp), ) connection.execute("DELETE FROM finding_occurrences WHERE scan_id = ?", (scan["id"],)) index_findings(connection, scan["id"], findings, timestamp) @@ -1717,7 +1718,7 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) scan = require_scan(connection, scan_id) return { "contract": scan_contract(scan), - "scanDir": str(scan_dir), + "scanDir": str(portable_path(scan_dir)), "scanId": scan_id, "scopeFileCount": scope_file_count, "targetId": target_id, @@ -2447,7 +2448,7 @@ def export_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> if path is None: raise SystemExit(f"Could not export Codex Security findings as {args.format.upper()}.") return { - "export": {"format": args.format, "path": str(path)}, + "export": {"format": args.format, "path": str(portable_path(path))}, "scan": scan_result(connection, scan), "workspace": workspace_state(connection, scan["workspace_id"]), } @@ -2994,12 +2995,12 @@ def scan_result( continue path = available_artifact_path(Path(scan["scan_dir"]), Path(row["path"])) if path is not None: - artifacts[row["kind"]] = str(path) + artifacts[row["kind"]] = str(portable_path(path)) sarif_path = available_artifact_path( Path(scan["scan_dir"]), Path(scan["scan_dir"]) / "exports" / "results.sarif" ) if sarif_path is not None: - artifacts["sarifReport"] = str(sarif_path) + artifacts["sarifReport"] = str(portable_path(sarif_path)) occurrence_rows = scan_history.finding_occurrence_rows( connection, scan["id"], offset=0, limit=FINDINGS_RESULT_LIMIT ) @@ -3486,7 +3487,7 @@ def patch_artifact_preview( def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None: try: resolved_scan_dir = require_canonical_scan_directory(scan_dir) - resolved = candidate.resolve(strict=True) + resolved = filesystem_path(candidate).resolve(strict=True) resolved.relative_to(resolved_scan_dir) except (FileNotFoundError, RuntimeError, SystemExit, ValueError): return None @@ -3513,7 +3514,8 @@ def artifact_path(scan_dir: Path, file_name: str, *, required: bool) -> Path | N def require_canonical_scan_directory(scan_dir: Path) -> Path: - scan_dir = scan_dir.absolute() + portable_scan_dir = portable_path(scan_dir.absolute()) + scan_dir = filesystem_path(portable_scan_dir) try: metadata = scan_dir.lstat() resolved = scan_dir.resolve(strict=True) @@ -3521,9 +3523,9 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path: raise SystemExit( "Scan directory must be an existing canonical non-symlink directory." ) from exc - if not stat.S_ISDIR(metadata.st_mode) or os.path.normcase(resolved) != os.path.normcase( - scan_dir - ): + if not stat.S_ISDIR(metadata.st_mode) or os.path.normcase( + portable_path(resolved) + ) != os.path.normcase(portable_scan_dir): raise SystemExit("Scan directory must be an existing canonical non-symlink directory.") # Re-check privacy on every resolution so a mid-scan rename/replace under a # shared parent cannot substitute another user's forged artifact tree. diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 1d7d0df5d8..36d333432c 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -501,20 +501,22 @@ describe("plugin runtime preparation", () => { expect( await readFile(join(longScanDirectory, "report.md"), "utf8"), ).toContain("# Security Review: example/repo"); - expect(JSON.parse(await readFile(longSarifOutput, "utf8"))).toMatchObject({ - version: "2.1.0", - runs: [ - { - results: [ - { - partialFingerprints: { - primaryLocationLineHash: expect.any(String), + expect(JSON.parse(await readFile(longSarifOutput, "utf8"))).toMatchObject( + { + version: "2.1.0", + runs: [ + { + results: [ + { + partialFingerprints: { + primaryLocationLineHash: expect.any(String), + }, }, - }, - ], - }, - ], - }); + ], + }, + ], + }, + ); const candidates = join(root, "long-path-candidates.jsonl"); const normalized = join(root, "long-path-normalized.jsonl"); @@ -654,6 +656,89 @@ describe("plugin runtime preparation", () => { scan: { targetPath: longRepository }, workspace: { targetPath: longRepository }, }); + + const longDeepScanRoot = join( + root, + `deep-${"d".repeat(100)}`, + `deep-${"e".repeat(100)}`, + ); + expect(longDeepScanRoot.length).toBeGreaterThan(260); + const deepStarted = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "begin-deep-scan", + "--thread-id", + "deep-long-path-test", + "--target-path", + longRepository, + "--scan-root", + longDeepScanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(deepStarted.status, deepStarted.stderr).toBe(0); + const deepScan = JSON.parse(deepStarted.stdout).deepScan as { + scanDir: string; + scanId: string; + }; + expect(deepScan.scanDir.length).toBeGreaterThan(260); + expect(deepScan.scanDir).not.toStartWith("\\\\?\\"); + + const deepPromptPath = join(deepScan.scanDir, "setup-prompt.md"); + const deepArtifactDirectory = join(deepScan.scanDir, "setup-artifacts"); + await writeFile(deepPromptPath, "# Setup\n"); + await mkdir(deepArtifactDirectory); + const workerId = "00000000-0000-4000-8000-000000000001"; + const workerUpdated = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "upsert-deep-scan-worker", + "--scan-id", + deepScan.scanId, + "--worker-id", + workerId, + "--kind", + "setup", + "--status", + "running", + "--prompt-path", + deepPromptPath, + "--artifact-dir", + deepArtifactDirectory, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(workerUpdated.status, workerUpdated.stderr).toBe(0); + expect(JSON.parse(workerUpdated.stdout)).toMatchObject({ + deepScan: { + workers: [ + { + artifactDir: deepArtifactDirectory, + id: workerId, + promptPath: deepPromptPath, + status: "running", + }, + ], + }, + }); } }); From 8dd0409212f69b0b60d82fb3937747a883dc49ba Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:29:17 -0700 Subject: [PATCH 04/32] fix(windows): query history through long paths --- .../scripts/workbench_scan_history.py | 29 ++++++++++------- sdk/typescript/tests-ts/runtime.test.ts | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 323fdbe78b..6e963f4655 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -16,6 +16,7 @@ from workbench_constants import FINDINGS_PAGE_MAX from workbench_scan_usage import stored_scan_cost_fields from workbench_target import git_output +from windows_paths import filesystem_path, portable_path def _same_repository( @@ -26,8 +27,8 @@ def _same_repository( ) -> bool: if before["target_id"] == after["target_id"]: return True - before_target = Path(before["target_path"]) - after_target = Path(after["target_path"]) + before_target = filesystem_path(Path(before["target_path"])) + after_target = filesystem_path(Path(after["target_path"])) before_git_dir = git_output( before_target, "rev-parse", "--path-format=absolute", "--git-common-dir" ) @@ -39,7 +40,8 @@ def _same_repository( if ( before_git_dir is not None and after_git_dir is not None - and Path(before_git_dir).resolve() == Path(after_git_dir).resolve() + and filesystem_path(Path(before_git_dir)).resolve() + == filesystem_path(Path(after_git_dir)).resolve() ): return True before_origin = _repository_origin(before_target) @@ -81,13 +83,14 @@ def list_scans( clauses: list[str] = [] values: list[Any] = [] if args is not None and args.repository: - repository = Path(args.repository).expanduser().resolve() + repository = filesystem_path(Path(args.repository).expanduser()).resolve() + repository_path = str(portable_path(repository)) requested_repository = connection.execute( """ SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, ? AS target_path """, - (str(repository), str(repository)), + (repository_path, repository_path), ).fetchone() requested_identity = ( git_output(repository, "rev-parse", "--path-format=absolute", "--git-common-dir"), @@ -101,14 +104,16 @@ def list_scans( if _same_repository(target, requested_repository, after_identity=requested_identity) ] repository_clauses = ["scans.target_path = ?"] - values.append(str(repository)) + values.append(repository_path) if related_target_ids: placeholders = ", ".join("?" for _ in related_target_ids) repository_clauses.append(f"scans.target_id IN ({placeholders})") values.extend(related_target_ids) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: - scan_root = str(Path(args.scan_root).expanduser().resolve()) + scan_root = str( + portable_path(filesystem_path(Path(args.scan_root).expanduser()).resolve()) + ) prefix = scan_root.rstrip(os.sep) + os.sep clauses.append("(scans.scan_dir = ? OR substr(scans.scan_dir, 1, ?) = ?)") values.extend((scan_root, len(prefix), prefix)) @@ -226,20 +231,22 @@ def list_unmatched_scan_pairs( backfill_finding_details: Callable[[sqlite3.Connection, sqlite3.Row], None], read_coverage: Callable[[sqlite3.Row], dict[str, Any]], ) -> dict[str, Any]: - repository = Path(args.repository).expanduser().resolve() + repository = filesystem_path(Path(args.repository).expanduser()).resolve() + repository_path = str(portable_path(repository)) requested = connection.execute( """ SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, ? AS target_path """, - (str(repository), str(repository)), + (repository_path, repository_path), ).fetchone() selected = [ scan for scan in connection.execute( "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" ) - if Path(scan["target_path"]).resolve() == repository or _same_repository(scan, requested) + if filesystem_path(Path(scan["target_path"])).resolve() == repository + or _same_repository(scan, requested) ] available = [] @@ -288,7 +295,7 @@ def list_unmatched_scan_pairs( ) return { "batches": batches, - "repository": str(repository), + "repository": repository_path, "scanCount": len(selected), "skippedPairs": skipped, "unavailableScans": len(selected) - len(available), diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 36d333432c..dc9d3c40dd 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -739,6 +739,37 @@ describe("plugin runtime preparation", () => { ], }, }); + + const listed = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "list-scans", + "--repository", + longRepository, + "--scan-root", + longDeepScanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(listed.status, listed.stderr).toBe(0); + expect(JSON.parse(listed.stdout)).toMatchObject({ + scans: [ + { + scanDir: deepScan.scanDir, + scanId: deepScan.scanId, + targetPath: longRepository, + }, + ], + }); } }); From 4c26065b76c3542265a70c207bbb4f5c2861ac6f Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:32:46 -0700 Subject: [PATCH 05/32] fix(windows): open long workbench state paths --- sdk/typescript/_bundled_plugin/scripts/workbench_db.py | 4 ++-- sdk/typescript/tests-ts/runtime.test.ts | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index d95911c2cc..a0303086f0 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -150,9 +150,9 @@ def stale_claim_before(seconds: int = CLAIM_LEASE_SECONDS) -> str: def state_dir() -> Path: state_dir = os.environ.get("CODEX_SECURITY_STATE_DIR") if state_dir: - return Path(state_dir).expanduser().resolve() + return filesystem_path(Path(state_dir).expanduser()).resolve() codex_home = Path(os.environ.get("CODEX_HOME", "~/.codex")).expanduser() - return (codex_home / "state" / "plugins" / "codex-security").resolve() + return filesystem_path(codex_home / "state" / "plugins" / "codex-security").resolve() def database_path() -> Path: diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index dc9d3c40dd..3556ee74e5 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -554,7 +554,12 @@ describe("plugin runtime preparation", () => { locations: [{ path: "app.ts", start_line: 1, role: "sink" }], }); - const stateDirectory = join(root, "long-path-state"); + const stateDirectory = join( + root, + `state-${"f".repeat(100)}`, + `state-${"0".repeat(100)}`, + ); + expect(stateDirectory.length).toBeGreaterThan(260); const scanRoot = join(root, "long-path-scans"); const workbench = join(PLUGIN_ROOT, "scripts", "workbench_db.py"); const mixedStarted = spawnSync( From c571062ca5719489cdec2f3d4838c693ef9ee850 Mon Sep 17 00:00:00 2001 From: faizan-oai <269039902+faizan-oai@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:35:17 -0700 Subject: [PATCH 06/32] fix(windows): read long rollout paths --- .../scripts/workbench_scan_usage.py | 7 ++++-- sdk/typescript/tests-ts/runtime.test.ts | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_usage.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_usage.py index be5bc92827..8d9a161506 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_usage.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_usage.py @@ -14,6 +14,9 @@ from pathlib import Path from typing import Any, Mapping +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import filesystem_path, portable_path + TOKEN_FIELDS = { "input_tokens": "inputTokens", "cached_input_tokens": "cachedInputTokens", @@ -349,11 +352,11 @@ def _rollout_path(value: object) -> Path | None: if not candidate.is_absolute(): return None try: - resolved = candidate.resolve(strict=True) + resolved = filesystem_path(candidate).resolve(strict=True) if not resolved.is_file(): return None - if resolved == candidate: + if portable_path(resolved) == portable_path(candidate.absolute()): return resolved if sys.platform == "darwin" and candidate.parts[1] in {"var", "tmp"}: diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 3556ee74e5..1ae9fc7304 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -560,6 +560,31 @@ describe("plugin runtime preparation", () => { `state-${"0".repeat(100)}`, ); expect(stateDirectory.length).toBeGreaterThan(260); + await mkdir(stateDirectory, { recursive: true }); + const longRolloutPath = join(stateDirectory, "rollout.jsonl"); + await writeFile(longRolloutPath, '{"type":"synthetic"}\n'); + const rolloutProbe = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, runpy, sys", + "module = runpy.run_path(sys.argv[1])", + "path = module['_rollout_path'](sys.argv[2])", + "print(json.dumps({'available': path is not None, 'contents': None if path is None else path.read_text(encoding='utf-8')}))", + ].join("\n"), + join(PLUGIN_ROOT, "scripts", "workbench_scan_usage.py"), + longRolloutPath, + ], + { encoding: "utf8" }, + ); + expect(rolloutProbe.status, rolloutProbe.stderr).toBe(0); + expect(JSON.parse(rolloutProbe.stdout)).toEqual({ + available: true, + contents: '{"type":"synthetic"}\n', + }); const scanRoot = join(root, "long-path-scans"); const workbench = join(PLUGIN_ROOT, "scripts", "workbench_db.py"); const mixedStarted = spawnSync( From 82f54aa15e510a8adcded0c19e414753c37221fa Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 20:09:16 -0700 Subject: [PATCH 07/32] fix(windows): exclude Python bytecode from packages --- sdk/typescript/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index ac27122a6c..419668d978 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -32,6 +32,7 @@ "bin", "dist", "_bundled_plugin", + "!_bundled_plugin/**/__pycache__/**", "LICENSE", "README.md" ], From b4d2f045f7565e6d05899fe74bd3cac723ad1647 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 21:11:49 -0700 Subject: [PATCH 08/32] fix(windows): preserve long paths through scan results --- .../scripts/generate_rank_input.py | 6 +- .../_bundled_plugin/scripts/workbench_db.py | 5 +- .../scripts/workbench_native_indexes.py | 3 +- sdk/typescript/tests-ts/runtime.test.ts | 74 +++++++++++++++++++ 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py index a371fcf7aa..ba5119c2e8 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py @@ -337,6 +337,7 @@ def resolve_scope( def write_jsonl(output: Path, rows: list[JsonRow]) -> None: + output = filesystem_path(output) output.parent.mkdir(parents=True, exist_ok=True) with output.open("w", encoding="utf-8") as handle: for row in rows: @@ -367,11 +368,12 @@ def load_scopes_file(scopes_file: Path) -> list[str]: def load_jsonl(path: Path, label: str, validator: RowValidator) -> list[JsonRow]: - if not path.exists(): + input_path = filesystem_path(path) + if not input_path.exists(): raise SystemExit(f"{label} missing: {path}") rows: list[JsonRow] = [] - with path.open(encoding="utf-8") as handle: + with input_path.open(encoding="utf-8") as handle: for line_number, raw_line in enumerate(handle, start=1): if not raw_line.strip(): raise SystemExit(f"{path}:{line_number}: blank JSONL rows are not allowed") diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index a0303086f0..e848fd1782 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -3491,7 +3491,10 @@ def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None: resolved.relative_to(resolved_scan_dir) except (FileNotFoundError, RuntimeError, SystemExit, ValueError): return None - if os.path.normcase(resolved) != os.path.normcase(candidate) or not candidate.is_file(): + if ( + os.path.normcase(portable_path(resolved)) != os.path.normcase(portable_path(candidate)) + or not resolved.is_file() + ): return None return resolved diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 8ce2492bc7..a7bf56a58d 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -14,6 +14,7 @@ import workbench_scan_history as scan_history from workbench_constants import FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, FINDINGS_PAGE_MAX from workbench_validation import bounded_output_text +from windows_paths import filesystem_path def list_global_findings( @@ -210,7 +211,7 @@ def list_repositories( targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")} repositories = [ { - "checkoutAvailable": Path(target["current_path"]).is_dir(), + "checkoutAvailable": filesystem_path(Path(target["current_path"])).is_dir(), "displayName": target["display_name"], "latestScan": latest_scan, "openFindingsCount": open_findings_by_target.get(target_id, 0), diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 87370e522f..e9492e306a 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -613,6 +613,34 @@ describe("plugin runtime preparation", () => { }, ); + const longRankInput = join(longScanDirectory, "rank_input.jsonl"); + const longReviewInput = join( + longScanDirectory, + "deep_review_input.jsonl", + ); + await writeFile( + longRankInput, + `${JSON.stringify({ path: "app.ts", area: "diff", preview: "export {};" })}\n`, + ); + const copiedReview = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"), + "copy-deep-review-input", + "--rank-input", + longRankInput, + "--out", + longReviewInput, + ], + { encoding: "utf8" }, + ); + expect(copiedReview.status, copiedReview.stderr).toBe(0); + expect( + JSON.parse((await readFile(longReviewInput, "utf8")).trim()), + ).toEqual({ path: "app.ts", area: "diff" }); + const candidates = join(root, "long-path-candidates.jsonl"); const normalized = join(root, "long-path-normalized.jsonl"); await writeFile( @@ -682,6 +710,31 @@ describe("plugin runtime preparation", () => { }); const scanRoot = join(root, "long-path-scans"); const workbench = join(PLUGIN_ROOT, "scripts", "workbench_db.py"); + const artifactProbe = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, runpy, sys", + "from pathlib import Path", + "module = runpy.run_path(sys.argv[1])", + "scan_dir = Path(sys.argv[2])", + "names = ('report.md', 'findings.json', 'exports/results.sarif')", + "print(json.dumps({name: module['available_artifact_path'](scan_dir, scan_dir / name) is not None for name in names}))", + ].join("\n"), + workbench, + longScanDirectory, + ], + { encoding: "utf8" }, + ); + expect(artifactProbe.status, artifactProbe.stderr).toBe(0); + expect(JSON.parse(artifactProbe.stdout)).toEqual({ + "report.md": true, + "findings.json": true, + "exports/results.sarif": true, + }); const mixedStarted = spawnSync( python!, [ @@ -782,6 +835,27 @@ describe("plugin runtime preparation", () => { workspace: { targetPath: longRepository }, }); + const repositories = spawnSync( + python!, + ["-I", "-B", workbench, "list-repositories"], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(repositories.status, repositories.stderr).toBe(0); + expect(JSON.parse(repositories.stdout)).toMatchObject({ + repositories: expect.arrayContaining([ + expect.objectContaining({ + checkoutAvailable: true, + targetPath: longRepository, + }), + ]), + }); + const longDeepScanRoot = join( root, `deep-${"d".repeat(100)}`, From 6dc0fd8ccafa231a1b97790ace23f40c8ddeb288 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 21:17:35 -0700 Subject: [PATCH 09/32] test(windows): run long-path coverage with bundled ripgrep --- sdk/typescript/tests-ts/runtime.test.ts | 44 ++++++++++++++++--------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index e9492e306a..945d88a5f0 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -321,15 +321,23 @@ describe("plugin runtime preparation", () => { }); test("generates canonical scoped security inventory paths", async () => { - if (Bun.which("rg") === null) { - const generator = await readFile( - join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), - "utf8", + const ripgrep = + Bun.which("rg") ?? + join( + dirname(dirname(resolveCodexCommand().command)), + "codex-path", + process.platform === "win32" ? "rg.exe" : "rg", ); - expect(generator).toContain('"--no-ignore"'); - expect(generator).toContain('"--path-separator"'); - return; - } + expect(existsSync(ripgrep)).toBe(true); + const inventoryOptions = { + encoding: "utf8" as const, + env: { + ...process.env, + PATH: [dirname(ripgrep), process.env["PATH"]] + .filter(Boolean) + .join(delimiter), + }, + }; const root = await temporaryDirectory("codex-security-scan-inventory-"); const repository = join(root, "repository"); @@ -377,9 +385,11 @@ describe("plugin runtime preparation", () => { destination, ] as const; for (const destination of [output, repeatedOutput]) { - const inventory = spawnSync(python!, generatorArguments(destination), { - encoding: "utf8", - }); + const inventory = spawnSync( + python!, + generatorArguments(destination), + inventoryOptions, + ); expect(inventory.status, inventory.stderr).toBe(0); } @@ -405,9 +415,11 @@ describe("plugin runtime preparation", () => { ); await writeFile(join(repository, "literal:colon.txt"), "colon\n"); const posixOutput = join(root, "inventory-posix-filenames.txt"); - const inventory = spawnSync(python!, generatorArguments(posixOutput), { - encoding: "utf8", - }); + const inventory = spawnSync( + python!, + generatorArguments(posixOutput), + inventoryOptions, + ); expect(inventory.status, inventory.stderr).toBe(0); const posixRows = (await readFile(posixOutput, "utf8")) .trimEnd() @@ -449,7 +461,7 @@ describe("plugin runtime preparation", () => { "--out", mixedScopeOutput, ], - { encoding: "utf8" }, + inventoryOptions, ); expect(mixedScopeInventory.status, mixedScopeInventory.stderr).toBe(0); expect(await readFile(mixedScopeOutput, "utf8")).toBe( @@ -513,7 +525,7 @@ describe("plugin runtime preparation", () => { const inventory = spawnSync( python!, generatorArguments(longOutput, longRepository), - { encoding: "utf8" }, + inventoryOptions, ); expect(inventory.status, inventory.stderr).toBe(0); expect(await readFile(longOutput, "utf8")).toBe( From 9a7f97c7f16ac47989806a42f77e94e31ecb9b06 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 21:21:23 -0700 Subject: [PATCH 10/32] fix(windows): handle legacy scan paths and scoped inventories --- .../scripts/generate_rank_input.py | 15 ++++- .../_bundled_plugin/scripts/workbench_db.py | 6 +- .../scripts/workbench_scan_start.py | 7 ++- sdk/typescript/tests-ts/runtime.test.ts | 57 +++++++++++++++++-- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py index ba5119c2e8..f20831022b 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py @@ -531,6 +531,12 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: if git_candidates is not None: candidates = git_candidates else: + absolute_search = os.name == "nt" and str(repo).startswith("\\\\?\\") + search_path = ( + str(scope_path) + if absolute_search + else str(portable_path(scope_path).relative_to(portable_path(repo))) + ) command = [ "rg", "--files", @@ -540,10 +546,15 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: "--glob", "!.git/**", "--", - str(portable_path(scope_path).relative_to(portable_path(repo))), + search_path, ] try: - result = subprocess.run(command, cwd=repo, capture_output=True, check=False) + result = subprocess.run( + command, + cwd=None if absolute_search else repo, + capture_output=True, + check=False, + ) except OSError as exc: ignore_names = (".gitignore", ".ignore", ".rgignore") ancestors = (scope_path, *scope_path.parents) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index e848fd1782..19beafdf49 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -3488,7 +3488,7 @@ def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None: try: resolved_scan_dir = require_canonical_scan_directory(scan_dir) resolved = filesystem_path(candidate).resolve(strict=True) - resolved.relative_to(resolved_scan_dir) + portable_path(resolved).relative_to(portable_path(resolved_scan_dir)) except (FileNotFoundError, RuntimeError, SystemExit, ValueError): return None if ( @@ -3501,10 +3501,10 @@ def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None: def artifact_path(scan_dir: Path, file_name: str, *, required: bool) -> Path | None: scan_dir = require_canonical_scan_directory(scan_dir) - candidate = scan_dir / file_name + candidate = filesystem_path(scan_dir / file_name) try: resolved = candidate.resolve(strict=True) - resolved.relative_to(scan_dir.resolve()) + portable_path(resolved).relative_to(portable_path(scan_dir)) except (FileNotFoundError, RuntimeError, ValueError) as exc: if not required and isinstance(exc, FileNotFoundError): return None diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index 2101e505d9..d8938cbe72 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -23,7 +23,7 @@ worktree_content_digest, ) from workbench_validation import optional_text, user_text -from windows_paths import portable_path +from windows_paths import extended_path, portable_path def safe_segment(value: str) -> str: @@ -108,7 +108,8 @@ def archive_scan( raise SystemExit("The archived scan must be a previous sibling of the scan directory.") previous_scan = connection.execute( - "SELECT id, status FROM scans WHERE scan_dir = ?", (str(portable_scan_dir),) + "SELECT id, status FROM scans WHERE scan_dir IN (?, ?)", + (str(portable_scan_dir), str(extended_path(portable_scan_dir))), ).fetchone() if previous_scan is None: return @@ -137,7 +138,7 @@ def archive_scan( ) for artifact in artifacts: try: - relative_path = Path(artifact["path"]).relative_to(portable_scan_dir) + relative_path = portable_path(Path(artifact["path"])).relative_to(portable_scan_dir) except ValueError: continue connection.execute( diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 945d88a5f0..21d2d2664c 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -329,11 +329,14 @@ describe("plugin runtime preparation", () => { process.platform === "win32" ? "rg.exe" : "rg", ); expect(existsSync(ripgrep)).toBe(true); + const pathVariable = + Object.keys(process.env).find((name) => name.toUpperCase() === "PATH") ?? + "PATH"; const inventoryOptions = { encoding: "utf8" as const, env: { ...process.env, - PATH: [dirname(ripgrep), process.env["PATH"]] + [pathVariable]: [dirname(ripgrep), process.env[pathVariable]] .filter(Boolean) .join(delimiter), }, @@ -532,6 +535,33 @@ describe("plugin runtime preparation", () => { "./SECURITY.md\n./app.ts\n", ); + const longScopes = join(root, "long-path-scopes.json"); + const longScopedOutput = join(root, "long-path-scoped-source.jsonl"); + await writeFile(longScopes, JSON.stringify(["."])); + const scopedSource = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"), + "make-repo-scope-input", + "--repo", + longRepository, + "--scopes-file", + longScopes, + "--out", + longScopedOutput, + ], + inventoryOptions, + ); + expect(scopedSource.status, scopedSource.stderr).toBe(0); + expect( + (await readFile(longScopedOutput, "utf8")) + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)), + ).toEqual([{ path: "SECURITY.md" }, { path: "app.ts" }]); + const policy = spawnSync( python!, [ @@ -722,6 +752,16 @@ describe("plugin runtime preparation", () => { }); const scanRoot = join(root, "long-path-scans"); const workbench = join(PLUGIN_ROOT, "scripts", "workbench_db.py"); + const nearLimitScanDirectory = join( + root, + "n".repeat(247 - root.length - 1), + ); + expect(nearLimitScanDirectory.length).toBe(247); + await mkdir(nearLimitScanDirectory); + await copyFile( + join(longScanDirectory, "scan-manifest.json"), + join(nearLimitScanDirectory, "scan-manifest.json"), + ); const artifactProbe = spawnSync( python!, [ @@ -734,10 +774,15 @@ describe("plugin runtime preparation", () => { "module = runpy.run_path(sys.argv[1])", "scan_dir = Path(sys.argv[2])", "names = ('report.md', 'findings.json', 'exports/results.sarif')", - "print(json.dumps({name: module['available_artifact_path'](scan_dir, scan_dir / name) is not None for name in names}))", + "artifacts = {name: module['available_artifact_path'](scan_dir, scan_dir / name) is not None for name in names}", + "near_limit = Path(sys.argv[3])", + "artifacts['nearLimitRequired'] = module['artifact_path'](near_limit, 'scan-manifest.json', required=True) is not None", + "artifacts['nearLimitAvailable'] = module['available_artifact_path'](near_limit, near_limit / 'scan-manifest.json') is not None", + "print(json.dumps(artifacts))", ].join("\n"), workbench, longScanDirectory, + nearLimitScanDirectory, ], { encoding: "utf8" }, ); @@ -746,6 +791,8 @@ describe("plugin runtime preparation", () => { "report.md": true, "findings.json": true, "exports/results.sarif": true, + nearLimitRequired: true, + nearLimitAvailable: true, }); const mixedStarted = spawnSync( python!, @@ -4917,15 +4964,17 @@ describe("runtime directories and plugin Python boundary", () => { "from pathlib import Path", "sys.path.insert(0, sys.argv[1])", "from workbench_scan_start import archive_scan", + "from windows_paths import extended_path", "scan_dir = Path(sys.argv[2])", + "stored_scan_dir = extended_path(scan_dir)", "archived_scan_dir = Path(sys.argv[3])", "connection = sqlite3.connect(':memory:')", "connection.row_factory = sqlite3.Row", "connection.execute('CREATE TABLE scans (id TEXT PRIMARY KEY, status TEXT NOT NULL, scan_dir TEXT NOT NULL, updated_at TEXT NOT NULL)')", "connection.execute('CREATE TABLE scan_artifacts (scan_id TEXT NOT NULL, kind TEXT NOT NULL, path TEXT NOT NULL, PRIMARY KEY (scan_id, kind))')", - "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?)', ('previous-scan', 'complete', str(scan_dir), 'before'))", + "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?)', ('previous-scan', 'complete', str(stored_scan_dir), 'before'))", "artifacts = {'coverage': 'coverage.json', 'findings': 'findings.json', 'manifest': 'scan-manifest.json', 'markdownReport': 'report.md'}", - "connection.executemany('INSERT INTO scan_artifacts VALUES (?, ?, ?)', [('previous-scan', kind, str(scan_dir / path)) for kind, path in artifacts.items()])", + "connection.executemany('INSERT INTO scan_artifacts VALUES (?, ?, ?)', [('previous-scan', kind, str(stored_scan_dir / path)) for kind, path in artifacts.items()])", "args = argparse.Namespace(archive_existing=True, archived_scan_dir=str(archived_scan_dir))", "archive_scan(connection, args, scan_dir, 'after', lambda path: path.resolve(strict=True))", "scan = connection.execute('SELECT scan_dir FROM scans WHERE id = ?', ('previous-scan',)).fetchone()", From 1fa404ff117da2c4bf3b7a3bb25a0f0eba49dce1 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 23:31:36 -0700 Subject: [PATCH 11/32] fix(windows): reconcile legacy target and Unicode paths --- .../_bundled_plugin/scripts/windows_paths.py | 3 +- .../scripts/windows_scan_local_files.py | 3 + .../_bundled_plugin/scripts/workbench_db.py | 8 +- .../scripts/workbench_target_state.py | 45 +++++++- sdk/typescript/tests-ts/runtime.test.ts | 108 ++++++++++++++++++ 5 files changed, 156 insertions(+), 11 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/windows_paths.py b/sdk/typescript/_bundled_plugin/scripts/windows_paths.py index 168874ce66..6e53f48b3c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/windows_paths.py +++ b/sdk/typescript/_bundled_plugin/scripts/windows_paths.py @@ -26,7 +26,8 @@ def filesystem_path(path: Path) -> Path: if os.name != "nt": return path value = os.path.abspath(path) - if value.startswith("\\\\?\\") or len(value) >= WINDOWS_DIRECTORY_PATH_LIMIT: + path_length = len(value.encode("utf-16-le")) // 2 + if value.startswith("\\\\?\\") or path_length >= WINDOWS_DIRECTORY_PATH_LIMIT: return extended_path(Path(value)) return Path(value) diff --git a/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py b/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py index 25a1010332..59c1b68b92 100644 --- a/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py +++ b/sdk/typescript/_bundled_plugin/scripts/windows_scan_local_files.py @@ -25,10 +25,13 @@ import ntpath import os import secrets +import sys from collections.abc import Iterator from ctypes import wintypes from pathlib import Path, PurePosixPath +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) from windows_paths import filesystem_path _msvcrt = importlib.import_module("msvcrt") if os.name == "nt" else None diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 19beafdf49..0d936f0c05 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -156,14 +156,14 @@ def state_dir() -> Path: def database_path() -> Path: - return state_dir() / "workbench.sqlite3" + return filesystem_path(state_dir() / "workbench.sqlite3") @contextmanager def scan_completion_lock(scan_id: str) -> Any: - lock_dir = state_dir() / "completion-locks" + lock_dir = filesystem_path(state_dir() / "completion-locks") lock_dir.mkdir(parents=True, exist_ok=True) - lock_path = lock_dir / f"{require_uuid(scan_id, 'scan-id')}.lock" + lock_path = filesystem_path(lock_dir / f"{require_uuid(scan_id, 'scan-id')}.lock") descriptor = os.open( lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), @@ -3750,7 +3750,7 @@ def main() -> None: elif args.command == "export-findings": result = export_findings(connection, args) elif args.command == "database-info": - result = {"databasePath": str(database_path())} + result = {"databasePath": str(portable_path(database_path()))} else: raise SystemExit(f"Unknown command: {args.command}") print(json.dumps(result, allow_nan=False, sort_keys=True)) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 47d2ea6d38..50abe067b7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -5,9 +5,14 @@ import argparse import hashlib import sqlite3 +import sys from datetime import datetime, timezone from pathlib import Path +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from windows_paths import extended_path, portable_path + def stable_target_id(target: Path) -> str: digest = hashlib.sha256(f"local-workspace\0{target}".encode()).hexdigest() @@ -36,13 +41,41 @@ def backfill_security_targets(connection: sqlite3.Connection) -> None: def ensure_security_target(connection: sqlite3.Connection, target_path: str) -> str: + target = portable_path(Path(target_path)) + target_path = str(target) + extended_target_path = str(extended_path(target)) existing = connection.execute( - "SELECT id FROM security_targets WHERE current_path = ?", - (target_path,), - ).fetchone() - if existing is not None: - return str(existing["id"]) - target_id = stable_target_id(Path(target_path)) + "SELECT id, current_path FROM security_targets WHERE current_path IN (?, ?)", + (target_path, extended_target_path), + ).fetchall() + if existing: + current = next((row for row in existing if row["current_path"] == target_path), existing[0]) + target_id = str(current["id"]) + for previous in existing: + previous_id = str(previous["id"]) + if previous_id == target_id: + continue + for table in ("workspaces", "scans"): + connection.execute( + f"UPDATE {table} SET target_id = ?, target_path = ? WHERE target_id = ?", + (target_id, target_path, previous_id), + ) + connection.execute("DELETE FROM security_targets WHERE id = ?", (previous_id,)) + if current["current_path"] != target_path: + timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + connection.execute( + "UPDATE security_targets SET current_path = ?, display_name = ?, updated_at = ? " + "WHERE id = ?", + (target_path, target.name, timestamp, target_id), + ) + if extended_target_path != target_path: + for table in ("workspaces", "scans"): + connection.execute( + f"UPDATE {table} SET target_path = ? WHERE target_id = ? AND target_path = ?", + (target_path, target_id, extended_target_path), + ) + return target_id + target_id = stable_target_id(target) timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") connection.execute( """ diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 21d2d2664c..2f9a3b26f8 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -370,6 +370,17 @@ describe("plugin runtime preparation", () => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); + const isolatedWindowsHelper = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "windows_scan_local_files.py"), + "--help", + ], + { encoding: "utf8" }, + ); + expect(isolatedWindowsHelper.status, isolatedWindowsHelper.stderr).toBe(0); const output = join(root, "inventory.txt"); const repeatedOutput = join(root, "inventory-repeated.txt"); const generatorArguments = ( @@ -562,6 +573,26 @@ describe("plugin runtime preparation", () => { .map((line) => JSON.parse(line)), ).toEqual([{ path: "SECURITY.md" }, { path: "app.ts" }]); + const emojiCount = Math.ceil( + (248 - root.length - "unicode-".length - 1) / 2, + ); + const unicodeRepository = join( + root, + `unicode-${"๐Ÿ˜€".repeat(emojiCount)}`, + ); + expect(unicodeRepository.length).toBeGreaterThanOrEqual(248); + expect(Array.from(unicodeRepository).length).toBeLessThan(248); + await mkdir(unicodeRepository); + await writeFile(join(unicodeRepository, "app.ts"), "export {};\n"); + const unicodeOutput = join(root, "inventory-unicode-long-path.txt"); + const unicodeInventory = spawnSync( + python!, + generatorArguments(unicodeOutput, unicodeRepository), + inventoryOptions, + ); + expect(unicodeInventory.status, unicodeInventory.stderr).toBe(0); + expect(await readFile(unicodeOutput, "utf8")).toBe("./app.ts\n"); + const policy = spawnSync( python!, [ @@ -762,6 +793,21 @@ describe("plugin runtime preparation", () => { join(longScanDirectory, "scan-manifest.json"), join(nearLimitScanDirectory, "scan-manifest.json"), ); + const nearLimitDatabase = spawnSync( + python!, + ["-I", "-B", workbench, "database-info"], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: nearLimitScanDirectory, + }, + }, + ); + expect(nearLimitDatabase.status, nearLimitDatabase.stderr).toBe(0); + expect(JSON.parse(nearLimitDatabase.stdout)).toEqual({ + databasePath: join(nearLimitScanDirectory, "workbench.sqlite3"), + }); const artifactProbe = spawnSync( python!, [ @@ -894,6 +940,68 @@ describe("plugin runtime preparation", () => { workspace: { targetPath: longRepository }, }); + const legacyTarget = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import sqlite3, sys", + "from pathlib import Path", + "sys.path.insert(0, sys.argv[1])", + "from windows_paths import extended_path, filesystem_path", + "database = filesystem_path(Path(sys.argv[2]) / 'workbench.sqlite3')", + "connection = sqlite3.connect(database)", + "target_path = sys.argv[3]", + "target_id = connection.execute('SELECT id FROM security_targets WHERE current_path = ?', (target_path,)).fetchone()[0]", + "legacy_path = str(extended_path(Path(target_path)))", + "connection.execute('UPDATE security_targets SET current_path = ? WHERE id = ?', (legacy_path, target_id))", + "for table in ('workspaces', 'scans'):", + " connection.execute(f'UPDATE {table} SET target_path = ? WHERE target_id = ?', (legacy_path, target_id))", + "connection.commit()", + "print(target_id)", + ].join("\n"), + join(PLUGIN_ROOT, "scripts"), + stateDirectory, + longRepository, + ], + { encoding: "utf8" }, + ); + expect(legacyTarget.status, legacyTarget.stderr).toBe(0); + const migratedTarget = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "start-headless-standard-scan", + "--thread-id", + "migrated-long-path-test", + "--target-path", + longRepository, + "--scope", + ".", + "--scan-root", + scanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(migratedTarget.status, migratedTarget.stderr).toBe(0); + expect(JSON.parse(migratedTarget.stdout)).toMatchObject({ + scan: { + targetId: legacyTarget.stdout.trim(), + targetPath: longRepository, + }, + workspace: { targetPath: longRepository }, + }); + const repositories = spawnSync( python!, ["-I", "-B", workbench, "list-repositories"], From 6391332eced0761c13e8b394c8764d48e9f73d9f Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 23:35:17 -0700 Subject: [PATCH 12/32] test(windows): assert persisted target identity at its contract boundary --- sdk/typescript/tests-ts/runtime.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 2f9a3b26f8..1a98736192 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -996,7 +996,7 @@ describe("plugin runtime preparation", () => { expect(migratedTarget.status, migratedTarget.stderr).toBe(0); expect(JSON.parse(migratedTarget.stdout)).toMatchObject({ scan: { - targetId: legacyTarget.stdout.trim(), + contract: { target: { targetId: legacyTarget.stdout.trim() } }, targetPath: longRepository, }, workspace: { targetPath: longRepository }, From e04622e0ccf2f0eef39abb7500751d6cca6605ce Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 23:50:06 -0700 Subject: [PATCH 13/32] fix(windows): preserve long-path ranking and sealed scan history --- .../scripts/config_preflight.py | 4 +- .../scripts/generate_rank_input.py | 28 +-- .../_bundled_plugin/scripts/workbench_db.py | 8 +- .../scripts/workbench_target_state.py | 12 +- sdk/typescript/tests-ts/runtime.test.ts | 178 ++++++++++++++++-- 5 files changed, 186 insertions(+), 44 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/config_preflight.py b/sdk/typescript/_bundled_plugin/scripts/config_preflight.py index e96d77380f..f73d9c3e90 100644 --- a/sdk/typescript/_bundled_plugin/scripts/config_preflight.py +++ b/sdk/typescript/_bundled_plugin/scripts/config_preflight.py @@ -125,7 +125,7 @@ def parse_args() -> argparse.Namespace: def read_toml(path: Path, *, required: bool) -> dict[str, Any]: try: - with path.open("rb") as file: + with filesystem_path(path).open("rb") as file: return tomllib.load(file) except FileNotFoundError: if required: @@ -235,7 +235,7 @@ def resolve_project_root(cwd: Path, config_layers: list[tuple[Path, dict[str, An if not markers: return cwd for candidate in (cwd, *cwd.parents): - if any((candidate / marker).exists() for marker in markers): + if any(filesystem_path(candidate / marker).exists() for marker in markers): return candidate return cwd diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py index f20831022b..363b37b76f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py @@ -346,6 +346,7 @@ def write_jsonl(output: Path, rows: list[JsonRow]) -> None: def write_json(output: Path, payload: dict[str, object]) -> None: + output = filesystem_path(output) output.parent.mkdir(parents=True, exist_ok=True) output.write_text( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", @@ -355,7 +356,7 @@ def write_json(output: Path, payload: dict[str, object]) -> None: def load_scopes_file(scopes_file: Path) -> list[str]: try: - loaded: object = json.loads(scopes_file.read_text(encoding="utf-8")) + loaded: object = json.loads(filesystem_path(scopes_file).read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc if ( @@ -608,8 +609,8 @@ def make_repo_scope_input(args: argparse.Namespace) -> None: def bind_repo_scopes(args: argparse.Namespace) -> None: scopes = load_scopes_file(Path(args.scopes_file).expanduser()) - manifest_path = Path(args.manifest).expanduser() - coverage_path = Path(args.coverage).expanduser() + manifest_path = filesystem_path(Path(args.manifest).expanduser()) + coverage_path = filesystem_path(Path(args.coverage).expanduser()) try: manifest: object = json.loads(manifest_path.read_text(encoding="utf-8")) coverage: object = json.loads(coverage_path.read_text(encoding="utf-8")) @@ -762,7 +763,7 @@ def make_rank_shards(args: argparse.Namespace) -> None: rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row) require_unique_paths(rows, "Rank input") - output_dir = Path(args.out_dir).expanduser() + output_dir = filesystem_path(Path(args.out_dir).expanduser()) output_dir.mkdir(parents=True, exist_ok=True) existing = sorted((*output_dir.glob(SHARD_INPUT_GLOB), *output_dir.glob(SHARD_OUTPUT_GLOB))) if existing: @@ -833,8 +834,8 @@ def make_rank_pool_plan(args: argparse.Namespace) -> None: if args.usable_worker_slots < 1: raise SystemExit("--usable-worker-slots must be at least 1") - shard_dir = Path(args.shard_dir).expanduser() - output = Path(args.out).expanduser() + shard_dir = filesystem_path(Path(args.shard_dir).expanduser()) + output = filesystem_path(Path(args.out).expanduser()) require_plan_shard_dir(output, shard_dir) input_shards = discover_input_shards(shard_dir) worker_count = min(len(input_shards), args.usable_worker_slots, RANK_POOL_WORKER_CAP) @@ -861,9 +862,10 @@ def make_rank_pool_plan(args: argparse.Namespace) -> None: def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]: - if not plan_path.exists(): + plan_file = filesystem_path(plan_path) + if not plan_file.exists(): raise SystemExit(f"Rank pool plan missing: {plan_path}") - plan_bytes = plan_path.read_bytes() + plan_bytes = plan_file.read_bytes() try: payload: object = json.loads(plan_bytes) except json.JSONDecodeError as exc: @@ -1006,8 +1008,8 @@ def validate_rank_pool_plan( def validate_rank_worker_command(args: argparse.Namespace) -> None: - plan_path = Path(args.plan).expanduser() - shard_dir = Path(args.shard_dir).expanduser() + plan_path = filesystem_path(Path(args.plan).expanduser()) + shard_dir = filesystem_path(Path(args.shard_dir).expanduser()) _, _, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir) slot = require_integer(args.slot, "--slot", minimum=1) @@ -1046,8 +1048,8 @@ def validate_rank_worker_command(args: argparse.Namespace) -> None: def validate_rank_pool_command(args: argparse.Namespace) -> None: - plan_path = Path(args.plan).expanduser() - shard_dir = Path(args.shard_dir).expanduser() + plan_path = filesystem_path(Path(args.plan).expanduser()) + shard_dir = filesystem_path(Path(args.shard_dir).expanduser()) input_shards, expected_output_names, workers, _ = validate_rank_pool_plan(plan_path, shard_dir) actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)} @@ -1109,7 +1111,7 @@ def merge_rank_outputs(args: argparse.Namespace) -> None: authoritative_rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row) require_unique_paths(authoritative_rows, "Rank input") - shard_dir = Path(args.shard_dir).expanduser() + shard_dir = filesystem_path(Path(args.shard_dir).expanduser()) input_shards = discover_input_shards(shard_dir) output_shards = sorted(shard_dir.glob(SHARD_OUTPUT_GLOB)) expected_output_names = { diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 0d936f0c05..d276ae6fbf 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -1218,7 +1218,7 @@ def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[ inventory = Path(artifacts["inScopeFilesPath"]) inventory_descriptor = open_scan_local_file_descriptor( scan_dir, - inventory.relative_to(scan_dir).as_posix(), + portable_path(inventory).relative_to(portable_path(scan_dir)).as_posix(), "Canonical Deep Scan in-scope inventory", ) with os.fdopen(inventory_descriptor, "rb") as source: @@ -1226,7 +1226,7 @@ def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[ in_scope = {re.sub(r"^(?:\./)+", "", line) for line in lines if line} descriptor = open_scan_local_file_descriptor( scan_dir, - ledger.relative_to(scan_dir).as_posix(), + portable_path(ledger).relative_to(portable_path(scan_dir)).as_posix(), "Canonical Deep Scan candidate ledger", ) with os.fdopen(descriptor, "r", encoding="utf-8") as source: @@ -3299,7 +3299,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] artifacts.append(report_path) poc_relative = report_relative.parent / "poc" - poc_root = scan_dir.joinpath(*poc_relative.parts) + poc_root = filesystem_path(scan_dir.joinpath(*poc_relative.parts)) try: if not stat.S_ISDIR(poc_root.stat(follow_symlinks=False).st_mode): return artifacts @@ -3321,7 +3321,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] for file_name in sorted(file_names): candidate = current_path / file_name try: - relative_path = candidate.relative_to(scan_dir).as_posix() + relative_path = portable_path(candidate).relative_to(portable_path(scan_dir)).as_posix() except ValueError: continue if not scan_local_regular_file(scan_dir, relative_path): diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 50abe067b7..9ec29a9001 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -51,16 +51,8 @@ def ensure_security_target(connection: sqlite3.Connection, target_path: str) -> if existing: current = next((row for row in existing if row["current_path"] == target_path), existing[0]) target_id = str(current["id"]) - for previous in existing: - previous_id = str(previous["id"]) - if previous_id == target_id: - continue - for table in ("workspaces", "scans"): - connection.execute( - f"UPDATE {table} SET target_id = ?, target_path = ? WHERE target_id = ?", - (target_id, target_path, previous_id), - ) - connection.execute("DELETE FROM security_targets WHERE id = ?", (previous_id,)) + if len(existing) > 1: + return target_id if current["current_path"] != target_path: timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") connection.execute( diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 1a98736192..2de142e66c 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -695,25 +695,82 @@ describe("plugin runtime preparation", () => { longRankInput, `${JSON.stringify({ path: "app.ts", area: "diff", preview: "export {};" })}\n`, ); - const copiedReview = spawnSync( - python!, - [ - "-I", - "-B", - join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"), - "copy-deep-review-input", - "--rank-input", - longRankInput, - "--out", - longReviewInput, - ], - { encoding: "utf8" }, - ); - expect(copiedReview.status, copiedReview.stderr).toBe(0); + const runRankCommand = (args: string[]) => { + const result = spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"), + ...args, + ], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + }; + runRankCommand([ + "copy-deep-review-input", + "--rank-input", + longRankInput, + "--out", + longReviewInput, + ]); expect( JSON.parse((await readFile(longReviewInput, "utf8")).trim()), ).toEqual({ path: "app.ts", area: "diff" }); + const shardDirectory = join(longScanDirectory, "rank_shards"); + const rankPlan = join(longScanDirectory, "rank_worker_assignments.json"); + const rankOutput = join(longScanDirectory, "rank_output.jsonl"); + runRankCommand([ + "make-rank-shards", + "--rank-input", + longRankInput, + "--out-dir", + shardDirectory, + ]); + runRankCommand([ + "make-rank-pool-plan", + "--shard-dir", + shardDirectory, + "--usable-worker-slots", + "1", + "--out", + rankPlan, + ]); + await writeFile( + join(shardDirectory, "rank-shard-0001.output.jsonl"), + `${JSON.stringify({ path: "app.ts", area: "diff", score: 1, include: true, reason: "review" })}\n`, + ); + runRankCommand([ + "validate-rank-worker", + "--plan", + rankPlan, + "--shard-dir", + shardDirectory, + "--slot", + "1", + ]); + runRankCommand([ + "validate-rank-pool", + "--plan", + rankPlan, + "--shard-dir", + shardDirectory, + ]); + runRankCommand([ + "merge-rank-outputs", + "--rank-input", + longRankInput, + "--shard-dir", + shardDirectory, + "--out", + rankOutput, + ]); + expect( + JSON.parse((await readFile(rankOutput, "utf8")).trim()), + ).toMatchObject({ path: "app.ts", area: "diff", score: 1 }); + const candidates = join(root, "long-path-candidates.jsonl"); const normalized = join(root, "long-path-normalized.jsonl"); await writeFile( @@ -793,6 +850,33 @@ describe("plugin runtime preparation", () => { join(longScanDirectory, "scan-manifest.json"), join(nearLimitScanDirectory, "scan-manifest.json"), ); + const nearLimitConfigDirectory = join(nearLimitScanDirectory, ".codex"); + await mkdir(nearLimitConfigDirectory); + await writeFile( + join(nearLimitConfigDirectory, "config.toml"), + 'model = "near-limit"\n', + ); + const configProbe = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, runpy, sys", + "from pathlib import Path", + "module = runpy.run_path(sys.argv[1])", + "root = Path(sys.argv[2])", + "path = module['project_config_paths'](root, root)[0]", + "print(json.dumps(module['read_toml'](path, required=True)))", + ].join("\n"), + join(PLUGIN_ROOT, "scripts", "config_preflight.py"), + nearLimitScanDirectory, + ], + { encoding: "utf8" }, + ); + expect(configProbe.status, configProbe.stderr).toBe(0); + expect(JSON.parse(configProbe.stdout)).toEqual({ model: "near-limit" }); const nearLimitDatabase = spawnSync( python!, ["-I", "-B", workbench, "database-info"], @@ -808,6 +892,14 @@ describe("plugin runtime preparation", () => { expect(JSON.parse(nearLimitDatabase.stdout)).toEqual({ databasePath: join(nearLimitScanDirectory, "workbench.sqlite3"), }); + const findingDirectory = join( + longScanDirectory, + "findings", + "candidate-1", + ); + await mkdir(join(findingDirectory, "poc"), { recursive: true }); + await writeFile(join(findingDirectory, "candidate-1.md"), "# Finding\n"); + await writeFile(join(findingDirectory, "poc", "repro.txt"), "proof\n"); const artifactProbe = spawnSync( python!, [ @@ -824,6 +916,7 @@ describe("plugin runtime preparation", () => { "near_limit = Path(sys.argv[3])", "artifacts['nearLimitRequired'] = module['artifact_path'](near_limit, 'scan-manifest.json', required=True) is not None", "artifacts['nearLimitAvailable'] = module['available_artifact_path'](near_limit, near_limit / 'scan-manifest.json') is not None", + "artifacts['findingArtifacts'] = module['finding_artifact_paths'](scan_dir, {'writeup': {'reportPath': 'findings/candidate-1/candidate-1.md'}})", "print(json.dumps(artifacts))", ].join("\n"), workbench, @@ -839,6 +932,10 @@ describe("plugin runtime preparation", () => { "exports/results.sarif": true, nearLimitRequired: true, nearLimitAvailable: true, + findingArtifacts: [ + "findings/candidate-1/candidate-1.md", + "findings/candidate-1/poc/repro.txt", + ], }); const mixedStarted = spawnSync( python!, @@ -951,11 +1048,19 @@ describe("plugin runtime preparation", () => { "from pathlib import Path", "sys.path.insert(0, sys.argv[1])", "from windows_paths import extended_path, filesystem_path", + "from workbench_target_state import ensure_security_target", "database = filesystem_path(Path(sys.argv[2]) / 'workbench.sqlite3')", "connection = sqlite3.connect(database)", "target_path = sys.argv[3]", "target_id = connection.execute('SELECT id FROM security_targets WHERE current_path = ?', (target_path,)).fetchone()[0]", "legacy_path = str(extended_path(Path(target_path)))", + "fixture = sqlite3.connect(':memory:')", + "fixture.row_factory = sqlite3.Row", + "fixture.executescript('CREATE TABLE security_targets (id TEXT PRIMARY KEY, current_path TEXT); CREATE TABLE workspaces (target_id TEXT, target_path TEXT); CREATE TABLE scans (target_id TEXT, target_path TEXT)')", + "fixture.executemany('INSERT INTO security_targets VALUES (?, ?)', ((target_id, target_path), ('legacy-target', legacy_path)))", + "fixture.execute('INSERT INTO scans VALUES (?, ?)', ('legacy-target', legacy_path))", + "assert ensure_security_target(fixture, target_path) == target_id", + "assert fixture.execute('SELECT target_id FROM scans').fetchone()[0] == 'legacy-target'", "connection.execute('UPDATE security_targets SET current_path = ? WHERE id = ?', (legacy_path, target_id))", "for table in ('workspaces', 'scans'):", " connection.execute(f'UPDATE {table} SET target_path = ? WHERE target_id = ?', (legacy_path, target_id))", @@ -1059,6 +1164,49 @@ describe("plugin runtime preparation", () => { expect(deepScan.scanDir.length).toBeGreaterThan(260); expect(deepScan.scanDir).not.toStartWith("\\\\?\\"); + const deepDiscovery = join(deepScan.scanDir, "artifacts", "02_discovery"); + await mkdir(deepDiscovery, { recursive: true }); + await writeFile(join(deepDiscovery, "in_scope_files.txt"), "app.ts\n"); + await writeFile( + join(deepDiscovery, "candidate_ledger.jsonl"), + `${JSON.stringify({ + candidate_id: "candidate-1", + summary: "candidate", + evidence: "evidence", + locations: [{ path: "app.ts" }], + })}\n`, + ); + const budgetProbe = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, runpy, sys", + "from pathlib import Path", + "from types import SimpleNamespace", + "module = runpy.run_path(sys.argv[1])", + "module['deep_scan'].configure(SimpleNamespace(require_canonical_scan_directory=module['require_canonical_scan_directory']))", + "scan = {'scan_dir': sys.argv[2]}", + "scan_dir = module['require_canonical_scan_directory'](Path(sys.argv[2]))", + "print(json.dumps(module['budget_exhausted_candidates'](scan, scan_dir)))", + ].join("\n"), + workbench, + deepScan.scanDir, + ], + { encoding: "utf8" }, + ); + expect(budgetProbe.status, budgetProbe.stderr).toBe(0); + expect(JSON.parse(budgetProbe.stdout)).toEqual([ + { + candidate_id: "candidate-1", + summary: "candidate", + evidence: "evidence", + locations: [{ path: "app.ts" }], + }, + ]); + const deepPromptPath = join(deepScan.scanDir, "setup-prompt.md"); const deepArtifactDirectory = join(deepScan.scanDir, "setup-artifacts"); await writeFile(deepPromptPath, "# Setup\n"); From ff8b06b61298de9b33f3170c22fd0c676f246d59 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 14 Aug 2026 23:55:47 -0700 Subject: [PATCH 14/32] fix(windows): retain legacy scan roots in workbench history --- .../scripts/workbench_scan_history.py | 15 +++- sdk/typescript/tests-ts/runtime.test.ts | 78 +++++++++++++------ 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 6e963f4655..ea0cb6e2ce 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -16,7 +16,7 @@ from workbench_constants import FINDINGS_PAGE_MAX from workbench_scan_usage import stored_scan_cost_fields from workbench_target import git_output -from windows_paths import filesystem_path, portable_path +from windows_paths import extended_path, filesystem_path, portable_path def _same_repository( @@ -115,8 +115,15 @@ def list_scans( portable_path(filesystem_path(Path(args.scan_root).expanduser()).resolve()) ) prefix = scan_root.rstrip(os.sep) + os.sep - clauses.append("(scans.scan_dir = ? OR substr(scans.scan_dir, 1, ?) = ?)") - values.extend((scan_root, len(prefix), prefix)) + extended_root = str(extended_path(Path(scan_root))) + extended_prefix = extended_root.rstrip(os.sep) + os.sep + clauses.append( + "(scans.scan_dir IN (?, ?) OR substr(scans.scan_dir, 1, ?) = ? " + "OR substr(scans.scan_dir, 1, ?) = ?)" + ) + values.extend( + (scan_root, extended_root, len(prefix), prefix, len(extended_prefix), extended_prefix) + ) if args is not None and args.target_id: clauses.append("scans.target_id = ?") values.append(args.target_id) @@ -195,7 +202,7 @@ def list_scans( }, "recipeAvailable": row["recipe_json"] is not None, "reasoningEffort": row["reasoning_effort"], - "scanDir": row["scan_dir"], + "scanDir": str(portable_path(Path(row["scan_dir"]))), "scanId": row["id"], "scope": row["scope"], "startedAt": row["started_at"], diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 2de142e66c..6890078c5a 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -1254,36 +1254,68 @@ describe("plugin runtime preparation", () => { }, }); - const listed = spawnSync( + const expectListed = (scanRoot: string) => { + const listed = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + "list-scans", + "--repository", + longRepository, + "--scan-root", + scanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(listed.status, listed.stderr).toBe(0); + expect(JSON.parse(listed.stdout)).toMatchObject({ + scans: [ + { + scanDir: deepScan.scanDir, + scanId: deepScan.scanId, + targetPath: longRepository, + }, + ], + }); + }; + expectListed(longDeepScanRoot); + + const legacyScan = spawnSync( python!, [ "-I", "-B", - workbench, - "list-scans", - "--repository", - longRepository, - "--scan-root", + "-c", + [ + "import sqlite3, sys", + "from pathlib import Path", + "sys.path.insert(0, sys.argv[1])", + "from windows_paths import extended_path, filesystem_path", + "database = filesystem_path(Path(sys.argv[2]) / 'workbench.sqlite3')", + "connection = sqlite3.connect(database)", + "connection.execute('UPDATE scans SET scan_dir = ? WHERE id = ?', (str(extended_path(Path(sys.argv[3]))), sys.argv[4]))", + "connection.commit()", + "print(extended_path(Path(sys.argv[5])))", + ].join("\n"), + join(PLUGIN_ROOT, "scripts"), + stateDirectory, + deepScan.scanDir, + deepScan.scanId, longDeepScanRoot, ], - { - encoding: "utf8", - env: { - ...process.env, - CODEX_SECURITY_STATE_DIR: stateDirectory, - }, - }, + { encoding: "utf8" }, ); - expect(listed.status, listed.stderr).toBe(0); - expect(JSON.parse(listed.stdout)).toMatchObject({ - scans: [ - { - scanDir: deepScan.scanDir, - scanId: deepScan.scanId, - targetPath: longRepository, - }, - ], - }); + expect(legacyScan.status, legacyScan.stderr).toBe(0); + expectListed(longDeepScanRoot); + expectListed(legacyScan.stdout.trim()); } }); From 4641e9b8cd945dcf45f4509442d0ad569964cc17 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 00:04:13 -0700 Subject: [PATCH 15/32] fix(windows): normalize deep scan children and legacy targets --- .../scripts/deep_scan_workbench.py | 50 +++++------ .../scripts/workbench_scan_history.py | 35 ++++---- sdk/typescript/tests-ts/runtime.test.ts | 86 +++++++++++++++++++ 3 files changed, 130 insertions(+), 41 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index e14e9b80f2..c5dc9d5eca 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -197,8 +197,8 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path: return dependencies().require_canonical_scan_directory(scan_dir) -def scan_directory_path(scan: sqlite3.Row) -> Path: - return filesystem_path(Path(scan["scan_dir"])) +def scan_directory_path(scan: sqlite3.Row, *parts: str) -> Path: + return filesystem_path(Path(scan["scan_dir"]).joinpath(*parts)) def safe_segment(value: str) -> str: @@ -304,7 +304,11 @@ def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, output = filesystem_path(Path(output_path)) if staged == output: raise SystemExit("A staged Deep Scan artifact must not be its published output path.") - backup = output.with_name(f".{output.name}.{uuid.uuid4()}.backup") if output.exists() else None + backup = ( + filesystem_path(output.with_name(f".{output.name}.{uuid.uuid4()}.backup")) + if output.exists() + else None + ) if backup is not None: os.replace(output, backup) try: @@ -331,10 +335,13 @@ def finish_staged_file(promotion: tuple[Path, Path, Path | None]) -> None: def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: - discovery_dir = scan_directory_path(scan) / "artifacts" / "02_discovery" artifacts = { - "inScopeFilesPath": discovery_dir / "in_scope_files.txt", - "candidateLedgerPath": discovery_dir / "candidate_ledger.jsonl", + "inScopeFilesPath": scan_directory_path( + scan, "artifacts", "02_discovery", "in_scope_files.txt" + ), + "candidateLedgerPath": scan_directory_path( + scan, "artifacts", "02_discovery", "candidate_ledger.jsonl" + ), } labels = { "inScopeFilesPath": "Canonical in-scope inventory path", @@ -373,13 +380,8 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A and run["status"] == "succeeded" and run["manifest_path"] is not None and run["manifest_path"] - != str(portable_path(scan_directory_path(scan) / "scan-manifest.json")) - and ( - scan_directory_path(scan) - / "artifacts" - / "02_discovery" - / "in_scope_files.txt" - ).exists() + != str(portable_path(scan_directory_path(scan, "scan-manifest.json"))) + and scan_directory_path(scan, "artifacts", "02_discovery", "in_scope_files.txt").exists() ): canonical_artifacts = canonical_discovery_artifacts(scan) if ( @@ -912,11 +914,11 @@ def coordinator_lease_is_live( seconds=DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS ) heartbeat_time = datetime.fromisoformat(str(run["updated_at"])) - heartbeat_path = ( - scan_directory_path(scan) - / "artifacts" - / "deep_discovery" - / f"coordinator-heartbeat-{run['coordinator_generation']}.json" + heartbeat_path = scan_directory_path( + scan, + "artifacts", + "deep_discovery", + f"coordinator-heartbeat-{run['coordinator_generation']}.json", ) try: heartbeat = json.loads(heartbeat_path.read_text(encoding="utf-8")) @@ -1073,7 +1075,7 @@ def recover_expired_coordinator( def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None: scan = require_scan(connection, scan_id) - ledger = scan_directory_path(scan) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl" + ledger = scan_directory_path(scan, "artifacts", "02_discovery", "candidate_ledger.jsonl") backups = sorted( ledger.parent.glob(f".{ledger.name}.*.backup"), key=lambda backup: backup.stat().st_mtime_ns, @@ -1092,7 +1094,7 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id (scan_id,), ) for reducer in reducers: - snapshot = filesystem_path(Path(reducer["artifact_dir"])) / "canonical" / ledger.name + snapshot = filesystem_path(Path(reducer["artifact_dir"]) / "canonical" / ledger.name) if not snapshot.exists(): continue published = ledger.exists() and ledger.samefile(snapshot) @@ -1557,7 +1559,7 @@ def commit_deep_scan_dedup_locked( "Staged candidate ledger path", kind="file", ) - discovery_dir = scan_directory_path(scan) / "artifacts" / "02_discovery" + discovery_dir = scan_directory_path(scan, "artifacts", "02_discovery") deep_scan_path( scan, str(discovery_dir / "in_scope_files.txt"), @@ -1678,7 +1680,7 @@ def finish_deep_scan_locked( ) ) standard_scan_manifest = manifest_path == str( - portable_path(scan_directory_path(scan) / "scan-manifest.json") + portable_path(scan_directory_path(scan, "scan-manifest.json")) ) failure_capped = False @@ -1690,7 +1692,7 @@ def finish_deep_scan_locked( for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, - str(scan_directory_path(scan) / artifact_name), + str(scan_directory_path(scan, artifact_name)), f"Canonical parent {artifact_name}", kind="file", ) @@ -1794,7 +1796,7 @@ def finish_deep_scan_locked( for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, - str(scan_directory_path(scan) / artifact_name), + str(scan_directory_path(scan, artifact_name)), f"Canonical parent {artifact_name}", kind="file", ) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index ea0cb6e2ce..8ca75f132c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -29,6 +29,8 @@ def _same_repository( return True before_target = filesystem_path(Path(before["target_path"])) after_target = filesystem_path(Path(after["target_path"])) + if portable_path(before_target) == portable_path(after_target): + return True before_git_dir = git_output( before_target, "rev-parse", "--path-format=absolute", "--git-common-dir" ) @@ -77,6 +79,17 @@ def _repository_origin(target: Path) -> tuple[str, str] | None: return (host.lower(), path) if host and path else None +def _requested_repository(connection: sqlite3.Connection, repository: Path) -> sqlite3.Row: + repository_path = str(portable_path(repository)) + return connection.execute( + """ + SELECT COALESCE((SELECT id FROM security_targets WHERE current_path IN (?, ?)), '') + AS target_id, ? AS target_path + """, + (repository_path, str(extended_path(Path(repository_path))), repository_path), + ).fetchone() + + def list_scans( connection: sqlite3.Connection, args: argparse.Namespace | None = None ) -> dict[str, Any]: @@ -85,13 +98,7 @@ def list_scans( if args is not None and args.repository: repository = filesystem_path(Path(args.repository).expanduser()).resolve() repository_path = str(portable_path(repository)) - requested_repository = connection.execute( - """ - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - """, - (repository_path, repository_path), - ).fetchone() + requested_repository = _requested_repository(connection, repository) requested_identity = ( git_output(repository, "rev-parse", "--path-format=absolute", "--git-common-dir"), _repository_origin(repository), @@ -103,8 +110,8 @@ def list_scans( ) if _same_repository(target, requested_repository, after_identity=requested_identity) ] - repository_clauses = ["scans.target_path = ?"] - values.append(repository_path) + repository_clauses = ["scans.target_path IN (?, ?)"] + values.extend((repository_path, str(extended_path(Path(repository_path))))) if related_target_ids: placeholders = ", ".join("?" for _ in related_target_ids) repository_clauses.append(f"scans.target_id IN ({placeholders})") @@ -207,7 +214,7 @@ def list_scans( "scope": row["scope"], "startedAt": row["started_at"], "targetId": row["target_id"], - "targetPath": row["target_path"], + "targetPath": str(portable_path(Path(row["target_path"]))), "targetRevision": row["target_revision"], "targetSummary": row["target_summary"], "updatedAt": max(row["updated_at"], row["progress_updated_at"]), @@ -240,13 +247,7 @@ def list_unmatched_scan_pairs( ) -> dict[str, Any]: repository = filesystem_path(Path(args.repository).expanduser()).resolve() repository_path = str(portable_path(repository)) - requested = connection.execute( - """ - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - """, - (repository_path, repository_path), - ).fetchone() + requested = _requested_repository(connection, repository) selected = [ scan for scan in connection.execute( diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 6890078c5a..056487a0e2 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -892,6 +892,67 @@ describe("plugin runtime preparation", () => { expect(JSON.parse(nearLimitDatabase.stdout)).toEqual({ databasePath: join(nearLimitScanDirectory, "workbench.sqlite3"), }); + const nearLimitDiscovery = join( + nearLimitScanDirectory, + "artifacts", + "02_discovery", + ); + const nearLimitWorkers = join( + nearLimitScanDirectory, + "artifacts", + "deep_discovery", + ); + const nearLimitReducer = join(nearLimitWorkers, "reducer"); + await mkdir(nearLimitDiscovery, { recursive: true }); + await mkdir(join(nearLimitReducer, "canonical"), { recursive: true }); + await writeFile( + join(nearLimitWorkers, "coordinator-heartbeat-2.json"), + JSON.stringify({ + coordinatorGeneration: 2, + updatedAt: "2026-01-01T00:00:10+00:00", + }), + ); + await writeFile( + join(nearLimitDiscovery, ".candidate_ledger.jsonl.synthetic.backup"), + "restored\n", + ); + await writeFile( + join(nearLimitReducer, "canonical", "candidate_ledger.jsonl"), + "pending\n", + ); + const nearLimitDeepScan = spawnSync( + python!, + [ + "-I", + "-B", + "-c", + [ + "import json, runpy, sqlite3, sys", + "from types import SimpleNamespace", + "module = runpy.run_path(sys.argv[1])", + "scan = {'scan_dir': sys.argv[2]}", + "module['configure'](SimpleNamespace(require_scan=lambda _connection, _scan_id: scan))", + "connection = sqlite3.connect(':memory:')", + "connection.row_factory = sqlite3.Row", + "connection.execute('CREATE TABLE deep_scan_workers (scan_id TEXT, status TEXT, artifact_dir TEXT, kind TEXT, updated_at TEXT)')", + "connection.execute('INSERT INTO deep_scan_workers VALUES (?, ?, ?, ?, ?)', ('scan', 'running', sys.argv[3], 'dedup', '2026-01-01T00:00:00+00:00'))", + "run = {'coordinator_generation': 2, 'updated_at': '2000-01-01T00:00:00+00:00'}", + "live = module['coordinator_lease_is_live'](connection, run, scan, '2026-01-01T00:00:20+00:00')", + "module['recover_candidate_ledger_publication'](connection, 'scan')", + "ledger = module['scan_directory_path'](scan, 'artifacts', '02_discovery', 'candidate_ledger.jsonl')", + "print(json.dumps({'heartbeatLive': live, 'recoveredLedger': ledger.read_text(encoding='utf-8')}))", + ].join("\n"), + join(PLUGIN_ROOT, "scripts", "deep_scan_workbench.py"), + nearLimitScanDirectory, + nearLimitReducer, + ], + { encoding: "utf8" }, + ); + expect(nearLimitDeepScan.status, nearLimitDeepScan.stderr).toBe(0); + expect(JSON.parse(nearLimitDeepScan.stdout)).toEqual({ + heartbeatLive: true, + recoveredLedger: "restored\n", + }); const findingDirectory = join( longScanDirectory, "findings", @@ -1074,6 +1135,31 @@ describe("plugin runtime preparation", () => { { encoding: "utf8" }, ); expect(legacyTarget.status, legacyTarget.stderr).toBe(0); + for (const repositoryPath of [ + longRepository, + `\\\\?\\${longRepository}`, + ]) { + const legacyHistory = spawnSync( + python!, + ["-I", "-B", workbench, "list-scans", "--repository", repositoryPath], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(legacyHistory.status, legacyHistory.stderr).toBe(0); + expect(JSON.parse(legacyHistory.stdout)).toMatchObject({ + scans: [ + { + targetId: legacyTarget.stdout.trim(), + targetPath: longRepository, + }, + ], + }); + } const migratedTarget = spawnSync( python!, [ From 5c9c09fd24b48a6bdab4f70b02bcddf6ae8139e2 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 00:13:54 -0700 Subject: [PATCH 16/32] fix(windows): extend generated temporary scan artifact paths --- .../scripts/deep_scan_workbench.py | 18 ++- .../scripts/generate_in_scope_files.py | 4 +- .../scripts/normalize_candidates.py | 4 +- .../scripts/workbench_scan_start.py | 6 +- sdk/typescript/tests-ts/runtime.test.ts | 111 +++++++++++++++--- 5 files changed, 108 insertions(+), 35 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index c5dc9d5eca..34402bda84 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -26,7 +26,7 @@ worktree_content_digest, ) from workbench_validation import optional_text, require_uuid, user_text -from windows_paths import filesystem_path, portable_path +from windows_paths import extended_path, filesystem_path, portable_path DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled") @@ -299,16 +299,16 @@ def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str: return str(output) +def temporary_artifact_path(path: Path, suffix: str) -> Path: + return filesystem_path(path.with_name(f".{path.name}.{uuid.uuid4()}.{suffix}")) + + def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]: staged = filesystem_path(Path(staged_path)) output = filesystem_path(Path(output_path)) if staged == output: raise SystemExit("A staged Deep Scan artifact must not be its published output path.") - backup = ( - filesystem_path(output.with_name(f".{output.name}.{uuid.uuid4()}.backup")) - if output.exists() - else None - ) + backup = temporary_artifact_path(output, "backup") if output.exists() else None if backup is not None: os.replace(output, backup) try: @@ -800,7 +800,7 @@ def begin_deep_scan_for_target( Path( tempfile.mkdtemp( prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", - dir=target_root, + dir=extended_path(target_root), ) ) ).resolve() @@ -1596,9 +1596,7 @@ def commit_deep_scan_dedup_locked( raise SystemExit("Dedup inputs are not in the claimed merging state.") if candidate_ledger_path and canonical_candidate_ledger_path: canonical_path = filesystem_path(Path(canonical_candidate_ledger_path)) - publication_copy = canonical_path.with_name( - f".{canonical_path.name}.{uuid.uuid4()}.publish" - ) + publication_copy = temporary_artifact_path(canonical_path, "publish") os.link(filesystem_path(Path(candidate_ledger_path)), publication_copy) promotion = promote_staged_file( str(publication_copy), diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py index 560ed0fcca..891695b996 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py @@ -12,7 +12,7 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from windows_paths import filesystem_path, portable_path +from windows_paths import extended_path, filesystem_path, portable_path class InventoryError(ValueError): @@ -248,7 +248,7 @@ def write_inventory(output: Path, rows: list[bytes]) -> int: try: with tempfile.NamedTemporaryFile( mode="wb", - dir=output.parent, + dir=extended_path(output.parent), prefix=f".{output.name}.", suffix=".tmp", delete=False, diff --git a/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py b/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py index 56e50b0c5f..74f30599ab 100644 --- a/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py +++ b/sdk/typescript/_bundled_plugin/scripts/normalize_candidates.py @@ -14,7 +14,7 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from windows_paths import filesystem_path, portable_path +from windows_paths import extended_path, filesystem_path, portable_path CWE = re.compile(r"(?i)CWE-(\d+)") ROLES = { @@ -326,7 +326,7 @@ def main() -> None: with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", - dir=output.parent, + dir=extended_path(output.parent), prefix=f".{output.name}.", suffix=".tmp", delete=False, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index d8938cbe72..51254c0fc3 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -130,7 +130,9 @@ def archive_scan( "The archived scan directory is required to preserve existing scan artifacts." ) archived_scan_dir = Path( - tempfile.mkdtemp(prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent) + tempfile.mkdtemp( + prefix=f"{scan_dir.name}.previous-", dir=extended_path(scan_dir.parent) + ) ).resolve() connection.execute( "UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?", @@ -176,7 +178,7 @@ def insert_running_scan( scan_dir = Path( tempfile.mkdtemp( prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", - dir=target_root, + dir=extended_path(target_root), ) ).resolve() connection.execute( diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 056487a0e2..365d62739c 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -782,23 +782,25 @@ describe("plugin runtime preparation", () => { evidence: "evidence", })}\n`, ); - const normalization = spawnSync( - python!, - [ - "-I", - "-B", - join(PLUGIN_ROOT, "scripts", "normalize_candidates.py"), - "--input", - candidates, - "--out", - normalized, - "--repo-root", - longRepository, - "--in-scope-files", - longOutput, - ], - { encoding: "utf8" }, - ); + const normalizeCandidates = (output: string) => + spawnSync( + python!, + [ + "-I", + "-B", + join(PLUGIN_ROOT, "scripts", "normalize_candidates.py"), + "--input", + candidates, + "--out", + output, + "--repo-root", + longRepository, + "--in-scope-files", + longOutput, + ], + { encoding: "utf8" }, + ); + const normalization = normalizeCandidates(normalized); expect(normalization.status, normalization.stderr).toBe(0); expect( JSON.parse((await readFile(normalized, "utf8")).trim()), @@ -806,6 +808,26 @@ describe("plugin runtime preparation", () => { cwe_ids: ["CWE-20"], locations: [{ path: "app.ts", start_line: 1, role: "sink" }], }); + const atomicDirectory = join(root, "t".repeat(224 - root.length - 1)); + await mkdir(atomicDirectory); + const atomicInventory = join(atomicDirectory, "inventory-output.jsonl"); + const atomicNormalized = join(atomicDirectory, "normalize-output.jsonl"); + expect(atomicInventory.length).toBe(247); + expect(atomicNormalized.length).toBe(247); + const atomicInventoryResult = spawnSync( + python!, + generatorArguments(atomicInventory, longRepository), + inventoryOptions, + ); + expect(atomicInventoryResult.status, atomicInventoryResult.stderr).toBe( + 0, + ); + expect(await readFile(atomicInventory, "utf8")).toContain("./app.ts\n"); + const atomicNormalization = normalizeCandidates(atomicNormalized); + expect(atomicNormalization.status, atomicNormalization.stderr).toBe(0); + expect( + JSON.parse((await readFile(atomicNormalized, "utf8")).trim()), + ).toMatchObject({ cwe_ids: ["CWE-20"] }); const stateDirectory = join( root, @@ -920,6 +942,10 @@ describe("plugin runtime preparation", () => { join(nearLimitReducer, "canonical", "candidate_ledger.jsonl"), "pending\n", ); + const atomicLedger = join(atomicDirectory, "candidate_ledger.json"); + const atomicStagedLedger = join(atomicDirectory, "staged.json"); + await writeFile(atomicLedger, "previous\n"); + await writeFile(atomicStagedLedger, "published\n"); const nearLimitDeepScan = spawnSync( python!, [ @@ -927,7 +953,8 @@ describe("plugin runtime preparation", () => { "-B", "-c", [ - "import json, runpy, sqlite3, sys", + "import json, os, runpy, sqlite3, sys", + "from pathlib import Path", "from types import SimpleNamespace", "module = runpy.run_path(sys.argv[1])", "scan = {'scan_dir': sys.argv[2]}", @@ -940,11 +967,18 @@ describe("plugin runtime preparation", () => { "live = module['coordinator_lease_is_live'](connection, run, scan, '2026-01-01T00:00:20+00:00')", "module['recover_candidate_ledger_publication'](connection, 'scan')", "ledger = module['scan_directory_path'](scan, 'artifacts', '02_discovery', 'candidate_ledger.jsonl')", - "print(json.dumps({'heartbeatLive': live, 'recoveredLedger': ledger.read_text(encoding='utf-8')}))", + "canonical = module['filesystem_path'](Path(sys.argv[4]))", + "publication = module['temporary_artifact_path'](canonical, 'publish')", + "os.link(module['filesystem_path'](Path(sys.argv[5])), publication)", + "promotion = module['promote_staged_file'](str(publication), str(canonical))", + "module['finish_staged_file'](promotion)", + "print(json.dumps({'heartbeatLive': live, 'recoveredLedger': ledger.read_text(encoding='utf-8'), 'publishedLedger': canonical.read_text(encoding='utf-8')}))", ].join("\n"), join(PLUGIN_ROOT, "scripts", "deep_scan_workbench.py"), nearLimitScanDirectory, nearLimitReducer, + atomicLedger, + atomicStagedLedger, ], { encoding: "utf8" }, ); @@ -952,6 +986,7 @@ describe("plugin runtime preparation", () => { expect(JSON.parse(nearLimitDeepScan.stdout)).toEqual({ heartbeatLive: true, recoveredLedger: "restored\n", + publishedLedger: "published\n", }); const findingDirectory = join( longScanDirectory, @@ -1214,6 +1249,44 @@ describe("plugin runtime preparation", () => { ]), }); + const nearLimitScanRoot = join(root, "r".repeat(215 - root.length - 1)); + await mkdir(nearLimitScanRoot); + for (const [command, threadId] of [ + ["start-headless-standard-scan", "near-limit-standard-root"], + ["begin-deep-scan", "near-limit-deep-root"], + ] as const) { + const nearLimitStarted = spawnSync( + python!, + [ + "-I", + "-B", + workbench, + command, + "--thread-id", + threadId, + "--target-path", + longRepository, + "--scope", + ".", + "--scan-root", + nearLimitScanRoot, + ], + { + encoding: "utf8", + env: { + ...process.env, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + }, + ); + expect(nearLimitStarted.status, nearLimitStarted.stderr).toBe(0); + const scan = JSON.parse(nearLimitStarted.stdout)[ + command === "begin-deep-scan" ? "deepScan" : "scan" + ] as { scanDir: string }; + expect(scan.scanDir.length).toBeGreaterThan(260); + expect(scan.scanDir).not.toStartWith("\\\\?\\"); + } + const longDeepScanRoot = join( root, `deep-${"d".repeat(100)}`, From c22f1f6b85527d159139b1dfaabfdb80915e3488 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 30 Aug 2026 11:29:40 -0700 Subject: [PATCH 17/32] test(windows): keep scope path doubles consistent --- .../tests-ts/workbench-windows-compatibility.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts b/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts index d731259da9..42d83e49d5 100644 --- a/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts +++ b/sdk/typescript/tests-ts/workbench-windows-compatibility.test.ts @@ -28,16 +28,19 @@ function probe(program: string, ...args: string[]): void { test("normalizes absolute Windows scopes without accepting escapes", () => { probe( [ - "import sys", + "import ntpath, sys", "from pathlib import PureWindowsPath", "from types import SimpleNamespace", "sys.path.insert(0, sys.argv[1])", "import workbench_db as workbench", + "import windows_paths", "class WindowsPath(PureWindowsPath):", " def resolve(self): return self", " def is_dir(self): return True", "workbench.Path = WindowsPath", "workbench.os = SimpleNamespace(name='nt')", + "windows_paths.Path = WindowsPath", + "windows_paths.os = SimpleNamespace(name='nt', path=ntpath)", "target = WindowsPath('C:/repository')", "assert workbench.require_scope(r'C:\\repository\\src', 'standard', target) == 'src'", "assert workbench.require_scope('src/nested', 'standard', target) == 'src/nested'", From 4e2b8c7d2ebf25eef0f485cbebde1df20e13b66a Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 4 Nov 2026 23:46:34 +0000 Subject: [PATCH 18/32] Keep Windows archive results portable and protect verbatim path identities --- .../scripts/deep_scan_workbench.py | 8 +-- .../codex-security/scripts/windows_paths.py | 53 +++++++++++++------ .../codex-security/scripts/workbench_db.py | 13 +++-- .../codex-security/scripts/workbench_setup.py | 11 +++- .../scripts/workbench_target.py | 9 ++-- .../tests/test_windows_paths.py | 51 ++++++++++++++++++ 6 files changed, 117 insertions(+), 28 deletions(-) diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index c86895f1d8..5a056bfcb0 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_saved_results as saved_results from deep_scan_config import resolve_deep_scan_setup -from windows_paths import filesystem_path, portable_path, portable_relative_path +from windows_paths import extended_path, filesystem_path, portable_path, portable_relative_path from workbench.handoff import require_current_continuation from workbench.storage import archived_deep_path, create_private_directory, saved_deep_workers from workbench_constants import PHASES @@ -1048,7 +1048,7 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id scan = dependencies().require_scan(connection, scan_id) ledger = scan_directory_path(scan, "artifacts", "02_discovery", "candidate_ledger.jsonl") backups = sorted( - ledger.parent.glob(f".{ledger.name}.*.backup"), + extended_path(ledger.parent).glob(f".{ledger.name}.*.backup"), key=lambda backup: backup.stat().st_mtime_ns, reverse=True, ) @@ -1640,7 +1640,9 @@ def fail_deep_scan_locked( kind="file", ) ) - elif recorded_manifest == str(Path(scan["scan_dir"]) / "scan-manifest.json"): + elif recorded_manifest is not None and portable_path( + Path(recorded_manifest) + ) == portable_path(scan_directory_path(scan, "scan-manifest.json")): manifest_path = recorded_manifest if run["status"] in {"failed", "interrupted"} or scan["status"] == "failed": if ( diff --git a/plugins/codex-security/scripts/windows_paths.py b/plugins/codex-security/scripts/windows_paths.py index 348e76a27c..9ccdfa5a0c 100644 --- a/plugins/codex-security/scripts/windows_paths.py +++ b/plugins/codex-security/scripts/windows_paths.py @@ -4,51 +4,72 @@ import argparse import os +import re import string from pathlib import Path WINDOWS_DIRECTORY_PATH_LIMIT = 248 +_NAMESPACE_PREFIXES = ("\\\\?\\", "\\\\.\\") +_DOS_DEVICE = re.compile(r"(?:AUX|CON|CONIN\$|CONOUT\$|NUL|PRN|(?:COM|LPT)[1-9ยนยฒยณ])", re.ASCII) + + +def _ordinary_component(name: str) -> bool: + # Verbatim paths bypass the DOS parsing of names, trailing dots/spaces and streams. + return bool( + name + and not name.endswith((" ", ".")) + and not any(ord(c) < 32 or c in '<>:"|?*' for c in name) + and not _DOS_DEVICE.fullmatch(name.partition(".")[0].rstrip(" ").upper()) + ) def extended_path(path: Path) -> Path: - """Use absolute Win32 extended-length spelling for a filesystem path.""" - if os.name != "nt": + """Use absolute Win32 extended-length spelling for an equivalent filesystem path.""" + if os.name != "nt" or str(path).startswith(_NAMESPACE_PREFIXES): return path value = os.path.abspath(path) - if value.startswith(("\\\\?\\", "\\\\.\\")): - return Path(value) - if value.startswith("\\\\"): - return Path("\\\\?\\UNC\\" + value[2:]) - return Path("\\\\?\\" + value) + candidate = Path("\\\\?\\UNC\\" + value[2:] if value.startswith("\\\\") else "\\\\?\\" + value) + return candidate if portable_path(candidate) != candidate else path def filesystem_path(path: Path) -> Path: """Use Win32 extended-length spelling when a path is near legacy limits.""" - if os.name != "nt": + if os.name != "nt" or str(path).startswith(_NAMESPACE_PREFIXES): return path value = os.path.abspath(path) path_length = len(value.encode("utf-16-le", errors="surrogatepass")) // 2 - if value.startswith("\\\\?\\") or path_length >= WINDOWS_DIRECTORY_PATH_LIMIT: - return extended_path(Path(value)) - return path + return extended_path(Path(value)) if path_length >= WINDOWS_DIRECTORY_PATH_LIMIT else path def portable_path(path: Path) -> Path: - """Remove Win32 extended-length spelling before persisting a path.""" + """Remove an extended-length prefix only when DOS addresses the same path.""" if os.name != "nt": return path value = str(path) if value[:8].lower() == "\\\\?\\unc\\": - return Path("\\\\" + value[8:]) - if value.startswith("\\\\?\\") and value[4:5] in string.ascii_letters and value[5:7] == ":\\": - return Path(value[4:]) + tail = value[8:].rstrip("\\") + components = tail.split("\\") + if len(components) >= 2 and all(_ordinary_component(part) for part in components): + return Path("\\\\" + value[8:]) + elif value.startswith("\\\\?\\") and value[4:5] in string.ascii_letters and value[5:7] == ":\\": + tail = value[7:] + if not tail or all(_ordinary_component(part) for part in tail.split("\\")): + return Path(value[4:]) return path +def uses_windows_namespace(path: Path) -> bool: + """Whether ordinary drive/UNC spelling cannot represent the selected object.""" + return os.name == "nt" and str(portable_path(path)).startswith(_NAMESPACE_PREFIXES) + + def portable_relative_path(path: Path, root: Path) -> Path: - return portable_path(path).relative_to(portable_path(root)) + if os.name == "nt" and path.anchor != root.anchor and path.is_absolute() and root.is_absolute(): + # An extended child can contain a verbatim-only name below an ordinary root. + return extended_path(path).relative_to(extended_path(root)) + return path.relative_to(root) def main() -> None: diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index fbaca5cb05..62b47f6f1a 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -54,7 +54,7 @@ open_scan_local_file_descriptor, ) from finding_preview import bounded_finding_details -from windows_paths import filesystem_path, portable_path, portable_relative_path +from windows_paths import extended_path, filesystem_path, portable_path, portable_relative_path from workbench import handoff from workbench.storage import ( create_private_directory, @@ -1083,7 +1083,8 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(portable_path(scan_dir / "scan-manifest.json")) + if portable_path(Path(run["manifest_path"])) + == portable_path(scan_dir / "scan-manifest.json") else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message) @@ -1624,7 +1625,11 @@ def register_cli_scan( connection.rollback() raise return { - **({"archivedScanDir": str(archived_scan_dir)} if archived_scan_dir is not None else {}), + **( + {"archivedScanDir": str(portable_path(archived_scan_dir))} + if archived_scan_dir is not None + else {} + ), "contract": scan_contract(scan), "lifecycle": json.loads(scan["lifecycle_json"]), "scanDir": str(portable_path(scan_dir)), @@ -2911,7 +2916,7 @@ def finding_artifact_paths(scan_dir: Path, details: dict[str, Any]) -> list[str] return artifacts for directories_seen, (current_directory, directory_names, file_names) in enumerate( - os.walk(poc_root, topdown=True, followlinks=False), start=1 + os.walk(extended_path(poc_root), topdown=True, followlinks=False), start=1 ): if directories_seen > FINDING_ARTIFACT_DIRECTORIES_LIMIT: break diff --git a/plugins/codex-security/scripts/workbench_setup.py b/plugins/codex-security/scripts/workbench_setup.py index a16a93ac6f..f1dbed067f 100644 --- a/plugins/codex-security/scripts/workbench_setup.py +++ b/plugins/codex-security/scripts/workbench_setup.py @@ -10,7 +10,12 @@ from typing import Any from deep_scan_config import resolve_deep_scan_setup -from windows_paths import filesystem_path, portable_path, portable_relative_path +from windows_paths import ( + filesystem_path, + portable_path, + portable_relative_path, + uses_windows_namespace, +) from workbench_cli import parse_args from workbench_constants import DIFF_TARGET_KINDS, EMPTY_GIT_TREE, EMPTY_GIT_TREES, optional_text from workbench_target import ( @@ -26,7 +31,11 @@ def require_target(value: str) -> Path: expanded = Path(value).expanduser() if not expanded.is_absolute(): raise SystemExit("Scan target must be an absolute local directory path.") + if uses_windows_namespace(expanded): + raise SystemExit("Scan target must have an ordinary Windows drive or UNC path.") target = filesystem_path(expanded).resolve() + if uses_windows_namespace(target): + raise SystemExit("Scan target must have an ordinary Windows drive or UNC path.") if not target.is_dir(): raise SystemExit(f"Scan target is not a readable local directory: {target}") return target diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py index 3b1881e758..daa122fb84 100644 --- a/plugins/codex-security/scripts/workbench_target.py +++ b/plugins/codex-security/scripts/workbench_target.py @@ -1169,7 +1169,7 @@ def ignored(directory: str, names: list[str]) -> list[str]: # Empty placeholders retain parent directories while a patch is reversed. for path in junctions: - (destination / path).mkdir() + filesystem_path(destination / path).mkdir() return junctions @@ -1178,8 +1178,9 @@ def restore_directory_junctions(source: Path, destination: Path, junctions: list destination = destination.resolve() for path in junctions: placeholder = require_remediation_target(str(destination / path)) - os.chmod(placeholder, stat.S_IMODE((source / path).lstat().st_mode)) - copy_directory_junction(source / path, placeholder) + source_junction = filesystem_path(source / path) + os.chmod(placeholder, stat.S_IMODE(source_junction.lstat().st_mode)) + copy_directory_junction(source_junction, placeholder) def copy_git_worktree_files( @@ -1260,7 +1261,7 @@ def copy_git_worktree_files( ): junctions.add(relative) for path in junctions: - (destination / path).mkdir(parents=True, exist_ok=True) + filesystem_path(destination / path).mkdir(parents=True, exist_ok=True) copied_target = filesystem_path(destination if pathspec == "." else destination / pathspec) copied_target.mkdir(parents=True, exist_ok=True) return copied_target, sorted(junctions) diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index 0ea94ac60b..2696c70c64 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -38,6 +38,57 @@ def test_filesystem_path_counts_utf16_units_and_preserves_spelling(windows_paths ) == PureWindowsPath("child") +@pytest.mark.parametrize( + "value", + [ + "//?/C:/checkout./source", + "//?/C:/checkout /source", + "//?/C:/AUX.txt/source", + "//?/C:/LPTยน .log", + "//?/C:/dir/file:stream", + "//?/UNC/server/share/checkout.", + "//?/UNC/./pipe/name", + "//?/Volume{123}/source", + "//./pipe/name", + ], +) +def test_verbatim_only_names_preserve_their_namespace(windows_paths, value): + path = PureWindowsPath(value) + assert windows_paths.uses_windows_namespace(path) + assert windows_paths.portable_path(path) == path + assert windows_paths.filesystem_path(path) == path + assert windows_paths.extended_path(path) == path + + +def test_windows_path_conversion_keeps_relative_spelling_and_literal_leaf(windows_paths): + name = PureWindowsPath("sub/source.py") + assert windows_paths.filesystem_path(name) == name + checkout = PureWindowsPath("C:/checkout") + literal = PureWindowsPath("//?/C:/checkout/reports.") + assert windows_paths.portable_relative_path(literal, checkout) == PureWindowsPath("reports.") + assert windows_paths.extended_path(checkout / "reports.") == checkout / "reports." + assert not windows_paths.uses_windows_namespace(windows_paths.extended_path(checkout)) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires Windows namespace I/O") +def test_target_rejects_verbatim_only_sibling_without_changing_its_identity(tmp_path): + paths = load_script("windows_paths") + setup = load_script("workbench_setup") + ordinary = tmp_path / "reports" + ordinary.mkdir() + literal = paths.extended_path(tmp_path) / "reports." + literal.mkdir() + try: + assert not literal.samefile(ordinary) + assert paths.portable_path(literal).samefile(literal) + assert setup.require_target(str(ordinary)).samefile(ordinary) + with pytest.raises(SystemExit, match="ordinary Windows drive or UNC"): + setup.require_target(str(literal)) + finally: + literal.rmdir() + + def test_target_lookup_preserves_both_legacy_ids_after_a_path_spelling_collision( windows_paths, monkeypatch, workbench_db ): From fb51d91b599c9dcd25a60710a9277d07217e0d5c Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 4 Nov 2026 23:51:03 +0000 Subject: [PATCH 19/32] Preserve Windows inventory scope when a long checkout cannot be a child cwd --- .../mcp-app/src/helpers/generate-inventory.ts | 17 ++-- .../mcp-app/src/helpers/inventory-git.ts | 56 ++++++++++- .../tests-ts/inventory-input.test.ts | 96 ++++++++++++++++++- 3 files changed, 154 insertions(+), 15 deletions(-) diff --git a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts index 4fdd1d00c6..830b9ebf23 100644 --- a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts +++ b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts @@ -10,7 +10,6 @@ import { dirname, isAbsolute, parse, relative, sep } from "node:path"; import { parseArgs } from "node:util"; import { mkdir, readFile, writeFile } from "./helper-files"; import { expandHome, windowsFiles, windowsJoin } from "./resolve-security-md"; -import { decodePosixBytes, encodePosixPath } from "./posix-path"; import { compareCodePoints, stringifyJson } from "./json"; import { print, reportCommandError, ArgumentError } from "./rank-worklists"; import { decodeUtf8 } from "./utf8"; @@ -33,6 +32,7 @@ import { blobSamples, changedPaths, directoryPaths, + decodeGitPath, encodeGitPath, ignoredTrackedPaths, runRipgrep, @@ -128,12 +128,12 @@ async function scopeCandidates(repo: string, scope: string): Promise { "!**/{.git,.svn}", "--glob", "!**/{.git,.svn}/**", - "--", - relative(repo, scope) || ".", ], repo, + relative(repo, scope) || ".", ); - } catch { + } catch (error) { + if (windows && environmentValue("RIPGREP_CONFIG_PATH")) throw error; const ignoreNames = [".gitignore", ".ignore", ".rgignore"]; const isIgnoreFile = (path: string) => { try { @@ -175,7 +175,7 @@ async function scopeCandidates(repo: string, scope: string): Promise { throw new Error( `Could not enumerate scoped repository files${result.signal ? ` (${result.signal})` : ""}: ${result.stderr.trim()}`, ); - return decodePosixBytes(result.stdout) + return decodeGitPath(result.stdout) .split("\0") .filter(Boolean) .map((path) => append(repo, path)); @@ -281,10 +281,9 @@ export async function generateInventory( "!**/{.git,.svn}", "--glob", "!**/{.git,.svn}/**", - "--", - requested, ], repo, + requested, ).catch((error: unknown) => { throw new Error( `could not run ripgrep: ${error instanceof Error ? error.message : String(error)}`, @@ -316,13 +315,13 @@ export async function generateInventory( encodeGitPath(paths.map((path) => prefix + path + "\0").join("")), ]); } - for (const path of decodePosixBytes(bytes).split("\0").filter(Boolean)) { + for (const path of decodeGitPath(bytes).split("\0").filter(Boolean)) { if (path.split("/").some((part) => sourceMetadata.has(part))) continue; if (/[\r\n]/u.test(path)) throw new Error( "Repository contains a path that cannot fit in the file inventory", ); - inventoryRows.push(encodePosixPath(path + "\n")); + inventoryRows.push(encodeGitPath(path + "\n")); } } else if (diff) { if ( diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts index d6df559617..da98903df1 100644 --- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts +++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts @@ -15,6 +15,7 @@ import { dirname, delimiter, isAbsolute, + parse, resolve, sep, } from "node:path"; @@ -353,19 +354,64 @@ function gitProcess(repo: string, args: string[]) { export async function runRipgrep( args: string[], repo: string, + scope: string, ): Promise { - if (!windows) return runTool("rg", args, repo); + const scopedArgs = [...args, "--", scope]; + if (!windows) return runTool("rg", scopedArgs, repo); const inherited = inheritedToolPaths([ "RIPGREP_CONFIG_PATH", "CODEX_SECURITY_GIT", ]); - const rawWindowsLookup = [repo, ...args, ...Object.values(inherited)].some( - rawToolValue, - ); + const rawWindowsLookup = [ + repo, + ...scopedArgs, + ...Object.values(inherited), + ].some(rawToolValue); const command = trustedTool("rg", repo, rawWindowsLookup); if (!command) throw Object.assign(new Error("spawn rg ENOENT"), { code: "ENOENT" }); - return runTool(command, args, repo); + if (!repo.startsWith("\\\\?\\") && repo.length < 260) + return runTool(command, scopedArgs, repo); + + const portable = (path: string) => { + const value = path.replaceAll("/", "\\"); + return /^\\\\\\?\\UNC\\/iu.test(value) + ? "\\\\" + value.slice(8) + : value.replace(/^\\\\\\?\\(?=[a-z]:\\)/iu, ""); + }; + const root = portable(repo); + if (root.length < 260 && sameFile(repo, root)) + return runTool(command, scopedArgs, root); + if (environmentValue("RIPGREP_CONFIG_PATH")) { + // Operator globs and ignore files can be relative to the working directory. + try { + return await runTool(command, scopedArgs, repo); + } catch (cause) { + throw new Error( + "Could not start ripgrep in the Windows repository directory required by RIPGREP_CONFIG_PATH.", + { cause }, + ); + } + } + // CreateProcessW can't use a long or extended current directory. Supply the + // scope as an operand and return the same relative names as a run from repo. + const result = await runTool( + command, + [...args, "--path-separator", "/", "--", resolve(repo, scope)], + parse(root).root, + ); + if (result.status !== 0) return result; + const prefix = /^\.([/\\]|$)/u.test(scope) ? "./" : ""; + const rows = decodeGitPath(result.stdout).split("\0").filter(Boolean); + const output = rows.map((path) => { + const absolute = portable(path); + if (!isAbsolute(absolute)) + throw new Error( + "Ripgrep returned a relative path for an absolute scope.", + ); + return prefix + inside(root, absolute) + "\0"; + }); + return { ...result, stdout: encodeGitPath(output.join("")) }; } export async function runTool( diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.ts index 5bebcd0c3a..d71bf68f2b 100644 --- a/sdk/typescript/tests-ts/inventory-input.test.ts +++ b/sdk/typescript/tests-ts/inventory-input.test.ts @@ -12,10 +12,22 @@ import { statSync, writeFileSync, } from "node:fs"; -import { delimiter, dirname, join, parse, relative, sep } from "node:path"; +import { + delimiter, + dirname, + join, + parse, + relative, + sep, + win32, +} from "node:path"; import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js"; import { nodeCommand } from "./support/shell.js"; import { git } from "./git-fixture.js"; +import { + hasWindowsLoopbackShare, + windowsLoopbackPath, +} from "./windows-helper-location.js"; import { PLUGIN_ROOT } from "./plugin-root.js"; import { resolveCodexCommand } from "../src/runtime.js"; import { bundledCodexSdkEnvironment } from "../src/codex-sdk-environment.js"; @@ -27,6 +39,88 @@ const pathNameBytes = process.platform === "darwin" ? Buffer.from("้›ช") : Buffer.from([0xff]); afterEach(temporary.cleanup); +for (const spelling of ["ordinary", "extended", "unc"] as const) { + test.skipIf( + process.platform !== "win32" || + (spelling === "unc" && !hasWindowsLoopbackShare), + )( + `long Windows ${spelling} inventory preserves ignores and scope-relative output`, + () => { + const f = fixture(); + rmSync(join(f.repo, ".git"), { recursive: true, force: true }); + let repo = f.repo; + while (repo.length < 270) repo = join(repo, "nested-๐ŸŒฒ๐ŸŒฒ๐ŸŒฒ๐ŸŒฒ๐ŸŒฒ๐ŸŒฒ"); + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, ".ignore"), "/src/ignored.py\n"); + writeFileSync(join(repo, "src", "visible.py"), "visible"); + writeFileSync(join(repo, "src", "ignored.py"), "ignored"); + const input = + spelling === "unc" + ? windowsLoopbackPath(repo) + : spelling === "extended" + ? win32.toNamespacedPath(repo) + : repo; + const scopes = join(f.root, "scopes.json"); + writeFileSync(scopes, '["src"]'); + for (const command of [ + "generate-in-scope-files", + "make-repo-scope-input", + ]) { + const result = spawnSync( + node, + [ + join(PLUGIN_ROOT, "mcp", "helpers.mjs"), + command, + "--repo", + input, + "--out", + f.out, + ...(command === "generate-in-scope-files" + ? ["--scope", "./src"] + : ["--scopes-file", scopes]), + ], + { env: f.toolEnvironment, encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + const output = readFileSync(f.out, "utf8").trim(); + if (command === "generate-in-scope-files") + expect(output).toBe("./src/visible.py"); + else expect(JSON.parse(output).path).toBe("src/visible.py"); + } + }, + ); +} + +test.skipIf(process.platform !== "win32")( + "an extended Windows repository keeps operator globs anchored to the checkout", + () => { + const f = fixture(); + f.write("src/visible.py"); + f.write("src/hidden.py"); + const config = join(f.root, "rg.conf"); + writeFileSync(config, "--glob\n!/src/hidden.py\n"); + const result = spawnSync( + node, + [ + join(PLUGIN_ROOT, "mcp", "helpers.mjs"), + "generate-in-scope-files", + "--repo", + win32.toNamespacedPath(f.repo), + "--scope", + "src", + "--out", + f.out, + ], + { + env: { ...f.toolEnvironment, RIPGREP_CONFIG_PATH: config }, + encoding: "utf8", + }, + ); + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(f.out, "utf8").trim()).toBe("src/visible.py"); + }, +); + function fixture() { const root = temporary.create("codex-security-inventory-"), repo = join(root, "repository"), From 8f711012157ebefb4be2499619c877cb5bea3fe7 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 4 Nov 2026 23:53:30 +0000 Subject: [PATCH 20/32] Use scan-relative names for Git snapshot order --- .../codex-security/scripts/workbench_target.py | 3 ++- .../codex-security/tests/test_windows_paths.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py index daa122fb84..375ec8c2e2 100644 --- a/plugins/codex-security/scripts/workbench_target.py +++ b/plugins/codex-security/scripts/workbench_target.py @@ -1013,7 +1013,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None: excluded = set(ignored.stdout.split(b"\0")) pending = [path for raw, path in candidates.items() if raw not in excluded] paths.extend(junctions.values()) - return sorted({str(path): path for path in paths}.values(), key=str) + entries = {str(portable_relative_path(path, target)): path for path in paths} + return [entries[name] for name in sorted(entries)] def source_directory_snapshot_paths( diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index 2696c70c64..592d67c5fb 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -3,6 +3,7 @@ import ntpath import os import sqlite3 +import subprocess from pathlib import Path, PureWindowsPath from types import SimpleNamespace @@ -171,3 +172,20 @@ def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Pa assert connection.execute( "SELECT path FROM scan_artifacts WHERE scan_id = ?", (scan_id,) ).fetchone() == (str(archived / "report.md"),) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows path spelling") +def test_git_snapshot_digest_is_independent_of_root_spelling(tmp_path: Path): + paths = load_script("windows_paths") + target = load_script("workbench_target") + subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True) + (tmp_path / "z.txt").write_text("short") + nested = tmp_path + while len(str(nested / "a.txt")) < paths.WINDOWS_DIRECTORY_PATH_LIMIT: + nested /= "alpha-" * 6 + paths.filesystem_path(nested).mkdir(parents=True, exist_ok=True) + paths.filesystem_path(nested / "a.txt").write_text("long") + assert target.directory_content_digest(tmp_path) == target.directory_content_digest( + paths.extended_path(tmp_path) + ) From 620d779e220accaa4bea5e0f700a046d53182059 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 5 Nov 2026 00:16:13 +0000 Subject: [PATCH 21/32] Resolve bundled path helpers when the SDK loads storage directly --- plugins/codex-security/scripts/workbench/storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/codex-security/scripts/workbench/storage.py b/plugins/codex-security/scripts/workbench/storage.py index 831c554c57..19c7bcfb47 100644 --- a/plugins/codex-security/scripts/workbench/storage.py +++ b/plugins/codex-security/scripts/workbench/storage.py @@ -4,9 +4,13 @@ import os import sqlite3 +import sys from pathlib import Path from typing import Any +# SDK hosts also load this file directly with runpy, outside the workbench package. +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from windows_paths import filesystem_path, portable_path From 99e90339dbbccfb967ff4e5a1993bf593bf7f7e9 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 5 Nov 2026 00:19:53 +0000 Subject: [PATCH 22/32] Compare Windows path namespaces consistently at containment checks --- .../codex-security/scripts/windows_paths.py | 14 +++++++++++-- .../codex-security/scripts/workbench_db.py | 16 +++++++++----- .../scripts/workbench_scan_start.py | 14 +++++++------ .../scripts/workbench_target.py | 21 +++++++++++-------- .../tests/test_windows_paths.py | 20 ++++++++++++++++++ 5 files changed, 63 insertions(+), 22 deletions(-) diff --git a/plugins/codex-security/scripts/windows_paths.py b/plugins/codex-security/scripts/windows_paths.py index 9ccdfa5a0c..c7f0e846f9 100644 --- a/plugins/codex-security/scripts/windows_paths.py +++ b/plugins/codex-security/scripts/windows_paths.py @@ -65,10 +65,20 @@ def uses_windows_namespace(path: Path) -> bool: return os.name == "nt" and str(portable_path(path)).startswith(_NAMESPACE_PREFIXES) -def portable_relative_path(path: Path, root: Path) -> Path: +def _comparable_paths(path: Path, root: Path) -> tuple[Path, Path]: if os.name == "nt" and path.anchor != root.anchor and path.is_absolute() and root.is_absolute(): # An extended child can contain a verbatim-only name below an ordinary root. - return extended_path(path).relative_to(extended_path(root)) + return extended_path(path), extended_path(root) + return path, root + + +def portable_is_relative_to(path: Path, root: Path) -> bool: + path, root = _comparable_paths(path, root) + return path.is_relative_to(root) + + +def portable_relative_path(path: Path, root: Path) -> Path: + path, root = _comparable_paths(path, root) return path.relative_to(root) diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 14a3aec610..e349670c1d 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -54,7 +54,13 @@ open_scan_local_file_descriptor, ) from finding_preview import bounded_finding_details -from windows_paths import extended_path, filesystem_path, portable_path, portable_relative_path +from windows_paths import ( + extended_path, + filesystem_path, + portable_is_relative_to, + portable_path, + portable_relative_path, +) from workbench import handoff from workbench.storage import ( create_private_directory, @@ -642,7 +648,7 @@ def save_workspace(connection: sqlite3.Connection, args: argparse.Namespace) -> def scan_target_root(scan_root: str | None, target: Path) -> Path: root = resolve_scan_root(scan_root) target_root = filesystem_path(root / safe_segment(target.name)).resolve() - if portable_path(target_root).is_relative_to(portable_path(target)): + if portable_is_relative_to(target_root, target): raise SystemExit("The scan artifact directory must be outside the selected target.") return target_root @@ -1465,9 +1471,9 @@ def add_warning() -> None: def require_cli_scan_directory(repository: Path, args: argparse.Namespace) -> Path: scan_dir = require_canonical_scan_directory(Path(args.scan_dir).expanduser()) - if portable_path(scan_dir).is_relative_to(portable_path(repository)) or portable_path( - repository - ).is_relative_to(portable_path(scan_dir)): + if portable_is_relative_to(scan_dir, repository) or portable_is_relative_to( + repository, scan_dir + ): raise SystemExit( "The scan artifact directory must be outside and not contain the selected target." ) diff --git a/plugins/codex-security/scripts/workbench_scan_start.py b/plugins/codex-security/scripts/workbench_scan_start.py index adda0faa2d..880b2c18d4 100644 --- a/plugins/codex-security/scripts/workbench_scan_start.py +++ b/plugins/codex-security/scripts/workbench_scan_start.py @@ -19,7 +19,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from filesystem_identity import serialize_filesystem_identity from finalize_scan_contract import write_scan_local_bytes -from windows_paths import extended_path, portable_path, portable_relative_path +from windows_paths import ( + extended_path, + portable_is_relative_to, + portable_path, + portable_relative_path, +) from workbench.storage import state_dir from workbench_feedback import get_scan_feedback from workbench_target import ( @@ -176,8 +181,7 @@ def archive_scan( ), ] if any( - portable_path(path).is_relative_to(portable_scan_dir) - for path in protected_directories + portable_is_relative_to(path, portable_scan_dir) for path in protected_directories ): raise SystemExit( "Cannot archive output containing the workbench state or active database." @@ -226,9 +230,7 @@ def archive_scan( ) def archive_path(value: str | None) -> str | None: - if value is None or not portable_path(Path(value)).is_relative_to( - portable_scan_dir - ): + if value is None or not portable_is_relative_to(Path(value), portable_scan_dir): return value return str(archive_dir / portable_relative_path(Path(value), scan_dir)) diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py index 4572b79e41..8f0e338aaf 100644 --- a/plugins/codex-security/scripts/workbench_target.py +++ b/plugins/codex-security/scripts/workbench_target.py @@ -21,7 +21,13 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) from filesystem_identity import stored_filesystem_identity_matches -from windows_paths import extended_path, filesystem_path, portable_path, portable_relative_path +from windows_paths import ( + extended_path, + filesystem_path, + portable_is_relative_to, + portable_path, + portable_relative_path, +) from windows_scan_local_files import copy_directory_junction from workbench_constants import GIT_REPOSITORY_ENVIRONMENT @@ -261,8 +267,7 @@ def trusted_git_executable(protected_root: Path) -> str | None: ): continue if any( - portable_path(path).is_relative_to(repository) - for path in (lexical, invocation, canonical) + portable_is_relative_to(path, repository) for path in (lexical, invocation, canonical) ): if configured is not None: raise SystemExit("CODEX_SECURITY_GIT must stay outside the protected repository.") @@ -432,7 +437,7 @@ def diff_scope_file_count(target: Path, diff_target: dict[str, str]) -> int | No else: try: resolved = path.resolve(strict=True) - if not portable_path(resolved).is_relative_to(portable_path(target)): + if not portable_is_relative_to(resolved, target): continue with resolved.open("rb") as stream: binary = _stream_is_binary(stream, os.fstat(stream.fileno()).st_size) @@ -756,9 +761,7 @@ def git_worktree_context(target: Path) -> tuple[Path, str]: pathspec = os.fsdecode(prefix.removesuffix(b"\n")).removesuffix("/") or "." scoped = filesystem_path(repository / pathspec).resolve() try: - if not portable_path(scoped).is_relative_to( - portable_path(repository) - ) or not scoped.samefile(target): + if not portable_is_relative_to(scoped, repository) or not scoped.samefile(target): raise ValueError("Git working tree does not contain the selected target") except (OSError, ValueError) as exc: raise SystemExit("Scan target must stay inside its Git working tree.") from exc @@ -1074,7 +1077,7 @@ def raise_walk_error(error: OSError) -> None: excluded_relative = [ portable_relative_path(path, target) for path in excluded - if portable_path(path).is_relative_to(portable_path(target)) + if portable_is_relative_to(path, target) ] paths = ( source_directory_snapshot_paths( @@ -1142,7 +1145,7 @@ def copy_directory_excluding( excluded_relative = [ portable_relative_path(path, source) for path in excluded - if portable_path(path).is_relative_to(portable_path(source)) + if portable_is_relative_to(path, source) ] junctions: list[Path] = [] diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index 592d67c5fb..3a8e3929c4 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -67,6 +67,8 @@ def test_windows_path_conversion_keeps_relative_spelling_and_literal_leaf(window checkout = PureWindowsPath("C:/checkout") literal = PureWindowsPath("//?/C:/checkout/reports.") assert windows_paths.portable_relative_path(literal, checkout) == PureWindowsPath("reports.") + assert windows_paths.portable_is_relative_to(literal / "tool.exe", checkout) + assert not windows_paths.portable_is_relative_to(literal, checkout / "reports") assert windows_paths.extended_path(checkout / "reports.") == checkout / "reports." assert not windows_paths.uses_windows_namespace(windows_paths.extended_path(checkout)) @@ -189,3 +191,21 @@ def test_git_snapshot_digest_is_independent_of_root_spelling(tmp_path: Path): assert target.directory_content_digest(tmp_path) == target.directory_content_digest( paths.extended_path(tmp_path) ) + + +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows path spelling") +def test_git_discovery_rejects_a_tool_under_a_verbatim_only_directory(tmp_path, monkeypatch): + paths = load_script("windows_paths") + target = load_script("workbench_target") + literal = paths.extended_path(tmp_path) / "tools." + literal.mkdir() + candidate = literal / "git.exe" + try: + candidate.touch() + monkeypatch.setenv("CODEX_SECURITY_GIT", str(candidate)) + with pytest.raises(SystemExit, match="outside the protected repository"): + target.trusted_git_executable(tmp_path) + finally: + candidate.unlink(missing_ok=True) + literal.rmdir() From 3f0a13eed4f1a6041d6fa2174154f3cb08fd662d Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 5 Nov 2026 00:38:43 +0000 Subject: [PATCH 23/32] Use native separators when reopening Windows inventory paths --- plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts index cecc2c27af..4f795d540c 100644 --- a/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts +++ b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts @@ -20,8 +20,9 @@ import { isMissingPathError } from "./helper-files"; export const windows = process.platform === "win32"; export const fsPath = (path: string) => windows ? toNamespacedPath(path) : encodePosixPath(path); +// Git and ripgrep return POSIX separators. Win32 verbatim paths require backslashes. export const append = (parent: string, child: string) => - `${parent || "."}${parent.endsWith(sep) ? "" : sep}${child}`; + `${parent || "."}${parent.endsWith(sep) ? "" : sep}${windows ? child.replaceAll("/", sep) : child}`; export const stat = (path: string) => windows ? windowsFiles().stat(Buffer.from(path, "utf16le")) From 9fa7731e222b630fdc4b1a27b14d29bf83946708 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 5 Nov 2026 01:11:01 +0000 Subject: [PATCH 24/32] Include path support in the compressed usage fixture --- sdk/typescript/tests-ts/workbench-compressed-usage.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts b/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts index a263dfcf8c..57890dd9af 100644 --- a/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts +++ b/sdk/typescript/tests-ts/workbench-compressed-usage.test.ts @@ -40,6 +40,7 @@ async function fixture() { const plugin = join(root, "plugin"); const paths = [ "scripts/workbench_scan_usage.py", + "scripts/windows_paths.py", "scripts/windows_scan_local_files.py", "scripts/read_codex_session.mjs", "scripts/workbench/__init__.py", From 1c153c9b1964828971c2237cf05abe39639facc5 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 5 Nov 2026 01:15:52 +0000 Subject: [PATCH 25/32] Recover saved worker outputs across equivalent Windows path spellings --- .../scripts/workbench/retained_coverage.py | 9 ++++-- .../scripts/workbench/saved_sources.py | 7 +++-- .../scripts/workbench_coverage_receipts.py | 3 +- .../tests/test_windows_paths.py | 29 +++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py index 7329c0e0db..eff12a08cb 100644 --- a/plugins/codex-security/scripts/workbench/retained_coverage.py +++ b/plugins/codex-security/scripts/workbench/retained_coverage.py @@ -21,6 +21,7 @@ write_scan_local_bytes, ) from review_state import uses_legacy_review +from windows_paths import portable_relative_path from workbench.retained_records import ( _append_previous_finding, _competing_rejection_receipts, @@ -266,7 +267,9 @@ def capture_head(relative: str, canonical: Path | None = None) -> dict[str, tupl latest_reducer: str | None = None if reducer is not None: try: - latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix() + latest_reducer = portable_relative_path( + Path(reducer["result_manifest_path"]), scan_dir + ).as_posix() paths[latest_reducer] = None reducer_paths.add(latest_reducer) except ValueError: @@ -312,7 +315,9 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None checkpoints(f"{output}/checkpoints", worker["id"], attempt) if worker["result_manifest_path"]: try: - current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() + current_path = portable_relative_path( + Path(worker["result_manifest_path"]), scan_dir + ).as_posix() paths[current_path] = worker["id"] current_results.add(current_path) except ValueError: diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py index d8c018a1e7..1c11793f2f 100644 --- a/plugins/codex-security/scripts/workbench/saved_sources.py +++ b/plugins/codex-security/scripts/workbench/saved_sources.py @@ -21,6 +21,7 @@ write_scan_local_bytes, ) from review_state import review_state, uses_legacy_review +from windows_paths import portable_relative_path from workbench_coverage_receipts import capture_receipt_checkpoints @@ -691,7 +692,7 @@ def _latest_iteration(workers: Iterable[Any]) -> Any | None: def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]: - output = Path(worker["artifact_dir"]).relative_to(scan_dir) + output = portable_relative_path(Path(worker["artifact_dir"]), scan_dir) attempts = (output.parent if output.name == "output" else output) / "attempts" archived = [ ((attempts / name).as_posix(), int(name.split("-")[1])) @@ -733,7 +734,9 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st ): try: yield ( - Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(), + portable_relative_path( + Path(worker["result_manifest_path"]), scan_dir + ).as_posix(), worker["kind"], ) except ValueError: diff --git a/plugins/codex-security/scripts/workbench_coverage_receipts.py b/plugins/codex-security/scripts/workbench_coverage_receipts.py index 4b27b39977..cf1a144366 100644 --- a/plugins/codex-security/scripts/workbench_coverage_receipts.py +++ b/plugins/codex-security/scripts/workbench_coverage_receipts.py @@ -16,6 +16,7 @@ open_scan_local_file_descriptor, path_within_scope, ) +from windows_paths import portable_relative_path from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id @@ -115,7 +116,7 @@ def resolve_coverage_receipts( directory = Path(relative).parent if directory.name == "checkpoints": directory = directory.parent - output = Path(worker["artifact_dir"]).relative_to(scan_dir) + output = portable_relative_path(Path(worker["artifact_dir"]), scan_dir) worker_prefix = (output.parent if output.name == "output" else output).as_posix() + "/" active_prefix = output.as_posix() + "/" if directory != output: diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index 3a8e3929c4..f6c8fd7598 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -209,3 +209,32 @@ def test_git_discovery_rejects_a_tool_under_a_verbatim_only_directory(tmp_path, finally: candidate.unlink(missing_ok=True) literal.rmdir() + + +@pytest.mark.cross_platform +@pytest.mark.parametrize("extended_worker", [False, True]) +def test_saved_worker_results_accept_mixed_scan_root_spellings(tmp_path, extended_worker): + paths = load_script("windows_paths") + sources = load_script("workbench/saved_sources") + ordinary = paths.portable_path(tmp_path) + extended = paths.extended_path(ordinary) + root, stored = (ordinary, extended) if extended_worker else (extended, ordinary) + worker = { + "kind": "discovery", + "artifact_dir": str(stored / "worker" / "output"), + "result_manifest_path": str(stored / "worker" / "last-result.json"), + "attempt": 2, + } + receipts = load_script("workbench_coverage_receipts") + assert ( + receipts.resolve_coverage_receipts( + root, {"receiptRefs": []}, worker, "worker/output/result.json" + ) + == [] + ) + assert sources._worker_outputs(root, worker) == [("worker/output", 2)] + assert ("worker/last-result.json", "discovery") in list( + sources._saved_result_paths(root, [worker]) + ) + with pytest.raises(ValueError): + sources._worker_outputs(root, {**worker, "artifact_dir": str(stored.parent / "sibling")}) From af727e0b5440821d0a25dee544774f49f7c03b44 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 6 Nov 2026 04:23:48 +0000 Subject: [PATCH 26/32] fix(windows): compare inventory tool paths using consistent namespace spelling --- .../mcp-app/src/helpers/inventory-git.ts | 28 ++++++++++----- .../mcp-app/tests/test_inventory_paths.ts | 35 +++++++++++++++++-- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts index 718056c416..4925c709aa 100644 --- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts +++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts @@ -10,7 +10,7 @@ import { } from "node:child_process"; import { randomUUID } from "node:crypto"; import type { Writable } from "node:stream"; -import { +import pathSyntax, { basename, dirname, delimiter, @@ -84,6 +84,23 @@ function selectedBundledRipgrep(candidate: string): boolean { } } +// Filesystem resolution can return different Win32 spellings for the same root. +export function toolPathInRepository( + root: string, + candidate: string, + paths = pathSyntax, +): boolean { + const name = paths.relative( + paths.toNamespacedPath(root), + paths.toNamespacedPath(candidate), + ); + return ( + !paths.isAbsolute(name) && + name !== ".." && + !name.startsWith(`..${paths.sep}`) + ); +} + function trustedTool( name: "git" | "rg", target: string, @@ -150,14 +167,7 @@ function trustedTool( ) continue; const inRepository = [resolve(candidate), invocation, resolved].some( - (path) => { - try { - inside(protectedRoot, path); - return true; - } catch { - return false; - } - }, + (path) => toolPathInRepository(protectedRoot, path), ); if (inRepository && !(name === "rg" && selectedBundledRipgrep(resolved))) { if (configured !== undefined) diff --git a/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts index 2dbe96d704..97f193d141 100644 --- a/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts +++ b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import childProcess from "node:child_process"; import { createRequire, syncBuiltinESMExports } from "node:module"; import { constants, tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, posix, win32 } from "node:path"; import { importSource } from "./import-module.ts"; import type { WindowsBinding } from "../../native/windows-binding.mjs"; @@ -12,8 +12,37 @@ const helperUrl = new URL( "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs", import.meta.url, ); -const { git, runTool } = await importSource("src/helpers/inventory-git.ts", { - define: { "import.meta.url": JSON.stringify(helperUrl.href) }, +const { git, runTool, toolPathInRepository } = await importSource( + "src/helpers/inventory-git.ts", + { + define: { "import.meta.url": JSON.stringify(helperUrl.href) }, + }, +); + +test("tool containment recognizes equivalent Windows repository paths", () => { + const contained = (root: string, path: string) => + toolPathInRepository(root, path, win32); + for (const root of [String.raw`C:\repo`, String.raw`\\server\share\repo`]) { + const extended = win32.toNamespacedPath(root); + const child = win32.join(root, "tools", "placeholder"); + assert.equal(contained(extended, child), true); + assert.equal(contained(root, win32.toNamespacedPath(child)), true); + assert.equal(contained(extended, root), true); + assert.equal(contained(root, `${root}-other`), false); + } + assert.equal(contained("C:\\repo", "D:\\repo\\tools"), false); + assert.equal( + contained(String.raw`\\server\share\repo`, String.raw`\\server\other\repo`), + false, + ); + const volume = String.raw`\\?\Volume{00000000-0000-0000-0000-000000000001}\repo`; + assert.equal(contained(volume, win32.join(volume, "tools")), true); + assert.equal(contained(volume, "C:\\repo\\tools"), false); + assert.equal(toolPathInRepository("/repo", "/repo/..notes", posix), true); + assert.equal( + toolPathInRepository("/repo", "/repo-other/tools", posix), + false, + ); }); for (const [name, details] of [ From 1da6ab0a3f8322a6dd29579c61f725b24bed10ce Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 15 Nov 2026 07:30:19 -0800 Subject: [PATCH 27/32] fix(windows): keep workbench SQLite filenames within the pager budget --- plugins/codex-security/scripts/windows_paths.py | 16 ++++++++++++++++ plugins/codex-security/scripts/workbench_db.py | 3 ++- .../scripts/workbench_publication.py | 4 ++-- .../codex-security/tests/test_windows_paths.py | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/plugins/codex-security/scripts/windows_paths.py b/plugins/codex-security/scripts/windows_paths.py index c7f0e846f9..f66699a372 100644 --- a/plugins/codex-security/scripts/windows_paths.py +++ b/plugins/codex-security/scripts/windows_paths.py @@ -9,6 +9,8 @@ from pathlib import Path WINDOWS_DIRECTORY_PATH_LIMIT = 248 +# The default Win32 VFS allows MAX_PATH * 4 UTF-8 bytes; the pager needs eight for "-journal". +_WIN32_SQLITE_FILENAME_BYTES = 260 * 4 - 8 _NAMESPACE_PREFIXES = ("\\\\?\\", "\\\\.\\") _DOS_DEVICE = re.compile(r"(?:AUX|CON|CONIN\$|CONOUT\$|NUL|PRN|(?:COM|LPT)[1-9ยนยฒยณ])", re.ASCII) @@ -60,6 +62,20 @@ def portable_path(path: Path) -> Path: return path +def sqlite_filename(path: Path) -> Path: + """Keep a usable ordinary filename when its equivalent prefix alone exceeds SQLite's limit.""" + if os.name == "nt" and len(str(path).encode("utf-8", errors="surrogatepass")) > ( + _WIN32_SQLITE_FILENAME_BYTES + ): + ordinary = portable_path(path) + if ( + len(str(ordinary).encode("utf-8", errors="surrogatepass")) + <= _WIN32_SQLITE_FILENAME_BYTES + ): + return ordinary + return path + + def uses_windows_namespace(path: Path) -> bool: """Whether ordinary drive/UNC spelling cannot represent the selected object.""" return os.name == "nt" and str(portable_path(path)).startswith(_NAMESPACE_PREFIXES) diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 6135f9b1b4..1e28d4ca4a 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -60,6 +60,7 @@ portable_is_relative_to, portable_path, portable_relative_path, + sqlite_filename, ) from workbench import handoff from workbench.storage import ( @@ -272,7 +273,7 @@ def connect(*, deferred: bool = False) -> sqlite3.Connection: raise for attempt in range(SQLITE_RETRY_ATTEMPTS): try: - connection = sqlite3.connect(path, timeout=5) + connection = sqlite3.connect(sqlite_filename(path), timeout=5) except sqlite3.OperationalError: print(guidance, file=sys.stderr) raise diff --git a/plugins/codex-security/scripts/workbench_publication.py b/plugins/codex-security/scripts/workbench_publication.py index 31e711c0b2..672867bb0a 100644 --- a/plugins/codex-security/scripts/workbench_publication.py +++ b/plugins/codex-security/scripts/workbench_publication.py @@ -28,7 +28,7 @@ write_sarif_projection, ) from validate_scan_contract import validate_contract -from windows_paths import portable_path +from windows_paths import portable_path, sqlite_filename def destination_key(destination: Any) -> str: @@ -226,7 +226,7 @@ def finding_issues(db: Any, payload: Any) -> dict[str, Any]: except PermissionError: pass # Let SQLite report the open failure used by the host's state fallback. if payload["action"] == "inspect": - database_uri = f"file:{quote(str(path), safe='')}?mode=ro" + database_uri = f"file:{quote(str(sqlite_filename(path)), safe='')}?mode=ro" try: connection = sqlite3.connect(database_uri, uri=True, timeout=5) except sqlite3.OperationalError as exc: diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index f6c8fd7598..de8abbcf67 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -39,6 +39,21 @@ def test_filesystem_path_counts_utf16_units_and_preserves_spelling(windows_paths ) == PureWindowsPath("child") +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_sqlite_uses_an_equivalent_filename_within_the_win32_pager_budget(windows_paths, root): + directory = PureWindowsPath(root, *(["็•Œ" * 80] * 4)) + filename = "workbench.sqlite3" + base_bytes = len(str(directory / filename).encode("utf-8")) + 1 + ordinary = directory / ("s" * (1030 - base_bytes)) / filename + extended = windows_paths.filesystem_path(ordinary) + assert len(str(ordinary).encode("utf-8")) == 1030 < len(str(extended).encode("utf-8")) + assert windows_paths.sqlite_filename(extended) == ordinary + smaller = windows_paths.filesystem_path(directory / filename) + assert windows_paths.sqlite_filename(smaller) == smaller + literal = extended.with_name("workbench.sqlite3.") + assert windows_paths.sqlite_filename(literal) == literal + + @pytest.mark.parametrize( "value", [ From 3871950ce150f4617a5266f715c2dc73fc597ffb Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:34:11 -0800 Subject: [PATCH 28/32] fix(windows): retain verbatim filenames during scan archiving --- .../scripts/workbench/storage.py | 31 +++++++++--- .../scripts/workbench_scan_start.py | 7 +-- .../tests/test_windows_paths.py | 48 +++++++++++++++++-- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/plugins/codex-security/scripts/workbench/storage.py b/plugins/codex-security/scripts/workbench/storage.py index f26243d890..c73fd802ad 100644 --- a/plugins/codex-security/scripts/workbench/storage.py +++ b/plugins/codex-security/scripts/workbench/storage.py @@ -11,7 +11,13 @@ # SDK hosts also load this file directly with runpy, outside the workbench package. if not __package__: sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from windows_paths import filesystem_path, portable_path +from windows_paths import ( + extended_path, + filesystem_path, + portable_path, + portable_relative_path, + uses_windows_namespace, +) def state_dir(*, canonical: bool = True) -> Path: @@ -41,19 +47,32 @@ def worker_usage_directory(scan_id: str) -> Path: return state_dir() / "codex-home" / "worker-usage" / scan_id +def relocate_scan_path(scan_dir: Path, archive_dir: Path, value: str | None) -> str | None: + """Move a stored scan-local path to the archive, preserving verbatim-only names.""" + if value is None: + return None + path = Path(value) + try: + relative = portable_relative_path(path, scan_dir) + except ValueError: + return value + if uses_windows_namespace(path): + return str(portable_path(extended_path(archive_dir) / relative)) + return str(portable_path(archive_dir) / relative) + + def archived_deep_path(scan_dir: Path, value: str | None) -> str | None: """Resolve old archive rows that still name the original sibling output.""" if value is None or ".previous-" not in scan_dir.name: return value - path = portable_path(Path(value)) - scan_dir = portable_path(scan_dir) - if not path.is_relative_to(scan_dir.parent): + try: + parts = portable_relative_path(Path(value), scan_dir.parent).parts + except ValueError: return value - parts = path.relative_to(scan_dir.parent).parts if not parts or ".." in parts or not scan_dir.name.startswith(f"{parts[0]}.previous-"): return value # Keep missing files inside the archive too. Never follow the replacement scan's output. - return str(scan_dir.joinpath(*parts[1:])) + return relocate_scan_path(scan_dir.parent / parts[0], scan_dir, value) def saved_deep_workers(connection: sqlite3.Connection, scan: sqlite3.Row) -> list[Any]: diff --git a/plugins/codex-security/scripts/workbench_scan_start.py b/plugins/codex-security/scripts/workbench_scan_start.py index 880b2c18d4..70009ffdbd 100644 --- a/plugins/codex-security/scripts/workbench_scan_start.py +++ b/plugins/codex-security/scripts/workbench_scan_start.py @@ -23,9 +23,8 @@ extended_path, portable_is_relative_to, portable_path, - portable_relative_path, ) -from workbench.storage import state_dir +from workbench.storage import relocate_scan_path, state_dir from workbench_feedback import get_scan_feedback from workbench_target import ( directory_content_digest, @@ -230,9 +229,7 @@ def archive_scan( ) def archive_path(value: str | None) -> str | None: - if value is None or not portable_is_relative_to(Path(value), portable_scan_dir): - return value - return str(archive_dir / portable_relative_path(Path(value), scan_dir)) + return relocate_scan_path(scan_dir, archive_dir, value) connection.create_function("archive_path", 1, archive_path) try: diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index de8abbcf67..eed0b9d481 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -4,6 +4,7 @@ import os import sqlite3 import subprocess +import sys from pathlib import Path, PureWindowsPath from types import SimpleNamespace @@ -132,9 +133,27 @@ def test_target_lookup_preserves_both_legacy_ids_after_a_path_spelling_collision } == {("original", str(alias)), ("current", str(path))} +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_archive_relocation_preserves_verbatim_artifact_names(windows_paths, monkeypatch, root): + monkeypatch.setitem(sys.modules, "windows_paths", windows_paths) + storage = load_script("workbench/storage") + monkeypatch.setattr(storage, "Path", PureWindowsPath) + + scan_dir = PureWindowsPath(root) / "scans" / "run" + archive = scan_dir.with_name("run.previous-test") + for name, archived_root in ( + ("result.json.", windows_paths.extended_path(archive)), + ("result.json", archive), + ): + stored = windows_paths.extended_path(scan_dir) / "worker" / name + expected = archived_root / "worker" / name + assert storage.archived_deep_path(archive, str(stored)) == str(expected) + assert storage.relocate_scan_path(scan_dir, archive, str(stored)) == str(expected) + + @pytest.mark.native_windows @pytest.mark.skipif(os.name != "nt", reason="exercise native Win32 files and SQLite paths") -def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Path): +def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Path, request): paths = load_script("windows_paths") root = tmp_path while len(str(root)) < 270: @@ -146,6 +165,13 @@ def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Pa scan_id = first["scanId"] fail_scan(state_dir, scan_id, "Archive the previous run") paths.filesystem_path(output / "report.md").write_text("# Previous review\n") + archived = root / "output.previous-test" + literal = paths.extended_path(output) / "coverage.json." + archived_literal = paths.extended_path(archived) / literal.name + request.addfinalizer(lambda: literal.unlink(missing_ok=True)) + request.addfinalizer(lambda: archived_literal.unlink(missing_ok=True)) + paths.filesystem_path(output / "coverage.json").write_text("ordinary") + literal.write_text("verbatim") database = paths.filesystem_path(state_dir / "workbench.sqlite3") with sqlite3.connect(database) as connection: target_id = connection.execute( @@ -169,11 +195,19 @@ def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Pa "2026-08-13T00:00:00Z", ), ) + connection.execute( + "INSERT INTO scan_artifacts (scan_id, kind, path, created_at) VALUES (?, ?, ?, ?)", + ( + scan_id, + "coverage", + str(literal), + "2026-08-13T00:00:00Z", + ), + ) for spelling in [repository, paths.extended_path(repository)]: listed = run_workbench(state_dir, "list-scans", "--repository", str(spelling))["scans"] assert [scan["scanId"] for scan in listed] == [scan_id] - archived = root / "output.previous-test" paths.filesystem_path(output).rename(paths.filesystem_path(archived)) paths.filesystem_path(output).mkdir() current = register_scan( @@ -186,9 +220,13 @@ def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Pa assert connection.execute( "SELECT scan_dir FROM scans WHERE id = ?", (scan_id,) ).fetchone() == (str(archived),) - assert connection.execute( - "SELECT path FROM scan_artifacts WHERE scan_id = ?", (scan_id,) - ).fetchone() == (str(archived / "report.md"),) + artifacts = dict( + connection.execute( + "SELECT kind, path FROM scan_artifacts WHERE scan_id = ?", (scan_id,) + ) + ) + assert artifacts["markdownReport"] == str(archived / "report.md") + assert Path(artifacts["coverage"]).read_text() == "verbatim" @pytest.mark.native_windows From 2c39822f0aaf6b8089ae4c125e9465b25ab5b896 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 15 Nov 2026 07:42:03 -0800 Subject: [PATCH 29/32] fix(windows): compare archive roots using equivalent DOS spellings --- .../scripts/workbench/storage.py | 4 +-- .../tests/test_windows_paths.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/plugins/codex-security/scripts/workbench/storage.py b/plugins/codex-security/scripts/workbench/storage.py index c73fd802ad..542a9bef09 100644 --- a/plugins/codex-security/scripts/workbench/storage.py +++ b/plugins/codex-security/scripts/workbench/storage.py @@ -53,7 +53,7 @@ def relocate_scan_path(scan_dir: Path, archive_dir: Path, value: str | None) -> return None path = Path(value) try: - relative = portable_relative_path(path, scan_dir) + relative = portable_relative_path(path, portable_path(scan_dir)) except ValueError: return value if uses_windows_namespace(path): @@ -66,7 +66,7 @@ def archived_deep_path(scan_dir: Path, value: str | None) -> str | None: if value is None or ".previous-" not in scan_dir.name: return value try: - parts = portable_relative_path(Path(value), scan_dir.parent).parts + parts = portable_relative_path(Path(value), portable_path(scan_dir.parent)).parts except ValueError: return value if not parts or ".." in parts or not scan_dir.name.startswith(f"{parts[0]}.previous-"): diff --git a/plugins/codex-security/tests/test_windows_paths.py b/plugins/codex-security/tests/test_windows_paths.py index eed0b9d481..4b42b13093 100644 --- a/plugins/codex-security/tests/test_windows_paths.py +++ b/plugins/codex-security/tests/test_windows_paths.py @@ -151,6 +151,32 @@ def test_archive_relocation_preserves_verbatim_artifact_names(windows_paths, mon assert storage.relocate_scan_path(scan_dir, archive, str(stored)) == str(expected) +@pytest.mark.parametrize("root", ["C:/", "//server/share/"]) +def test_archive_relocation_respects_ordinary_and_literal_parent_spellings( + windows_paths, monkeypatch, root +): + monkeypatch.setitem(sys.modules, "windows_paths", windows_paths) + storage = load_script("workbench/storage") + monkeypatch.setattr(storage, "Path", PureWindowsPath) + + parent = PureWindowsPath(root) / "scans" + source = windows_paths.extended_path(parent) / "run" + archive = source.with_name("run.previous-old") + stored = parent / "run" / "worker" / "result.json." + expected = parent / archive.name / "worker" / stored.name + assert storage.relocate_scan_path(source, archive, str(stored)) == str(expected) + assert storage.archived_deep_path(archive, str(stored)) == str(expected) + + literal_parent = windows_paths.extended_path(parent.parent) / "scans." + source = literal_parent / "run" + archive = source.with_name("run.previous-old") + ordinary = parent.with_name("scans.") / "run" / "worker" / "result.json" + literal = source / "worker" / "result.json" + for path, expected in ((ordinary, ordinary), (literal, archive / "worker" / literal.name)): + assert storage.relocate_scan_path(source, archive, str(path)) == str(expected) + assert storage.archived_deep_path(archive, str(path)) == str(expected) + + @pytest.mark.native_windows @pytest.mark.skipif(os.name != "nt", reason="exercise native Win32 files and SQLite paths") def test_long_scan_archive_and_history_accept_legacy_extended_paths(tmp_path: Path, request): From 32682484c0bd6f4c9e66062ead6e5d2719da18d0 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sun, 15 Nov 2026 07:51:11 -0800 Subject: [PATCH 30/32] test(windows): record and inspect receipts at the SQLite filename limit --- .../tests/test_finding_issue_receipts.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/plugins/codex-security/tests/test_finding_issue_receipts.py b/plugins/codex-security/tests/test_finding_issue_receipts.py index 81244f719c..26119c94f0 100644 --- a/plugins/codex-security/tests/test_finding_issue_receipts.py +++ b/plugins/codex-security/tests/test_finding_issue_receipts.py @@ -86,6 +86,43 @@ def legacy_database(path: Path, validated: dict, destination: dict) -> None: connection.commit() +@pytest.mark.native_windows +@pytest.mark.skipif(os.name != "nt", reason="requires native Windows SQLite") +def test_record_and_inspect_at_the_windows_sqlite_filename_limit(tmp_path: Path) -> None: + scan_dir, validated = source(tmp_path) + name = "workbench.sqlite3" + state = tmp_path + # Keep each directory component short while crossing SQLite's limit with the prefix. + while 1030 - len(str(state / name).encode("utf-8")) > 180: + state /= "็•Œ" * 40 + padding = 1030 - len(str(state / name).encode("utf-8")) - 1 + state /= "s" * padding + path = state / name + assert len(str(path).encode("utf-8")) == 1030 + native_path = load_script("windows_paths").extended_path(path) + native_path.parent.mkdir(parents=True) + + # Establish that this SQLite build can write the ordinary filename with WAL. + try: + with closing(sqlite3.connect(path)) as control: + assert control.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + except sqlite3.OperationalError as exc: + if str(exc) == "unable to open database file": + pytest.skip("native SQLite cannot write the ordinary 1,030-byte control filename") + raise + + receipt = receipt_for(validated) + expected = [{**receipt, "scanId": validated["manifest"]["scan"]["id"]}] + assert issues(state, scan_dir, "record", receipts=[receipt])["receipts"] == expected + with closing(sqlite3.connect(path)) as control: + assert control.execute("SELECT external_id FROM finding_issue_receipts").fetchall() == [ + ("SEC-101",) + ] + written = native_path.read_bytes() + assert issues(state, scan_dir, "inspect")["receipts"] == expected + assert native_path.read_bytes() == written + + def test_inspection_of_portable_source_does_not_create_state(tmp_path: Path) -> None: scan_dir, validated = source(tmp_path) state = tmp_path / "state" From 6daf01f8c93cbbb3eb8777dfbf6d38d4b514c692 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sun, 15 Nov 2026 07:51:37 -0800 Subject: [PATCH 31/32] chore(deps): update the evals lockfile to the patched Mongoose release The branch-head dependency audit flags the older optional dependency. Keep the existing Promptfoo features and resolve the fixed release within its declared range. --- evals/triage-finding/pnpm-lock.yaml | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/evals/triage-finding/pnpm-lock.yaml b/evals/triage-finding/pnpm-lock.yaml index 691230e275..14e6ded965 100644 --- a/evals/triage-finding/pnpm-lock.yaml +++ b/evals/triage-finding/pnpm-lock.yaml @@ -2339,8 +2339,8 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - kareem@3.3.0: - resolution: {integrity: sha512-kpSuLD3/7RenBnjnJdOHXCKC8dTd1JzeOiJhN0necWWci6cC+qX+VuwPnMVgb+a4+KNJSfgqahpnfWaeDXCimw==} + kareem@3.4.1: + resolution: {integrity: sha512-UlgCAFNAdk/HLSsbfkmbH8pJqpEN52Ek8RG53Ys1i3Q3gbCQruSrWpplZlgIGyDHX/LNoLW97ei3OG43s7UhUA==} engines: {node: '>=18.0.0'} keyv-file@5.3.5: @@ -2464,8 +2464,8 @@ packages: resolution: {integrity: sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==} engines: {node: '>=20.19.0'} - mongodb@7.5.0: - resolution: {integrity: sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==} + mongodb@7.7.0: + resolution: {integrity: sha512-XG1xXEqyouAV8GCb3PuRaujofeXKpAJAPvXYWZgHl6Dz4QHSJ1nRLl0+GpqaLTwKvC0RFE9truOur34RNFY1nw==} engines: {node: '>=20.19.0'} peerDependencies: '@aws-sdk/credential-providers': ^3.806.0 @@ -2491,16 +2491,16 @@ packages: socks: optional: true - mongoose@9.9.4: - resolution: {integrity: sha512-Gc7buf0ExrOZ3t8MD8tVzTek7Qr5d+tEwj4GRFmHCtTZvpaDb38EVsXgCkYacPL9ICAftgS+FGe+dQHLRLHhcw==} + mongoose@9.11.4: + resolution: {integrity: sha512-+xME0fduXl1sWF+NZpJHZaNcwQSt+X2vbJ/h7GRSZ/ltLycAkGZSEABhzic0+q3IMF66KqF0viEE5+frchKCAQ==} engines: {node: '>=20.19.0'} mpath@0.9.0: resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} engines: {node: '>=4.0.0'} - mquery@6.0.0: - resolution: {integrity: sha512-b2KQNsmgtkscfeDgkYMcWGn9vZI9YoXh802VDEwE6qc50zxBFQ0Oo8ROkawbPAsXCY1/Z1yp0MagqsZStPWJjw==} + mquery@6.0.1: + resolution: {integrity: sha512-/Nijmq3JcpHwDM+5CwBp5yEl6fUa6jLsIhPsoQywlGXU/ZrKIvSpBzfaGnW9BLqcne/CoUwd1ch7MRqptDgFTA==} engines: {node: '>=20.19.0'} ms@2.0.0: @@ -5903,7 +5903,7 @@ snapshots: safe-buffer: 5.2.1 optional: true - kareem@3.3.0: + kareem@3.4.1: optional: true keyv-file@5.3.5: @@ -6039,7 +6039,7 @@ snapshots: whatwg-url: 14.2.0 optional: true - mongodb@7.5.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): + mongodb@7.7.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): dependencies: '@mongodb-js/saslprep': 1.5.0 bson: 7.3.2 @@ -6050,13 +6050,13 @@ snapshots: socks: 2.8.10 optional: true - mongoose@9.9.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): + mongoose@9.11.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10): dependencies: '@standard-schema/spec': 1.1.0 - kareem: 3.3.0 - mongodb: 7.5.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) + kareem: 3.4.1 + mongodb: 7.7.0(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) mpath: 0.9.0 - mquery: 6.0.0 + mquery: 6.0.1 ms: 2.1.3 sift: 17.1.3 transitivePeerDependencies: @@ -6072,7 +6072,7 @@ snapshots: mpath@0.9.0: optional: true - mquery@6.0.0: + mquery@6.0.1: optional: true ms@2.0.0: {} @@ -6094,7 +6094,7 @@ snapshots: apparatus: 0.0.10 dotenv: 17.4.2 memjs: 1.3.2 - mongoose: 9.9.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) + mongoose: 9.11.4(@aws-sdk/credential-providers@3.1136.0)(gcp-metadata@9.0.3)(socks@2.8.10) pg: 8.23.0 redis: 5.12.1(@opentelemetry/api@1.9.1) safe-stable-stringify: 2.5.0 From dd156058dedc99d960a4f107c081fdd2cdbe57e3 Mon Sep 17 00:00:00 2001 From: Jayanth Yarlagadda <72214836+JayYarlagadda@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:53:14 -0800 Subject: [PATCH 32/32] fix(ci): load the installed Ubuntu AppArmor profile Backport the setup script from #2418 (upstream commit 228020a8b96291e825398a8d60ee264d65a7c940). --- github-action/scripts/setup-ubuntu-sandbox.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/github-action/scripts/setup-ubuntu-sandbox.sh b/github-action/scripts/setup-ubuntu-sandbox.sh index 17db9d0ce6..24ed38009e 100644 --- a/github-action/scripts/setup-ubuntu-sandbox.sh +++ b/github-action/scripts/setup-ubuntu-sandbox.sh @@ -7,4 +7,9 @@ $nrconf{override_rc}{qr(^php8\.3-fpm\.service$)} = 0; EOF sudo apt-get update sudo apt-get install --yes bubblewrap apparmor-profiles -sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict +bwrap_profile=/usr/share/apparmor/extra-profiles/bwrap-userns-restrict +# Ubuntu 26.04 installs this profile under /etc/apparmor.d. +if [[ ! -f "$bwrap_profile" ]]; then + bwrap_profile=/etc/apparmor.d/bwrap-userns-restrict +fi +sudo apparmor_parser -r "$bwrap_profile"