feat: add opt-in line-oriented graph.json output

graphify/cli.py

@@ -957,8 +957,8 @@ def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int
         data["hyperedges"] = kept_hyper
     from graphify.export import backup_if_protected as _backup
     _backup(graph_path.parent)
-    from graphify.paths import write_json_atomic
-    write_json_atomic(graph_path, data, compact=True)
+    from graphify.paths import write_graph_json_atomic
+    write_graph_json_atomic(graph_path, data)
     return n_removed
 
 
@@ -3520,8 +3520,8 @@ def _load_graph(p: str):
             _ts, _tt = _link.pop("_src", None), _link.pop("_tgt", None)
             if _ts is not None and _tt is not None:
                 _link["source"], _link["target"] = _ts, _tt
-        from graphify.paths import write_json_atomic
-        write_json_atomic(_current_path, out_data, compact=True)
+        from graphify.paths import write_graph_json_atomic
+        write_graph_json_atomic(_current_path, out_data)
         sys.exit(0)
 
     elif cmd == "merge-graphs":
@@ -3711,8 +3711,8 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
         # dual-slot shape so every writer agrees.
         out_data["hyperedges"] = merged.graph.get("hyperedges", [])
         out_path.parent.mkdir(parents=True, exist_ok=True)
-        from graphify.paths import write_json_atomic as _wja
-        _wja(out_path, out_data, compact=True)
+        from graphify.paths import write_graph_json_atomic as _wgja
+        _wgja(out_path, out_data)
         print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges")
         print(f"Written to: {out_path}")
 
@@ -5500,14 +5500,13 @@ def _invalidate_file_manifest_for_db_graph() -> None:
                     sys.exit(1)
             _backup(graphify_out)
             _invalidate_file_manifest_for_db_graph()
-            from graphify.paths import write_json_atomic as _write_json_atomic
+            from graphify.paths import write_graph_json_atomic as _write_graph_json_atomic
             from graphify.extract import RUN_ONLY_EXTRACTION_KEYS as _RUN_ONLY_KEYS
             # merge_raw_extraction above has consumed them; graph.json must not
             # carry this run's absolute input paths.
-            _write_json_atomic(
+            _write_graph_json_atomic(
                 graph_json_path,
                 {k: v for k, v in merged.items() if k not in _RUN_ONLY_KEYS},
-                compact=True,
             )
             try:
                 # Record the scan root so a later build_merge / update runbook can

graphify/export.py

@@ -515,9 +515,9 @@ def _canonical(item: dict, lead: tuple[str, ...]) -> dict:
     commit = built_at_commit if built_at_commit is not None else _git_head(Path(output_path).resolve().parent)
     if commit:
         data["built_at_commit"] = commit
-    from graphify.paths import write_json_atomic
+    from graphify.paths import write_graph_json_atomic
     # Atomic write: a crash/ENOSPC mid-write must not truncate a good graph.json.
-    write_json_atomic(output_path, data, compact=True)
+    write_graph_json_atomic(output_path, data)
     return True
 
 

graphify/paths.py

@@ -254,6 +254,100 @@ def write_json_atomic(
         path, lambda f: json.dump(obj, f, indent=indent, ensure_ascii=ensure_ascii, separators=separators)
     )
 
+
+_GRAPH_JSON_ARRAY_KEYS = ("nodes", "links", "hyperedges")
+
+
+def graph_json_lines_enabled() -> bool:
+    """Whether graph.json should be written one element per line.
+
+    Off by default: #3413 chose a single compact line so a committed graph does
+    not report tens of thousands of changed lines. Set
+    ``GRAPHIFY_GRAPH_JSON_LINES=1`` (or ``true``/``yes``) to opt into the
+    line-oriented form described in #4942.
+    """
+    return os.environ.get("GRAPHIFY_GRAPH_JSON_LINES", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+
+
+def _graph_json_item(item: object) -> str:
+    return json.dumps(
+        item, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+    )
+
+
+def _graph_json_lines_text(obj: dict) -> str:
+    keys = [key for key in sorted(obj) if key not in _GRAPH_JSON_ARRAY_KEYS]
+    keys.extend(key for key in _GRAPH_JSON_ARRAY_KEYS if key in obj)
+    lines = ["{"]
+    for index, key in enumerate(keys):
+        separator = "," if index < len(keys) - 1 else ""
+        value = obj[key]
+        if key == "nodes" and isinstance(value, list):
+            value = sorted(
+                value,
+                key=lambda item: (
+                    str(item.get("id", "")),
+                    _graph_json_item(item),
+                ),
+            )
+        elif key == "links" and isinstance(value, list):
+            value = sorted(
+                value,
+                key=lambda item: (
+                    str(item.get("source", "")),
+                    str(item.get("target", "")),
+                    str(item.get("relation", "")),
+                    _graph_json_item(item),
+                ),
+            )
+        elif key == "hyperedges" and isinstance(value, list):
+            value = sorted(value, key=_graph_json_item)
+        if key in _GRAPH_JSON_ARRAY_KEYS and isinstance(value, list):
+            if not value:
+                lines.append(f"{json.dumps(key)}:[]{separator}")
+                continue
+            lines.append(f"{json.dumps(key)}:[")
+            for item_index, item in enumerate(value):
+                lines.append(
+                    _graph_json_item(item)
+                    + ("," if item_index < len(value) - 1 else "")
+                )
+            lines.append(f"]{separator}")
+            continue
+        lines.append(f"{json.dumps(key)}:{_graph_json_item(value)}{separator}")
+    lines.append("}")
+    return "\n".join(lines) + "\n"
+
+
+def graph_json_text(obj: dict) -> str:
+    """Serialize a project graph using the configured layout.
+
+    The default is the compact one-line form from #3413; with
+    ``GRAPHIFY_GRAPH_JSON_LINES`` set it is the line-oriented form from #4942.
+    """
+    if graph_json_lines_enabled():
+        return _graph_json_lines_text(obj)
+    return json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n"
+
+
+def write_graph_json_atomic(path: "str | Path", obj: dict) -> None:
+    """Atomically write a project graph, honoring ``GRAPHIFY_GRAPH_JSON_LINES``.
+
+    The default remains the single compact line from #3413. With the flag set,
+    the file is still valid JSON with the same elements, but top-level keys are
+    sorted, nodes are ordered by ``id``, links by ``(source, target, relation)``
+    and hyperedges by their canonical record, with one element per line and a
+    trailing newline. Nested records stay compact.
+    """
+    if not graph_json_lines_enabled():
+        write_json_atomic(path, obj, compact=True)
+        return
+    _atomic_replace(path, lambda f: f.write(_graph_json_lines_text(obj)))
+
 # Directory segments that, when they appear as a whole path component, mark the
 # whole path as a test location. Matched against path *segments* (not raw
 # substrings) so "src/contest.py" / "latest/x.py" / "src/greatest/x.py" do NOT

graphify/watch.py

@@ -1830,8 +1830,12 @@ def _is_single_line(path: Path) -> bool:
         return not f.readline()
 
 
-def _json_text(data: dict) -> str:
-    return json.dumps(data, ensure_ascii=False, separators=(",", ":")) + "\n"
+def _is_indented_json(path: Path) -> bool:
+    """True for the legacy ``indent=2`` layout, whose second line is indented."""
+    with path.open(encoding="utf-8") as f:
+        f.readline()
+        second = f.readline()
+    return second.startswith("  ")
 
 
 def _stabilize_rebuild_cwd(watch_path: Path) -> bool:
@@ -1995,6 +1999,8 @@ def _current_build_state(
         )
     except OSError:
         rc_digest = ""
+    from graphify.paths import graph_json_lines_enabled
+
     return {
         "epoch": _extractor_epoch(),
         "scan_root": str(watch_root),
@@ -2004,6 +2010,7 @@ def _current_build_state(
         "gitignore": bool(gitignore),
         "excludes": sorted(excludes or []),
         "code_only": bool(code_only),
+        "graph_json_lines": graph_json_lines_enabled(),
         "rcfile": rc_digest,
         # Env knobs that change the written output rather than just the timing:
         # the viz cap removes graph.html outright, and the ambiguous-edge scale
@@ -2650,9 +2657,23 @@ def _add_deleted_source(path: Path) -> None:
             from graphify.extractors.terraform import refresh_terraform_paths
             wanted = refresh_terraform_paths(wanted, code_files, changed_paths)
             if not wanted and not deleted_paths:
-                print("[graphify watch] No tracked code files in change set - skipping rebuild.")
-                return True
-            extract_targets = wanted
+                from graphify.paths import graph_json_lines_enabled
+                try:
+                    _prev_state = json.loads(
+                        (out / _BUILD_STATE_NAME).read_text(encoding="utf-8")
+                    )
+                except (OSError, ValueError):
+                    _prev_state = {}
+                if _prev_state.get("graph_json_lines") == graph_json_lines_enabled():
+                    print("[graphify watch] No tracked code files in change set - skipping rebuild.")
+                    return True
+                # Only the graph layout changed: fall through to a full extraction
+                # so graph.json is rewritten in the requested layout (#4942).
+                extract_targets = [
+                    p for p in code_files if p not in semantic_doc_files
+                ]
+            else:
+                extract_targets = wanted
         else:
             # Full rebuild: skip the AST quick-scan for semantic-backed docs
             # (#1915). They remain in code_files for corpus membership and
@@ -2925,8 +2946,14 @@ def _stamp_build_state() -> None:
             return False
 
         # The equality skips below compare parsed data, not layout; an existing
-        # multi-line graph.json must still be rewritten compact (#3413).
-        legacy_layout = existing_graph.exists() and not _is_single_line(existing_graph)
+        # graph in a different layout must still be rewritten (#3413, #4942).
+        from graphify.paths import graph_json_lines_enabled
+        if graph_json_lines_enabled():
+            legacy_layout = existing_graph.exists() and (
+                _is_single_line(existing_graph) or _is_indented_json(existing_graph)
+            )
+        else:
+            legacy_layout = existing_graph.exists() and not _is_single_line(existing_graph)
 
         _relativize_source_files(result, project_root, scope=watch_root)
         # Source files re-extracted this run — their symbol sets may legitimately
@@ -2990,7 +3017,8 @@ def _stamp_build_state() -> None:
             # attribute-less phantom (#2873). Any other edge without two declared
             # endpoints is dropped, as build_from_json drops it.
             _finalize_raw_graph_endpoints(candidate_graph_data)
-            candidate_graph_text = _json_text(candidate_graph_data)
+            from graphify.paths import graph_json_text
+            candidate_graph_text = graph_json_text(candidate_graph_data)
             same_graph = False
             if existing_graph.exists():
                 try:

tests/test_graph_json_lines.py

@@ -0,0 +1,96 @@
+"""#4942: opt-in line-oriented graph.json.
+
+The default stays the single compact line from #3413. With
+``GRAPHIFY_GRAPH_JSON_LINES=1`` every project-graph writer emits one node or
+link per line, sorted deterministically, while the parsed content is unchanged.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import networkx as nx
+
+from graphify.export import to_json
+from graphify.watch import _rebuild_code
+
+
+def _graph() -> nx.Graph:
+    G = nx.Graph()
+    # Insert out of id order: the writer must sort nodes and links itself.
+    G.add_node("b", label="b", file_type="code", source_file="a.py", community=0)
+    G.add_node("a", label="a", file_type="code", source_file="a.py", community=0)
+    G.add_edge("b", "a", relation="calls", confidence="EXTRACTED", source_file="a.py")
+    return G
+
+
+def _assert_line_oriented(path: Path) -> dict:
+    text = path.read_text(encoding="utf-8")
+    assert text.endswith("\n")
+    data = json.loads(text)
+    lines = text.splitlines()
+    assert lines[0] == "{" and lines[-1] == "}"
+
+    records = {line[:-1] if line.endswith(",") else line for line in lines}
+    for node in data["nodes"]:
+        assert json.dumps(node, ensure_ascii=False, sort_keys=True, separators=(",", ":")) in records
+    for link in data["links"]:
+        assert json.dumps(link, ensure_ascii=False, sort_keys=True, separators=(",", ":")) in records
+
+    assert [node["id"] for node in data["nodes"]] == sorted(
+        node["id"] for node in data["nodes"]
+    )
+    assert [
+        (link["source"], link["target"], link["relation"]) for link in data["links"]
+    ] == sorted(
+        (link["source"], link["target"], link["relation"]) for link in data["links"]
+    )
+    return data
+
+
+def test_to_json_honors_graph_json_lines(tmp_path, monkeypatch):
+    monkeypatch.setenv("GRAPHIFY_GRAPH_JSON_LINES", "1")
+    out = tmp_path / "graph.json"
+
+    assert to_json(_graph(), {0: ["a", "b"]}, str(out), force=True)
+
+    data = _assert_line_oriented(out)
+    assert {node["id"] for node in data["nodes"]} == {"a", "b"}
+    assert len(data["links"]) == 1
+
+
+def test_to_json_defaults_to_one_compact_line(tmp_path, monkeypatch):
+    monkeypatch.delenv("GRAPHIFY_GRAPH_JSON_LINES", raising=False)
+    out = tmp_path / "graph.json"
+
+    assert to_json(_graph(), {0: ["a", "b"]}, str(out), force=True)
+
+    text = out.read_text(encoding="utf-8")
+    assert len(text.splitlines()) == 1
+    assert json.loads(text)["nodes"]
+
+
+def test_graph_json_lines_accepts_false_values(tmp_path, monkeypatch):
+    monkeypatch.setenv("GRAPHIFY_GRAPH_JSON_LINES", "false")
+    out = tmp_path / "graph.json"
+
+    assert to_json(_graph(), {0: ["a", "b"]}, str(out), force=True)
+
+    assert len(out.read_text(encoding="utf-8").splitlines()) == 1
+
+
+
+def test_enabling_the_flag_rewrites_an_existing_compact_graph(tmp_path, monkeypatch):
+    (tmp_path / "app.py").write_text(
+        "def alpha():\n    return beta()\n\n\ndef beta():\n    return 1\n",
+        encoding="utf-8",
+    )
+    graph = tmp_path / "graphify-out" / "graph.json"
+
+    monkeypatch.delenv("GRAPHIFY_GRAPH_JSON_LINES", raising=False)
+    assert _rebuild_code(tmp_path, acquire_lock=False) is True
+    assert len(graph.read_text(encoding="utf-8").splitlines()) == 1
+
+    monkeypatch.setenv("GRAPHIFY_GRAPH_JSON_LINES", "1")
+    assert _rebuild_code(tmp_path, acquire_lock=False) is True
+    _assert_line_oriented(graph)