fix(feed): render orphan ghosts only when JSONL stalls past proxy

docs/design/README.md

@@ -0,0 +1,46 @@
+# `docs/design/` — sources of truth for non-obvious system designs
+
+Files in this directory describe **what cc-shell's most complicated subsystems are and why they are shaped the way they are.** They are referenced by inline code comments and treated as authoritative. They are *not* changelogs, *not* running notes, *not* blow-by-blow histories of how a fix was derived.
+
+## DO NOT change these files lightly
+
+A design doc here gets touched **only when the actual system changes shape.** Renaming a constant, fixing a bug, or shipping a comment cleanup does not change the design — those land as code commits without a doc edit.
+
+You should change a file in `docs/design/` when:
+
+- A subsystem's responsibilities or invariants change.
+- A new component enters or leaves the picture.
+- A previously-load-bearing rule is replaced by a different one.
+- A "deliberately out of scope" item is brought into scope.
+
+You should NOT change a file in `docs/design/` when:
+
+- Fixing a bug whose root cause is consistent with the documented design.
+- Refactoring within the existing rules.
+- Adjusting a tunable (a TTL, a threshold, a cap) that the doc already describes by purpose rather than by value.
+- Rewording a comment for clarity.
+
+When in doubt: the doc is wrong if and only if its description no longer matches reality. If reality matches and the wording is just unclear, that's a code-comment problem, not a design-doc problem.
+
+## What goes here vs. elsewhere
+
+| Kind of doc | Lives in |
+|---|---|
+| Source-of-truth design explanation, evergreen | `docs/design/<topic>.md` (this directory) |
+| Date-stamped plan, executed once | `docs/superpowers/plans/YYYY-MM-DD-<slug>.md` |
+| Long-form diagnostic / forensic write-up of a specific incident | `docs/superpowers/plans/YYYY-MM-DD-<slug>-findings.md` |
+| Style or convention reference | `docs/<topic>-style.md` (root of `docs/`) |
+| README for the repo / a package | `README.md` next to the code |
+
+If you are documenting *what is true today and is expected to stay true*, it belongs here. If you are documenting *what we did in a particular moment*, it belongs in `plans/`.
+
+## How to add a new file
+
+1. Pick a short, durable filename: one subsystem, one file. `ghost-system.md`, not `2026-05-07-ghost-fix.md`.
+2. Write it in plain English first, then the technical rules. Aim for the level a senior engineer joining the team next month needs — not a domain expert, not a beginner.
+3. End with a `## Warning` section explaining what care must be taken when modifying the subsystem, and what the load-bearing invariants are. The whole point of these docs is to make future-you stop and read before pulling a thread.
+4. Reference the doc from inline comments at the top of every file involved in the subsystem. One line per file is enough: `// See docs/design/<topic>.md for the canonical explanation.` Don't repeat the doc's contents in code comments — code comments explain *this file's role* in the design, the doc explains *the design.*
+
+## Current files
+
+- [ghost-system.md](ghost-system.md) — The provisional-record bridge between the proxy semantic stream and the durable JSONL transcript. Covers the five-rule render predicate, reconciliation, the orphan TTL, the sidecar shape filter, and the dual-owner relationship with `SemanticStreamingTurn`.

scripts/test-ghost-fallback.ts

@@ -4,20 +4,27 @@ import {
   createGhost,
   orphanGhost,
   supersedeGhost,
+  type ClaudeContentBlock,
   type ClaudeEntry,
 } from 'agent-transcript-parser/ghost'
 
 import { orphanStale } from '../src/renderer/src/workspace/ghosts'
 import { selectMergedEntries } from '../src/renderer/src/workspace/mergedEntries'
 import { emptyRuntime } from '../src/renderer/src/workspace/workspaceState'
 
-function baseEntry(uuid: string): ClaudeEntry {
+// Wall-clock anchor for tests. Concrete values keep the predicate
+// inputs explicit and avoid Date.now() drift between assertions.
+const T_OLD_JSONL = 1_700_000_000_000
+const T_GHOST_NEWER = T_OLD_JSONL + 5_000
+const T_GHOST_OLDER = T_OLD_JSONL - 5_000
+
+function baseEntry(uuid: string, timestamp = '2026-04-24T00:00:00.000Z'): ClaudeEntry {
   return {
     type: 'assistant',
     uuid,
     parentUuid: null,
     sessionId: 'session-1',
-    timestamp: '2026-04-24T00:00:00.000Z',
+    timestamp,
     message: {
       id: `msg-${uuid}`,
       role: 'assistant',
@@ -26,69 +33,220 @@ function baseEntry(uuid: string): ClaudeEntry {
   } as ClaudeEntry
 }
 
-function textGhost(turnId: string, text: string) {
+function ghostWithContent(
+  turnId: string,
+  content: ClaudeContentBlock[],
+  now: number,
+) {
   return createGhost({
     sessionId: 'session-1',
     turnId,
     blockIndex: 0,
     role: 'assistant',
-    content: [{ type: 'text', text }],
+    content,
+    now,
   })
 }
 
+function shortTextGhost(turnId: string, text: string, now: number) {
+  return ghostWithContent(turnId, [{ type: 'text', text }], now)
+}
+
+function longTextGhost(turnId: string, now: number) {
+  // 250 chars — past the SIDECAR_GHOST_TEXT_MAX threshold (200), so
+  // the sidecar shape filter will let it through.
+  const text = 'A real assistant turn that streamed enough text that it could not be a one-line title-gen sidecar leak. The shape filter is a 200-char cap; this string deliberately blows past that to verify the structural rule applies.'
+  return ghostWithContent(turnId, [{ type: 'text', text }], now)
+}
+
+function toolUseGhost(turnId: string, now: number) {
+  // Tool_use ghosts are explicitly NOT sidecar-shaped (the shape
+  // filter only matches single-text-block assistant ghosts), so a
+  // tool_use orphan past lastJsonlEntryAt should render even if
+  // short.
+  return ghostWithContent(
+    turnId,
+    [
+      {
+        type: 'tool_use',
+        id: `tool-${turnId}`,
+        name: 'Read',
+        input: { file_path: '/tmp/example.ts' },
+      },
+    ],
+    now,
+  )
+}
+
+// ---------------------------------------------------------------------------
+// 1. Empty ghost map → return entries by identity (reference-stable
+//    short-circuit; Feed memos rely on this).
+// ---------------------------------------------------------------------------
 {
   const runtime = emptyRuntime()
   runtime.entries = [baseEntry('real-1')]
   const merged = selectMergedEntries(runtime, null)
   assert.equal(merged, runtime.entries)
 }
 
+// ---------------------------------------------------------------------------
+// 2. Ghost not yet orphaned → hidden (JSONL might still arrive
+//    within TTL; SemanticStreamingTurn covers the live current
+//    turn, this ghost shouldn't double-render).
+// ---------------------------------------------------------------------------
 {
   const runtime = emptyRuntime()
   runtime.entries = [baseEntry('real-1')]
-  const ghost = textGhost('turn-a', 'not ready yet')
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = longTextGhost('turn-a', T_GHOST_NEWER)
   runtime.ghosts = new Map([[ghost.uuid, ghost]])
+  const merged = selectMergedEntries(runtime, null)
+  assert.equal(merged, runtime.entries, 'unorphaned ghost must not render')
+}
 
+// ---------------------------------------------------------------------------
+// 3. Superseded ghost → hidden (JSONL caught up; reconcileUpstream
+//    has the row already).
+// ---------------------------------------------------------------------------
+{
+  const runtime = emptyRuntime()
+  runtime.entries = [baseEntry('real-1')]
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = orphanGhost(
+    supersedeGhost(longTextGhost('turn-a', T_GHOST_NEWER), 'real-1', T_GHOST_NEWER + 1),
+    T_GHOST_NEWER + 2,
+  )
+  runtime.ghosts = new Map([[ghost.uuid, ghost]])
   const merged = selectMergedEntries(runtime, null)
-  assert.equal(merged, runtime.entries)
+  assert.equal(merged, runtime.entries, 'superseded ghost must not render')
 }
 
+// ---------------------------------------------------------------------------
+// 4. Orphan ghost for the live current turn → hidden.
+//    SemanticStreamingTurn owns the live turn render; surfacing a
+//    ghost for the same turn would double-render.
+// ---------------------------------------------------------------------------
 {
   const runtime = emptyRuntime()
   runtime.entries = [baseEntry('real-1')]
-  const ghost = orphanGhost(textGhost('turn-a', 'jsonl never wrote'), 2000)
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = orphanGhost(
+    longTextGhost('turn-a', T_GHOST_NEWER),
+    T_GHOST_NEWER + 1,
+  )
   runtime.ghosts = new Map([[ghost.uuid, ghost]])
+  const merged = selectMergedEntries(runtime, 'turn-a')
+  assert.equal(merged, runtime.entries, 'current-turn ghost must not render')
+}
 
+// ---------------------------------------------------------------------------
+// 5. Orphan ghost OLDER than lastJsonlEntryAt → hidden. This is the
+//    "stale orphan from earlier in the session, JSONL kept writing
+//    past it" case: a sidecar leak that happened before later real
+//    JSONL entries. Predicate-4 catches it structurally.
+// ---------------------------------------------------------------------------
+{
+  const runtime = emptyRuntime()
+  runtime.entries = [baseEntry('real-1')]
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = orphanGhost(
+    longTextGhost('turn-a', T_GHOST_OLDER),
+    T_GHOST_OLDER + 1,
+  )
+  runtime.ghosts = new Map([[ghost.uuid, ghost]])
   const merged = selectMergedEntries(runtime, null)
-  assert.equal(merged.length, 2)
-  assert.equal(merged[1]?.uuid, ghost.uuid)
+  assert.equal(merged, runtime.entries, 'orphan older than JSONL tail must not render')
 }
 
+// ---------------------------------------------------------------------------
+// 6. Orphan ghost NEWER than lastJsonlEntryAt with sidecar shape
+//    (single short text block, ≤200 chars) → hidden. This is the
+//    tail-sidecar case the timestamp predicate alone cannot
+//    catch: predict-next-prompt fires after the last real JSONL
+//    entry, no later real turn supersedes it. Shape filter is the
+//    backstop.
+// ---------------------------------------------------------------------------
 {
   const runtime = emptyRuntime()
   runtime.entries = [baseEntry('real-1')]
-  const ghost = orphanGhost(textGhost('turn-a', 'current live turn'), 2000)
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = orphanGhost(
+    shortTextGhost('turn-a', 'short ack', T_GHOST_NEWER),
+    T_GHOST_NEWER + 1,
+  )
   runtime.ghosts = new Map([[ghost.uuid, ghost]])
+  const merged = selectMergedEntries(runtime, null)
+  assert.equal(merged, runtime.entries, 'sidecar-shaped tail orphan must not render')
+}
 
-  const merged = selectMergedEntries(runtime, 'turn-a')
-  assert.equal(merged, runtime.entries)
+// ---------------------------------------------------------------------------
+// 7. Orphan ghost NEWER than lastJsonlEntryAt with substantive text
+//    (>200 chars) → renders. This is the main case ghost was built
+//    for: JSONL stopped writing while proxy kept going for an
+//    in-flight assistant turn.
+// ---------------------------------------------------------------------------
+{
+  const runtime = emptyRuntime()
+  runtime.entries = [baseEntry('real-1')]
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
+  const ghost = orphanGhost(
+    longTextGhost('turn-a', T_GHOST_NEWER),
+    T_GHOST_NEWER + 1,
+  )
+  runtime.ghosts = new Map([[ghost.uuid, ghost]])
+  const merged = selectMergedEntries(runtime, null)
+  assert.equal(merged.length, 2, 'substantive orphan past JSONL tail must render')
+  assert.equal(merged[1]?.uuid, ghost.uuid)
 }
 
+// ---------------------------------------------------------------------------
+// 8. Orphan tool_use ghost NEWER than lastJsonlEntryAt → renders
+//    even though short. tool_use is structurally not a sidecar
+//    shape — Claude Code's auxiliary calls all return text-only
+//    bodies, never tool_use blocks.
+// ---------------------------------------------------------------------------
 {
   const runtime = emptyRuntime()
   runtime.entries = [baseEntry('real-1')]
+  runtime.lastJsonlEntryAt = T_OLD_JSONL
   const ghost = orphanGhost(
-    supersedeGhost(textGhost('turn-a', 'already committed'), 'real-1', 2000),
-    3000,
+    toolUseGhost('turn-a', T_GHOST_NEWER),
+    T_GHOST_NEWER + 1,
   )
   runtime.ghosts = new Map([[ghost.uuid, ghost]])
+  const merged = selectMergedEntries(runtime, null)
+  assert.equal(merged.length, 2, 'tool_use orphan past JSONL tail must render')
+  assert.equal(merged[1]?.uuid, ghost.uuid)
+}
 
+// ---------------------------------------------------------------------------
+// 9. lastJsonlEntryAt === null (fresh session, no JSONL ever) AND
+//    orphan ghost present → render decision falls through the
+//    timestamp gate (rule 4 only fires when lastJsonlEntryAt is
+//    non-null). The shape filter still applies, so the ghost
+//    renders only if it's not sidecar-shaped.
+// ---------------------------------------------------------------------------
+{
+  const runtime = emptyRuntime()
+  runtime.entries = [baseEntry('real-1')]
+  runtime.lastJsonlEntryAt = null
+  const ghost = orphanGhost(
+    longTextGhost('turn-a', T_GHOST_NEWER),
+    T_GHOST_NEWER + 1,
+  )
+  runtime.ghosts = new Map([[ghost.uuid, ghost]])
   const merged = selectMergedEntries(runtime, null)
-  assert.equal(merged, runtime.entries)
+  assert.equal(merged.length, 2, 'orphan with null lastJsonlEntryAt must render if shape passes')
+  assert.equal(merged[1]?.uuid, ghost.uuid)
 }
 
+// ---------------------------------------------------------------------------
+// 10. orphanStale TTL behavior. Confirms the helper still produces
+//     reference-stable no-ops and that an orphan flag is set when
+//     the threshold elapses.
+// ---------------------------------------------------------------------------
 {
-  const fresh = textGhost('turn-a', 'fresh')
+  const fresh = shortTextGhost('turn-a', 'fresh', T_OLD_JSONL)
   const freshMap = new Map([[fresh.uuid, fresh]])
   assert.equal(orphanStale(freshMap, fresh._atp.updatedAt + 500, 1000), freshMap)
 
@@ -97,4 +255,4 @@ function textGhost(turnId: string, text: string) {
   assert.ok(staleMap.get(fresh.uuid)?._atp.orphanedAt !== undefined)
 }
 
-console.log('ghost fallback tests passed')
+console.log('ghost render predicate tests passed')

src/main/ghostJournal.ts

@@ -6,6 +6,13 @@
 // a cc-shell-owned JSONL file so the live view survives a crash, a
 // window reload, or an Electron restart.
 //
+// See docs/design/ghost-system.md for the canonical explanation
+// of the ghost subsystem — what it does, when ghosts surface in
+// the rendered feed, and why this file exists at all. DO NOT
+// change the persistence model (one file per session, append-
+// only, 100 ms batch) without re-reading that doc; resume + the
+// reduceGhostLog last-write-wins rule depend on it.
+//
 // -----------------------------------------------------------------------------
 // Why a separate file and not upstream's JSONL
 // -----------------------------------------------------------------------------

src/providers/claude/runtime/claudeSession.ts

@@ -256,14 +256,16 @@ export class ClaudeSession extends EventEmitter {
       // that opted into proxy specifically want the richer stream.
       proxy: this.useProxy
         ? {
-            // Sidecar Haiku filtering — see ClaudeProxyAdapter for the
-            // full rationale. Without this, Claude Code's auxiliary
-            // Haiku calls (session title generation, compaction
-            // summaries, hook agents, teleport title-and-branch) leak
-            // into the visible transcript as orphan ghost entries
-            // because Claude Code never writes them to the JSONL
-            // rollout, so the renderer's ghost-supersede path can
-            // never fire.
+            // Sidecar Haiku filtering — see ClaudeProxyAdapter for
+            // the full rationale, and docs/design/ghost-system.md
+            // for how this interacts with the renderer-side shape
+            // filter (rule 5 of the predicate). Without this, Claude
+            // Code's auxiliary Haiku calls (session title generation,
+            // compaction summaries, hook agents, teleport
+            // title-and-branch) leak into the visible transcript as
+            // orphan ghost entries because Claude Code never writes
+            // them to the JSONL rollout, so the renderer's
+            // ghost-supersede path can never fire.
             //
             // We default to assuming the user is on a non-Haiku
             // primary model. Why this is OK as a default: