Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/tools/fusion/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Worker spend is priced per completion with the prompt-cache discount (`estimateU

[contract-example.ts](contract-example.ts) runs a provide's optional `example` (ATO-1362): `{ call, expect, page? }` becomes one more `verify.run` spec after the declared `checks` (a page probe for browser code, else `node` loading `in`), attributed to the provide's task, so a result that does not match what the contract declared fails that task at acceptance. No example, no extra run; one with nowhere to run is a contract warning.

[judge.ts](judge.ts) is the trimmed judge (ATO-944, `judge: "on"`): an acceptance list per task, rendered read-only into its brief (`briefSuffix`); after the waves, facts (check outcomes, provide presence, declared files) decide their items and can only be red or green on their own; a fresh-context judge (no transcript; files, or the reply when no file was declared) rules on the rest, and only a literal `pass: true` passes. A red list re-runs the task once with the findings; the contract is re-evaluated on the retried rows only. Unreadable or impossible judging leaves the task `needs_orchestrator`, never `ok`.
[judge.ts](judge.ts) is the trimmed judge (ATO-944, `judge: "on"`): an acceptance list per task, rendered read-only into its brief (`briefSuffix`); after the waves, facts (check outcomes, provide presence, declared files) decide their items and can only be red or green on their own; a fresh-context judge (no transcript; files, or the reply when no file was declared) rules on the rest, and only a literal `pass: true` passes. A red list re-runs the task once with the findings. After that repair, every task's contract and acceptance list is re-evaluated against current facts and files, including tasks that passed before: a repair can break another task. Earlier acceptance evidence is replaced, while worker execution outcomes are retained; there is still only one worker retry, and the fresh judge calls count toward the existing turn budget. Unreadable or impossible judging leaves the task `needs_orchestrator`, never `ok`.

[replan.ts](replan.ts) is automatic replanning (ATO-941, `autoReplan: N`): dependents of a provider that did not close are held (`not started: held for the replan`), attempts per task are counted per turn in `FusionTurnBudget`, and `details.replan` lists pending and exhausted tasks; `src/agent/fusion-replan.ts` holds the planner's next step to `fusion.delegate` while any are pending, for at most two steps without a fan-out.

Expand Down
31 changes: 15 additions & 16 deletions src/tools/fusion/fusion-delegate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1376,25 +1376,27 @@ export function buildFusionDelegateTool(
// into the rows BEFORE the head line counts them: a task whose
// declared check failed is `failed` in the table the orchestrator
// reads, not `ok` with a footnote.
// Keep the worker outcomes separate from acceptance evidence: a
// repair can invalidate any task's earlier checks or findings.
const workerResults = results;
let contract: ContractReport | undefined;
// Check outcomes in `contract.checks` order, for the judge.
let checkOutcomes: readonly (ContractCheckOutcome | undefined)[] = [];
// Folds the disk and the check runner into `rows`. Run once after
// the waves and, for the tasks a judge sent back, once after their
// retry — on the retried rows only, so nothing is applied twice.
// the waves and again after a judge's retry, on the whole fan-out
// from its worker outcomes so no old evidence is carried forward.
const evaluateContract = async (
rows: WorkerTaskResult[],
allRows: WorkerTaskResult[],
): Promise<WorkerTaskResult[]> => {
if (parsed.contract === undefined) return rows;
// `allRows` goes in so a task that never ran is not accused of
// Every row goes in so a task that never ran is not accused of
// failing to provide: a cancelled fan-out leaves the disk empty
// for reasons that are not the worker's.
const provides = await inspectContractProvides(
parsed.contract,
parsed.tasks,
ctx.workingDir,
{ results: allRows, signal: ctx.signal },
{ results: rows, signal: ctx.signal },
);
let out = applyContractFindings(rows, provides.findings);
// The declared checks, then one call per provide `example`
Expand Down Expand Up @@ -1430,7 +1432,7 @@ export function buildFusionDelegateTool(
};
return out;
};
results = await evaluateContract(results, results);
results = await evaluateContract(workerResults);

// The judge (ATO-944): a task closes on a green acceptance list, a
// red list is re-run once with the findings, and a list still red
Expand Down Expand Up @@ -1522,17 +1524,14 @@ export function buildFusionDelegateTool(
true,
),
);
const merged = results.map(
const merged = workerResults.map(
(r) => retried.find((x) => x.id === r.id) ?? r,
);
const evaluated = await evaluateContract(
merged.filter((r) => retryIds.has(r.id)),
merged,
);
results = merged.map(
(r) => evaluated.find((x) => x.id === r.id) ?? r,
);
second = await judgeRows(results.filter((r) => retryIds.has(r.id)));
results = await evaluateContract(merged);
// A repair can break a previously green task, including an
// item only the judge can assess. Recheck every current list;
// this does not grant another worker retry or bypass budgets.
second = await judgeRows(results);
}
results = results.map((r) => {
if (retryIds.has(r.id)) {
Expand All @@ -1550,7 +1549,7 @@ export function buildFusionDelegateTool(
],
};
}
const verdict = first.get(r.id);
const verdict = (retryIds.size > 0 ? second : first).get(r.id);
return verdict === undefined ? r : applyJudgeVerdict(r, verdict, false);
});
}
Expand Down
158 changes: 157 additions & 1 deletion src/tools/fusion/judge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { FanoutScopeRegistry } from "../../approval/fanout-scope.js";
import type { RunTurnResult } from "../../agent/agent-loop.js";
import type { ResolvedRunMode } from "../../llm/run-mode/index.js";
import { createEmptySessionState } from "../../session/session-state.js";
import { FUSION_WORKER_ID_PREFIX } from "../../session/fusion-worker-session.js";
import { FUSION_WORKER_ID_PREFIX, readFusionWorkerMeta } from "../../session/fusion-worker-session.js";
import { StructuredLogger } from "../../tracing/structured-logger.js";
import {
buildAcceptanceChecklist,
Expand All @@ -20,6 +20,8 @@ import {
} from "./fusion-delegate.js";
import type { EscalationComplete } from "./escalation.js";
import type { DelegateTask } from "./delegate-args.js";
import type { WorkerTaskResult } from "./worker-result.js";
import { runChecks as runVerifyChecks } from "../verify/run-verify.js";

const logger = new StructuredLogger({ level: "error", sinks: [] });
const TARGET = { providerId: "cloud", model: "judge-model" };
Expand Down Expand Up @@ -350,3 +352,157 @@ describe("a node closes only on a green list (ATO-944)", () => {
rmSync(dir, { recursive: true, force: true });
});
});

describe("the whole acceptance list is current after a repair (#539)", () => {
const tasks = [
{ id: "a", title: "A", instructions: "write a.txt containing alpha", files: ["a.txt"] },
{ id: "b", title: "B", instructions: "write b.txt containing beta", files: ["b.txt"] },
];
const checks = tasks.map((task) => ({
task: task.id,
kind: "command",
cmd: process.execPath,
args: [
"-e",
`const fs = require('node:fs'); const assert = require('node:assert/strict'); ` +
`assert.equal(fs.readFileSync('${task.id}.txt', 'utf8'), '${task.id === "a" ? "alpha" : "beta"}');`,
],
checks: ["exit 0"],
}));

function workers(
dir: string,
complete: EscalationComplete,
onRepair: () => void,
over: Partial<FusionDelegateDeps> = {},
) {
const attempts = { a: 0, b: 0 };
const runs: string[] = [];
const d = deps(dir, complete, over);
d.runTurn = async (session, message) => {
const id = readFusionWorkerMeta(session.metadata)?.taskId;
if (id !== "a" && id !== "b") throw new Error(`unexpected worker ${id}`);
runs.push(id);
attempts[id] += 1;
d.briefs.push(message);
writeFileSync(join(dir, `${id}.txt`), id === "a" ? "alpha" : attempts.b === 1 ? "pending" : "beta");
if (id === "b" && attempts.b > 1) onRepair();
return { session, reason: "reply", stepCount: 1 } as RunTurnResult;
};
return { d, runs };
}

it("fails a previously green task when another task's repair breaks its real executable check", async () => {
const dir = mkdtempSync(join(tmpdir(), "judge-recheck-script-"));
try {
const checked: boolean[][] = [];
const { d, runs } = workers(
dir,
answer([{ id: "a3", pass: true, why: "the file matches its instruction" }]),
() => writeFileSync(join(dir, "a.txt"), "broken by the repair"),
{
runChecks: async (specs, ctx) => {
const result = await runVerifyChecks(specs, ctx);
checked.push(result.results.map((r) => r.ok));
return result;
},
},
);
const result = await buildFusionDelegateTool(d).run(
{ tasks, contract: { checks } },
{ workingDir: dir, sessionId: "s-p", stepIndex: 0, signal: new AbortController().signal },
);
expect(checked).toEqual([[true, false], [false, true]]);
expect(runs).toEqual(["a", "b", "b"]);
const rows = result.details.tasks as WorkerTaskResult[];
expect(rows.find((r) => r.id === "a")).toMatchObject({
status: "failed",
checks: { total: 1, failed: 1 },
judge: { green: false, retried: false },
});
expect(rows.find((r) => r.id === "b")).toMatchObject({
status: "ok",
checks: { total: 1, failed: 0 },
judge: { green: true, retried: true },
});
expect(result.details.outcome).toBe("partial");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it.each([false, true])("judges the latest evidence for every task after repair (earlier task damaged: %s)", async (damaged) => {
const dir = mkdtempSync(join(tmpdir(), "judge-recheck-model-"));
try {
const evidenceSeen: Array<{ task: string; pass: boolean }> = [];
const { d, runs } = workers(
dir,
async ({ prompt }) => {
const task = prompt.includes("## Task\nwrite a.txt") ? "a" : "b";
const pass = prompt.includes(`### ${task}.txt\n${task === "a" ? "alpha" : "beta"}`);
evidenceSeen.push({ task, pass });
return { content: JSON.stringify({ items: [{ id: "a2", pass, why: pass ? "expected content" : "wrong content" }] }) };
},
() => {
if (damaged) writeFileSync(join(dir, "a.txt"), "broken by the repair");
},
);
const result = await buildFusionDelegateTool(d).run(
{ tasks },
{ workingDir: dir, sessionId: "s-p", stepIndex: 0, signal: new AbortController().signal },
);
expect(runs).toEqual(["a", "b", "b"]);
expect(evidenceSeen.filter((e) => e.task === "a").map((e) => e.pass)).toEqual([true, !damaged]);
expect(evidenceSeen.filter((e) => e.task === "b").map((e) => e.pass)).toEqual([false, true]);
const rows = result.details.tasks as WorkerTaskResult[];
expect(rows.find((r) => r.id === "a")).toMatchObject({
status: damaged ? "failed" : "ok",
judge: { green: !damaged, retried: false },
});
expect(rows.find((r) => r.id === "b")).toMatchObject({
status: "ok",
judge: { green: true, retried: true },
});
expect(result.details.outcome).toBe(damaged ? "partial" : "all_ok");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("does not keep earlier green evidence when the check runner fails after repair", async () => {
const dir = mkdtempSync(join(tmpdir(), "judge-recheck-unavailable-"));
try {
let checkRuns = 0;
const { d, runs } = workers(
dir,
answer([{ id: "a3", pass: true, why: "the file matches its instruction" }]),
() => {},
{
runChecks: async (specs, ctx) => {
checkRuns += 1;
if (checkRuns > 1) throw new Error("verification unavailable after repair");
return runVerifyChecks(specs, ctx);
},
},
);
const result = await buildFusionDelegateTool(d).run(
{ tasks, contract: { checks } },
{ workingDir: dir, sessionId: "s-p", stepIndex: 0, signal: new AbortController().signal },
);
expect(checkRuns).toBe(2);
expect(runs).toEqual(["a", "b", "b"]);
const a = (result.details.tasks as WorkerTaskResult[]).find((r) => r.id === "a")!;
expect(a.status).not.toBe("ok");
expect(a.judge?.green).toBe(false);
expect(a.judge?.items).toContainEqual(expect.objectContaining({
pass: false,
by: "fact",
why: "the check did not run",
}));
expect(a.checks?.failed).not.toBe(0);
expect(result.summary).toContain("verification unavailable after repair");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});