diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts index ff81c9b7b6ded..dfa1ad58545ee 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts @@ -75,7 +75,10 @@ const handlers = { Effect.mapError( (cause) => new OrchestratorMcpFailure({ - code: cause.reason === "unsupported" ? "invalid_request" : "orchestration_error", + code: + cause.reason === "unsupported" || cause.reason === "unmatched" + ? "invalid_request" + : "orchestration_error", message: cause.message, }), ), diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index ab846252c9e58..7056e07303c31 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -462,7 +462,13 @@ it.effect("waitForThread returns the cut run once its continuation is declined", }), ); -it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( +it.effect.each([ + "live", + "no-match", + "no-match-clear", + "no-session", + "unsupported-runtime", +] as const)( "marks background work awaited on the thread's active provider thread with state %s", (state) => Effect.gen(function* () { @@ -538,7 +544,11 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( const result = yield* Effect.gen(function* () { const service = yield* ThreadManagementService.ThreadManagementService; return yield* service - .awaitBackgroundTasks({ threadId, taskIds: ["tests", "finished"], awaited: true }) + .awaitBackgroundTasks({ + threadId, + taskIds: state.startsWith("no-match") ? ["finished"] : ["tests", "finished"], + awaited: state !== "no-match-clear", + }) .pipe(Effect.result); }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer))); @@ -547,6 +557,14 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( success: { matched: ["tests"], unmatched: ["finished"] }, }); expect(calls).toEqual([{ providerThreadId, awaited: true }]); + } else if (state === "no-match") { + expect(result).toMatchObject({ + failure: { _tag: "ThreadManagementBackgroundWaitError", reason: "unmatched" }, + }); + expect(calls).toEqual([{ providerThreadId, awaited: true }]); + } else if (state === "no-match-clear") { + expect(result).toMatchObject({ success: { matched: [], unmatched: ["finished"] } }); + expect(calls).toEqual([{ providerThreadId, awaited: false }]); } else { expect(result).toMatchObject({ failure: { diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index e1a1abdc1819b..1dd01ae504ca5 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -332,7 +332,7 @@ export class ThreadManagementBackgroundWaitError extends Schema.TaggedError !matchedIds.has(taskId)) }; }); diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 66e5b2c72dea9..b70386d67c0a8 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. - `experience`: the real mistake, in a sentence. `source`: where it happened. -- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. +- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. With `world.backgroundTasks`, `t3_background_wait` answers from T3's own handler and ThreadManagementService, so its result and errors are the checkout's; a prior wait with `"result": "default"` gets that result too. - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. - `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). diff --git a/evals/mcp-tool-use/backgroundWait.ts b/evals/mcp-tool-use/backgroundWait.ts new file mode 100644 index 0000000000000..24377c4247f8a --- /dev/null +++ b/evals/mcp-tool-use/backgroundWait.ts @@ -0,0 +1,154 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * T3's real `t3_background_wait` handler and ThreadManagementService over a Claude session. + * Only the adapter is the eval's: like Claude's, it matches task IDs against the session's + * running background work, which the case lists in `world.backgroundTasks`. + */ +import { + EnvironmentId, + type OrchestrationV2ProviderThread, + type OrchestrationV2ThreadProjection, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + ThreadId, +} from "@t3tools/contracts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as References from "effect/References"; +// oxlint-disable-next-line t3code/no-raw-mcp-registration -- the eval calls the tool McpHttpServer registered and registers none. +import { McpSchema, McpServer } from "effect/ai"; + +import * as McpHttpServer from "../../apps/server/src/mcp/McpHttpServer.ts"; +import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts"; +import { liveThreadShell } from "../../apps/server/src/mcp/McpToolAccess.testkit.ts"; +import * as OrchestratorMcpService from "../../apps/server/src/mcp/OrchestratorMcpService.ts"; +import * as ThreadMetadataMcpService from "../../apps/server/src/mcp/ThreadMetadataMcpService.ts"; +import * as OrchestratorHandlers from "../../apps/server/src/mcp/toolkits/orchestrator/handlers.ts"; +import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts"; +import * as CommandReceiptStore from "../../apps/server/src/orchestration-v2/CommandReceiptStore.ts"; +import * as Orchestrator from "../../apps/server/src/orchestration-v2/Orchestrator.ts"; +import * as ProviderSessionManager from "../../apps/server/src/orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadManagementService from "../../apps/server/src/orchestration-v2/ThreadManagementService.ts"; +import * as PeerForwarding from "../../apps/server/src/peer/PeerForwarding.ts"; +import * as PeerLinkRequests from "../../apps/server/src/peer/PeerLinkRequests.ts"; +import * as PeerLinks from "../../apps/server/src/peer/PeerLinks.ts"; +import * as RemoteDelegation from "../../apps/server/src/peer/RemoteDelegation.ts"; +import * as SqlitePersistence from "../../apps/server/src/persistence/Sqlite.ts"; +import * as ServerSettings from "../../apps/server/src/serverSettings.ts"; + +const threadId = ThreadId.make("mcp-tool-use-eval"); +const providerThreadId = ProviderThreadId.make("provider-thread:mcp-tool-use-eval"); +const providerSessionId = ProviderSessionId.make("provider-session:mcp-tool-use-eval"); +const providerInstanceId = ProviderInstanceId.make("claudeAgent"); +const driver = ProviderDriverKind.make("claudeAgent"); + +/** Calls T3's `t3_background_wait` with `running` as the session's running background task IDs. */ +export const makeBackgroundWait = Effect.fn("evaluateMcpToolUse.makeBackgroundWait")(function* ( + running: ReadonlyArray, +) { + const now = yield* DateTime.now; + const providerThread = { + id: providerThreadId, + driver, + providerInstanceId, + providerSessionId, + appThreadId: threadId, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + pendingBackgroundTasks: running.map((taskId) => ({ taskId, kind: "command" })), + createdAt: now, + updatedAt: now, + } as unknown as OrchestrationV2ProviderThread; + const runtime = { + driver, + providerSessionId, + awaitBackgroundTasks: (input: { readonly taskIds: ReadonlyArray }) => + Effect.succeed({ matched: [...new Set(input.taskIds)].filter((id) => running.includes(id)) }), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime; + const real = yield* Layer.build( + ThreadManagementService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null, activeProviderThreadId: providerThreadId }, + providerThreads: [providerThread], + } as unknown as OrchestrationV2ThreadProjection), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + get: () => Effect.succeedSome(runtime), + }), + ServerSettings.layerTest(), + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), + ), + ), + ), + // The receipt store's in-memory migrations log every step. + ).pipe(Effect.provideService(References.MinimumLogLevel, "Error")); + const service = Context.get(real, ThreadManagementService.ThreadManagementService); + // The caller's thread is live on the Claude instance, mid-turn. + const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + ...service, + getThreadShell: () => Effect.succeed({ ...liveThreadShell(threadId), providerInstanceId }), + }); + const context = yield* Layer.build( + McpHttpServer.toolkitRegistration(OrchestratorToolkit, OrchestratorHandlers.layer).pipe( + Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), + Layer.provide(Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({})), + Layer.provide(Layer.mock(ThreadMetadataMcpService.ThreadMetadataMcpService)({})), + Layer.provide(threads), + Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), + Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), + Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), + Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})), + ), + ); + type Entry = Parameters[0]; + // The registry keeps each tool's handler; its public type lists only definitions. + const tools = Context.get(context, McpServer.McpServer).tools as ReadonlyArray; + const entry = tools.find((tool) => tool.tool.name === "t3_background_wait"); + if (entry === undefined) return yield* Effect.die(new Error("T3 has no t3_background_wait.")); + const scope: McpInvocationContext.McpThreadInvocationScope = { + environmentId: EnvironmentId.make("00000000-0000-4000-8000-0000000000e0"), + requestNamespace: "mcp-tool-use-eval", + thread: { threadId, providerSessionId: "mcp-tool-use-eval", providerInstanceId }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 0, + }; + const request = McpSchema.McpRequestContext.of({ + clientId: 0, + protocolVersion: "2025-06-18", + clientCapabilities: {}, + }); + return Effect.fn("evaluateMcpToolUse.backgroundWait")(function* (input: unknown) { + const result = yield* entry.handle(input).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpRequestContext, request), + // An MCP client shows a protocol error, such as bad parameters, as the tool's error. + Effect.catch((error) => + Effect.succeed( + new McpSchema.CallToolResult({ + isError: true, + content: [{ type: "text", text: error.message }], + }), + ), + ), + ); + if (!("content" in result)) + return yield* Effect.die(new Error("Background wait asked for input.")); + const text = result.content + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("\n"); + if (text === "Tool execution failed due to an internal server error.") { + return yield* Effect.die(new Error("t3_background_wait hit a defect in T3 or the eval.")); + } + return { text, isError: result.isError === true }; + }); +}); +export type BackgroundWait = Effect.Success>; diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 520bd2abf10d3..28e89df5597e8 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,6 +34,8 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id": "background-wait-unmatched-id", "area": "orchestration", "kind": "guard", "pattern": "waits on a task ID that matches no background work, then ends the turn as if waiting", "experience": "One agent called t3_background_wait with the literal ID 'placeholder', another with a Claude subagent ID. The tool returned a normal result listing the ID under `unmatched`, so the agent said it would continue when the work came back and ended its turn. Nothing ever woke it.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["placeholder"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"calls": [{"tool": "t3_background_wait", "args": {"taskIds": ["bq7m2x9k"]}}]}, "source": {"issue": "#29772"}, "notes": "The seeded wait gets T3's real result for an ID the session does not know. The tests are still running, so the only way to keep the thread waiting is a wait on bq7m2x9k. In replay, Sonnet 5.5 and Opus 5.5 also recover from main's success result, so this guards the recovery rather than reproducing the incident. Claude only: t3_background_wait is a Claude tool.", "maxSteps": 8} +{"id": "background-wait-matched-id", "area": "orchestration", "kind": "control", "pattern": "waits on the right task ID, then ends the turn", "experience": "With the background command's own ID registered, ending the turn is correct: T3 keeps the thread working and wakes it when the tests finish.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["bq7m2x9k"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"maxCalls": {"t3_background_wait": 1, "Bash": 3}, "notCalls": ["ScheduleWakeup"]}, "source": {"issue": "#29772"}, "notes": "Same thread as background-wait-unmatched-id, with a wait T3 matched. Guards against a fix that makes agents distrust a good wait and poll; a few looks at the output are ordinary. Claude only.", "maxSteps": 8} {"id": "settle-this-thread-after-cleanup", "area": "orchestration", "kind": "bad", "pattern": "settle read as plain English for finish", "experience": "Asked to clean up scratch files and settle this thread, the agent deleted the files and ended the turn without calling t3_thread_organize (#26749).", "rule": "A request to settle this thread calls t3_thread_organize with action settle.", "privacy": "commit-safe", "notes": "From #26749. Claude defers MCP tools, so only the tool's name is visible until ToolSearch loads it; 'settle' appears only in the t3_thread_organize description.", "setup": {"role": "top", "world": {"files": {"scratch/run-1.log": "ok\n", "scratch/run-2.log": "ok\n", "src/index.ts": "export const x = 1;\n"}}, "mocks": {"t3_thread_organize": {"settlesWhenTurnEnds": true}}}, "prompt": "Clean up the scratch files and settle this thread.", "expect": {"calls": [{"tool": "t3_thread_organize", "args": {"action": "settle"}}], "notCalls": ["t3_thread_update"]}} {"id": "settle-this-thread-bare", "area": "orchestration", "kind": "bad", "pattern": "settle read as plain English for finish", "experience": "After a finished Q&A, asked only to settle this thread, the agent replied in text and ended the turn without calling t3_thread_organize (#26749).", "rule": "A request to settle this thread calls t3_thread_organize with action settle.", "privacy": "commit-safe", "notes": "From #26749. The prior turn is general knowledge with no claims about the repo, so the only work left is settling.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "In general, what's the difference between an HTTP 401 and a 403?"}, {"role": "assistant", "text": "A 401 means the request has no valid credentials, so the client should authenticate and try again. A 403 means the server knows who you are but won't let you do this, so retrying with the same credentials won't help."}], "mocks": {"t3_thread_organize": {"settlesWhenTurnEnds": true}}}, "prompt": "Makes sense, thanks. Settle this thread.", "expect": {"calls": [{"tool": "t3_thread_organize", "args": {"action": "settle"}}], "notCalls": ["t3_thread_update"]}} {"id": "settle-on-a-choice-is-not-organize", "area": "orchestration", "kind": "guard", "pattern": "settle read as plain English for finish", "experience": "Guard for the #26749 instruction: 'settle on' a choice is ordinary English and must not settle the thread.", "rule": "Only a request to settle a thread calls t3_thread_organize.", "privacy": "commit-safe", "notes": "Catches over-triggering on the settle bullet in the orchestration instructions.", "setup": {"role": "top", "world": {"files": {"src/db.ts": "export const driver = process.env.DB_DRIVER ?? 'sqlite';\n"}}}, "prompt": "We keep going back and forth on the database for this thread. Let's settle on Postgres: make it the default driver in src/db.ts.", "expect": {"notCalls": ["t3_thread_organize"]}} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 718cf5e87202d..ec1a121fee485 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -39,6 +39,7 @@ import { t3Default, t3SystemAppend, t3SubagentContext, + withBackgroundWait, withDevices, } from "./run.ts"; import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts"; @@ -1146,6 +1147,42 @@ it.effect("builds device cases from product-backed worlds and synthetic prior tu }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); +// Models recover from either result, so this keeps the guard case failing on the old success. +it.effect("seeds T3's error for a prior background wait whose IDs match nothing", () => + Effect.gen(function* () { + const cases = NodeFS.readFileSync( + new URL("./cases/orchestration.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)); + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false }); + for (const [id, isError] of [ + ["background-wait-unmatched-id", true], + ["background-wait-matched-id", false], + ] as const) { + const testCase = cases.find((c) => c.id === id)!; + const [wait] = (testCase.setup.priorTurns ?? []) + .flatMap((turn) => turn.toolCalls ?? []) + .filter((call) => call.tool === "t3_background_wait"); + assert.isDefined(wait, id); + const sample = makeSample(testCase, base); + yield* withBackgroundWait(sample); + assert.equal(sample.priorErrors.has(wait), isError, id); + const blocks = buildMessages(sample).flatMap((message) => message.content); + const waitUse = blocks.find( + (block) => block.type === "tool_use" && String(block.name).endsWith("t3_background_wait"), + ); + const replayed = blocks.find( + (block) => block.type === "tool_result" && block.tool_use_id === waitUse?.id, + ); + assert.isDefined(replayed, id); + assert.equal(replayed?.is_error === true, isError, id); + } + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + const secretCases = () => NodeFS.readFileSync(new URL("./cases/secrets.ndjson", import.meta.url), "utf8") .trim() diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 0ca13fe14fea8..d724268351c08 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -72,6 +72,7 @@ import { } from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts"; import { proxyConfig } from "../shared/proxy.ts"; import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts"; +import { type BackgroundWait, makeBackgroundWait } from "./backgroundWait.ts"; import { DeviceExpect, gradeDevice } from "./device-grading.ts"; import { FileLinksExpect, gradeFileLinks } from "./file-link-grading.ts"; import { @@ -168,7 +169,10 @@ const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const; const PriorToolCall = Schema.Struct({ tool: Schema.String, input: Schema.Unknown, - /** For a device tool, `"default"` runs the call on the world's devices for T3's result. */ + /** + * For a device tool or t3_background_wait, `"default"` runs the call on the world's devices or + * background tasks for T3's result. + */ result: Schema.Unknown, /** Repository-relative file whose current contents supply this tool result. */ resultFile: Schema.optional(Schema.String), @@ -273,6 +277,11 @@ const McpToolUseWorld = Schema.StructWithRest( deviceInvalidCapture: Schema.optional(Schema.Boolean), /** Devices another thread's agent-device session holds, by device id: that thread's id. */ deviceOwners: Schema.optional(Schema.Record(Schema.String, Schema.String)), + /** + * Task IDs of the background commands and Monitors still running on the agent's Claude + * session. With them, T3's own t3_background_wait handler answers from this world. + */ + backgroundTasks: Schema.optional(Schema.Array(Schema.String)), /** PRs already linked to the thread, with the state T3 last saw. */ linkedPullRequests: Schema.optional( Schema.Array( @@ -860,6 +869,10 @@ type Sample = { devices?: DeviceTools; /** T3's results for prior device calls whose case result is "default". */ readonly priorResults: Map; + /** Prior calls whose T3 result is an error, so the agent sees it as one. */ + readonly priorErrors: Set; + /** T3's t3_background_wait, once `withBackgroundWait` builds it for `world.backgroundTasks`. */ + backgroundWait?: BackgroundWait; /** Calls so far per mocked tool, for mocks that answer in sequence. */ readonly mockCalls: Map; }; @@ -906,6 +919,7 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => { secretFiles: [], secretRequests: 0, priorResults: new Map(), + priorErrors: new Set(), mockCalls: new Map(), }; }; @@ -937,6 +951,25 @@ export const withDevices = Effect.fn("evaluateMcpToolUse.withDevices")(function* } }); +/** + * Builds T3's t3_background_wait for a world with background tasks, and answers prior waits whose + * result is "default" with T3's result. + */ +export const withBackgroundWait = Effect.fn("evaluateMcpToolUse.withBackgroundWait")(function* ( + sample: Sample, +) { + const running = sample.testCase.setup.world?.backgroundTasks; + if (running === undefined) return; + const backgroundWait = yield* makeBackgroundWait(running); + sample.backgroundWait = backgroundWait; + for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) { + if (call.result !== "default" || call.tool !== "t3_background_wait") continue; + const result = yield* backgroundWait(call.input); + sample.priorResults.set(call, result.text); + if (result.isError) sample.priorErrors.add(call); + } +}); + // Main's history under every case's world. const MAIN_COMMITS = [ { sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" }, @@ -1024,8 +1057,13 @@ export const buildMessages = (sample: Sample) => { const calls = (turn.toolCalls ?? []).map((call) => { const tool = findTool(base, call.tool); if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`); - if (call.result === "default" && isDeviceTool(call.tool) && !sample.priorResults.has(call)) { - throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run withDevices.`); + const builder = isDeviceTool(call.tool) + ? "withDevices" + : call.tool === "t3_background_wait" + ? "withBackgroundWait" + : undefined; + if (call.result === "default" && builder && !sample.priorResults.has(call)) { + throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run ${builder}.`); } return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call }; }); @@ -1047,7 +1085,8 @@ export const buildMessages = (sample: Sample) => { normalizeToolName(tool.name) === "orchestrator_capabilities" ? capabilities(sample, result) : (sample.priorResults.get(prior) ?? result); - push("user", toolResult(id, replayed)); + const answer = toolResult(id, replayed); + push("user", sample.priorErrors.has(prior) ? { ...answer, is_error: true } : answer); } if (turn.text) push(turn.role, text(turn.text)); } @@ -1640,6 +1679,10 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun const result = yield* sample.devices.call(name, input, failure); return result.isError ? failed(result.text) : toolResult(call.id, result.text); } + if (sample.backgroundWait && isT3Name(call.name) && name === "t3_background_wait" && fromWorld) { + const result = yield* sample.backgroundWait(input); + return result.isError ? failed(result.text) : toolResult(call.id, result.text); + } // A pasted secret is saved without a card, so a mocked card outcome does not apply. if (mock !== undefined && mock !== "default" && !pastedSecret(name, input)) { if (name === "request_secret") savedSecretFile(sample, mock); @@ -2142,6 +2185,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( ) { const sample = { ...makeSample(testCase, base), shellUsage: spent.shellUsage }; yield* withDevices(sample); + yield* withBackgroundWait(sample); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const messages = buildMessages(sample); const input = base.codex ? codexMessages(messages) : undefined; @@ -2562,7 +2606,7 @@ const main = Effect.gen(function* () { // One JSON line per case. Without --case this also checks that every case builds. for (const testCase of selected) { const sample = makeSample(testCase, baseFor(testCase)); - yield* Effect.scoped(withDevices(sample)); + yield* Effect.scoped(Effect.andThen(withDevices(sample), withBackgroundWait(sample))); const messages = buildMessages(sample); const body = sample.base.codex ? codexRequest(sample.base, codexMessages(messages))