Skip to content
Merged
24 changes: 24 additions & 0 deletions apps/server/src/orchestration-v2/EventSink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
type OrchestrationV2Run,
OrchestrationV2DomainEvent,
type OrchestrationV2ProviderThread,
type OrchestrationV2TurnItem,
OrchestrationV2StoredEvent,
ProviderThreadId,
RunAttemptId,
Expand Down Expand Up @@ -127,6 +128,11 @@ export interface EventSinkV2Shape {
readonly runId: RunId;
readonly activeAttemptId: RunAttemptId;
readonly expectedStatus: OrchestrationV2Run["status"];
/** Also refuse while this run interrupt request is stored without its result. */
readonly unlessInterruptRequested?: {
readonly requestId: OrchestrationV2TurnItem["id"];
readonly resultId: OrchestrationV2TurnItem["id"];
};
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
readonly effects?: ReadonlyArray<EffectOutbox.PendingOrchestrationEffectV2>;
}) => Effect.Effect<
Expand Down Expand Up @@ -525,6 +531,24 @@ const layerBase: Layer.Layer<
storedEvents: [] as ReadonlyArray<OrchestrationV2StoredEvent>,
};
}
if (input.unlessInterruptRequested !== undefined) {
const { requestId, resultId } = input.unlessInterruptRequested;
const pending = yield* sql<{ readonly pending: number }>`
SELECT EXISTS (
SELECT 1 FROM orchestration_v2_projection_turn_items
WHERE thread_id = ${input.threadId} AND turn_item_id = ${requestId}
) AND NOT EXISTS (
SELECT 1 FROM orchestration_v2_projection_turn_items
WHERE thread_id = ${input.threadId} AND turn_item_id = ${resultId}
) AS pending
`;
if (pending[0]?.pending === 1) {
return {
committed: false as const,
storedEvents: [] as ReadonlyArray<OrchestrationV2StoredEvent>,
};
}
}

const normalized = yield* normalizeEvents(
input.guardPendingUserInputCancellations === true
Expand Down
101 changes: 101 additions & 0 deletions apps/server/src/orchestration-v2/FoundationPersistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1984,6 +1984,107 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
}).pipe(Effect.provide(Layer.fresh(layerTest))),
);

it.effect("refuses a run-current write once an unpaired interrupt request is stored", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:foundation-interrupt-guard");
const runId = RunId.make("run:foundation-interrupt-guard");
const attemptId = RunAttemptId.make("run-attempt:foundation-interrupt-guard");
const requestId = TurnItemId.make("turn-item:foundation-interrupt-guard:request");
const resultId = TurnItemId.make("turn-item:foundation-interrupt-guard:result");
const run: OrchestrationV2Run = {
id: runId,
threadId,
ordinal: 1,
providerInstanceId,
modelSelection,
providerThreadId: null,
userMessageId: MessageId.make("message:foundation-interrupt-guard"),
rootNodeId: null,
activeAttemptId: attemptId,
status: "running",
queuePosition: null,
requestedAt: now,
startedAt: now,
completedAt: null,
checkpointId: null,
contextHandoffId: null,
};
yield* eventSink.write({
events: [
threadCreatedEvent({
id: "event:foundation-interrupt-guard:thread",
thread: makeThread(threadId, now),
now,
}),
{
id: EventId.make("event:foundation-interrupt-guard:run"),
type: "run.created",
threadId,
runId,
providerInstanceId,
occurredAt: now,
payload: run,
},
],
});
const signalItem = (id: TurnItemId, type: "run_interrupt_request" | "run_interrupt_result") =>
({
id: EventId.make(`event:${id}`),
type: "turn-item.updated",
threadId,
runId,
occurredAt: now,
payload: {
id,
threadId,
runId,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
status: "completed",
title: "Interrupt",
startedAt: now,
completedAt: now,
updatedAt: now,
type,
message: "Stopped",
},
}) as OrchestrationV2DomainEvent;
const completeRun = (suffix: string) =>
eventSink.writeIfRunCurrent({
threadId,
runId,
activeAttemptId: attemptId,
expectedStatus: "running",
unlessInterruptRequested: { requestId, resultId },
events: [
{
id: EventId.make(`event:foundation-interrupt-guard:waiting:${suffix}`),
type: "run.updated",
threadId,
runId,
providerInstanceId,
occurredAt: now,
payload: { ...run, status: "waiting" },
},
],
});

yield* eventSink.write({ events: [signalItem(requestId, "run_interrupt_request")] });
const refused = yield* completeRun("refused");
assert.isFalse(refused.committed);
assert.deepEqual(refused.storedEvents, []);

yield* eventSink.write({ events: [signalItem(resultId, "run_interrupt_result")] });
assert.isTrue((yield* completeRun("paired")).committed);
}).pipe(Effect.provide(Layer.fresh(layerTest))),
);

it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
Expand Down
8 changes: 8 additions & 0 deletions apps/server/src/orchestration-v2/ProviderTurnStartService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ export const layer: Layer.Layer<
readonly attemptId: OrchestrationV2RunAttempt["id"];
readonly providerThreadId: OrchestrationV2ProviderThread["id"];
readonly runOrdinal: number;
readonly worktreePath: string | null;
readonly inheritedBackgroundTurnItems: ReturnType<
typeof RunExecutionService.selectInheritedBackgroundTurnItems
>;
Expand Down Expand Up @@ -284,6 +285,11 @@ export const layer: Layer.Layer<
}),
)
.pipe(Effect.catchCause(() => Effect.succeed(false))),
hasWorkspaceMoved: () =>
projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe(
Effect.map((current) => current.thread.worktreePath !== input.worktreePath),
Effect.catchCause(() => Effect.succeed(false)),
),
loadRunInterruptReason: () =>
projectionStore
.getThreadRecords(input.threadId, ["turnItems"], {
Expand Down Expand Up @@ -695,6 +701,7 @@ export const layer: Layer.Layer<
attemptId: attempt.id,
providerThreadId: providerThread.id,
runOrdinal: run.ordinal,
worktreePath: projection.thread.worktreePath,
inheritedBackgroundTurnItems,
});
const { isCurrentAttemptInStatus } = runControls;
Expand Down Expand Up @@ -1709,6 +1716,7 @@ export const layer: Layer.Layer<
shouldFinalizeRun: runControls.shouldFinalizeRun,
hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest,
loadRunInterruptReason: runControls.loadRunInterruptReason,
hasWorkspaceMoved: runControls.hasWorkspaceMoved,
message: {
messageId: message.id,
text: userText,
Expand Down
172 changes: 172 additions & 0 deletions apps/server/src/orchestration-v2/RunExecutionService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2580,6 +2580,143 @@ it.effect(
}),
);

it.effect("settles native subagents when a workspace handoff completes the root run", () =>
Effect.gen(function* () {
const ids = backgroundScenarioIds("subagent-workspace-handoff");
const providerInstanceId = ProviderInstanceId.make("codex");
const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
const ingestionDone = yield* Deferred.make<void>();
const layerTest = RunExecutionService.layer.pipe(
Layer.provide(
Layer.mergeAll(
McpAppModelContext.layerEmpty,
PreviewAgentActivity.layer,
Layer.mock(CheckpointService.CheckpointServiceV2)({
captureBaseline: () => Effect.void,
}),
layerTurnItemPositionsTest,
mockEventSink({
write: () => Effect.succeed([]),
writeWithEffects: (input) =>
Ref.update(written, (current) => [...current, ...input.events]).pipe(Effect.as([])),
writeIfRunCurrent: (input) =>
Ref.update(written, (current) => [...current, ...input.events]).pipe(
Effect.as({ committed: true, storedEvents: [] }),
),
}),
IdAllocator.layer,
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
ingestNormalized: () => Effect.succeed([]),
}),
ServerSettings.layerTest(),
),
),
);
const fixture = {
ids,
providerInstanceId,
childThreadId: ids.childThreadId,
driver,
status: "running",
} as const;
const runningSubagent = makeRunOwnedSubagentFixture(fixture);
const runningTurnItem = makeRunOwnedSubagentTurnItemFixture(fixture);
const runningNode = makeRunOwnedSubagentNodeFixture({ ids, status: "running" });
const runningChildNode = makeRunOwnedSubagentChildNodeFixture({ ids, status: "running" });

yield* Effect.gen(function* () {
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
yield* runExecution.startRootRun({
commandId: CommandId.make("command:subagent-workspace-handoff"),
appThread: rootAppThread(ids.threadId),
providerSessionId: ProviderSessionId.make("session:subagent-workspace-handoff"),
session: {
events: Stream.empty,
subscribeEvents: Effect.succeed({
// The provider event stream stays open: only settled children may end ingestion.
events: Stream.concat(
Stream.fromIterable([
childThreadCreatedEvent(ids),
{ type: "subagent.updated", driver, subagent: runningSubagent },
{ type: "node.updated", driver, node: runningNode },
{ type: "node.updated", driver, node: runningChildNode },
{ type: "turn_item.updated", driver, turnItem: runningTurnItem },
rootTerminalEvent(ids, "interrupted"),
] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
Stream.never,
),
close: Deferred.succeed(ingestionDone, undefined),
}),
startTurn: () => Effect.void,
} as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
run: {
id: ids.runId,
threadId: ids.threadId,
ordinal: 1,
providerInstanceId,
} as OrchestrationV2Run,
rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
checkpointScope: {
id: CheckpointScopeId.make("checkpoint-scope:subagent-workspace-handoff"),
} as OrchestrationV2CheckpointScope,
providerThread: { id: ids.providerThreadId, driver } as OrchestrationV2ProviderThread,
attempt: {
id: ids.attemptId,
providerTurnId: ids.rootProviderTurnId,
} as OrchestrationV2RunAttempt,
attemptId: ids.attemptId,
providerTurnOrdinal: 1,
shouldFinalizeRun: () => Effect.succeed(true),
hasWorkspaceMoved: () => Effect.succeed(true),
message: {
messageId: MessageId.make("message:subagent-workspace-handoff:user"),
text: "Spawn a subagent then move to a worktree.",
attachments: [],
createdBy: "user",
creationSource: "web",
},
modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
runtimePolicy: {
runtimeMode: "full-access",
interactionMode: "default",
cwd: process.cwd(),
approvalPolicy: "never",
sandboxPolicy: {
type: "readOnly",
access: { type: "fullAccess" },
networkAccess: false,
},
},
});
}).pipe(Effect.provide(layerTest));

const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
assert.isTrue(Option.isSome(closed), "settled children must end event ingestion");

const events = yield* Ref.get(written);
const statusOf = (type: OrchestrationV2DomainEvent["type"], id: string) =>
events.flatMap((event) => {
if (event.type !== type) return [];
const payload = (
event as { readonly payload: { readonly id: string; readonly status: string } }
).payload;
return payload.id === id ? [payload.status] : [];
});
assert.deepEqual(statusOf("subagent.updated", runningSubagent.id), ["interrupted"]);
assert.deepEqual(statusOf("turn-item.updated", runningTurnItem.id), ["interrupted"]);
assert.deepEqual(statusOf("node.updated", runningNode.id), ["interrupted"]);
assert.deepEqual(statusOf("node.updated", runningChildNode.id), ["interrupted"]);
assert.deepEqual(statusOf("run.updated", ids.runId), ["waiting"]);
assert.deepEqual(statusOf("run-attempt.updated", ids.attemptId), ["completed"]);
assert.isFalse(
events.some(
(event) =>
event.type === "turn-item.updated" && event.payload.type === "run_interrupt_result",
),
);
}),
);

it.effect("omits interrupt results and subagent cascade for a superseded attempt", () =>
Effect.gen(function* () {
const { written, observed } = yield* captureRootRunTermination({
Expand Down Expand Up @@ -2619,6 +2756,37 @@ it.effect("emits run_interrupt_result when superseded attempt still has a hard-s
}),
);

it.effect("keeps a Stop that commits while a workspace handoff completes the run", () =>
Effect.gen(function* () {
const stopCommitted = yield* Ref.make(false);
const { written, finalEvents, committedEffects } = yield* captureRootRunTermination({
key: "stop-races-workspace-handoff",
shouldFinalizeRun: () => Effect.succeed(true),
// Stop commits after the handoff is detected and before the terminal write.
hasWorkspaceMoved: () => Ref.set(stopCommitted, true).pipe(Effect.as(true)),
// The sink refuses a guarded write once the Stop request is stored.
beforeTerminalWrite: (request) =>
Ref.get(stopCommitted).pipe(
Effect.map((stopped) => !(stopped && request.unlessInterruptRequested !== undefined)),
),
});
const statuses = (type: OrchestrationV2DomainEvent["type"]) =>
finalEvents.flatMap((event) =>
event.type === type ? [(event.payload as { readonly status: string }).status] : [],
);
assert.deepEqual(statuses("run.updated"), ["interrupted"]);
assert.deepEqual(statuses("run-attempt.updated"), ["interrupted"]);
assert.deepEqual(
written.map((item) => item.type),
["run_interrupt_result"],
);
assert.deepEqual(
committedEffects.map((effect) => effect.request.type),
["checkpoint.capture"],
);
}),
);

it.effect("does not overwrite Stop when ownership changes after the finalization read", () =>
Effect.gen(function* () {
const { written, observed, submittedEffects, committedEffects } =
Expand Down Expand Up @@ -3180,6 +3348,7 @@ function captureRootRunTermination(input: {
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
readonly rejectTerminalWrite?: boolean;
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"];
readonly seedOpenSubagent?: boolean;
Expand Down Expand Up @@ -3376,6 +3545,9 @@ function captureRootRunTermination(input: {
: {
hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest,
}),
...(input.hasWorkspaceMoved === undefined
? {}
: { hasWorkspaceMoved: input.hasWorkspaceMoved }),
message: {
messageId: MessageId.make(`message:${input.key}`),
text: "interrupt projection",
Expand Down
Loading
Loading