Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions src/agents/run_internal/agent_runner_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,19 +205,24 @@ def _extract_tool_call_id(raw: Any) -> str | None:
return candidate if isinstance(candidate, str) else None


def _latest_response_tool_call_ids(run_state: RunState[Any]) -> set[str]:
"""Return the call or item identifiers carried by the most recent model response."""
if not run_state._model_responses:
return set()
return {
call_id
for item in run_state._model_responses[-1].output
if (call_id := _extract_tool_call_id(item)) is not None
}


def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | None) -> set[str]:
"""Return tool call IDs whose local outputs have not reached a server conversation."""
if run_state is None:
return set()

if isinstance(run_state._current_step, NextStepRunAgain):
if not run_state._model_responses:
return set()
return {
call_id
for item in run_state._model_responses[-1].output
if (call_id := _extract_tool_call_id(item)) is not None
}
return _latest_response_tool_call_ids(run_state)

if not isinstance(run_state._current_step, NextStepInterruption):
return set()
Expand All @@ -226,7 +231,14 @@ def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | No
if processed_response is None:
return set()

tool_call_ids: set[str] = set()
# An interrupted turn stops before its next model request, so every tool output it built
# locally is still unsent. Some of those outputs have no pending tool run left to enumerate
# below, such as the error synthesized for a call to a tool that does not exist, and that
# classification is not part of the serialized `RunState`. The latest model response is the
# source of truth that survives both cases. Seeding from it stays safe because
# `hydrate_from_state` records everything that response reported as server-owned before it
# consults this set, so a server-owned item is never resent.
tool_call_ids = _latest_response_tool_call_ids(run_state)
tool_run_groups = (
processed_response.handoffs,
processed_response.functions,
Expand Down
143 changes: 143 additions & 0 deletions tests/test_run_impl_resume_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,149 @@ async def fake_resolve_interrupted_turn(**kwargs: object) -> SingleStepResult:
assert server_managed_values == [True]


def _sent_tool_outputs(model: ScriptedModel, *, first_call_index: int) -> list[tuple[str, str]]:
"""Collect the tool outputs the model received, from `first_call_index` onward."""
outputs: list[tuple[str, str]] = []
for call in model.calls[first_call_index:]:
for item in cast(list[dict[str, Any]], call.input):
if item.get("type") == "function_call_output":
outputs.append((str(item.get("call_id")), str(item.get("output"))))
return outputs


async def _run_server_managed(
agent: Agent[Any],
agent_input: Any,
*,
run_config: RunConfig,
use_conversation_id: bool,
streaming: bool,
) -> Any:
"""Run the agent under one of the server-managed continuation modes."""
kwargs: dict[str, Any] = (
{"conversation_id": "conv-resume"}
if use_conversation_id
else {"auto_previous_response_id": True}
)
if streaming:
streamed = Runner.run_streamed(agent, agent_input, run_config=run_config, **kwargs)
async for _ in streamed.stream_events():
pass
return streamed
return await Runner.run(agent, agent_input, run_config=run_config, **kwargs)


@pytest.mark.parametrize("streaming", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.parametrize("serialize_state", [False, True], ids=["live_state", "serialized_state"])
@pytest.mark.parametrize(
"use_conversation_id", [False, True], ids=["auto_previous_response_id", "conversation_id"]
)
@pytest.mark.asyncio
async def test_resumed_server_managed_run_sends_tool_not_found_output(
streaming: bool,
serialize_state: bool,
use_conversation_id: bool,
) -> None:
"""A resumed server-managed run must send the output built for a missing tool.

The interrupted turn answers the unknown tool locally while another call waits for
approval. The server already owns both calls, so resuming has to deliver both outputs.
"""

@function_tool(name_override="needs_ok", needs_approval=True)
async def needs_ok(text: str) -> str:
return text

model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[needs_ok])
model.extend(
[
[
get_function_tool_call(
"needs_ok", json.dumps({"text": "one"}), call_id="call-approval"
),
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
],
[get_text_message("done")],
]
)
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")

async def run_once(agent_input: Any) -> Any:
return await _run_server_managed(
agent,
agent_input,
run_config=run_config,
use_conversation_id=use_conversation_id,
streaming=streaming,
)

first = await run_once("Use needs_ok and missing_tool")
state = first.to_state()
if serialize_state:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
interruptions = state.get_interruptions()
assert [item.raw_item.call_id for item in interruptions] == ["call-approval"]
state.approve(interruptions[0])

resumed = await run_once(state)

assert resumed.final_output == "done"
delivered = _sent_tool_outputs(model, first_call_index=1)
assert sorted(call_id for call_id, _ in delivered) == ["call-approval", "call-missing"]
assert "missing_tool" in dict(delivered)["call-missing"]


@pytest.mark.asyncio
async def test_resumed_server_managed_run_sends_each_tool_output_once() -> None:
"""Staged approvals must deliver every output exactly once to a server-managed conversation.

Approving one of two gated calls resumes and interrupts again without a model request, so
the same model response stays current across both resumes.
"""

@function_tool(name_override="needs_ok", needs_approval=True)
async def needs_ok(text: str) -> str:
return f"ok:{text}"

model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[needs_ok])
model.extend(
[
[
get_function_tool_call("needs_ok", json.dumps({"text": "a"}), call_id="call-a"),
get_function_tool_call("needs_ok", json.dumps({"text": "b"}), call_id="call-b"),
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
],
[get_text_message("done")],
]
)
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")

async def run_once(agent_input: Any) -> Any:
return await _run_server_managed(
agent,
agent_input,
run_config=run_config,
use_conversation_id=False,
streaming=False,
)

result = await run_once("Use needs_ok twice and missing_tool")
for expected_model_calls in (1, 2):
state = await RunState.from_json(agent, json.loads(json.dumps(result.to_state().to_json())))
interruptions = state.get_interruptions()
assert interruptions
state.approve(interruptions[0])
result = await run_once(state)
# Approving only the first gated call resumes without asking the model again.
assert len(model.calls) == expected_model_calls

assert result.final_output == "done"
delivered = _sent_tool_outputs(model, first_call_index=1)
assert sorted(call_id for call_id, _ in delivered) == ["call-a", "call-b", "call-missing"]


@pytest.mark.asyncio
async def test_resumed_approval_does_not_duplicate_session_items() -> None:
async def test_tool() -> str:
Expand Down
96 changes: 96 additions & 0 deletions tests/test_server_conversation_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() ->
]
interrupted_state = SimpleNamespace(
_current_step=NextStepInterruption(interruptions=[]),
_model_responses=[model_response],
_last_processed_response=SimpleNamespace(
handoffs=[],
functions=[
Expand Down Expand Up @@ -211,6 +212,101 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() ->
] == ["call_DIAG", "call_CLEANUP1", "call_CLEANUP2"]


def test_hydrate_from_state_preserves_unsent_output_without_a_pending_tool_run() -> None:
"""An interrupted turn's local output is unsent even with no pending tool run to enumerate.

A call to a tool that does not exist is answered locally and leaves no pending tool run in
the resumed `ProcessedResponse`, so the latest model response is what identifies its output
as still unsent.
"""
agent = Agent(name="test")
approval_call = ResponseFunctionToolCall(
id="fc_101",
type="function_call",
call_id="call_APPROVAL",
name="run_cleanup",
arguments='{"target": "temp_files"}',
status="completed",
)
missing_tool_call = ResponseFunctionToolCall(
id="fc_102",
type="function_call",
call_id="call_MISSING",
name="not_a_registered_tool",
arguments="{}",
status="completed",
)
model_response = ModelResponse(
output=[approval_call, missing_tool_call],
usage=Usage(),
response_id="resp_101",
)
missing_tool_output = ToolCallOutputItem(
agent=agent,
raw_item={
"type": "function_call_output",
"call_id": "call_MISSING",
"output": "Tool not_a_registered_tool not found.",
},
output="Tool not_a_registered_tool not found.",
)
generated_items: list[RunItem] = [
ToolCallItem(agent=agent, raw_item=approval_call),
ToolCallItem(agent=agent, raw_item=missing_tool_call),
ToolApprovalItem(agent=agent, raw_item=approval_call, tool_name="run_cleanup"),
missing_tool_output,
]
interrupted_state = SimpleNamespace(
_current_step=NextStepInterruption(interruptions=[]),
_model_responses=[model_response],
_last_processed_response=SimpleNamespace(
handoffs=[],
functions=[SimpleNamespace(tool_call=approval_call)],
computer_actions=[],
custom_tool_calls=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
),
)

tracker = OpenAIServerConversationTracker(previous_response_id="resp_101")
tracker.hydrate_from_state(
original_input="Clean up and call the missing tool.",
generated_items=generated_items,
model_responses=[model_response],
unsent_tool_call_ids=get_unsent_tool_call_ids_for_interrupted_state(
cast(Any, interrupted_state)
),
)

assert "call_MISSING" not in tracker.server_tool_call_ids

prepared = tracker.prepare_input(
"Clean up and call the missing tool.",
[
ToolCallItem(agent=agent, raw_item=approval_call),
ToolCallItem(agent=agent, raw_item=missing_tool_call),
missing_tool_output,
ToolCallOutputItem(
agent=agent,
raw_item={
"type": "function_call_output",
"call_id": "call_APPROVAL",
"output": "Cleanup finished.",
},
output="Cleanup finished.",
),
],
)

assert [
item.get("call_id")
for item in prepared
if isinstance(item, dict) and item.get("type") == "function_call_output"
] == ["call_MISSING", "call_APPROVAL"]


def test_hydrate_from_state_does_not_track_string_initial_input_by_object_identity() -> None:
tracker = OpenAIServerConversationTracker(
conversation_id="conv-init-string", previous_response_id=None
Expand Down