diff --git a/src/agents/run_internal/agent_runner_helpers.py b/src/agents/run_internal/agent_runner_helpers.py index 6662f71e26..103d2af444 100644 --- a/src/agents/run_internal/agent_runner_helpers.py +++ b/src/agents/run_internal/agent_runner_helpers.py @@ -205,19 +205,24 @@ def _extract_tool_call_id(raw: Any) -> str | None: return candidate if isinstance(candidate, str) else None +def _latest_response_tool_call_ids(run_state: RunState[Any]) -> set[str]: + """Return the call or item identifiers carried by the most recent model response.""" + if not run_state._model_responses: + return set() + return { + call_id + for item in run_state._model_responses[-1].output + if (call_id := _extract_tool_call_id(item)) is not None + } + + def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | None) -> set[str]: """Return tool call IDs whose local outputs have not reached a server conversation.""" if run_state is None: return set() if isinstance(run_state._current_step, NextStepRunAgain): - if not run_state._model_responses: - return set() - return { - call_id - for item in run_state._model_responses[-1].output - if (call_id := _extract_tool_call_id(item)) is not None - } + return _latest_response_tool_call_ids(run_state) if not isinstance(run_state._current_step, NextStepInterruption): return set() @@ -226,7 +231,14 @@ def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | No if processed_response is None: return set() - tool_call_ids: set[str] = set() + # An interrupted turn stops before its next model request, so every tool output it built + # locally is still unsent. Some of those outputs have no pending tool run left to enumerate + # below, such as the error synthesized for a call to a tool that does not exist, and that + # classification is not part of the serialized `RunState`. The latest model response is the + # source of truth that survives both cases. Seeding from it stays safe because + # `hydrate_from_state` records everything that response reported as server-owned before it + # consults this set, so a server-owned item is never resent. + tool_call_ids = _latest_response_tool_call_ids(run_state) tool_run_groups = ( processed_response.handoffs, processed_response.functions, diff --git a/tests/test_run_impl_resume_paths.py b/tests/test_run_impl_resume_paths.py index 482a13edd8..5d1aa4640a 100644 --- a/tests/test_run_impl_resume_paths.py +++ b/tests/test_run_impl_resume_paths.py @@ -918,6 +918,149 @@ async def fake_resolve_interrupted_turn(**kwargs: object) -> SingleStepResult: assert server_managed_values == [True] +def _sent_tool_outputs(model: ScriptedModel, *, first_call_index: int) -> list[tuple[str, str]]: + """Collect the tool outputs the model received, from `first_call_index` onward.""" + outputs: list[tuple[str, str]] = [] + for call in model.calls[first_call_index:]: + for item in cast(list[dict[str, Any]], call.input): + if item.get("type") == "function_call_output": + outputs.append((str(item.get("call_id")), str(item.get("output")))) + return outputs + + +async def _run_server_managed( + agent: Agent[Any], + agent_input: Any, + *, + run_config: RunConfig, + use_conversation_id: bool, + streaming: bool, +) -> Any: + """Run the agent under one of the server-managed continuation modes.""" + kwargs: dict[str, Any] = ( + {"conversation_id": "conv-resume"} + if use_conversation_id + else {"auto_previous_response_id": True} + ) + if streaming: + streamed = Runner.run_streamed(agent, agent_input, run_config=run_config, **kwargs) + async for _ in streamed.stream_events(): + pass + return streamed + return await Runner.run(agent, agent_input, run_config=run_config, **kwargs) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["non_streamed", "streamed"]) +@pytest.mark.parametrize("serialize_state", [False, True], ids=["live_state", "serialized_state"]) +@pytest.mark.parametrize( + "use_conversation_id", [False, True], ids=["auto_previous_response_id", "conversation_id"] +) +@pytest.mark.asyncio +async def test_resumed_server_managed_run_sends_tool_not_found_output( + streaming: bool, + serialize_state: bool, + use_conversation_id: bool, +) -> None: + """A resumed server-managed run must send the output built for a missing tool. + + The interrupted turn answers the unknown tool locally while another call waits for + approval. The server already owns both calls, so resuming has to deliver both outputs. + """ + + @function_tool(name_override="needs_ok", needs_approval=True) + async def needs_ok(text: str) -> str: + return text + + model = ScriptedModel() + agent = Agent(name="test", model=model, tools=[needs_ok]) + model.extend( + [ + [ + get_function_tool_call( + "needs_ok", json.dumps({"text": "one"}), call_id="call-approval" + ), + get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"), + ], + [get_text_message("done")], + ] + ) + run_config = RunConfig(tool_not_found_behavior="return_error_to_model") + + async def run_once(agent_input: Any) -> Any: + return await _run_server_managed( + agent, + agent_input, + run_config=run_config, + use_conversation_id=use_conversation_id, + streaming=streaming, + ) + + first = await run_once("Use needs_ok and missing_tool") + state = first.to_state() + if serialize_state: + state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json()))) + interruptions = state.get_interruptions() + assert [item.raw_item.call_id for item in interruptions] == ["call-approval"] + state.approve(interruptions[0]) + + resumed = await run_once(state) + + assert resumed.final_output == "done" + delivered = _sent_tool_outputs(model, first_call_index=1) + assert sorted(call_id for call_id, _ in delivered) == ["call-approval", "call-missing"] + assert "missing_tool" in dict(delivered)["call-missing"] + + +@pytest.mark.asyncio +async def test_resumed_server_managed_run_sends_each_tool_output_once() -> None: + """Staged approvals must deliver every output exactly once to a server-managed conversation. + + Approving one of two gated calls resumes and interrupts again without a model request, so + the same model response stays current across both resumes. + """ + + @function_tool(name_override="needs_ok", needs_approval=True) + async def needs_ok(text: str) -> str: + return f"ok:{text}" + + model = ScriptedModel() + agent = Agent(name="test", model=model, tools=[needs_ok]) + model.extend( + [ + [ + get_function_tool_call("needs_ok", json.dumps({"text": "a"}), call_id="call-a"), + get_function_tool_call("needs_ok", json.dumps({"text": "b"}), call_id="call-b"), + get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"), + ], + [get_text_message("done")], + ] + ) + run_config = RunConfig(tool_not_found_behavior="return_error_to_model") + + async def run_once(agent_input: Any) -> Any: + return await _run_server_managed( + agent, + agent_input, + run_config=run_config, + use_conversation_id=False, + streaming=False, + ) + + result = await run_once("Use needs_ok twice and missing_tool") + for expected_model_calls in (1, 2): + state = await RunState.from_json(agent, json.loads(json.dumps(result.to_state().to_json()))) + interruptions = state.get_interruptions() + assert interruptions + state.approve(interruptions[0]) + result = await run_once(state) + # Approving only the first gated call resumes without asking the model again. + assert len(model.calls) == expected_model_calls + + assert result.final_output == "done" + delivered = _sent_tool_outputs(model, first_call_index=1) + assert sorted(call_id for call_id, _ in delivered) == ["call-a", "call-b", "call-missing"] + + @pytest.mark.asyncio async def test_resumed_approval_does_not_duplicate_session_items() -> None: async def test_tool() -> str: diff --git a/tests/test_server_conversation_tracker.py b/tests/test_server_conversation_tracker.py index 3474a3d235..5e4ea87ed7 100644 --- a/tests/test_server_conversation_tracker.py +++ b/tests/test_server_conversation_tracker.py @@ -149,6 +149,7 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() -> ] interrupted_state = SimpleNamespace( _current_step=NextStepInterruption(interruptions=[]), + _model_responses=[model_response], _last_processed_response=SimpleNamespace( handoffs=[], functions=[ @@ -211,6 +212,101 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() -> ] == ["call_DIAG", "call_CLEANUP1", "call_CLEANUP2"] +def test_hydrate_from_state_preserves_unsent_output_without_a_pending_tool_run() -> None: + """An interrupted turn's local output is unsent even with no pending tool run to enumerate. + + A call to a tool that does not exist is answered locally and leaves no pending tool run in + the resumed `ProcessedResponse`, so the latest model response is what identifies its output + as still unsent. + """ + agent = Agent(name="test") + approval_call = ResponseFunctionToolCall( + id="fc_101", + type="function_call", + call_id="call_APPROVAL", + name="run_cleanup", + arguments='{"target": "temp_files"}', + status="completed", + ) + missing_tool_call = ResponseFunctionToolCall( + id="fc_102", + type="function_call", + call_id="call_MISSING", + name="not_a_registered_tool", + arguments="{}", + status="completed", + ) + model_response = ModelResponse( + output=[approval_call, missing_tool_call], + usage=Usage(), + response_id="resp_101", + ) + missing_tool_output = ToolCallOutputItem( + agent=agent, + raw_item={ + "type": "function_call_output", + "call_id": "call_MISSING", + "output": "Tool not_a_registered_tool not found.", + }, + output="Tool not_a_registered_tool not found.", + ) + generated_items: list[RunItem] = [ + ToolCallItem(agent=agent, raw_item=approval_call), + ToolCallItem(agent=agent, raw_item=missing_tool_call), + ToolApprovalItem(agent=agent, raw_item=approval_call, tool_name="run_cleanup"), + missing_tool_output, + ] + interrupted_state = SimpleNamespace( + _current_step=NextStepInterruption(interruptions=[]), + _model_responses=[model_response], + _last_processed_response=SimpleNamespace( + handoffs=[], + functions=[SimpleNamespace(tool_call=approval_call)], + computer_actions=[], + custom_tool_calls=[], + local_shell_calls=[], + shell_calls=[], + apply_patch_calls=[], + ), + ) + + tracker = OpenAIServerConversationTracker(previous_response_id="resp_101") + tracker.hydrate_from_state( + original_input="Clean up and call the missing tool.", + generated_items=generated_items, + model_responses=[model_response], + unsent_tool_call_ids=get_unsent_tool_call_ids_for_interrupted_state( + cast(Any, interrupted_state) + ), + ) + + assert "call_MISSING" not in tracker.server_tool_call_ids + + prepared = tracker.prepare_input( + "Clean up and call the missing tool.", + [ + ToolCallItem(agent=agent, raw_item=approval_call), + ToolCallItem(agent=agent, raw_item=missing_tool_call), + missing_tool_output, + ToolCallOutputItem( + agent=agent, + raw_item={ + "type": "function_call_output", + "call_id": "call_APPROVAL", + "output": "Cleanup finished.", + }, + output="Cleanup finished.", + ), + ], + ) + + assert [ + item.get("call_id") + for item in prepared + if isinstance(item, dict) and item.get("type") == "function_call_output" + ] == ["call_MISSING", "call_APPROVAL"] + + def test_hydrate_from_state_does_not_track_string_initial_input_by_object_identity() -> None: tracker = OpenAIServerConversationTracker( conversation_id="conv-init-string", previous_response_id=None