From 4d1a13abde47568fb17ac93d55ebb260e3aae32e Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 06:53:24 -0800 Subject: [PATCH 1/3] feat: Filter tool declarations and recheck eligibility before execution --- src/agents/__init__.py | 2 + src/agents/agent.py | 9 +- src/agents/mcp/server.py | 174 +++---- src/agents/mcp/util.py | 39 +- src/agents/models/openai_responses.py | 11 +- src/agents/run.py | 5 + src/agents/run_config.py | 52 ++- src/agents/run_context.py | 11 + src/agents/run_internal/run_loop.py | 50 +- src/agents/run_internal/tool_declarations.py | 259 ++++++++++ src/agents/run_internal/tool_execution.py | 31 +- src/agents/run_internal/turn_resolution.py | 24 + src/agents/run_state.py | 16 + src/agents/tool.py | 27 ++ tests/test_mutable_tool_hooks.py | 57 ++- tests/test_tool_declarations.py | 467 +++++++++++++++++++ 16 files changed, 1103 insertions(+), 131 deletions(-) create mode 100644 src/agents/run_internal/tool_declarations.py create mode 100644 tests/test_tool_declarations.py diff --git a/src/agents/__init__.py b/src/agents/__init__.py index 7e6885c971..3af8702257 100644 --- a/src/agents/__init__.py +++ b/src/agents/__init__.py @@ -120,6 +120,7 @@ RunConfig, Runner, ToolCallHookContext, + ToolDeclaration, ToolErrorFormatter, ToolErrorFormatterArgs, ToolExecutionConfig, @@ -499,6 +500,7 @@ def enable_verbose_stdout_logging() -> None: "ReasoningItemIdPolicy", "ToolExecutionConfig", "ToolCallHookContext", + "ToolDeclaration", "ToolErrorFormatter", "ToolErrorFormatterArgs", "ToolNotFoundBehavior", diff --git a/src/agents/agent.py b/src/agents/agent.py index 48fea87c0d..d87e7b526d 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -54,6 +54,7 @@ ToolOriginType, _build_handled_function_tool_error_handler, _build_wrapped_function_tool, + _function_tool_enabled, _log_function_tool_invocation, _parse_function_tool_json_input, default_tool_error_function, @@ -303,13 +304,7 @@ async def _check_tool_enabled(tool: Tool) -> bool: if not isinstance(tool, FunctionTool): return True - attr = tool.is_enabled - if isinstance(attr, bool): - return attr - res = attr(run_context, get_public_agent(self)) - if inspect.isawaitable(res): - return bool(await res) - return bool(res) + return await _function_tool_enabled(tool, run_context, self) results = await gather_with_cancel(*(_check_tool_enabled(t) for t in tools)) enabled: list[Tool] = [t for t, ok in zip(tools, results, strict=False) if ok] diff --git a/src/agents/mcp/server.py b/src/agents/mcp/server.py index 57fa9e1d65..f645a87018 100644 --- a/src/agents/mcp/server.py +++ b/src/agents/mcp/server.py @@ -841,6 +841,94 @@ def _is_tool_list_schema(value: object) -> bool: return _to_bool(require_approval, location="require_approval") + tool_filter: ToolFilter = None + + async def _apply_tool_filter( + self, + tools: list[MCPTool], + run_context: RunContextWrapper[Any] | None = None, + agent: AgentBase | None = None, + ) -> list[MCPTool]: + """Apply the tool filter to the list of tools.""" + if self.tool_filter is None: + return tools + + # Handle static tool filter + if isinstance(self.tool_filter, dict): + return self._apply_static_tool_filter(tools, self.tool_filter) + + # Handle callable tool filter (dynamic filter) + else: + if run_context is None or agent is None: + raise UserError("run_context and agent are required for dynamic tool filtering") + return await self._apply_dynamic_tool_filter(tools, run_context, agent) + + def _apply_static_tool_filter( + self, tools: list[MCPTool], static_filter: ToolFilterStatic + ) -> list[MCPTool]: + """Apply static tool filtering based on allowlist and blocklist.""" + filtered_tools = tools + + # Apply allowed_tool_names filter (whitelist) + if "allowed_tool_names" in static_filter: + allowed_names = static_filter["allowed_tool_names"] + filtered_tools = [t for t in filtered_tools if t.name in allowed_names] + + # Apply blocked_tool_names filter (blacklist) + if "blocked_tool_names" in static_filter: + blocked_names = static_filter["blocked_tool_names"] + filtered_tools = [t for t in filtered_tools if t.name not in blocked_names] + + return filtered_tools + + async def _apply_dynamic_tool_filter( + self, + tools: list[MCPTool], + run_context: RunContextWrapper[Any], + agent: AgentBase, + ) -> list[MCPTool]: + """Apply dynamic tool filtering using a callable filter function.""" + + # Ensure we have a callable filter + if not callable(self.tool_filter): + raise ValueError("Tool filter must be callable for dynamic filtering") + tool_filter_func = self.tool_filter + + # Create filter context + filter_context = ToolFilterContext( + run_context=run_context, + agent=agent, + server_name=self.name, + ) + + filtered_tools = [] + for tool, detached in zip(tools, _snapshot_tools(tools), strict=True): + try: + # Inspect a detached copy so a mutating filter cannot corrupt the cache. + result = tool_filter_func(filter_context, detached) + + if inspect.isawaitable(result): + should_include = await result + else: + should_include = result + + if should_include: + filtered_tools.append(tool) + except Exception as e: + if _debug.DONT_LOG_TOOL_DATA: + message = "Error applying MCP tool filter" + else: + server_name = get_mcp_server_log_name(self.name) + message = ( + f"Error applying MCP tool filter to tool '{tool.name}' " + f"on server '{server_name}'" + ) + log_tool_action_error(logger, message, e) + # On error, exclude the tool for safety + continue + + return filtered_tools + def _get_needs_approval_for_tool( self, tool: MCPTool, @@ -1038,92 +1126,6 @@ async def _list_prompts_page( ), ) - async def _apply_tool_filter( - self, - tools: list[MCPTool], - run_context: RunContextWrapper[Any] | None = None, - agent: AgentBase | None = None, - ) -> list[MCPTool]: - """Apply the tool filter to the list of tools.""" - if self.tool_filter is None: - return tools - - # Handle static tool filter - if isinstance(self.tool_filter, dict): - return self._apply_static_tool_filter(tools, self.tool_filter) - - # Handle callable tool filter (dynamic filter) - else: - if run_context is None or agent is None: - raise UserError("run_context and agent are required for dynamic tool filtering") - return await self._apply_dynamic_tool_filter(tools, run_context, agent) - - def _apply_static_tool_filter( - self, tools: list[MCPTool], static_filter: ToolFilterStatic - ) -> list[MCPTool]: - """Apply static tool filtering based on allowlist and blocklist.""" - filtered_tools = tools - - # Apply allowed_tool_names filter (whitelist) - if "allowed_tool_names" in static_filter: - allowed_names = static_filter["allowed_tool_names"] - filtered_tools = [t for t in filtered_tools if t.name in allowed_names] - - # Apply blocked_tool_names filter (blacklist) - if "blocked_tool_names" in static_filter: - blocked_names = static_filter["blocked_tool_names"] - filtered_tools = [t for t in filtered_tools if t.name not in blocked_names] - - return filtered_tools - - async def _apply_dynamic_tool_filter( - self, - tools: list[MCPTool], - run_context: RunContextWrapper[Any], - agent: AgentBase, - ) -> list[MCPTool]: - """Apply dynamic tool filtering using a callable filter function.""" - - # Ensure we have a callable filter - if not callable(self.tool_filter): - raise ValueError("Tool filter must be callable for dynamic filtering") - tool_filter_func = self.tool_filter - - # Create filter context - filter_context = ToolFilterContext( - run_context=run_context, - agent=agent, - server_name=self.name, - ) - - filtered_tools = [] - for tool, detached in zip(tools, _snapshot_tools(tools), strict=True): - try: - # Inspect a detached copy so a mutating filter cannot corrupt the cache. - result = tool_filter_func(filter_context, detached) - - if inspect.isawaitable(result): - should_include = await result - else: - should_include = result - - if should_include: - filtered_tools.append(tool) - except Exception as e: - if _debug.DONT_LOG_TOOL_DATA: - message = "Error applying MCP tool filter" - else: - server_name = get_mcp_server_log_name(self.name) - message = ( - f"Error applying MCP tool filter to tool '{tool.name}' " - f"on server '{server_name}'" - ) - log_tool_action_error(logger, message, e) - # On error, exclude the tool for safety - continue - - return filtered_tools - @abc.abstractmethod def create_streams( self, diff --git a/src/agents/mcp/util.py b/src/agents/mcp/util.py index d51a773342..d7eafdc27f 100644 --- a/src/agents/mcp/util.py +++ b/src/agents/mcp/util.py @@ -30,6 +30,7 @@ ToolOutputTextDict, _build_handled_function_tool_error_handler, _build_wrapped_function_tool, + _ToolEligibilityError, default_tool_error_function, ) from ..tool_context import ToolContext @@ -54,7 +55,7 @@ ToolOutput = Union[str, ToolOutputItem, list[ToolOutputItem]] # noqa: UP007 if TYPE_CHECKING: - from mcp.types import Tool as MCPTool + from mcp.types import CallToolResult, Tool as MCPTool from ..agent import AgentBase from .server import MCPServer @@ -716,19 +717,45 @@ async def invoke_mcp_tool( f"Invalid JSON input for tool {tool_name_for_display}: expected a JSON object" ) + async def call_peer() -> CallToolResult: + if merged_meta is None: + return await server.call_tool(tool.name, json_data) + return await server.call_tool(tool.name, json_data, meta=merged_meta) + + invoke = call_peer if _debug.DONT_LOG_TOOL_DATA: logger.debug("Invoking MCP tool") else: logger.debug("Invoking MCP tool %s with input %s", tool_name_for_display, input_json) try: + if ( + isinstance(context, ToolContext) + and (record := context._tool_invocations.get(context.tool_call_id)) is not None + and record.declaration is not None + ): + binding = record.declaration + + async def check_and_call() -> CallToolResult: + # Check inside the scheduled task, after metadata resolution. + try: + if context.run_config is None or binding.server is not server: + raise UserError("MCP recipient or policy changed.") + if not await server._apply_tool_filter([tool], context, binding.agent): + raise UserError("MCP tool filter refused the call.") + await binding.check(context, context.run_config) + if tool.name != binding.view.mcp_tool_name: + raise UserError("MCP tool name changed.") + except Exception as error: + raise _ToolEligibilityError( + "Prepared MCP tool is no longer eligible." + ) from error + return await call_peer() + + invoke = check_and_call resolved_meta = await cls._resolve_meta(server, context, tool.name, json_data) merged_meta = cls._merge_mcp_meta(resolved_meta, meta) - call_task = asyncio.create_task( - server.call_tool(tool.name, json_data) - if merged_meta is None - else server.call_tool(tool.name, json_data, meta=merged_meta) - ) + call_task = asyncio.create_task(invoke()) try: done, _ = await asyncio.wait({call_task}, return_when=asyncio.FIRST_COMPLETED) finished_task = done.pop() diff --git a/src/agents/models/openai_responses.py b/src/agents/models/openai_responses.py index b63c257fa6..48bfa45d54 100644 --- a/src/agents/models/openai_responses.py +++ b/src/agents/models/openai_responses.py @@ -70,6 +70,7 @@ Tool, ToolSearchTool, WebSearchTool, + _builtin_tool_choice_name, has_required_tool_search_surface, validate_responses_programmatic_tool_calling_configuration, validate_responses_tool_search_configuration, @@ -1962,11 +1963,9 @@ def convert_tool_choice( return "auto" elif tool_choice == "none": return "none" - elif tool_choice in { - "computer", - "computer_use", - "computer_use_preview", - } and cls._has_computer_tool(tools): + elif _builtin_tool_choice_name( + tool_choice + ) == "computer_use_preview" and cls._has_computer_tool(tools): return cls._convert_builtin_computer_tool_choice( tool_choice=tool_choice, model=model, @@ -1990,7 +1989,7 @@ def convert_tool_choice( "web_search_preview", }: # WebSearchTool advertises web_search for both search selectors. - hosted_type = "web_search" if tool_choice == "web_search_preview" else tool_choice + hosted_type = _builtin_tool_choice_name(tool_choice) if not any(tool["type"] == hosted_type for tool in tool_params) and any( tool["type"] == "function" and tool["name"] == tool_choice diff --git a/src/agents/run.py b/src/agents/run.py index 57ca88bbe2..be16199273 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -55,6 +55,7 @@ RunConfig, RunOptions, ToolCallHookContext, + ToolDeclaration, ToolErrorFormatter, ToolErrorFormatterArgs, ToolExecutionConfig, @@ -166,6 +167,7 @@ take_held_session_write, update_run_state_after_resume, ) +from .run_internal.tool_declarations import validate_resume_policy from .run_internal.tool_use_tracker import ( AgentToolUseTracker, hydrate_tool_use_tracker, @@ -210,6 +212,7 @@ "ReasoningItemIdPolicy", "ToolExecutionConfig", "ToolCallHookContext", + "ToolDeclaration", "ToolErrorFormatter", "ToolErrorFormatterArgs", "ToolNotFoundBehavior", @@ -699,6 +702,7 @@ async def _run_impl( context=context, ) context_wrapper._resolve_function_approval_owners(starting_agent) + validate_resume_policy(run_state, context_wrapper, run_config) context = context_wrapper.context reject_unrecoverable_terminal_state(run_state) @@ -2725,6 +2729,7 @@ def run_streamed( context=context, ) context_wrapper._resolve_function_approval_owners(starting_agent) + validate_resume_policy(run_state, context_wrapper, run_config) # Override max_turns with the state's max_turns to preserve it across resumption max_turns = run_state._max_turns diff --git a/src/agents/run_config.py b/src/agents/run_config.py index c65d9cbd75..bdeafcdbd0 100644 --- a/src/agents/run_config.py +++ b/src/agents/run_config.py @@ -10,6 +10,7 @@ from pydantic import TypeAdapter from typing_extensions import NotRequired, TypedDict +from . import run_context from ._config_coercion import ( _declared_dataclass_type, coerce_dataclass_config, @@ -158,11 +159,22 @@ class ToolCallHookContext(Generic[TContext]): """ +@dataclass(frozen=True) +class ToolDeclaration: + """Resolved function name, original description and optional namespace and raw MCP names.""" + + name: str + description: str + namespace: str | None = None + mcp_server_name: str | None = None + mcp_tool_name: str | None = None + + @dataclass class ToolExecutionConfig: """Configure tool execution and run-wide preparation callbacks. - Callbacks cover SDK-executed tools and native tools for which Codex emits tool hooks. + Input and output callbacks cover SDK-executed tools and native tools with Codex hooks. Codex reports native shell and patch inputs as `{"command": ...}`. A native output hook can return the original result or replacement text for the model. API-hosted tools execute outside the client hook lifecycle. @@ -183,6 +195,41 @@ class ToolExecutionConfig: The same guardrails still run again immediately before tool execution after approval. """ + prepare_tool_declaration: ( + Callable[[run_context.RunContextWrapper[Any], ToolDeclaration], Awaitable[str | None]] + | None + ) = field(default=None, kw_only=True) + """Experimental. Describe an enabled function tool or return None to withhold it. + + Return declaration.description to keep the description. The Runner applies the decision + after name resolution, to both the model request and the tools available for that turn. + It checks the decision and current eligibility again before execution, including after + queueing or approval. Keep the callback free of side effects. Only the request uses the + returned description. + + For example, wait for an inspection to succeed before offering a publish tool. + + ```python + async def prepare(ctx, declaration): + if declaration.name == "publish": + return "Publish the inspected report" if ctx.context.inspected else None + return declaration.description + + config = ToolExecutionConfig(prepare_tool_declaration=prepare) + ``` + + Set `inspected` only after success. The Runner refreshes the catalog on the next + request. This does not force a selection or order calls in a batch. Withholding + the named choice or emptying a required catalog raises UserError before sampling. + + Covers direct function tools and MCP tools discovered through agent.mcp_servers. + Preconverted MCP functions in agent.tools raise UserError. Handoffs, deferred tools, + programmatic callers, agents as tools and provider-managed tools are outside this policy. + Use the same policy on resume. A serialized run trusts the application's current tool + implementation and authorization policy. Enforce atomic authorization at the service + that owns the operation. + """ + prepare_tool_input: Callable[[ToolCallHookContext[Any]], Awaitable[dict[str, Any]]] | None = ( None ) @@ -219,7 +266,7 @@ class ToolExecutionConfig: sdk_only_tool_hooks: bool = False """Run hooks only for SDK-executed tools, preserving classic behavior. - When either preparation callback is set, the Codex provider disables native workspace tools. + With input or output preparation, the Codex provider disables native workspace tools. """ def __post_init__(self) -> None: @@ -718,6 +765,7 @@ def _coerce_run_config(value: RunConfig | dict[str, Any]) -> RunConfig: "SandboxRunConfig", "ToolExecutionConfig", "ToolCallHookContext", + "ToolDeclaration", "ToolErrorFormatter", "ToolErrorFormatterArgs", "_default_trace_include_sensitive_data", diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 77ae747a8e..7cf5c0e6ff 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -43,6 +43,7 @@ if TYPE_CHECKING: from .items import ToolApprovalItem, TResponseInputItem + from .run_internal.tool_declarations import PreparedTool else: # Keep runtime annotations resolvable for TypeAdapter users (e.g., Temporal's # Pydantic data converter) without importing items.py and introducing cycles. @@ -157,6 +158,11 @@ class _ToolInvocationRecord: completed: bool = False # The input admitted under this ID, paired with the digest of the original proposal. input_rewrite: tuple[str, dict[str, Any]] | None = None + requires_declaration: bool = False + + def __post_init__(self) -> None: + # Never serialize the live executor and policy into a checkpoint. + self.declaration: PreparedTool | None = None @dataclass(eq=False) @@ -749,6 +755,7 @@ def _mark_tool_call_completed( return record.executed = True record.completed = True + record.declaration = None def _mark_tool_invocation_executed( self, @@ -1642,6 +1649,9 @@ def _rebuild_tool_invocations( raise validation_error_factory( "RunState tool invocation contains invalid lifecycle data." ) + requires_declaration = serialized_invocation.get("requires_declaration", False) + if not isinstance(requires_declaration, bool): + raise validation_error_factory("RunState tool declaration requirement is invalid.") input_rewrite = None if "input_rewrite" in serialized_invocation: rewrite = serialized_invocation["input_rewrite"] @@ -1662,6 +1672,7 @@ def _rebuild_tool_invocations( executed=executed, completed=completed, input_rewrite=input_rewrite, + requires_declaration=requires_declaration, ) def _mark_restored_unbound_approval_call_ids(self) -> None: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 659642845a..44747601ff 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -17,6 +17,7 @@ from openai.types.responses import ( Response, ResponseCompletedEvent, + ResponseFunctionToolCall, ResponseOutputItemDoneEvent, ) from openai.types.responses.response_output_item import ResponseOutputItem @@ -207,6 +208,7 @@ ) from .streaming import stream_step_items_to_queue, stream_step_result_to_queue from .tool_actions import ApplyPatchAction, ComputerAction, LocalShellAction, ShellAction +from .tool_declarations import prepare_declarations, validate_prepared_choice from .tool_execution import ( coerce_shell_call, execute_apply_patch_calls, @@ -236,6 +238,7 @@ validate_run_hooks, ) from .turn_resolution import ( + _snapshot_function_calls, check_for_final_output_from_tools, execute_final_output, execute_handoffs, @@ -2519,6 +2522,9 @@ async def raise_if_input_guardrail_tripwire_known() -> None: handoffs, collision_policy=run_config.tool_name_collision_policy, ) + all_tools, model_tools, prepared_tools, omitted = await prepare_declarations( + all_tools, public_agent, context_wrapper, run_config + ) if agent_span is not None: agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] agent_span.span_data.tools = [ @@ -2544,6 +2550,7 @@ async def raise_if_input_guardrail_tripwire_known() -> None: has_tools=bool(all_tools or handoffs), ) model_settings = maybe_reset_tool_choice(public_agent, tool_use_tracker, model_settings) + validate_prepared_choice(omitted, model_settings.tool_choice, bool(all_tools or handoffs)) final_response: ModelResponse | None = None streamed_response_output: list[ResponseOutputItem] = [] @@ -2656,7 +2663,7 @@ def get_model_stream() -> AsyncIterator[TResponseStreamEvent]: filtered.instructions, filtered.input, model_settings, - all_tools, + model_tools, output_schema, handoffs, get_model_tracing_impl( @@ -2702,12 +2709,28 @@ def get_model_stream() -> AsyncIterator[TResponseStreamEvent]: if isinstance(event_response, Response): stream_response_id = event_response.id stream_request_id = getattr(event_response, "_request_id", None) - streamed_result._event_queue.put_nowait(RawResponsesStreamEvent(data=event)) + if ( + isinstance(event, ResponseCompletedEvent) + and not event.response.output + and streamed_response_output + ): + # Some backends leave completed output empty after item.done events. + event.response.output = list(streamed_response_output) + + raw_event = event + if run_config.tool_execution is not None: + item = getattr(event, "item", None) + if isinstance(item, ResponseFunctionToolCall): + raw_event = event.model_copy(update={"item": deepcopy(item)}) + elif isinstance(event_response, Response) and event_response.output: + output = _snapshot_function_calls(event_response.output) + if output is not event_response.output: + response_view = event_response.model_copy(update={"output": output}) + raw_event = event.model_copy(update={"response": response_view}) + streamed_result._event_queue.put_nowait(RawResponsesStreamEvent(data=raw_event)) terminal_response: Response | None = None - is_completed_event = False if isinstance(event, ResponseCompletedEvent): - is_completed_event = True terminal_response = event.response elif getattr(event, "type", None) in {"response.incomplete", "response.failed"}: event_type = cast(str, event.type) @@ -2723,11 +2746,6 @@ def get_model_stream() -> AsyncIterator[TResponseStreamEvent]: raise response_error_event_failure_error(cast(str, event.type), event) if terminal_response is not None: - if is_completed_event and not terminal_response.output and streamed_response_output: - # Some streaming backends emit output items during item.done events while - # leaving the terminal response output empty. Preserve those items so the - # runner can resolve the completed step correctly. - terminal_response.output = list(streamed_response_output) # Always fold retry attempts into usage, even when the terminal response omits # provider usage (common for some Chat Completions / LiteLLM streams). Skipping # apply_retry_attempt_usage here would drop failed-attempt accounting and diverge @@ -2801,7 +2819,8 @@ async def after_invocation_validation( ) tool_execution = run_config.tool_execution if tool_execution is not None and ( - tool_execution.prepare_tool_input is not None + tool_execution.prepare_tool_declaration is not None + or tool_execution.prepare_tool_input is not None or tool_execution.prepare_tool_output is not None ): # A stream consumer must not change the invocation seen by preparation hooks. @@ -2836,6 +2855,7 @@ def on_tool_error_selected() -> None: new_response=final_response, output_schema=output_schema, all_tools=all_tools, + prepared_tools=prepared_tools, handoffs=handoffs, hooks=hooks, context_wrapper=context_wrapper, @@ -2927,6 +2947,9 @@ async def run_single_turn( handoffs, collision_policy=run_config.tool_name_collision_policy, ) + all_tools, model_tools, prepared_tools, omitted = await prepare_declarations( + all_tools, public_agent, context_wrapper, run_config + ) if agent_span is not None: agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] agent_span.span_data.tools = [ @@ -2946,7 +2969,7 @@ async def run_single_turn( system_prompt, input, output_schema, - all_tools, + model_tools, handoffs, hooks, context_wrapper, @@ -2958,6 +2981,7 @@ async def run_single_turn( session_items_to_rewind=session_items_to_rewind, prompt_cache_key_resolver=prompt_cache_key_resolver, defer_llm_end_hooks=True, + omitted=omitted, reasoning_item_id_policy=reasoning_item_id_policy, ) @@ -2989,6 +3013,7 @@ async def after_invocation_validation( new_response=new_response, output_schema=output_schema, all_tools=all_tools, + prepared_tools=prepared_tools, handoffs=handoffs, hooks=hooks, context_wrapper=context_wrapper, @@ -3021,6 +3046,8 @@ async def get_new_response( prompt_cache_key_resolver: PromptCacheKeyResolver | None = None, defer_llm_end_hooks: bool = False, reasoning_item_id_policy: ReasoningItemIdPolicy | None = None, + *, + omitted: set[str] | None = None, ) -> ModelResponse: """Call the model and return the raw response, handling retries and hooks.""" public_agent = bindings.public_agent @@ -3052,6 +3079,7 @@ async def get_new_response( has_tools=bool(all_tools or handoffs), ) model_settings = maybe_reset_tool_choice(public_agent, tool_use_tracker, model_settings) + validate_prepared_choice(omitted, model_settings.tool_choice, bool(all_tools or handoffs)) if server_conversation_tracker is not None: server_conversation_tracker.validate_pending_input_filter(filtered.input) diff --git a/src/agents/run_internal/tool_declarations.py b/src/agents/run_internal/tool_declarations.py new file mode 100644 index 0000000000..65d7b23721 --- /dev/null +++ b/src/agents/run_internal/tool_declarations.py @@ -0,0 +1,259 @@ +"""Prepare a narrowing view of the resolved Runner tool inventory.""" + +from __future__ import annotations + +import copy +from types import MethodType +from typing import TYPE_CHECKING, Any + +from .._tool_identity import get_function_tool_qualified_name +from ..exceptions import ModelBehaviorError, UserError +from ..run_config import RunConfig, ToolDeclaration +from ..tool import ( + FunctionTool, + HostedMCPTool, + Tool, + _builtin_tool_choice_name, + _function_tool_enabled, +) + +if TYPE_CHECKING: + from ..agent import Agent + from ..model_settings import ToolChoice + from ..run_context import RunContextWrapper + from ..run_state import RunState + from .run_steps import ProcessedResponse + + +def _covered(tool: FunctionTool) -> bool: + return ( + not tool.defer_loading + and tool._agent_instance is None + and not (tool.allowed_callers and "programmatic" in tool.allowed_callers) + ) + + +def _same_callback(first: object, second: object) -> bool: + return first is second or ( + isinstance(first, MethodType) + and isinstance(second, MethodType) + and first.__func__ is second.__func__ + and first.__self__ is second.__self__ + ) + + +def validate_resume_policy( + state: RunState[Any], context: RunContextWrapper[Any], config: RunConfig +) -> None: + """Reject missing or replaced policy before claiming a paused session.""" + from .run_steps import NextStepInterruption + + if not isinstance(state._current_step, NextStepInterruption): + return + for item in state._current_step.interruptions: + call_id = item.call_id + record = context._tool_invocations.get(call_id) if call_id else None + if not call_id or record is None or not record.requires_declaration: + continue + if ( + context.get_approval_status( + item.tool_name or "", + call_id, + tool_namespace=item.tool_namespace, + existing_pending=item, + current_invocation=item, + ) + is False + ): + continue + if config.tool_execution is None or config.tool_execution.prepare_tool_declaration is None: + raise UserError("Restore prepare_tool_declaration to resume this tool call.") + if record.declaration is not None: + record.declaration.validate(config) + + +class PreparedTool: + """In-memory authority for a sampled function. Never persisted with the call.""" + + def __init__(self, tool: FunctionTool, agent: Agent[Any], config: RunConfig): + settings = config.tool_execution + if settings is None or settings.prepare_tool_declaration is None: + raise UserError("Restore prepare_tool_declaration to resume this tool call.") + self.policy = settings.prepare_tool_declaration + self.tool = tool + self.agent = agent + self.binding = tool._mcp_tool_binding + index = self.binding[2] if self.binding else None + if self.binding and ( + index is None + or not 0 <= index < len(agent.mcp_servers) + or any(item is tool for item in agent.tools) + ): + raise UserError("Use agent.mcp_servers for MCP tools with prepare_tool_declaration.") + self.server = agent.mcp_servers[index] if index is not None else None + self.executor = tool.on_invoke_tool + self.approval = tool.needs_approval + self.peer_call = self.server.call_tool if self.server else None + self.peer_approval = self.server._needs_approval_policy if self.server else None + self.schema = copy.deepcopy(tool.params_json_schema) + self.strict = tool.strict_json_schema + self.view = ToolDeclaration( + name=tool.name, + description=tool.description, + namespace=tool._tool_namespace, + mcp_server_name=self.server.name if self.server else None, + mcp_tool_name=self.binding[1] if self.binding else None, + ) + + def __deepcopy__(self, memo: dict[int, Any]) -> PreparedTool: + # Checkpoints copy invocation state but must retain the original authority. + return self + + def validate(self, config: RunConfig) -> None: + settings = config.tool_execution + tool = self.tool + if ( + settings is None + or not _same_callback(settings.prepare_tool_declaration, self.policy) + or tool.on_invoke_tool is not self.executor + or tool.needs_approval is not self.approval + or tool.strict_json_schema != self.strict + or tool.params_json_schema != self.schema + or tool.name != self.view.name + or tool._tool_namespace != self.view.namespace + or tool._mcp_tool_binding != self.binding + or not _covered(tool) + ): + raise UserError("The prepared tool or its policy changed before execution.") + if self.server is not None: + assert self.binding is not None and self.binding[2] is not None + index = self.binding[2] + if ( + index >= len(self.agent.mcp_servers) + or self.agent.mcp_servers[index] is not self.server + or self.server.name != self.view.mcp_server_name + or not _same_callback(self.server.call_tool, self.peer_call) + or self.server._needs_approval_policy is not self.peer_approval + or ( + isinstance(self.peer_approval, dict) + and bool(self.peer_approval.get(self.binding[1])) != self.approval + ) + ): + raise UserError("The prepared MCP recipient changed before execution.") + elif all(item is not tool for item in self.agent.tools): + raise UserError("The prepared function is no longer configured.") + + async def describe(self, context: RunContextWrapper[Any]) -> str | None: + result = await self.policy(context, self.view) + if result is not None and not isinstance(result, str): + raise UserError("prepare_tool_declaration must return a string or None.") + return result + + async def check(self, context: RunContextWrapper[Any], config: RunConfig) -> None: + if await self.describe(context) is None or not await _function_tool_enabled( + self.tool, context, self.agent + ): + raise ModelBehaviorError(f"Tool {self.view.name} is no longer eligible.") + self.validate(config) + + +async def prepare_declarations( + tools: list[Tool], agent: Agent[Any], context: RunContextWrapper[Any], config: RunConfig +) -> tuple[list[Tool], list[Tool], dict[int, PreparedTool], set[str]]: + if config.tool_execution is None or config.tool_execution.prepare_tool_declaration is None: + return tools, tools, {}, set() + omitted: set[str] = set() + admitted: list[Tool] = [] + presented: list[Tool] = [] + prepared = { + id(tool): PreparedTool(tool, agent, config) + for tool in tools + if isinstance(tool, FunctionTool) and _covered(tool) + } + for tool in tools: + binding = prepared.get(id(tool)) + if binding is not None: + description = await binding.describe(context) + if description is None: + if qualified := get_function_tool_qualified_name(tool): + omitted.add(qualified) + del prepared[id(tool)] + continue + declaration = copy.copy(binding.tool) + declaration.params_json_schema = copy.deepcopy(binding.schema) + declaration.description = description + presented.append(declaration) + else: + presented.append(tool) + admitted.append(tool) + for binding in prepared.values(): + binding.validate(config) + if omitted: + # A retained built-in takes precedence over a same-named local function. + builtins = { + tool.tool_config["type"] if isinstance(tool, HostedMCPTool) else tool.name + for tool in admitted + if not isinstance(tool, FunctionTool) + } + omitted = {name for name in omitted if _builtin_tool_choice_name(name) not in builtins} + return admitted, presented, prepared, omitted + + +def validate_prepared_choice(omitted: set[str] | None, choice: ToolChoice, has_tools: bool) -> None: + if not omitted or not isinstance(choice, str) or choice in ("auto", "none"): + return + conflict = not has_tools if choice == "required" else choice in omitted + if conflict: + raise UserError(f"tool_choice {choice!r} is unavailable after preparing tool declarations.") + + +def bind_declarations( + prepared: dict[int, PreparedTool], + response: ProcessedResponse, + context: RunContextWrapper[Any], + config: RunConfig, +) -> None: + for run in response.functions: + binding = prepared.get(id(run.function_tool)) + if binding is not None: + binding.validate(config) + record = context._tool_invocations.get(run.tool_call.call_id) + if record is None: + raise UserError("A prepared tool call requires a unique call ID.") + record.requires_declaration = True + record.declaration = binding + + +async def resume_declaration( + tool: FunctionTool | None, + call_id: str, + agent: Agent[Any], + context: RunContextWrapper[Any], + config: RunConfig, +) -> None: + record = context._tool_invocations.get(call_id) + if record is None: + return + if not record.requires_declaration and ( + config.tool_execution is None + or config.tool_execution.prepare_tool_declaration is None + or tool is None + or not _covered(tool) + ): + return + if tool is None or not _covered(tool): + raise UserError("The prepared tool is no longer available.") + current = PreparedTool(tool, agent, config) + previous = record.declaration + if previous is not None: + previous.validate(config) + if ( + previous.server is not current.server + or (previous.server is None and previous.tool is not tool) + or previous.schema != current.schema + or previous.binding != current.binding + ): + raise UserError("Cannot resume a prepared call with a replacement tool.") + await current.check(context, config) + record.requires_declaration = True + record.declaration = current diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index 84d0855cd2..d6e1dfaf73 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -24,7 +24,6 @@ from .. import _debug from .._function_tool_arguments import FunctionToolApproval -from .._public_agent import get_public_agent from .._tool_identity import ( FunctionToolLookupKey, NamedToolLookupKey, @@ -84,6 +83,7 @@ ToolOrigin, _computer_tool_uses_run_scoped_initializer, _consume_function_tool_default_failure, + _function_tool_enabled, _invoke_function_tool_with_metadata, _is_programmatic_tool_call, default_tool_error_function, @@ -577,21 +577,12 @@ async def resolve_enabled_function_tools( ) -> list[FunctionTool]: """Resolve enabled function tools without triggering MCP tool discovery.""" - async def _check_tool_enabled(tool: FunctionTool) -> bool: - attr = tool.is_enabled - if isinstance(attr, bool): - return attr - result = attr(context_wrapper, get_public_agent(agent)) - if inspect.isawaitable(result): - return bool(await result) - return bool(result) - function_tools = [tool for tool in agent.tools if isinstance(tool, FunctionTool)] if not function_tools: return [] enabled_results = await gather_with_cancel( - *(_check_tool_enabled(tool) for tool in function_tools) + *(_function_tool_enabled(tool, context_wrapper, agent) for tool in function_tools) ) return [tool for tool, enabled in zip(function_tools, enabled_results, strict=False) if enabled] @@ -2064,6 +2055,11 @@ async def _execute_single_tool_body( scope_id=self.tool_state_scope_id, ) is_nested_continuation = bool(self._get_nested_tool_interruptions(pending_nested_result)) + record = self.context_wrapper._tool_invocations.get(tool_call.call_id) + if record is not None and record.requires_declaration: + if record.declaration is None or record.declaration.tool is not func_tool: + raise UserError("The prepared tool binding is missing or changed.") + await record.declaration.check(self.context_wrapper, self.config) if not is_nested_continuation: self.context_wrapper._mark_tool_invocation_executed( tool_call, @@ -2090,6 +2086,16 @@ async def _execute_single_tool_body( ), ) + if ( + record is not None + and record.requires_declaration + and ( + self.context_wrapper._tool_invocations.get(tool_call.call_id) is not record + or tool_context.tool_call_id != tool_call.call_id + ) + ): + raise UserError("The prepared tool call ID changed before execution.") + invoke_task = asyncio.create_task( self._invoke_tool_and_run_post_invoke( outer_task=outer_task, @@ -2119,6 +2125,9 @@ async def _invoke_tool_and_run_post_invoke( tool_context: ToolContext[Any], agent_hooks: Any, ) -> Any: + record = self.context_wrapper._tool_invocations.get(tool_call.call_id) + if record is not None and record.declaration is not None: + await record.declaration.check(self.context_wrapper, self.config) bypass_output_schema = False try: invocation_result = await _invoke_function_tool_with_metadata( diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 04fad384e2..5add6f2faf 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -96,6 +96,7 @@ ToolSearchCallItem, ToolSearchOutputItem, TResponseInputItem, + TResponseOutputItem, coerce_tool_search_call_raw_item, coerce_tool_search_output_raw_item, ) @@ -171,6 +172,7 @@ ) from .session_persistence import extend_held_session_write from .tool_caller import ensure_programmatic_tool_call_parent, ensure_tool_caller_allowed +from .tool_declarations import PreparedTool, bind_declarations, resume_declaration from .tool_execution import ( build_litellm_json_tool_call, coerce_apply_patch_operations, @@ -2499,6 +2501,14 @@ def _rebind_function_run( stale_function = stale_functions.get(call_id) original_binding = processed_response.mcp_tool_bindings.get(call_id) current_function = current_functions.get(call_id) + if approval_status is not False: + await resume_declaration( + current_function.function_tool if current_function else None, + call_id, + public_agent, + context_wrapper, + run_config, + ) if current_function is not None: current_binding = current_function.function_tool._mcp_tool_binding missing_mcp_binding = ( @@ -4093,6 +4103,14 @@ def _preflight_response_invocations( } +def _snapshot_function_calls(items: list[TResponseOutputItem]) -> list[TResponseOutputItem]: + if not any(isinstance(item, ResponseFunctionToolCall) for item in items): + return items + return [ + deepcopy(item) if isinstance(item, ResponseFunctionToolCall) else item for item in items + ] + + async def get_single_step_result_from_response( *, bindings: AgentBindings[TContext], @@ -4108,6 +4126,7 @@ async def get_single_step_result_from_response( tool_use_tracker, error_handlers: RunErrorHandlers[TContext] | None = None, server_manages_conversation: bool = False, + prepared_tools: dict[int, PreparedTool] | None = None, after_invocation_validation: Callable[[ProcessedResponse | None], Awaitable[bool]] | None = None, before_side_effects: Callable[[], Awaitable[None]] | None = None, @@ -4140,6 +4159,9 @@ def classify() -> ProcessedResponse: ): # Keep the tool and its callbacks on the same arguments if a consumer edits a raw event. new_response.output = deepcopy(new_response.output) + elif settings is not None: + # A model adapter can retain the items it returned. + new_response.output = _snapshot_function_calls(new_response.output) new_bindings: dict[str, tuple[str, dict[str, Any]]] = {} try: processed_response = classify() @@ -4187,6 +4209,8 @@ def classify() -> ProcessedResponse: existing_items=pre_step_items, ) + if prepared_tools: + bind_declarations(prepared_tools, processed_response, context_wrapper, run_config) for call_id, rewrite in new_bindings.items(): context_wrapper._tool_invocations[call_id].input_rewrite = rewrite diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 20c8ce2755..2adfd82689 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -311,6 +311,7 @@ def _default_run_state_validation_error( "and distinguishes provider response IDs from stored Responses IDs for compaction." ), "1.23": ( + "Requires a declaration policy on resume for calls admitted under one. " "Binds prepared tool inputs to their original proposals across retries and resume, " "and records required ResumableSession ownership for paused runs." ), @@ -1526,6 +1527,8 @@ def _serialize_tool_invocations(self) -> dict[str, dict[str, Any]]: "proposal_fingerprint": fingerprint, "arguments": copy.deepcopy(arguments), } + if invocation.requires_declaration and not invocation.completed: + record["requires_declaration"] = True serialized[call_id] = record return serialized @@ -4358,6 +4361,17 @@ async def _build_run_state_from_json( raise validation_error_factory( "Tool input rewrites require RunState schema 1.23.", UserError ) + if ( + (schema_major, schema_minor) < (1, 23) + and isinstance(saved_invocations, Mapping) + and any( + isinstance(record, Mapping) and "requires_declaration" in record + for record in saved_invocations.values() + ) + ): + raise validation_error_factory( + "Prepared declarations require RunState schema 1.23.", UserError + ) if (schema_major, schema_minor) >= (1, 15): context._rebuild_tool_invocations( saved_invocations, @@ -5706,6 +5720,8 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]: "RunState tool invocation contains invalid lifecycle data.", "Tool input rewrites require RunState schema 1.23.", "RunState tool input rewrite is invalid.", + "Prepared declarations require RunState schema 1.23.", + "RunState tool declaration requirement is invalid.", "Hosted MCP approval decisions require a non-empty request id.", ( "Persistent hosted MCP approval decisions require a non-empty server_label " diff --git a/src/agents/tool.py b/src/agents/tool.py index 66d70c20c9..a430b61215 100644 --- a/src/agents/tool.py +++ b/src/agents/tool.py @@ -800,6 +800,27 @@ def _build_wrapped_function_tool( ) +def _builtin_tool_choice_name(choice: str) -> str: + """Resolve built-in selector aliases to their advertised SDK names.""" + if choice == "web_search_preview": + return "web_search" + if choice in ("computer", "computer_use"): + return "computer_use_preview" + return choice + + +async def _function_tool_enabled( + tool: FunctionTool, context: RunContextWrapper[Any], agent: AgentBase[Any] +) -> bool: + from ._public_agent import get_public_agent + + enabled = tool.is_enabled + if isinstance(enabled, bool): + return enabled + result = enabled(context, get_public_agent(agent)) + return bool(await result if inspect.isawaitable(result) else result) + + def get_function_tool_origin(function_tool: FunctionTool) -> ToolOrigin | None: """Return scalar origin metadata for a function tool.""" if not function_tool._emit_tool_origin: @@ -2098,6 +2119,10 @@ def _coerce_tool_error_for_failure_error_function(error: BaseException) -> Excep return Exception(str(error) or error.__class__.__name__) +class _ToolEligibilityError(UserError): + """Policy refusals must bypass tool failure formatters.""" + + async def maybe_invoke_function_tool_failure_error_function( *, function_tool: FunctionTool, @@ -2105,6 +2130,8 @@ async def maybe_invoke_function_tool_failure_error_function( error: BaseException, ) -> str | None: """Invoke the configured failure formatter, if one exists.""" + if isinstance(error, _ToolEligibilityError): + return None failure_error_function = resolve_function_tool_failure_error_function(function_tool, context) if failure_error_function is None: return None diff --git a/tests/test_mutable_tool_hooks.py b/tests/test_mutable_tool_hooks.py index b96b2f01d1..58d04feee3 100644 --- a/tests/test_mutable_tool_hooks.py +++ b/tests/test_mutable_tool_hooks.py @@ -1,7 +1,7 @@ import asyncio import json import traceback -from typing import Any +from typing import Any, get_type_hints import pytest from openai.types.responses import ( @@ -9,7 +9,7 @@ ResponseCustomToolCall, ResponseFunctionWebSearch, ) -from pydantic import BaseModel, Field, Json, SkipValidation, field_validator +from pydantic import BaseModel, Field, Json, SkipValidation, TypeAdapter, field_validator from agents import ( Agent, @@ -838,3 +838,56 @@ async def pass_through(call, result): ) output = next(i for i in model.calls[1].input if i.get("type") == "function_call_output") assert json.loads(output["output"]) == {"reading": "pending"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("point", "fallback"), + [ + ("response.output_item.done", True), + ("response.completed", False), + ("response.completed", True), + ], +) +async def test_declaration_preserves_streamed_call_before_admission(point, fallback): + displayed = asyncio.Event() + + @function_tool + def report(color: str): + return color + + async def allow(ctx, declaration): + return declaration.description + + class PausedModel(ScriptedModel): + async def stream_response(self, *args, **kwargs): + async for event in super().stream_response(*args, **kwargs): + if event.type == "response.completed" and fallback: + event.response.output = [] + yield event + if event.type == point: + await asyncio.wait_for(displayed.wait(), timeout=5) + + model = PausedModel([[function_call("report", {"color": "Blue"}, call_id="one")]]) + result = Runner.run_streamed( + Agent(name="host", model=model, tools=[report], tool_use_behavior="stop_on_first_tool"), + "go", + run_config=RunConfig(tool_execution=ToolExecutionConfig(prepare_tool_declaration=allow)), + ) + async for event in result.stream_events(): + if event.type == "raw_response_event" and event.data.type == point: + item = ( + event.data.item + if point == "response.output_item.done" + else event.data.response.output[0] + ) + item.arguments = '{"color":"blue"}' + displayed.set() + assert result.final_output == "Blue" + + +def test_tool_execution_annotations_resolve_for_settings_adapters(): + assert "prepare_tool_declaration" in get_type_hints(ToolExecutionConfig) + config = TypeAdapter(ToolExecutionConfig).validate_python({"max_function_tool_concurrency": 2}) + assert config.max_function_tool_concurrency == 2 + assert config.prepare_tool_declaration is None diff --git a/tests/test_tool_declarations.py b/tests/test_tool_declarations.py new file mode 100644 index 0000000000..5ac894ca8f --- /dev/null +++ b/tests/test_tool_declarations.py @@ -0,0 +1,467 @@ +import pytest + +from agents import ( + Agent, + ModelBehaviorError, + ModelSettings, + RunConfig, + RunHooks, + Runner, + RunState, + ToolDeclaration, + ToolExecutionConfig, + UserError, + function_tool, +) +from agents.testing import ScriptedModel, assistant_message, function_call + +from .mcp.helpers import FakeMCPServer + +pytestmark = [pytest.mark.asyncio, pytest.mark.parametrize("streamed", [False, True])] + + +async def run(streamed, agent, config, prompt="go", **kwargs): + if streamed: + result = Runner.run_streamed(agent, prompt, run_config=config, **kwargs) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, prompt, run_config=config, **kwargs) + + +async def describe(ctx, declaration): + return declaration.description + + +def configure(policy=describe, **kwargs): + return RunConfig(tool_execution=ToolExecutionConfig(prepare_tool_declaration=policy, **kwargs)) + + +def call(name): + return function_call(name, {}, call_id=name) + + +@pytest.mark.parametrize("success", [False, True]) +async def test_unlock_after_success_and_preserve_original_tool(streamed, success): + effects = [] + seen = [] + + @function_tool + async def inspect_site(): + """Inspect the site.""" + if not success: + raise ValueError("Inspection failed") + effects.append("inspected") + + peer = FakeMCPServer(server_name="Site reports", tool_filter=lambda ctx, t: bool(effects)) + peer.add_tool("publish.report", {}) + + async def prepare(ctx, declaration): + seen.append(declaration) + return "Ready for " + declaration.name + + model = ScriptedModel( + [ + [call("inspect_site")], + [call("mcp_Site_reports__publish_report")], + [assistant_message("done")], + ] + ) + agent = Agent( + name="host", + tools=[inspect_site], + mcp_servers=[peer], + mcp_config={"include_server_in_tool_names": True}, + model=model, + model_settings=ModelSettings(tool_choice="required"), + ) + if success: + assert (await run(streamed, agent, configure(prepare))).final_output == "done" + assert peer.tool_calls == ["publish.report"] + assert any( + d.mcp_server_name == "Site reports" and d.mcp_tool_name == "publish.report" + for d in seen + ) + else: + with pytest.raises(ModelBehaviorError): + await run(streamed, agent, configure(prepare)) + assert not peer.tool_calls + assert all(d.name == "inspect_site" for d in seen) + assert [t.name for t in model.calls[0].tools] == ["inspect_site"] + assert model.calls[0].tools[0].description == "Ready for inspect_site" + assert model.calls[0].tools[0].params_json_schema == inspect_site.params_json_schema + assert model.calls[1].model_settings.tool_choice is None + assert inspect_site.description == "Inspect the site." + + +@pytest.mark.parametrize("choice", [None, "publish", "required"]) +async def test_omission_limits_sampling_and_routing(streamed, choice): + peer = FakeMCPServer() + peer.add_tool("publish", {}) + + async def omit(ctx, declaration): + return None + + model = ScriptedModel([[call("publish")]]) + agent = Agent( + name="host", + mcp_servers=[peer], + model=model, + model_settings=ModelSettings(tool_choice=choice), + ) + with pytest.raises(ModelBehaviorError if choice is None else UserError): + await run(streamed, agent, configure(omit)) + assert not peer.tool_calls + if choice is None: + assert model.calls[0].tools == [] + else: + assert not model.calls + + +async def test_cannot_replace_tool_or_edit_schema_through_policy(streamed): + @function_tool(needs_approval=True) + def publish(): + pytest.fail("unapproved call") + + async def replace(ctx, declaration): + assert isinstance(declaration, ToolDeclaration) + assert not hasattr(declaration, "params_json_schema") + return publish + + model = ScriptedModel([]) + with pytest.raises(UserError, match="string or None"): + await run(streamed, Agent(name="host", tools=[publish], model=model), configure(replace)) + assert not model.calls + + +@pytest.mark.parametrize("route", ["local", "mcp"]) +async def test_queued_tool_rechecks_original_predicate(streamed, route): + effects = [] + + @function_tool + async def revoke(): + effects.append("revoked") + + @function_tool(is_enabled=lambda ctx, agent: not effects) + async def publish(): + effects.append("published") + + peer = FakeMCPServer(tool_filter=lambda ctx, t: not effects) + peer.add_tool("publish", {}) + model = ScriptedModel([[call("revoke"), call("publish")]]) + agent = Agent( + name="host", + tools=[revoke, publish] if route == "local" else [revoke], + mcp_servers=[peer] if route == "mcp" else [], + model=model, + ) + with pytest.raises((UserError, ModelBehaviorError), match="eligible"): + await run(streamed, agent, configure(max_function_tool_concurrency=1)) + assert effects == ["revoked"] + assert not peer.tool_calls + + +@pytest.mark.parametrize("route", ["local", "mcp"]) +@pytest.mark.parametrize("change", ["memory", "none", "missing", "revoked", "replacement"]) +async def test_approval_binding_across_pause(streamed, route, change): + effects = [] + eligible = True + + @function_tool(needs_approval=True) + def publish(): + effects.append("local") + + def server(): + peer = FakeMCPServer(require_approval="always") + peer.add_tool("publish", {}) + return peer + + async def prepare(ctx, declaration): + return "Publish" if eligible else None + + peer = server() + config = configure(prepare) + model = ScriptedModel([[call("publish")], [assistant_message("done")]]) + agent = Agent( + name="host", + tools=[publish] if route == "local" else [], + mcp_servers=[peer] if route == "mcp" else [], + model=model, + ) + first = await run(streamed, agent, config) + state = first.to_state() + state.approve(first.interruptions[0]) + if change not in ("replacement", "memory"): + state = await RunState.from_json(agent, state.to_json()) + if change in ("none", "memory"): + result = await run(streamed, agent, config, state) + assert result.final_output == "done" + assert effects + peer.tool_calls == (["local"] if route == "local" else ["publish"]) + return + if change == "replacement": + if route == "local": + agent.tools = [ + function_tool(lambda: effects.append("replacement"), name_override="publish") + ] + else: + agent.mcp_servers = [server()] + if change == "revoked": + eligible = False + with pytest.raises((UserError, ModelBehaviorError)): + await run(streamed, agent, RunConfig() if change == "missing" else config, state) + assert not effects and not peer.tool_calls + assert all(not s.tool_calls for s in agent.mcp_servers) + + +async def test_recheck_after_mcp_metadata(streamed): + async def metadata(ctx): + peer.tool_filter = {"blocked_tool_names": ["publish"]} + + peer = FakeMCPServer(tool_meta_resolver=metadata) + peer.add_tool("publish", {}) + agent = Agent(name="host", mcp_servers=[peer], model=ScriptedModel([[call("publish")]])) + with pytest.raises(UserError, match="eligible"): + await run(streamed, agent, configure()) + assert not peer.tool_calls + + +async def test_recheck_after_tool_start(streamed): + @function_tool + def publish(): + pytest.fail("revoked") + + class Hooks(RunHooks): + async def on_tool_start(self, context, agent, tool): + tool.is_enabled = False + + agent = Agent(name="host", tools=[publish], model=ScriptedModel([[call("publish")]])) + with pytest.raises(ModelBehaviorError, match="eligible"): + await run(streamed, agent, configure(), hooks=Hooks()) + + +async def test_selected_description_does_not_replace_approval_or_argument_edit(streamed): + calls = [] + + @function_tool(needs_approval=True) + def publish(value: str): + calls.append(value) + + async def edit(call): + assert call.tool is publish + calls.append("edit") + return {"value": "approved"} + + async def policy(ctx, declaration): + return "Publish the reviewed value." + + model = ScriptedModel( + [ + [function_call("publish", {"value": "draft"}, call_id="one")], + [assistant_message("done")], + ] + ) + agent = Agent(name="host", tools=[publish], model=model) + config = configure(policy, prepare_tool_input=edit) + result = await run(streamed, agent, config) + assert calls == ["edit"] + assert model.calls[0].tools[0].description == "Publish the reviewed value." + state = result.to_state() + state.approve(result.interruptions[0]) + restored = await RunState.from_json(agent, state.to_json()) + await run(streamed, agent, config, restored) + assert calls == ["edit", "approved"] + + +async def test_pending_policy_cannot_be_saved_under_an_older_schema(streamed): + @function_tool(needs_approval=True) + def publish(): + pytest.fail("not approved") + + agent = Agent(name="host", tools=[publish], model=ScriptedModel([[call("publish")]])) + first = await run(streamed, agent, configure()) + data = first.to_state().to_json() + data["$schemaVersion"] = "1.22" + with pytest.raises(UserError, match="schema 1.23"): + await RunState.from_json(agent, data) + + +async def test_omitting_collision_winner_does_not_expose_loser(streamed): + @function_tool + def publish(): + pytest.fail("withheld") + + @function_tool + def inspect_site(): + return "ready" + + peer = FakeMCPServer() + peer.add_tool("publish", {}) + + async def policy(ctx, declaration): + return None if declaration.name == "publish" else declaration.description + + model = ScriptedModel([[call("publish")]]) + agent = Agent(name="host", tools=[publish, inspect_site], mcp_servers=[peer], model=model) + with pytest.raises(ModelBehaviorError): + await run(streamed, agent, configure(policy)) + assert [tool.name for tool in model.calls[0].tools] == ["inspect_site"] + assert not peer.tool_calls + + +@pytest.mark.parametrize("field", ["approval", "schema"]) +async def test_authority_edits_during_preparation_fail_before_sampling(streamed, field): + @function_tool(needs_approval=True) + def publish(): + pytest.fail("unauthorized") + + async def policy(ctx, declaration): + if field == "schema": + publish.params_json_schema.clear() + else: + publish.needs_approval = False + return "Publish" + + model = ScriptedModel([[call("publish")]]) + with pytest.raises(UserError, match="changed"): + await run(streamed, Agent(name="host", tools=[publish], model=model), configure(policy)) + assert not model.calls + + +@pytest.mark.parametrize("approved", [False, True]) +async def test_missing_policy_preserves_managed_approval(streamed, approved): + from agents.memory.resumable_session import ResumableSession + + from .memory.test_resumable_session import MemoryStore + + store = MemoryStore() + session = await ResumableSession.create(store) + peer = FakeMCPServer(require_approval="always") + peer.add_tool("publish", {}) + model = ScriptedModel([[call("publish")], [assistant_message("done")]]) + agent = Agent(name="host", mcp_servers=[peer], model=model) + await run(streamed, agent, configure(), session=session) + saved = store.records[session.session_id] + restored = await ResumableSession.resume(store, session.session_id) + state = await restored.load_state(agent) + pending = state.get_interruptions()[0] + if approved: + state.approve(pending) + with pytest.raises(UserError, match="prepare_tool_declaration"): + await run(streamed, agent, RunConfig(), state) + assert store.records[session.session_id] == saved + assert not peer.tool_calls and len(model.calls) == 1 + restored = await ResumableSession.resume(store, session.session_id) + state = await restored.load_state(agent) + state.approve(state.get_interruptions()[0]) + else: + state.reject(pending) + result = await run(streamed, agent, configure() if approved else RunConfig(), state) + assert result.final_output == "done" + assert peer.tool_calls == (["publish"] if approved else []) + + +@pytest.mark.parametrize("choice", ["required", "web_search_preview"]) +async def test_prompt_managed_choice_passes_through(streamed, choice): + model = ScriptedModel([[assistant_message("done")]]) + agent = Agent( + name="host", + model=model, + prompt={"id": "pmpt_example"}, + model_settings=ModelSettings(tool_choice=choice), + ) + await run(streamed, agent, configure()) + assert model.calls[0].model_settings.tool_choice == choice + + +async def test_approval_requires_same_bound_policy_owner(streamed): + class Owner: + async def decide(self, ctx, declaration): + return declaration.description + + owner = Owner() + effects = [] + + @function_tool(needs_approval=True) + def publish(): + effects.append("published") + + model = ScriptedModel([[call("publish")], [assistant_message("done")]]) + agent = Agent(name="host", model=model, tools=[publish]) + first = await run(streamed, agent, configure(owner.decide)) + state = first.to_state() + state.approve(first.interruptions[0]) + with pytest.raises(UserError, match="policy changed"): + await run(streamed, agent, configure(Owner().decide), state) + assert not effects + await run(streamed, agent, configure(owner.decide), state) + assert effects == ["published"] + + +@pytest.mark.parametrize("batch", [False, True]) +async def test_preconverted_tools_require_agent_discovery(streamed, batch): + from agents import RunContextWrapper + from agents.mcp import MCPUtil + + peer = FakeMCPServer() + peer.add_tool("publish", {}) + model = ScriptedModel([[call("publish")], [assistant_message("done")]]) + agent = Agent(name="host", model=model) + agent.tools = ( + await MCPUtil.get_all_function_tools([peer], False, RunContextWrapper(None), agent) + if batch + else [MCPUtil.to_function_tool(peer.tools[0], peer, False)] + ) + with pytest.raises(UserError, match="agent.mcp_servers"): + await run(streamed, agent, configure()) + assert not peer.tool_calls and not model.calls + assert (await run(streamed, agent, RunConfig())).final_output == "done" + assert peer.tool_calls == ["publish"] + + +async def test_named_choice_tracks_the_qualified_target(streamed): + from agents import tool_namespace + + @function_tool + def report(): + return "ready" + + async def keep_local(ctx, declaration): + return None if declaration.namespace else declaration.description + + tools = [report, *tool_namespace(name="archive", description="Archive", tools=[report])] + model = ScriptedModel([[call("report")]]) + agent = Agent( + name="host", + tools=tools, + model=model, + tool_use_behavior="stop_on_first_tool", + model_settings=ModelSettings(tool_choice="archive.report"), + ) + with pytest.raises(UserError, match="tool_choice"): + await run(streamed, agent, configure(keep_local)) + assert not model.calls + agent.model_settings.tool_choice = "report" + assert (await run(streamed, agent, configure(keep_local))).final_output == "ready" + + +async def test_named_choice_can_still_select_hosted_search(streamed): + from agents import WebSearchTool + + @function_tool + def web_search_preview(): + pytest.fail("withheld local function") + + async def omit(ctx, declaration): + return None + + search = WebSearchTool() + model = ScriptedModel([[assistant_message("done")]]) + agent = Agent( + name="host", + tools=[web_search_preview, search], + model=model, + model_settings=ModelSettings(tool_choice="web_search_preview"), + ) + await run(streamed, agent, configure(omit)) + assert model.calls[0].tools == [search] + assert model.calls[0].model_settings.tool_choice == "web_search_preview" From 464021ccecea061e08ee97a2df3b13394f7f194e Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 07:29:12 -0800 Subject: [PATCH 2/3] fix: Preserve sandbox tool authority across prepared approvals --- src/agents/run_config.py | 7 +- src/agents/run_context.py | 7 + src/agents/run_internal/run_loop.py | 4 +- src/agents/run_internal/tool_declarations.py | 108 +++++++++++-- src/agents/run_internal/turn_resolution.py | 2 +- src/agents/run_state.py | 6 +- .../capabilities/tools/apply_patch_tool.py | 20 +-- src/agents/sandbox/runtime.py | 7 +- .../sandbox/runtime_agent_preparation.py | 9 +- src/agents/sandbox/sandbox_agent.py | 10 +- tests/sandbox/test_provider_capabilities.py | 45 +++++- tests/test_tool_declarations.py | 146 +++++++++++++++++- 12 files changed, 331 insertions(+), 40 deletions(-) diff --git a/src/agents/run_config.py b/src/agents/run_config.py index bdeafcdbd0..6884280f37 100644 --- a/src/agents/run_config.py +++ b/src/agents/run_config.py @@ -222,7 +222,12 @@ async def prepare(ctx, declaration): request. This does not force a selection or order calls in a batch. Withholding the named choice or emptying a required catalog raises UserError before sampling. - Covers direct function tools and MCP tools discovered through agent.mcp_servers. + Covers direct functions, sandbox capability functions and MCP tools discovered through + agent.mcp_servers. Keep the same sandbox capabilities during a run. On in-memory + resume a fresh capability tool or the SDK sandbox patch adapter may rebind to + the current session if the tool type, method, schema and approval policy still match. + A pending capability call requires the same capability type, position and tool key + after restore. An approval cannot move between a direct tool and a capability. Preconverted MCP functions in agent.tools raise UserError. Handoffs, deferred tools, programmatic callers, agents as tools and provider-managed tools are outside this policy. Use the same policy on resume. A serialized run trusts the application's current tool diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 7cf5c0e6ff..f6e228d77e 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -159,6 +159,7 @@ class _ToolInvocationRecord: # The input admitted under this ID, paired with the digest of the original proposal. input_rewrite: tuple[str, dict[str, Any]] | None = None requires_declaration: bool = False + capability_source: str | None = None def __post_init__(self) -> None: # Never serialize the live executor and policy into a checkpoint. @@ -1652,6 +1653,11 @@ def _rebuild_tool_invocations( requires_declaration = serialized_invocation.get("requires_declaration", False) if not isinstance(requires_declaration, bool): raise validation_error_factory("RunState tool declaration requirement is invalid.") + source = serialized_invocation.get("capability_source") + if "capability_source" in serialized_invocation and ( + not requires_declaration or not is_tool_invocation_digest(source) + ): + raise validation_error_factory("RunState capability source is invalid.") input_rewrite = None if "input_rewrite" in serialized_invocation: rewrite = serialized_invocation["input_rewrite"] @@ -1673,6 +1679,7 @@ def _rebuild_tool_invocations( completed=completed, input_rewrite=input_rewrite, requires_declaration=requires_declaration, + capability_source=source, ) def _mark_restored_unbound_approval_call_ids(self) -> None: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 44747601ff..0e4a68d640 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -2523,7 +2523,7 @@ async def raise_if_input_guardrail_tripwire_known() -> None: collision_policy=run_config.tool_name_collision_policy, ) all_tools, model_tools, prepared_tools, omitted = await prepare_declarations( - all_tools, public_agent, context_wrapper, run_config + all_tools, bindings, context_wrapper, run_config ) if agent_span is not None: agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] @@ -2948,7 +2948,7 @@ async def run_single_turn( collision_policy=run_config.tool_name_collision_policy, ) all_tools, model_tools, prepared_tools, omitted = await prepare_declarations( - all_tools, public_agent, context_wrapper, run_config + all_tools, bindings, context_wrapper, run_config ) if agent_span is not None: agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] diff --git a/src/agents/run_internal/tool_declarations.py b/src/agents/run_internal/tool_declarations.py index 65d7b23721..96f7f3edba 100644 --- a/src/agents/run_internal/tool_declarations.py +++ b/src/agents/run_internal/tool_declarations.py @@ -6,7 +6,8 @@ from types import MethodType from typing import TYPE_CHECKING, Any -from .._tool_identity import get_function_tool_qualified_name +from .._tool_identity import get_function_tool_lookup_key_for_tool, get_function_tool_qualified_name +from .._tool_invocation import _fingerprint from ..exceptions import ModelBehaviorError, UserError from ..run_config import RunConfig, ToolDeclaration from ..tool import ( @@ -18,10 +19,11 @@ ) if TYPE_CHECKING: - from ..agent import Agent from ..model_settings import ToolChoice from ..run_context import RunContextWrapper from ..run_state import RunState + from ..sandbox.capabilities.tools import SandboxApplyPatchTool + from .agent_bindings import AgentBindings from .run_steps import ProcessedResponse @@ -75,24 +77,35 @@ def validate_resume_policy( class PreparedTool: """In-memory authority for a sampled function. Never persisted with the call.""" - def __init__(self, tool: FunctionTool, agent: Agent[Any], config: RunConfig): + def __init__(self, tool: FunctionTool, bindings: AgentBindings[Any], config: RunConfig): settings = config.tool_execution if settings is None or settings.prepare_tool_declaration is None: raise UserError("Restore prepare_tool_declaration to resume this tool call.") self.policy = settings.prepare_tool_declaration self.tool = tool - self.agent = agent + self.agent = agent = bindings.public_agent + self.execution = bindings.execution_agent + self.capability = self._capability_source() + self.source_digest = ( + _fingerprint( + {"capability": self.capability, "key": get_function_tool_lookup_key_for_tool(tool)} + ) + if self.capability is not None + else None + ) self.binding = tool._mcp_tool_binding index = self.binding[2] if self.binding else None if self.binding and ( index is None or not 0 <= index < len(agent.mcp_servers) - or any(item is tool for item in agent.tools) + or any(item is tool for item in self.execution.tools) ): raise UserError("Use agent.mcp_servers for MCP tools with prepare_tool_declaration.") self.server = agent.mcp_servers[index] if index is not None else None self.executor = tool.on_invoke_tool self.approval = tool.needs_approval + self.patch = self._patch_adapter() + self.operation_approval = self.patch.needs_approval if self.patch else self.approval self.peer_call = self.server.call_tool if self.server else None self.peer_approval = self.server._needs_approval_policy if self.server else None self.schema = copy.deepcopy(tool.params_json_schema) @@ -117,6 +130,7 @@ def validate(self, config: RunConfig) -> None: or not _same_callback(settings.prepare_tool_declaration, self.policy) or tool.on_invoke_tool is not self.executor or tool.needs_approval is not self.approval + or (self.patch is not None and self.patch.needs_approval is not self.operation_approval) or tool.strict_json_schema != self.strict or tool.params_json_schema != self.schema or tool.name != self.view.name @@ -140,9 +154,73 @@ def validate(self, config: RunConfig) -> None: ) ): raise UserError("The prepared MCP recipient changed before execution.") - elif all(item is not tool for item in self.agent.tools): + elif ( + self._capability_source() != self.capability + or all(item is not tool for item in self.execution.tools) + # Execution clones can retain a direct tool removed from the public agent. + or ( + self.capability is None + and self.execution is not self.agent + and all(item is not tool for item in self.agent.tools) + ) + ): raise UserError("The prepared function is no longer configured.") + def _capability_source(self) -> tuple[int, str] | None: + if self.agent is self.execution: + return None + from ..sandbox import SandboxAgent + + if isinstance(self.execution, SandboxAgent) and isinstance(self.agent, SandboxAgent): + for index, tool in self.execution._sandbox_capability_tools or (): + if tool is self.tool: + sources = self.execution._sandbox_capability_sources + if ( + index >= len(sources) + or index >= len(self.agent.capabilities) + or sources[index] is not self.agent.capabilities[index] + ): + raise UserError("The prepared capability is no longer configured.") + return index, sources[index].type + return None + + def _patch_adapter(self) -> SandboxApplyPatchTool | None: + if self.capability is None or not isinstance(self.executor, MethodType): + return None + from ..sandbox.capabilities.tools import SandboxApplyPatchTool + + owner = self.executor.__self__ + if ( + type(owner) is SandboxApplyPatchTool + and self.executor.__func__ is SandboxApplyPatchTool._invoke_function + and isinstance(self.approval, MethodType) + and self.approval.__func__ is SandboxApplyPatchTool._function_needs_approval + and self.approval.__self__ is owner + ): + return owner + return None + + def _same_function(self, current: PreparedTool) -> bool: + if self.tool is current.tool: + return True + # Sandbox resume recreates session-bound tools. Allow the same method to bind + # to the fresh tool only under the original capability and approval policy. + return ( + self.capability is not None + and self.agent is current.agent + and type(self.tool) is type(current.tool) + and type(self.patch) is type(current.patch) + and self.view.name == current.view.name + and self.view.namespace == current.view.namespace + and self.strict == current.strict + and _same_callback(self.operation_approval, current.operation_approval) + and isinstance(self.executor, MethodType) + and isinstance(current.executor, MethodType) + and self.executor.__self__ is (self.patch or self.tool) + and current.executor.__self__ is (current.patch or current.tool) + and self.executor.__func__ is current.executor.__func__ + ) + async def describe(self, context: RunContextWrapper[Any]) -> str | None: result = await self.policy(context, self.view) if result is not None and not isinstance(result, str): @@ -158,7 +236,10 @@ async def check(self, context: RunContextWrapper[Any], config: RunConfig) -> Non async def prepare_declarations( - tools: list[Tool], agent: Agent[Any], context: RunContextWrapper[Any], config: RunConfig + tools: list[Tool], + bindings: AgentBindings[Any], + context: RunContextWrapper[Any], + config: RunConfig, ) -> tuple[list[Tool], list[Tool], dict[int, PreparedTool], set[str]]: if config.tool_execution is None or config.tool_execution.prepare_tool_declaration is None: return tools, tools, {}, set() @@ -166,7 +247,7 @@ async def prepare_declarations( admitted: list[Tool] = [] presented: list[Tool] = [] prepared = { - id(tool): PreparedTool(tool, agent, config) + id(tool): PreparedTool(tool, bindings, config) for tool in tools if isinstance(tool, FunctionTool) and _covered(tool) } @@ -221,13 +302,14 @@ def bind_declarations( if record is None: raise UserError("A prepared tool call requires a unique call ID.") record.requires_declaration = True + record.capability_source = binding.source_digest record.declaration = binding async def resume_declaration( tool: FunctionTool | None, call_id: str, - agent: Agent[Any], + bindings: AgentBindings[Any], context: RunContextWrapper[Any], config: RunConfig, ) -> None: @@ -243,13 +325,17 @@ async def resume_declaration( return if tool is None or not _covered(tool): raise UserError("The prepared tool is no longer available.") - current = PreparedTool(tool, agent, config) + current = PreparedTool(tool, bindings, config) + if record.capability_source != current.source_digest: + raise UserError( + "Cannot resume a prepared call with a different or missing capability source." + ) previous = record.declaration if previous is not None: previous.validate(config) if ( previous.server is not current.server - or (previous.server is None and previous.tool is not tool) + or (previous.server is None and not previous._same_function(current)) or previous.schema != current.schema or previous.binding != current.binding ): diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 5add6f2faf..236d8c0b9a 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -2505,7 +2505,7 @@ def _rebind_function_run( await resume_declaration( current_function.function_tool if current_function else None, call_id, - public_agent, + bindings, context_wrapper, run_config, ) diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 2adfd82689..06f485d635 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -1529,6 +1529,8 @@ def _serialize_tool_invocations(self) -> dict[str, dict[str, Any]]: } if invocation.requires_declaration and not invocation.completed: record["requires_declaration"] = True + if invocation.capability_source is not None: + record["capability_source"] = invocation.capability_source serialized[call_id] = record return serialized @@ -4365,7 +4367,8 @@ async def _build_run_state_from_json( (schema_major, schema_minor) < (1, 23) and isinstance(saved_invocations, Mapping) and any( - isinstance(record, Mapping) and "requires_declaration" in record + isinstance(record, Mapping) + and ("requires_declaration" in record or "capability_source" in record) for record in saved_invocations.values() ) ): @@ -5722,6 +5725,7 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]: "RunState tool input rewrite is invalid.", "Prepared declarations require RunState schema 1.23.", "RunState tool declaration requirement is invalid.", + "RunState capability source is invalid.", "Hosted MCP approval decisions require a non-empty request id.", ( "Persistent hosted MCP approval decisions require a non-empty server_label " diff --git a/src/agents/sandbox/capabilities/tools/apply_patch_tool.py b/src/agents/sandbox/capabilities/tools/apply_patch_tool.py index 7c8454bdf9..7afdf62f49 100644 --- a/src/agents/sandbox/capabilities/tools/apply_patch_tool.py +++ b/src/agents/sandbox/capabilities/tools/apply_patch_tool.py @@ -215,14 +215,6 @@ def as_function_tool(self) -> FunctionTool: "use needs_approval and RunState approval decisions." ) - async def invoke(ctx: ToolContext[Any], arguments: str) -> str: - return await self._on_invoke_tool(ctx, json.loads(arguments)["patch"]) - - async def needs_approval( - ctx: RunContextWrapper[Any], arguments: dict[str, Any], call_id: str - ) -> bool: - return await self._needs_custom_approval(ctx, arguments["patch"], call_id) - return FunctionTool( name=self.name, description=self.description.replace( @@ -235,10 +227,18 @@ async def needs_approval( "required": ["patch"], "additionalProperties": False, }, - on_invoke_tool=invoke, - needs_approval=needs_approval, + on_invoke_tool=self._invoke_function, + needs_approval=self._function_needs_approval, ) + async def _invoke_function(self, ctx: ToolContext[Any], arguments: str) -> str: + return await self._on_invoke_tool(ctx, json.loads(arguments)["patch"]) + + async def _function_needs_approval( + self, ctx: RunContextWrapper[Any], arguments: dict[str, Any], call_id: str + ) -> bool: + return await self._needs_custom_approval(ctx, arguments["patch"], call_id) + @property def operation_needs_approval(self) -> bool | ApplyPatchApprovalFunction: return self.needs_approval diff --git a/src/agents/sandbox/runtime.py b/src/agents/sandbox/runtime.py index 87c0d37ffc..128501203b 100644 --- a/src/agents/sandbox/runtime.py +++ b/src/agents/sandbox/runtime.py @@ -238,7 +238,8 @@ async def prepare_agent( with span_cm: self._session_manager.acquire_agent(current_agent) cached_preparation = self._prepared_agents.get(id(current_agent)) - prepared_capabilities = clone_capabilities(current_agent.capabilities) + sources = tuple(current_agent.capabilities) + prepared_capabilities = clone_capabilities(sources) session = await self._session_manager.ensure_session( agent=current_agent, capabilities=prepared_capabilities, @@ -276,6 +277,9 @@ async def prepare_agent( resolved_model=resolved_model, workspace_scope=self._workspace_scope, ) + cast( + SandboxAgent[TContext], prepared_agent + )._sandbox_capability_sources = cached_agent._sandbox_capability_sources self._prepared_agents[id(current_agent)] = _SandboxPreparedAgentCache( agent=prepared_agent, session=session, @@ -314,6 +318,7 @@ async def prepare_agent( resolved_model=resolved_model, workspace_scope=self._workspace_scope, ) + cast(SandboxAgent[TContext], prepared_agent)._sandbox_capability_sources = sources self._prepared_agents[id(current_agent)] = _SandboxPreparedAgentCache( agent=prepared_agent, session=session, diff --git a/src/agents/sandbox/runtime_agent_preparation.py b/src/agents/sandbox/runtime_agent_preparation.py index b6ea8cf9f7..ae5874b6ba 100644 --- a/src/agents/sandbox/runtime_agent_preparation.py +++ b/src/agents/sandbox/runtime_agent_preparation.py @@ -113,7 +113,10 @@ def prepare_sandbox_agent( for capability in capabilities: if isinstance(capability, Memory) and isinstance(model, Model): capability._model_instance = model - capability_tools = [tool for capability in capabilities for tool in capability.tools()] + indexed_tools = [ + (index, tool) for index, cap in enumerate(capabilities) for tool in cap.tools() + ] + capability_tools = [tool for _, tool in indexed_tools] namespace = model_capabilities.get("tool_namespace") if namespace: capability_tools = [ @@ -172,7 +175,9 @@ def prepare_sandbox_agent( tools=[*agent.tools, *capability_tools], capabilities=capabilities, ) - cast(SandboxAgent[TContext], prepared_agent)._sandbox_capability_tools = capability_tools + cast(SandboxAgent[TContext], prepared_agent)._sandbox_capability_tools = [ + (index, tool) for (index, _), tool in zip(indexed_tools, capability_tools, strict=True) + ] set_public_agent(prepared_agent, agent) return prepared_agent diff --git a/src/agents/sandbox/sandbox_agent.py b/src/agents/sandbox/sandbox_agent.py index 8ff5f3ba7c..2021c5f109 100644 --- a/src/agents/sandbox/sandbox_agent.py +++ b/src/agents/sandbox/sandbox_agent.py @@ -58,9 +58,12 @@ class SandboxAgent(Agent[TContext]): """User identity used for model-facing sandbox tools such as shell, file reads, and patches.""" _sandbox_concurrency_guard: object | None = field(default=None, init=False, repr=False) - _sandbox_capability_tools: list[Tool] | None = field( + _sandbox_capability_tools: list[tuple[int, Tool]] | None = field( default=None, init=False, repr=False, compare=False ) + _sandbox_capability_sources: tuple[Capability, ...] = field( + default=(), init=False, repr=False, compare=False + ) if TYPE_CHECKING: @@ -110,7 +113,10 @@ async def get_all_tools(self, run_context: RunContextWrapper[TContext]) -> list[ if self._sandbox_capability_tools is not None: # Refresh application tools without recreating the session-bound capability tools. public_agent = get_public_agent(self) - self.tools = [*public_agent.tools, *self._sandbox_capability_tools] + self.tools = [ + *public_agent.tools, + *(tool for _, tool in self._sandbox_capability_tools), + ] return await super().get_all_tools(run_context) def __post_init__(self) -> None: diff --git a/tests/sandbox/test_provider_capabilities.py b/tests/sandbox/test_provider_capabilities.py index 3eb43cb91b..eae3c8b495 100644 --- a/tests/sandbox/test_provider_capabilities.py +++ b/tests/sandbox/test_provider_capabilities.py @@ -6,7 +6,7 @@ import pytest -from agents import ModelSettings, RunConfig, Runner +from agents import ModelSettings, RunConfig, Runner, RunState, ToolExecutionConfig, UserError from agents.sandbox import Manifest, MemoryGenerateConfig, SandboxAgent, SandboxRunConfig from agents.sandbox.capabilities import Filesystem, Memory from agents.sandbox.capabilities.tools import SandboxApplyPatchTool @@ -28,8 +28,11 @@ class _FunctionSandboxModel(ScriptedModel): @pytest.mark.asyncio -@pytest.mark.parametrize("approve", [True, False]) -async def test_function_sandbox_patch_uses_runner_approval_and_session_files(approve: bool) -> None: +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("decision", ["approve", "reject", "prepared", "restored", "changed"]) +async def test_function_sandbox_patch_uses_runner_approval_and_session_files( + streamed: bool, decision: str +) -> None: client = FilesystemTestSandboxClient() session = await client.create(manifest=Manifest()) model = _FunctionSandboxModel( @@ -51,8 +54,10 @@ async def test_function_sandbox_patch_uses_runner_approval_and_session_files(app ] ) + requires_approval = True + def configure_tools(tools) -> None: - tools.apply_patch.needs_approval = True + tools.apply_patch.needs_approval = requires_approval agent = SandboxAgent( name="sandbox", @@ -60,20 +65,44 @@ def configure_tools(tools) -> None: capabilities=[Filesystem(configure_tools=configure_tools)], ) config = RunConfig(sandbox=SandboxRunConfig(session=session), tracing_disabled=True) + + async def describe(ctx, declaration): + return declaration.description + + if decision in ("prepared", "restored", "changed"): + config.tool_execution = ToolExecutionConfig(prepare_tool_declaration=describe) + + async def run(prompt): + if streamed: + result = Runner.run_streamed(agent, prompt, run_config=config) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, prompt, run_config=config) + try: async with session: - first = await Runner.run(agent, "create note", run_config=config) + first = await run("create note") assert len(first.interruptions) == 1 with pytest.raises(WorkspaceReadNotFoundError): await session.read(Path("note.txt")) state = first.to_state() - if approve: + if decision != "reject": state.approve(first.interruptions[0]) else: state.reject(first.interruptions[0]) - result = await Runner.run(agent, state, run_config=config) + if decision == "restored": + state = await RunState.from_json(agent, state.to_json()) + if decision == "changed": + requires_approval = False + with pytest.raises(UserError, match="replacement tool"): + await run(state) + with pytest.raises(WorkspaceReadNotFoundError): + await session.read(Path("note.txt")) + return + result = await run(state) assert result.final_output == "done" - if approve: + if decision != "reject": with await session.read(Path("note.txt")) as handle: assert handle.read() == b"hello" else: diff --git a/tests/test_tool_declarations.py b/tests/test_tool_declarations.py index 5ac894ca8f..e9bc48f061 100644 --- a/tests/test_tool_declarations.py +++ b/tests/test_tool_declarations.py @@ -13,7 +13,9 @@ UserError, function_tool, ) -from agents.testing import ScriptedModel, assistant_message, function_call +from agents.sandbox import ExecResult, SandboxAgent +from agents.sandbox.capabilities import Filesystem, Shell +from agents.testing import ScriptedModel, assistant_message, function_call, scripted_sandbox_session from .mcp.helpers import FakeMCPServer @@ -465,3 +467,145 @@ async def omit(ctx, declaration): await run(streamed, agent, configure(omit)) assert model.calls[0].tools == [search] assert model.calls[0].model_settings.tool_choice == "web_search_preview" + + +@pytest.mark.parametrize("resume", [None, "allow", "restore", "replace", "move", "executor"]) +async def test_prepared_sandbox_capability(streamed, resume): + replace_executor = False + + def enabled(ctx, owner): + assert owner is agent + return True + + def shell_tools(tools): + tools.exec_command.needs_approval = resume is not None + if replace_executor: + tools.exec_command.on_invoke_tool = lambda *_: pytest.fail("Replacement ran") + tools.exec_command.is_enabled = enabled + + session = scripted_sandbox_session( + [{"method": "exec", "result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0)}] + ) + model = ScriptedModel( + [ + [function_call("exec_command", {"cmd": "pwd"}, call_id="one")], + [assistant_message("done")], + ] + ) + agent = SandboxAgent( + name="host", model=model, capabilities=[Shell(configure_tools=shell_tools)] + ) + config = RunConfig(sandbox={"session": session}, tool_execution=configure().tool_execution) + result = await run(streamed, agent, config) + if resume: + assert not session.calls + state = result.to_state() + state.approve(result.interruptions[0]) + if resume == "restore": + state = await RunState.from_json(agent, state.to_json()) + elif resume != "allow": + if resume == "replace": + agent.capabilities = [Shell(configure_tools=shell_tools)] + if resume == "move": + agent.capabilities = [Filesystem(), *agent.capabilities] + state = await RunState.from_json(agent, state.to_json()) + replace_executor = resume == "executor" + with pytest.raises( + UserError, match="no longer configured|replacement tool|capability source" + ): + await run(streamed, agent, config, state) + assert not session.calls + return + result = await run(streamed, agent, config, state) + assert result.final_output == "done" + session.assert_complete() + + +async def test_prepared_sandbox_checks_live_public_tools(streamed): + def enabled(ctx, owner): + assert owner is agent + return True + + @function_tool(is_enabled=enabled) + def publish(): + pytest.fail("Revoked tool ran") + + class Hooks(RunHooks): + async def on_tool_start(self, context, owner, tool): + assert owner is agent + agent.tools = [] + + agent = SandboxAgent( + name="host", tools=[publish], capabilities=[], model=ScriptedModel([[call("publish")]]) + ) + config = RunConfig( + sandbox={"session": scripted_sandbox_session()}, tool_execution=configure().tool_execution + ) + with pytest.raises(UserError, match="no longer configured"): + await run(streamed, agent, config, hooks=Hooks()) + + +@pytest.mark.parametrize("direct", [False, True]) +async def test_prepared_sandbox_approval_cannot_change_source(streamed, direct): + effects = [] + + @function_tool(needs_approval=True) + def exec_command(cmd: str): + effects.append(cmd) + + def require_approval(tools): + tools.exec_command.needs_approval = True + + shell = Shell(configure_tools=require_approval) + session = scripted_sandbox_session( + [{"method": "exec", "result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0)}] + ) + model = ScriptedModel( + [ + [function_call("exec_command", {"cmd": "pwd"}, call_id="one")], + [assistant_message("done")], + ] + ) + agent = SandboxAgent( + name="host", + model=model, + capabilities=[] if direct else [shell], + tools=[exec_command] if direct else [], + ) + config = RunConfig(sandbox={"session": session}, tool_execution=configure().tool_execution) + result = await run(streamed, agent, config) + state = result.to_state() + state.approve(result.interruptions[0]) + data = state.to_json() + agent.tools = [] if direct else [exec_command] + agent.capabilities = [shell] if direct else [] + restored = await RunState.from_json(agent, data) + with pytest.raises(UserError, match="source"): + await run(streamed, agent, config, restored) + assert not session.calls and not effects + + +async def test_prepared_view_image_requires_capability_source(streamed): + def require_approval(tools): + tools.view_image.needs_approval = True + + session = scripted_sandbox_session([{"method": "read", "result": b""}]) + agent = SandboxAgent( + name="host", + model=ScriptedModel([[function_call("view_image", {"path": "image.png"}, call_id="one")]]), + capabilities=[Filesystem(configure_tools=require_approval)], + ) + config = RunConfig(sandbox={"session": session}, tool_execution=configure().tool_execution) + result = await run(streamed, agent, config) + state = result.to_state() + state.approve(result.interruptions[0]) + data = state.to_json() + record = data["context"]["tool_invocations"]["one"] + record["capability_source"] = None + with pytest.raises(UserError, match="capability source"): + await RunState.from_json(agent, data) + del record["capability_source"] + restored = await RunState.from_json(agent, data) + with pytest.raises(UserError, match="capability source"): + await run(streamed, agent, config, restored) + assert not session.calls From 8a268391a4d57d2f316bcfabfd85866e53601ab9 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 08:01:16 -0800 Subject: [PATCH 3/3] fix: Reject mixed function callers when preparing tool declarations --- src/agents/run_config.py | 2 + src/agents/run_internal/tool_declarations.py | 28 +++++-- src/agents/run_internal/turn_resolution.py | 9 +- tests/test_tool_declarations.py | 88 ++++++++++++++++++++ 4 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/agents/run_config.py b/src/agents/run_config.py index 6884280f37..f6c2541fe0 100644 --- a/src/agents/run_config.py +++ b/src/agents/run_config.py @@ -228,6 +228,8 @@ async def prepare(ctx, declaration): the current session if the tool type, method, schema and approval policy still match. A pending capability call requires the same capability type, position and tool key after restore. An approval cannot move between a direct tool and a capability. + During selection, functions allowing both direct and programmatic callers raise UserError. + A function cannot gain direct access after selection for the same request. Preconverted MCP functions in agent.tools raise UserError. Handoffs, deferred tools, programmatic callers, agents as tools and provider-managed tools are outside this policy. Use the same policy on resume. A serialized run trusts the application's current tool diff --git a/src/agents/run_internal/tool_declarations.py b/src/agents/run_internal/tool_declarations.py index 96f7f3edba..af57af63ab 100644 --- a/src/agents/run_internal/tool_declarations.py +++ b/src/agents/run_internal/tool_declarations.py @@ -28,11 +28,14 @@ def _covered(tool: FunctionTool) -> bool: - return ( - not tool.defer_loading - and tool._agent_instance is None - and not (tool.allowed_callers and "programmatic" in tool.allowed_callers) - ) + if tool.allowed_callers and "programmatic" in tool.allowed_callers: + if "direct" in tool.allowed_callers: + raise UserError( + "prepare_tool_declaration cannot handle both direct and programmatic callers " + "on the same function. Configure one caller type." + ) + return False + return not tool.defer_loading and tool._agent_instance is None def _same_callback(first: object, second: object) -> bool: @@ -235,14 +238,20 @@ async def check(self, context: RunContextWrapper[Any], config: RunConfig) -> Non self.validate(config) +def validate_prepared_inventory(tools: list[Tool], prepared: dict[int, PreparedTool]) -> None: + for tool in tools: + if isinstance(tool, FunctionTool) and id(tool) not in prepared and _covered(tool): + raise UserError("The function was not prepared for this model request.") + + async def prepare_declarations( tools: list[Tool], bindings: AgentBindings[Any], context: RunContextWrapper[Any], config: RunConfig, -) -> tuple[list[Tool], list[Tool], dict[int, PreparedTool], set[str]]: +) -> tuple[list[Tool], list[Tool], dict[int, PreparedTool] | None, set[str]]: if config.tool_execution is None or config.tool_execution.prepare_tool_declaration is None: - return tools, tools, {}, set() + return tools, tools, None, set() omitted: set[str] = set() admitted: list[Tool] = [] presented: list[Tool] = [] @@ -267,6 +276,11 @@ async def prepare_declarations( else: presented.append(tool) admitted.append(tool) + validate_prepared_inventory(admitted, prepared) + # Detach the caller list before request hooks can change the execution inventory. + for index, tool in enumerate(presented): + if isinstance(tool, FunctionTool) and tool.allowed_callers == ["programmatic"]: + presented[index] = copy.copy(tool) for binding in prepared.values(): binding.validate(config) if omitted: diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 236d8c0b9a..8d516b1f99 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -172,7 +172,12 @@ ) from .session_persistence import extend_held_session_write from .tool_caller import ensure_programmatic_tool_call_parent, ensure_tool_caller_allowed -from .tool_declarations import PreparedTool, bind_declarations, resume_declaration +from .tool_declarations import ( + PreparedTool, + bind_declarations, + resume_declaration, + validate_prepared_inventory, +) from .tool_execution import ( build_litellm_json_tool_call, coerce_apply_patch_operations, @@ -4137,6 +4142,8 @@ async def get_single_step_result_from_response( item_agent = bindings.public_agent def classify() -> ProcessedResponse: + if prepared_tools is not None: + validate_prepared_inventory(all_tools, prepared_tools) return process_model_response( agent=item_agent, all_tools=all_tools, diff --git a/tests/test_tool_declarations.py b/tests/test_tool_declarations.py index e9bc48f061..f064b67093 100644 --- a/tests/test_tool_declarations.py +++ b/tests/test_tool_declarations.py @@ -609,3 +609,91 @@ def require_approval(tools): with pytest.raises(UserError, match="capability source"): await run(streamed, agent, config, restored) assert not session.calls + + +@pytest.mark.parametrize("phase", ["initial", "live", "restored"]) +async def test_prepared_tools_reject_mixed_callers(streamed, phase): + effects = [] + + @function_tool(needs_approval=True, allowed_callers=["direct"]) + def publish(): + effects.append("published") + + async def prepare(ctx, declaration): + return None if phase == "initial" else declaration.description + + model = ScriptedModel([[call("publish")]]) + agent = Agent(name="host", tools=[publish], model=model) + config = configure(prepare) + prompt = "go" + if phase != "initial": + result = await run(streamed, agent, config) + prompt = result.to_state() + prompt.approve(result.interruptions[0]) + if phase == "restored": + prompt = await RunState.from_json(agent, prompt.to_json()) + + assert publish.allowed_callers is not None + publish.allowed_callers.append("programmatic") + with pytest.raises(UserError, match="both direct and programmatic"): + await run(streamed, agent, config, prompt) + assert not effects + assert len(model.calls) == (0 if phase == "initial" else 1) + + +async def test_programmatic_tool_without_direct_policy(streamed): + @function_tool(allowed_callers=["programmatic"]) + def publish(): + pytest.fail("Unexpected invocation") + + async def prepare(ctx, declaration): + pytest.fail("Programmatic-only tool reached declaration policy") + + model = ScriptedModel([[assistant_message("done")], [assistant_message("done")]]) + agent = Agent(name="host", tools=[publish], model=model) + await run(streamed, agent, configure(prepare)) + assert model.calls[0].tools[0].allowed_callers == ["programmatic"] + publish.allowed_callers = ["programmatic", "direct"] + await run(streamed, agent, RunConfig()) + assert model.calls[1].tools[0].allowed_callers == ["programmatic", "direct"] + + +@pytest.mark.parametrize("callers", [None, ["direct"], ["direct", "programmatic"]]) +async def test_unprepared_function_cannot_become_direct_after_selection(streamed, callers): + @function_tool(allowed_callers=["programmatic"]) + def publish(): + pytest.fail("Unprepared function ran") + + async def withhold(ctx, declaration): + return None + + class Hooks(RunHooks): + async def on_llm_start(self, context, agent, system_prompt, input_items): + publish.allowed_callers = callers + + model = ScriptedModel([[call("publish")]]) + agent = Agent(name="host", tools=[publish], model=model) + with pytest.raises(UserError, match="not prepared|both direct and programmatic"): + await run(streamed, agent, configure(withhold), hooks=Hooks()) + assert len(model.calls) == 1 + assert model.calls[0].tools[0].allowed_callers == ["programmatic"] + + +async def test_caller_change_during_declaration_stops_before_sampling(streamed): + @function_tool(allowed_callers=["programmatic"]) + def publish(): + pytest.fail("Unprepared function ran") + + @function_tool + def inspect(): + pass + + async def prepare(ctx, declaration): + publish.allowed_callers = ["direct"] + return None + + model = ScriptedModel([[assistant_message("done")]]) + agent = Agent(name="host", tools=[inspect, publish], model=model) + with pytest.raises(UserError, match="not prepared"): + await run(streamed, agent, configure(prepare)) + assert not model.calls