From 9abdfa00f88faa28e56733d7710655c14f6fa7ac Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:26:38 -0800 Subject: [PATCH 01/33] test(models): restore AnyLLM module bindings after provider tests --- tests/models/test_any_llm_model.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/models/test_any_llm_model.py b/tests/models/test_any_llm_model.py index 7101bb3b6c..e0c20d0b1e 100644 --- a/tests/models/test_any_llm_model.py +++ b/tests/models/test_any_llm_model.py @@ -121,6 +121,8 @@ def create(provider_name: str, api_key: str | None = None, api_base: str | None # together; otherwise a later ``from agents.extensions.models import any_llm_model`` # would still see the stub-backed module while ``sys.modules`` has the original. parent_package = importlib.import_module("agents.extensions.models") + monkeypatch.setitem(sys.modules, "agents.extensions.models.any_llm_model", None) + monkeypatch.setattr(parent_package, "any_llm_model", None, raising=False) monkeypatch.delitem(sys.modules, "agents.extensions.models.any_llm_model", raising=False) monkeypatch.delattr(parent_package, "any_llm_model", raising=False) monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm) @@ -130,24 +132,36 @@ def create(provider_name: str, api_key: str | None = None, api_base: str | None return module, create_calls -def test_import_any_llm_module_restores_module_bindings_on_teardown() -> None: +@pytest.mark.parametrize("preexisting", [False, True]) +def test_import_any_llm_module_restores_module_bindings_on_teardown( + monkeypatch: pytest.MonkeyPatch, preexisting: bool +) -> None: pytest.importorskip( "any_llm", reason="`any-llm-sdk` is only available when the optional dependency is installed.", ) parent_package = importlib.import_module("agents.extensions.models") - original = importlib.import_module("agents.extensions.models.any_llm_model") + if preexisting: + original = importlib.import_module("agents.extensions.models.any_llm_model") + else: + monkeypatch.delitem(sys.modules, "agents.extensions.models.any_llm_model", raising=False) + monkeypatch.delattr(parent_package, "any_llm_model", raising=False) + original = None - with pytest.MonkeyPatch.context() as monkeypatch: + with pytest.MonkeyPatch.context() as scoped_patch: module, _ = _import_any_llm_module( - monkeypatch, FakeAnyLLMProvider(supports_responses=False) + scoped_patch, FakeAnyLLMProvider(supports_responses=False) ) assert module is not original assert sys.modules["agents.extensions.models.any_llm_model"] is module assert parent_package.any_llm_model is module - assert sys.modules["agents.extensions.models.any_llm_model"] is original - assert parent_package.any_llm_model is original + if preexisting: + assert sys.modules["agents.extensions.models.any_llm_model"] is original + assert parent_package.any_llm_model is original + else: + assert "agents.extensions.models.any_llm_model" not in sys.modules + assert not hasattr(parent_package, "any_llm_model") def _chat_completion(text: str) -> ChatCompletion: From 7de7610f5f3c23dfbe8902dd95c60fb3f8ff3435 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:26:53 -0800 Subject: [PATCH 02/33] feat(tracing): add per-run capture and an OpenTelemetry processor --- pyproject.toml | 2 + src/agents/extensions/tracing/__init__.py | 5 + .../extensions/tracing/opentelemetry.py | 427 ++++++++++++++ src/agents/models/openai_responses.py | 168 ++++-- src/agents/run.py | 11 +- src/agents/run_internal/error_handlers.py | 2 +- .../run_internal/model_provider_lifecycle.py | 8 + src/agents/run_internal/model_retry.py | 535 +++++++++--------- src/agents/run_internal/tool_actions.py | 137 ++--- src/agents/run_internal/tool_execution.py | 83 ++- src/agents/tracing/_content.py | 149 +++++ src/agents/tracing/_model.py | 208 +++++++ src/agents/tracing/config.py | 46 ++ src/agents/tracing/processor_interface.py | 12 + src/agents/tracing/provider.py | 22 + src/agents/tracing/span_data.py | 72 ++- src/agents/tracing/spans.py | 96 +++- src/agents/util/_error_tracing.py | 11 +- .../models/test_response_operation_tracing.py | 445 +++++++++++++++ tests/test_local_shell_tool.py | 5 + tests/tracing/test_content_capture.py | 209 +++++++ tests/tracing/test_opentelemetry.py | 446 +++++++++++++++ tests/tracing/test_tool_operation_tracing.py | 80 +++ uv.lock | 8 +- 24 files changed, 2778 insertions(+), 409 deletions(-) create mode 100644 src/agents/extensions/tracing/__init__.py create mode 100644 src/agents/extensions/tracing/opentelemetry.py create mode 100644 src/agents/tracing/_content.py create mode 100644 src/agents/tracing/_model.py create mode 100644 tests/models/test_response_operation_tracing.py create mode 100644 tests/tracing/test_content_capture.py create mode 100644 tests/tracing/test_opentelemetry.py create mode 100644 tests/tracing/test_tool_operation_tracing.py diff --git a/pyproject.toml b/pyproject.toml index 073c44e4b0..3a6fb49c3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ Homepage = "https://openai.github.io/openai-agents-python/" Repository = "https://github.com/openai/openai-agents-python" [project.optional-dependencies] +opentelemetry = ["opentelemetry-api>=1.45.0,<2"] # Protobuf exclusions retain patched 5.x and 6.x releases (GHSA-7gcm-g887-7qv7). voice = ["numpy>=2.2.0, <3; python_version>='3.10'", "websockets>=15.0, <18"] viz = ["graphviz>=0.17"] @@ -95,6 +96,7 @@ temporal = [ [dependency-groups] dev = [ + "opentelemetry-sdk>=1.45.0,<2", "mypy", "ruff==0.17.1", "pytest>=9.0.3", diff --git a/src/agents/extensions/tracing/__init__.py b/src/agents/extensions/tracing/__init__.py new file mode 100644 index 0000000000..4710b5e900 --- /dev/null +++ b/src/agents/extensions/tracing/__init__.py @@ -0,0 +1,5 @@ +"""Optional integrations for Agents SDK tracing.""" + +from .opentelemetry import OpenTelemetryTracingProcessor + +__all__ = ["OpenTelemetryTracingProcessor"] diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py new file mode 100644 index 0000000000..93dba6b328 --- /dev/null +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -0,0 +1,427 @@ +"""Translate native SDK spans into live OpenTelemetry spans. + +Install ``openai-agents[opentelemetry]`` and register this processor with +``agents.add_trace_processor``. Configure exporters and sampling on the application's +OpenTelemetry tracer provider; this processor does not own that provider's lifecycle. + +The GenAI mapping follows semantic-conventions-genai commit +``622b2b6073b15e5ed886c3c12a65a892b9cd5c70``. These conventions are in development. +""" + +from __future__ import annotations + +import json +import re +import threading +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +from datetime import datetime +from typing import Any + +from opentelemetry import context as otel_context, trace as otel_trace +from opentelemetry.trace import SpanKind, StatusCode, TracerProvider + +from ...logger import logger +from ...tracing.processor_interface import TracingProcessor +from ...tracing.span_data import ( + AgentSpanData, + CustomSpanData, + FunctionSpanData, + GenerationSpanData, + GuardrailSpanData, + HandoffSpanData, + ResponseSpanData, + SpanData, + TaskSpanData, + TurnSpanData, +) +from ...tracing.spans import Span +from ...tracing.traces import Trace +from ...version import __version__ + +_scope_owner: ContextVar[object | None] = ContextVar("agents_otel_scope_owner", default=None) + + +def _timestamp(value: str | None) -> int | None: + if value is None: + return None + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + seconds = int(parsed.replace(microsecond=0).timestamp()) + fraction = re.search(r"T\d{2}:\d{2}:\d{2}\.(\d+)", value) + nanos = int(fraction.group(1).ljust(9, "0")[:9]) if fraction else 0 + return seconds * 1_000_000_000 + nanos + + +def _json_value(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + pass + return value + + +def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: + """Project Responses text and tool content into the GenAI message schema.""" + if isinstance(value, str): + return [ + { + "role": "assistant" if output else "user", + "parts": [{"type": "text", "content": value}], + } + ] + if not isinstance(value, Sequence): + return [] + messages: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, Mapping): + continue + kind = item.get("type", "message") + role = item.get("role", "assistant" if output else "user") + parts: list[dict[str, Any]] = [] + if kind in ("input_text", "output_text", "text"): + part = {"type": "text", "content": item.get("text", "")} + for flag in ("partial", "truncated"): + if isinstance(item.get(flag), bool): + part[flag] = item[flag] + parts.append(part) + elif item.get("truncated") is True: + parts.append({"type": "openai.omitted", "reason": "truncated"}) + elif kind in ("function_call", "custom_tool_call"): + role = "assistant" + parts.append( + { + "type": "tool_call", + "name": item.get("name", ""), + "id": item.get("call_id"), + "arguments": _json_value(item.get("arguments", item.get("input"))), + } + ) + elif kind in ("function_call_output", "custom_tool_call_output"): + role = "tool" + parts.append( + { + "type": "tool_call_response", + "id": item.get("call_id"), + "response": item.get("output"), + } + ) + elif kind == "message": + content = item.get("content") + if isinstance(content, str): + parts.append({"type": "text", "content": content}) + elif isinstance(content, Sequence): + for part in content: + if not isinstance(part, Mapping): + continue + if part.get("type") in ("input_text", "output_text", "text"): + parts.append({"type": "text", "content": part.get("text", "")}) + else: + # Preserve part order without exporting media or hidden reasoning. + parts.append({"type": "openai.omitted", "reason": "unsupported_content"}) + else: + parts.append({"type": "openai.omitted", "reason": "unsupported_content"}) + if output and messages: + messages[0]["parts"].extend(parts) + else: + messages.append({"role": role, "parts": parts}) + return messages + + +def _message_schema(value: Any, role: str) -> list[dict[str, Any]]: + """Preserve the message schema when the shared budget replaces a container.""" + messages: list[dict[str, Any]] = [] + for item in value if isinstance(value, list) else [{}]: + if not isinstance(item, Mapping) or "parts" not in item: + messages.append( + {"role": role, "parts": [{"type": "openai.omitted", "reason": "truncated"}]} + ) + continue + parts = [ + part + if isinstance(part, Mapping) and "type" in part + else {"type": "openai.omitted", "reason": "truncated"} + for part in item["parts"] + ] + messages.append({"role": item.get("role", role), "parts": parts}) + return messages + + +def _identity(data: SpanData) -> tuple[str, str, SpanKind]: + if isinstance(data, TaskSpanData): + return f"invoke_runner {data.name}", "invoke_runner", SpanKind.INTERNAL + if isinstance(data, AgentSpanData): + return f"invoke_agent {data.name}", "invoke_agent", SpanKind.INTERNAL + if isinstance(data, TurnSpanData): + return "openai.agents.turn", "openai.agents.turn", SpanKind.INTERNAL + if isinstance(data, FunctionSpanData): + if data.operation.outcome in ("paused", "rejected"): + return ( + f"openai.agents.tool_approval {data.name}", + "openai.agents.tool_approval", + SpanKind.INTERNAL, + ) + return f"execute_tool {data.name}", "execute_tool", SpanKind.INTERNAL + if isinstance(data, (GenerationSpanData, ResponseSpanData)): + model = data.operation.model or getattr(data, "model", None) + if isinstance(data, ResponseSpanData) and data.response is not None: + model = model or data.response.model + return f"chat {model}" if model else "chat", "chat", SpanKind.CLIENT + if isinstance(data, HandoffSpanData): + return "openai.agents.handoff", "openai.agents.handoff", SpanKind.INTERNAL + if isinstance(data, GuardrailSpanData): + return f"openai.agents.guardrail {data.name}", "openai.agents.guardrail", SpanKind.INTERNAL + if isinstance(data, CustomSpanData): + if data.name == "tool_approval": + name = data.data.get("tool_name") + return ( + f"openai.agents.tool_approval {name}" if name else "openai.agents.tool_approval", + "openai.agents.tool_approval", + SpanKind.INTERNAL, + ) + return data.name, "openai.agents.custom", SpanKind.INTERNAL + return f"openai.agents.{data.type}", f"openai.agents.{data.type}", SpanKind.INTERNAL + + +class OpenTelemetryTracingProcessor(TracingProcessor): + """Export native SDK spans through the application's OpenTelemetry provider. + + Content permission belongs to each SDK operation. Legacy SDK payload fields do not + grant permission to export content through this integration. ``force_flush`` and + ``shutdown`` do not flush or shut down the caller-owned OpenTelemetry provider. + """ + + def __init__(self, *, tracer_provider: TracerProvider | None = None) -> None: + self._tracer = otel_trace.get_tracer( + "openai.agents", __version__, tracer_provider=tracer_provider + ) + self._lock = threading.RLock() + self._spans: dict[tuple[str, str], otel_trace.Span] = {} + self._parents: dict[tuple[str, str], otel_trace.Span] = {} + self._traces: dict[str, dict[int, Trace]] = {} + + def on_trace_start(self, trace: Trace) -> None: + # A native trace may group several Runner invocations; task spans represent runs. + with self._lock: + self._traces.setdefault(trace.trace_id, {})[id(trace)] = trace + + def on_trace_end(self, trace: Trace) -> None: + with self._lock: + owners = self._traces.get(trace.trace_id) + if owners is None or id(trace) not in owners: + return + owners.pop(id(trace)) + if owners: + return + self._traces.pop(trace.trace_id, None) + unfinished = [ + self._spans.pop(key) for key in list(self._spans) if key[0] == trace.trace_id + ] + for key in [key for key in self._parents if key[0] == trace.trace_id]: + self._parents.pop(key, None) + self._end_interrupted(unfinished) + + def on_span_start(self, span: Span[Any]) -> None: + key = (span.trace_id, span.span_id) + with self._lock: + if key in self._spans: + return + parent = self._parents.get((span.trace_id, span.parent_id or "")) + context = otel_trace.set_span_in_context(parent) if parent is not None else None + name, operation, kind = _identity(span.span_data) + attributes = self._attributes(span) + attributes["gen_ai.operation.name"] = operation + attributes["openai.agents.span.type"] = span.span_data.type + otel_span = self._tracer.start_span( + name, + context=context, + kind=kind, + attributes=attributes, + start_time=_timestamp(span.started_at), + ) + self._spans[key] = otel_span + # Keep only identity after completion, not exported content or SDK resources. + self._parents[key] = otel_trace.NonRecordingSpan(otel_span.get_span_context()) + + def on_span_end(self, span: Span[Any]) -> None: + with self._lock: + otel_span = self._spans.pop((span.trace_id, span.span_id), None) + if span.trace_id not in self._traces and not any( + key[0] == span.trace_id for key in self._spans + ): + for key in [key for key in self._parents if key[0] == span.trace_id]: + self._parents.pop(key, None) + if otel_span is None: + return + try: + otel_span.set_attributes(self._attributes(span)) + if span.error is not None: + # Error messages may contain payloads even when content capture is disabled. + otel_span.set_attribute("error.type", "_OTHER") + otel_span.set_status(StatusCode.ERROR) + operation = getattr(span.span_data, "operation", None) + if operation is not None and operation.outcome in ( + "error", + "failed", + "cancelled", + "interrupted", + ): + otel_span.set_attribute("error.type", "_OTHER") + otel_span.set_status(StatusCode.ERROR) + finally: + otel_span.end(end_time=_timestamp(span.ended_at)) + + def on_span_scope_enter(self, span: Span[Any]) -> object | None: + """Activate a span only for the SDK execution scope that owns its context.""" + with self._lock: + otel_span = self._spans.get((span.trace_id, span.span_id)) + if otel_span is None: + return None + owner_token = _scope_owner.set(object()) + return owner_token, otel_context.attach(otel_trace.set_span_in_context(otel_span)) + + def on_span_scope_exit(self, span: Span[Any], token: Any) -> None: + if token is not None: + owner_token, context_token = token + try: + _scope_owner.reset(owner_token) + except ValueError: + # Async-generator finalization cannot reset the originating task's context. + return + otel_context.detach(context_token) + + def _attributes(self, span: Span[Any]) -> dict[str, Any]: + data = span.span_data + attributes: dict[str, Any] = {} + if isinstance(data, (GenerationSpanData, ResponseSpanData)): + if isinstance(data, ResponseSpanData) or data.operation.api_type == "responses": + attributes["gen_ai.provider.name"] = "openai" + attributes["openai.api.type"] = "responses" + model = data.operation.model or getattr(data, "model", None) + if isinstance(data, ResponseSpanData) and data.response is not None: + model = model or data.response.model + if model: + attributes["gen_ai.request.model"] = model + usage = data.usage + if usage: + for source, destination in ( + ("input_tokens", "gen_ai.usage.input_tokens"), + ("output_tokens", "gen_ai.usage.output_tokens"), + ): + if isinstance(usage.get(source), int): + attributes[destination] = usage[source] + if data.operation.response_model: + attributes["gen_ai.response.model"] = data.operation.response_model + if data.operation.response_id: + attributes["gen_ai.response.id"] = data.operation.response_id + if data.operation.previous_response_id: + attributes["openai.response.previous_response_id"] = ( + data.operation.previous_response_id + ) + if isinstance(data, (AgentSpanData, TurnSpanData)): + attributes["gen_ai.agent.name"] = ( + data.name if isinstance(data, AgentSpanData) else data.agent_name + ) + if isinstance(data, TurnSpanData): + attributes["openai.agents.turn.number"] = data.turn + if isinstance(data, FunctionSpanData): + attributes["gen_ai.tool.name"] = data.name + attributes["gen_ai.tool.type"] = "function" + if isinstance(data, (GenerationSpanData, ResponseSpanData, FunctionSpanData)): + operation = data.operation + if operation.outcome: + attributes["openai.agents.operation.outcome"] = operation.outcome + if operation.estimated_end: + attributes["openai.agents.operation.estimated_end"] = True + if operation.operation_id: + attributes["openai.agents.operation.id"] = operation.operation_id + if operation.tool_call_id: + attributes["gen_ai.tool.call.id"] = operation.tool_call_id + if operation.detail == "content": + attributes.update(self._content_attributes(data)) + if isinstance(data, CustomSpanData) and data.name == "codex.tracing.coverage": + for key in ("provider", "coverage", "reason"): + if isinstance(data.data.get(key), str): + attributes[f"openai.agents.operation.{key}"] = data.data[key] + if isinstance(data, CustomSpanData) and data.name == "tool_approval": + for source, target in ( + ("tool_name", "gen_ai.tool.name"), + ("tool_call_id", "gen_ai.tool.call.id"), + ("outcome", "openai.agents.operation.outcome"), + ): + if isinstance(data.data.get(source), str): + attributes[target] = data.data[source] + return attributes + + def _content_attributes( + self, data: GenerationSpanData | ResponseSpanData | FunctionSpanData + ) -> dict[str, Any]: + from ...tracing._content import bound_content + + values: dict[str, Any] = {} + if isinstance(data, FunctionSpanData): + if data.input is not None: + values["gen_ai.tool.call.arguments"] = bound_content(_json_value(data.input)) + if data.output is not None: + values["gen_ai.tool.call.result"] = bound_content(data.output) + else: + inputs = data.input + output = ( + data.output + if isinstance(data, GenerationSpanData) + else ( + [item.model_dump(exclude_none=True) for item in data.response.output] + if data.response is not None + else None + ) + ) + request: dict[str, Any] = {} + if inputs is not None: + request["messages"] = _messages(inputs) + if output is not None: + values["gen_ai.output.messages"] = _message_schema( + bound_content(_messages(output, output=True)), "assistant" + ) + if isinstance(data, GenerationSpanData) and data.model_config: + instructions = data.model_config.get("instructions") + if isinstance(instructions, str): + request["instructions"] = [{"type": "text", "content": instructions}] + request = bound_content(request) + if "messages" in request: + values["gen_ai.input.messages"] = _message_schema(request["messages"], "user") + if "instructions" in request: + values["gen_ai.system_instructions"] = request["instructions"] + if isinstance(data, GenerationSpanData) and data.tools is not None: + values["gen_ai.tool.definitions"] = bound_content(data.tools) + return values + + def force_flush(self) -> None: + """Leave exporter flushing to the application's OpenTelemetry provider.""" + pass + + @staticmethod + def _end_interrupted(spans: list[otel_trace.Span]) -> None: + for span in spans: + try: + try: + span.set_attributes( + { + "openai.agents.operation.outcome": "interrupted", + "openai.agents.operation.estimated_end": True, + "error.type": "_OTHER", + } + ) + span.set_status(StatusCode.ERROR) + finally: + span.end() + except Exception: + logger.warning("Error while ending an interrupted OpenTelemetry span") + + def shutdown(self) -> None: + """End outstanding spans without shutting down the application's provider.""" + with self._lock: + spans = list(self._spans.values()) + self._spans.clear() + self._parents.clear() + self._traces.clear() + self._end_interrupted(spans) diff --git a/src/agents/models/openai_responses.py b/src/agents/models/openai_responses.py index 7e013e30be..5b8a8ec048 100644 --- a/src/agents/models/openai_responses.py +++ b/src/agents/models/openai_responses.py @@ -74,7 +74,11 @@ validate_responses_programmatic_tool_calling_configuration, validate_responses_tool_search_configuration, ) -from ..tracing import SpanError, response_span +from ..tracing._content import bound_content +from ..tracing._model import ModelSpan, finish_model_span, model_span, record_model_input +from ..tracing.scope import Scope +from ..tracing.span_data import GenerationSpanData, ResponseSpanData +from ..tracing.util import time_iso from ..usage import ( Usage, _attach_raw_usage_snapshot, @@ -184,6 +188,68 @@ def _materialize_responses_tool_params( return typed_tools +def _record_response_id(span: ModelSpan, response_id: str) -> None: + span.span_data.operation.response_id = response_id + if isinstance(span.span_data, ResponseSpanData): + span.span_data._response_id = response_id + + +def _record_response_operation( + span: ModelSpan, + response: Response, + input: str | list[TResponseInputItem], + tracing: ModelTracing, + *, + outcome: Literal["completed", "failed"] = "completed", +) -> None: + data = span.span_data + data.operation.response_model = getattr(response, "model", None) + data.operation.outcome = outcome + if tracing.include_data(): + data.operation.response_id = response.id + if isinstance(data, ResponseSpanData): + if tracing.include_data() and outcome == "completed": + data.response = response + data.input = input + elif data.operation.detail == "content": + data.output = bound_content([item.model_dump(mode="json") for item in response.output]) + + +def _record_prepared_request(request: Mapping[str, Any]) -> None: + span = Scope.get_current_span() + if span is None or not isinstance(span.span_data, (GenerationSpanData, ResponseSpanData)): + return + data = span.span_data + model = request.get("model") + data.operation.model = model if isinstance(model, str) else None + if isinstance(data, GenerationSpanData): + data.model = data.operation.model + if not isinstance(data, GenerationSpanData) or data.operation.detail != "content": + return + record_model_input(data, dict(request)) + + +def _record_failed_response(response: Response, *, response_id_trusted: bool) -> None: + span = Scope.get_current_span() + if span is None or not isinstance(span.span_data, (GenerationSpanData, ResponseSpanData)): + return + data = span.span_data + if isinstance(data, GenerationSpanData): + _record_response_operation( + span, + response, + [], + ModelTracing.ENABLED + if data.operation.detail == "content" + else ModelTracing.ENABLED_WITHOUT_DATA, + outcome="failed", + ) + if response_id_trusted: + data.operation.response_id = response.id + else: + data.operation.response_model = getattr(response, "model", None) + + async def _refresh_openai_client_api_key_if_supported(client: Any) -> None: """Backward-compatible wrapper around shared WebSocket client credential refresh.""" await refresh_openai_client_api_key_if_supported(client) @@ -622,7 +688,7 @@ async def get_response( conversation_id: str | None = None, prompt: ResponsePromptParam | None = None, ) -> ModelResponse: - with response_span(disabled=tracing.is_disabled()) as span_response: + with model_span(tracing, self.model) as span_response: try: redacted_response_id_endpoint_is_trusted = ( not tracing.include_data() @@ -658,14 +724,12 @@ async def get_response( if response.usage is not None or usage.requests: span_response.span_data.usage = model_usage_to_span_usage(usage) - if tracing.include_data(): - span_response.span_data.response = response - span_response.span_data.input = input - elif ( + _record_response_operation(span_response, response, input, tracing) + if ( redacted_response_id_endpoint_is_trusted and self._uses_official_openai_endpoint() ): - span_response.span_data._response_id = response.id + _record_response_id(span_response, response.id) except asyncio.CancelledError: record_current_task_model_timeout_on_span( span_response, @@ -676,15 +740,11 @@ async def get_response( except Exception as e: if isinstance(e, ModelBehaviorError) and e._terminal_usage is not None: span_response.span_data.usage = model_usage_to_span_usage(e._terminal_usage) - span_response.set_error( - SpanError( - message="Error getting response", - data={ - "error": str(e) - if tracing.include_data() - else "Error details are redacted.", - }, - ) + record_model_error_on_span( + span_response, + message="Error getting response", + error=e, + trace_include_sensitive_data=tracing.include_data(), ) message = "Error getting response" if not _debug.DONT_LOG_MODEL_DATA: @@ -720,7 +780,8 @@ async def stream_response( """ Yields a partial message as it is generated, as well as the usage information. """ - with response_span(disabled=tracing.is_disabled()) as span_response: + with model_span(tracing, self.model) as span_response: + terminal_ended_at: str | None = None try: redacted_response_id_endpoint_is_trusted = ( not tracing.include_data() @@ -748,12 +809,13 @@ async def stream_response( async for chunk in stream: chunk_type = getattr(chunk, "type", None) if isinstance(chunk, ResponseCompletedEvent): + terminal_ended_at = time_iso() final_response = chunk.response if ( redacted_response_id_endpoint_is_trusted and self._uses_official_openai_endpoint() ): - span_response.span_data._response_id = chunk.response.id + _record_response_id(span_response, chunk.response.id) if model_settings.preserve_raw_usage is True: _attach_raw_usage_snapshot(chunk.response, chunk.response.usage) usage = _usage_from_response(chunk.response) @@ -761,17 +823,30 @@ async def stream_response( # Record before yielding the terminal event because consumers may # close the generator immediately after receiving it. span_response.span_data.usage = model_usage_to_span_usage(usage) - if tracing.include_data(): - # Same reason as usage: a consumer that stops at the terminal - # event closes the generator and never reaches the post-loop - # assignment, so record the model I/O before yielding. - span_response.span_data.response = chunk.response - span_response.span_data.input = input + _record_response_operation( + span_response, chunk.response, input, tracing + ) elif chunk_type in { "response.failed", "response.incomplete", }: + terminal_ended_at = time_iso() terminal_response = getattr(chunk, "response", None) + if isinstance(terminal_response, Response): + _record_response_operation( + span_response, + terminal_response, + input, + tracing, + outcome="failed", + ) + if ( + redacted_response_id_endpoint_is_trusted + and self._uses_official_openai_endpoint() + ): + span_response.span_data.operation.response_id = ( + terminal_response.id + ) terminal_failure_error = response_terminal_failure_error( cast(str, chunk_type), terminal_response @@ -783,6 +858,7 @@ async def stream_response( terminal_failure_error._terminal_usage ) elif chunk_type in {"error", "response.error"}: + terminal_ended_at = time_iso() terminal_failure_error = response_error_event_failure_error( cast(str, chunk_type), chunk, @@ -796,6 +872,7 @@ async def stream_response( }: yielded_terminal_event = True if terminal_failure_error is not None: + span_response.span_data.operation.outcome = "failed" # A consumer that stops at this event closes the # generator, which raises GeneratorExit at the yield # below and skips the raise after the loop, so the @@ -831,9 +908,8 @@ async def stream_response( if terminal_failure_error is not None: raise terminal_failure_error - if final_response is not None and tracing.include_data(): - span_response.span_data.response = final_response - span_response.span_data.input = input + if final_response is not None: + _record_response_operation(span_response, final_response, input, tracing) except asyncio.CancelledError: record_current_task_model_timeout_on_span( span_response, @@ -842,18 +918,17 @@ async def stream_response( ) raise except Exception as e: - span_response.set_error( - SpanError( - message="Error streaming response", - data={ - "error": str(e) - if tracing.include_data() - else "Error details are redacted.", - }, - ) + record_model_error_on_span( + span_response, + message="Error streaming response", + error=e, + trace_include_sensitive_data=tracing.include_data(), ) log_model_action_error(logger, "Error streaming response", e) raise + finally: + if terminal_ended_at is not None: + finish_model_span(span_response, ended_at=terminal_ended_at) @overload async def _fetch_response( @@ -910,11 +985,23 @@ async def _fetch_response( stream=stream, prompt=prompt, ) + prepared_request = dict(create_kwargs) + extra_body = prepared_request.get("extra_body") + if isinstance(extra_body, Mapping): + prepared_request.update(extra_body) + _record_prepared_request(prepared_request) client = self._get_client() if not stream: + response_id_trusted = self._uses_official_openai_endpoint() response = await client.responses.create(**create_kwargs) if getattr(response, "status", None) in {"failed", "incomplete"}: + _record_failed_response( + response, + response_id_trusted=( + response_id_trusted and self._uses_official_openai_endpoint() + ), + ) raise response_terminal_failure_error(f"response.{response.status}", response) _mark_transport_request_without_usage(response) return cast(Response, response) @@ -1344,6 +1431,7 @@ async def _fetch_response( final_response: Response | None = None terminal_event_type: str | None = None + response_id_trusted = self._uses_official_openai_endpoint() async for event in self._iter_websocket_response_events(create_kwargs): event_type = getattr(event, "type", None) if isinstance(event, ResponseCompletedEvent): @@ -1352,6 +1440,13 @@ async def _fetch_response( elif event_type in {"response.incomplete", "response.failed"}: terminal_event_type = cast(str, event_type) terminal_response = getattr(event, "response", None) + if isinstance(terminal_response, Response): + _record_failed_response( + terminal_response, + response_id_trusted=( + response_id_trusted and self._uses_official_openai_endpoint() + ), + ) raise response_terminal_failure_error( terminal_event_type, terminal_response if isinstance(terminal_response, Response) else None, @@ -1390,6 +1485,7 @@ async def _iter_websocket_response_events( request_frame, ws_url, request_headers = await self._prepare_websocket_request( create_kwargs ) + _record_prepared_request(request_frame) retry_pre_event_disconnect = _should_retry_pre_event_websocket_disconnect() while True: connection: Any = None diff --git a/src/agents/run.py b/src/agents/run.py index 3a70a0f694..76050a7b3f 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -180,7 +180,7 @@ task_span, turn_span, ) -from .tracing.config import include_task_and_turn_spans +from .tracing.config import include_task_and_turn_spans, operation_tracing_scope from .tracing.context import TraceCtxManager, create_trace_for_run from .tracing.span_data import AgentSpanData, TaskSpanData from .util import _error_tracing @@ -582,6 +582,9 @@ async def run( redacted_error: BaseException | None = None # Mask caller tracing before any run-owned preparation or cleanup can execute. with contextlib.ExitStack() as trace_stack: + trace_stack.enter_context( + operation_tracing_scope(run_config.tracing, run_config.trace_include_sensitive_data) + ) try: try: if run_config.tracing_disabled: @@ -2646,6 +2649,7 @@ def run_streamed( # Synchronous setup can invoke application providers before the task starts. # Restore caller context before creating the independently masked background task. with ( + operation_tracing_scope(run_config.tracing, run_config.trace_include_sensitive_data), TraceCtxManager( workflow_name=run_config.workflow_name, trace_id=run_config.trace_id, @@ -2655,7 +2659,7 @@ def run_streamed( disabled=True, ) if run_config.tracing_disabled - else contextlib.nullcontext() + else contextlib.nullcontext(), ): # Handle RunState input is_resumed_state = isinstance(input, RunState) @@ -2889,6 +2893,9 @@ async def run_loop() -> None: else starting_agent ) with ( + operation_tracing_scope( + run_config.tracing, run_config.trace_include_sensitive_data + ), # Mask the inherited span in this task; the streaming loop owns the trace. custom_span(name="Agent run", disabled=True) if run_config.tracing_disabled diff --git a/src/agents/run_internal/error_handlers.py b/src/agents/run_internal/error_handlers.py index f55e8b9929..ccb92c95a4 100644 --- a/src/agents/run_internal/error_handlers.py +++ b/src/agents/run_internal/error_handlers.py @@ -93,7 +93,7 @@ def attach_generic_agent_error( return detail = ( _format_agent_error_detail(exc) - if trace_include_sensitive_data + if _error_tracing.include_error_text(trace_include_sensitive_data) else REDACTED_TRACE_ERROR_MESSAGE ) _error_tracing.attach_error_to_span( diff --git a/src/agents/run_internal/model_provider_lifecycle.py b/src/agents/run_internal/model_provider_lifecycle.py index a3c22e8d53..17d32326c1 100644 --- a/src/agents/run_internal/model_provider_lifecycle.py +++ b/src/agents/run_internal/model_provider_lifecycle.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from copy import copy from typing import Any from ..logger import log_model_action_warning, logger @@ -16,6 +17,13 @@ def _normalize_run_config_for_runner( isinstance(value, dict) and "model_provider" not in value ) run_config = RunConfig() if value is None else _coerce_run_config(value) + detail = run_config.tracing.get("operation_detail") if run_config.tracing else None + if detail not in (None, "metadata", "content"): + raise ValueError("operation_detail must be 'metadata' or 'content'") + if detail == "metadata" and run_config.trace_include_sensitive_data: + # Route every existing capture decision through the run's sensitive-data gate. + run_config = copy(run_config) + run_config.trace_include_sensitive_data = False return run_config, owns_model_provider diff --git a/src/agents/run_internal/model_retry.py b/src/agents/run_internal/model_retry.py index 358567e0ce..4f23b729a8 100644 --- a/src/agents/run_internal/model_retry.py +++ b/src/agents/run_internal/model_retry.py @@ -35,6 +35,7 @@ _coerce_backoff_settings, retry_policy_retries_safe_transport_errors, ) +from ..tracing._model import ModelRetrySpans, model_retry_scope from ..usage import RequestUsage, Usage from ..util._error_tracing import mark_model_timeout_task @@ -619,91 +620,96 @@ async def get_response_with_retry( conversation_id=conversation_id, ) - while True: - try: - # Keep provider retries on the initial attempt, but disable them on explicit - # no-retry settings and on any replay attempt that the runner manages itself. - with ( - provider_managed_retries_disabled( - _should_disable_provider_managed_retries( - retry_settings, - attempt=request_attempt, - stateful_request=stateful_request, - replay_unsafe_request=replay_unsafe_request, - ) - ), - websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), - ): - response = await _await_model_attempt(get_response(), timeout) - response.usage = apply_retry_attempt_usage( - response.usage, request_attempt - 1, reported_usage - ) - return response - except Exception as error: - if isinstance(error, ModelBehaviorError) and error._terminal_usage is not None: - reported_usage[request_attempt] = error._terminal_usage - if ( - not replay_unsafe_request - and _is_conversation_locked_error(error) - and _should_preserve_conversation_locked_compatibility(retry_settings) - ): - # Preserve the historical conversation_locked retry path for backward - # compatibility, including when callers enable retry policies for unrelated - # failures. Callers can explicitly opt out of this compatibility behavior with - # max_retries=0. - if compatibility_retries_taken < COMPATIBILITY_CONVERSATION_LOCKED_RETRIES: - compatibility_retries_taken += 1 - delay = 1.0 * (2 ** (compatibility_retries_taken - 1)) - logger.debug( - "Conversation locked, retrying in %ss (attempt %s/%s).", - delay, - compatibility_retries_taken, - COMPATIBILITY_CONVERSATION_LOCKED_RETRIES, + model_spans = ModelRetrySpans() + try: + while True: + try: + # Keep provider retries on the initial attempt, but disable them on explicit + # no-retry settings and on any replay attempt that the runner manages itself. + with ( + provider_managed_retries_disabled( + _should_disable_provider_managed_retries( + retry_settings, + attempt=request_attempt, + stateful_request=stateful_request, + replay_unsafe_request=replay_unsafe_request, + ) + ), + websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), + model_retry_scope(model_spans, request_attempt), + ): + response = await _await_model_attempt(get_response(), timeout) + response.usage = apply_retry_attempt_usage( + response.usage, request_attempt - 1, reported_usage + ) + return response + except Exception as error: + if isinstance(error, ModelBehaviorError) and error._terminal_usage is not None: + reported_usage[request_attempt] = error._terminal_usage + if ( + not replay_unsafe_request + and _is_conversation_locked_error(error) + and _should_preserve_conversation_locked_compatibility(retry_settings) + ): + # Preserve the historical conversation_locked retry path for backward + # compatibility, including when callers enable retry policies for unrelated + # failures. Callers can explicitly opt out of this compatibility behavior with + # max_retries=0. + if compatibility_retries_taken < COMPATIBILITY_CONVERSATION_LOCKED_RETRIES: + compatibility_retries_taken += 1 + delay = 1.0 * (2 ** (compatibility_retries_taken - 1)) + logger.debug( + "Conversation locked, retrying in %ss (attempt %s/%s).", + delay, + compatibility_retries_taken, + COMPATIBILITY_CONVERSATION_LOCKED_RETRIES, + ) + await rewind() + await _sleep_for_retry(delay) + request_attempt += 1 + continue + + provider_advice = get_retry_advice( + ModelRetryAdviceRequest( + error=error, + attempt=policy_attempt, + stream=False, + previous_response_id=previous_response_id, + conversation_id=conversation_id, ) - await rewind() - await _sleep_for_retry(delay) - request_attempt += 1 - continue - - provider_advice = get_retry_advice( - ModelRetryAdviceRequest( + ) + decision = await _evaluate_retry( error=error, attempt=policy_attempt, + max_retries=( + max(retry_settings.max_retries or 0, 0) if retry_settings is not None else 0 + ), + retry_policy=retry_settings.policy if retry_settings is not None else None, + retry_backoff=retry_settings.backoff if retry_settings is not None else None, stream=False, + replay_unsafe_request=replay_unsafe_request, + emitted_retry_unsafe_event=False, + provider_advice=provider_advice, previous_response_id=previous_response_id, conversation_id=conversation_id, ) - ) - decision = await _evaluate_retry( - error=error, - attempt=policy_attempt, - max_retries=( - max(retry_settings.max_retries or 0, 0) if retry_settings is not None else 0 - ), - retry_policy=retry_settings.policy if retry_settings is not None else None, - retry_backoff=retry_settings.backoff if retry_settings is not None else None, - stream=False, - replay_unsafe_request=replay_unsafe_request, - emitted_retry_unsafe_event=False, - provider_advice=provider_advice, - previous_response_id=previous_response_id, - conversation_id=conversation_id, - ) - if not decision.retry: - raise - - logger.debug( - "Retrying failed model request in %ss (attempt %s/%s).", - decision.delay, - policy_attempt, - retry_settings.max_retries - if retry_settings is not None and retry_settings.max_retries is not None - else 0, - ) - await rewind() - await _sleep_for_retry(decision.delay or 0.0) - request_attempt += 1 - policy_attempt += 1 + if not decision.retry: + raise + + logger.debug( + "Retrying failed model request in %ss (attempt %s/%s).", + decision.delay, + policy_attempt, + retry_settings.max_retries + if retry_settings is not None and retry_settings.max_retries is not None + else 0, + ) + await rewind() + await _sleep_for_retry(decision.delay or 0.0) + request_attempt += 1 + policy_attempt += 1 + finally: + model_spans.finish() async def stream_response_with_retry( @@ -730,192 +736,205 @@ async def stream_response_with_retry( conversation_id=conversation_id, ) - while True: - emitted_retry_unsafe_event = False - event: TResponseStreamEvent | None = None - stream: AsyncIterator[TResponseStreamEvent] | None = None - stream_owner: asyncio.Task[None] | None = None - deadline = asyncio.get_running_loop().time() + timeout if timeout is not None else None - try: - disable_provider_managed_retries = _should_disable_provider_managed_retries( - retry_settings, - attempt=request_attempt, - stateful_request=stateful_request, - replay_unsafe_request=replay_unsafe_request, - ) - stream_requests: asyncio.Queue[None] | None = None - stream_results: ( - asyncio.Queue[tuple[str, TResponseStreamEvent | BaseException | None]] | None - ) = None - if timeout is None: - # Pull stream events under the retry-disable context, but yield them outside it - # so unrelated model calls made by the consumer do not inherit this setting. - with ( - provider_managed_retries_disabled(disable_provider_managed_retries), - websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), - ): - stream = get_stream() - else: - stream_requests = asyncio.Queue() - stream_results = asyncio.Queue() - with ( - provider_managed_retries_disabled(disable_provider_managed_retries), - websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), - ): - stream_owner = asyncio.create_task( - _run_stream_attempt_in_one_task( - get_stream, - stream_requests, - stream_results, - ) - ) + model_spans = ModelRetrySpans() + try: + while True: + emitted_retry_unsafe_event = False + event: TResponseStreamEvent | None = None + stream: AsyncIterator[TResponseStreamEvent] | None = None + stream_owner: asyncio.Task[None] | None = None + deadline = asyncio.get_running_loop().time() + timeout if timeout is not None else None try: - while True: - try: - if timeout is None: - assert stream is not None - with ( - provider_managed_retries_disabled(disable_provider_managed_retries), - websocket_pre_event_retries_disabled( - disable_websocket_pre_event_retry - ), - ): - event = await stream.__anext__() - else: - assert stream_owner is not None - assert stream_requests is not None - assert stream_results is not None - assert deadline is not None - stream_requests.put_nowait(None) - remaining = max(deadline - asyncio.get_running_loop().time(), 0.0) - try: - result_kind, result_value = await _await_model_attempt( - stream_results.get(), - remaining, - timeout_error_seconds=timeout, - ) - except ModelTimeoutError as error: - await _cancel_and_drain_model_attempt_task( - stream_owner, - error, - cancel_cleanup=True, - ) - raise - except asyncio.CancelledError: - await _cancel_and_drain_model_attempt_task(stream_owner) - raise - if result_kind == "done": - remaining = max(deadline - asyncio.get_running_loop().time(), 0.0) - await _await_model_attempt( - stream_owner, - remaining, - timeout_error_seconds=timeout, - ) - return - if result_kind == "error": + disable_provider_managed_retries = _should_disable_provider_managed_retries( + retry_settings, + attempt=request_attempt, + stateful_request=stateful_request, + replay_unsafe_request=replay_unsafe_request, + ) + stream_requests: asyncio.Queue[None] | None = None + stream_results: ( + asyncio.Queue[tuple[str, TResponseStreamEvent | BaseException | None]] | None + ) = None + if timeout is None: + # Pull stream events under the retry-disable context, but yield them outside it + # so unrelated model calls made by the consumer do not inherit this setting. + with ( + provider_managed_retries_disabled(disable_provider_managed_retries), + websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), + model_retry_scope(model_spans, request_attempt), + ): + stream = get_stream() + else: + stream_requests = asyncio.Queue() + stream_results = asyncio.Queue() + with ( + provider_managed_retries_disabled(disable_provider_managed_retries), + websocket_pre_event_retries_disabled(disable_websocket_pre_event_retry), + model_retry_scope(model_spans, request_attempt), + ): + stream_owner = asyncio.create_task( + _run_stream_attempt_in_one_task( + get_stream, + stream_requests, + stream_results, + ) + ) + try: + while True: + try: + if timeout is None: + assert stream is not None + with ( + provider_managed_retries_disabled( + disable_provider_managed_retries + ), + websocket_pre_event_retries_disabled( + disable_websocket_pre_event_retry + ), + model_retry_scope(model_spans, request_attempt), + ): + event = await stream.__anext__() + else: + assert stream_owner is not None + assert stream_requests is not None + assert stream_results is not None + assert deadline is not None + stream_requests.put_nowait(None) remaining = max(deadline - asyncio.get_running_loop().time(), 0.0) - await _await_model_attempt( - stream_owner, - remaining, - timeout_error_seconds=timeout, - ) - assert isinstance(result_value, BaseException) - raise result_value - assert result_kind == "event" - assert result_value is not None - assert not isinstance(result_value, BaseException) - event = result_value - except StopAsyncIteration: - if stream_owner is None: - await _close_async_iterator_quietly(stream) - return - if _stream_event_blocks_retry(event): - emitted_retry_unsafe_event = True - if failed_retry_attempts_out is not None: - failed_retry_attempts_out[:] = [ - failed_policy_attempts + compatibility_retries_taken - ] - yield event - finally: - if stream_owner is not None and not stream_owner.done(): - await _cancel_and_drain_model_attempt_task(stream_owner) - return - except _StreamAttemptSkipped as skipped: - raise skipped.reason from None - except BaseException as error: - if isinstance(error, ModelTimeoutError): - # The timed owner has already been cancelled and drained. Do not retain its - # task, result queues, or previous event in the public timeout traceback frame. - stream = None - stream_owner = None - stream_requests = None - stream_results = None - event = None - result_value = None - if stream_owner is None: - await _close_async_iterator_quietly(stream) - if isinstance(error, asyncio.CancelledError | GeneratorExit): - raise - if not isinstance(error, Exception): - raise - if ( - not replay_unsafe_request - and _is_conversation_locked_error(error) - and _should_preserve_conversation_locked_compatibility(retry_settings) - ): - if compatibility_retries_taken < COMPATIBILITY_CONVERSATION_LOCKED_RETRIES: - compatibility_retries_taken += 1 - delay = 1.0 * (2 ** (compatibility_retries_taken - 1)) - logger.debug( - ( - "Conversation locked during streamed request, retrying in %ss " - "(attempt %s/%s)." - ), - delay, - compatibility_retries_taken, - COMPATIBILITY_CONVERSATION_LOCKED_RETRIES, + try: + result_kind, result_value = await _await_model_attempt( + stream_results.get(), + remaining, + timeout_error_seconds=timeout, + ) + except ModelTimeoutError as error: + await _cancel_and_drain_model_attempt_task( + stream_owner, + error, + cancel_cleanup=True, + ) + raise + except asyncio.CancelledError: + await _cancel_and_drain_model_attempt_task(stream_owner) + raise + if result_kind == "done": + remaining = max( + deadline - asyncio.get_running_loop().time(), 0.0 + ) + await _await_model_attempt( + stream_owner, + remaining, + timeout_error_seconds=timeout, + ) + return + if result_kind == "error": + remaining = max( + deadline - asyncio.get_running_loop().time(), 0.0 + ) + await _await_model_attempt( + stream_owner, + remaining, + timeout_error_seconds=timeout, + ) + assert isinstance(result_value, BaseException) + raise result_value + assert result_kind == "event" + assert result_value is not None + assert not isinstance(result_value, BaseException) + event = result_value + except StopAsyncIteration: + if stream_owner is None: + await _close_async_iterator_quietly(stream) + return + if _stream_event_blocks_retry(event): + emitted_retry_unsafe_event = True + if failed_retry_attempts_out is not None: + failed_retry_attempts_out[:] = [ + failed_policy_attempts + compatibility_retries_taken + ] + yield event + finally: + if stream_owner is not None and not stream_owner.done(): + await _cancel_and_drain_model_attempt_task(stream_owner) + return + except _StreamAttemptSkipped as skipped: + raise skipped.reason from None + except BaseException as error: + if isinstance(error, ModelTimeoutError): + # The timed owner has already been cancelled and drained. Do not retain its + # task, result queues, or previous event in the public timeout traceback frame. + stream = None + stream_owner = None + stream_requests = None + stream_results = None + event = None + result_value = None + if stream_owner is None: + await _close_async_iterator_quietly(stream) + if isinstance(error, asyncio.CancelledError | GeneratorExit): + raise + if not isinstance(error, Exception): + raise + if ( + not replay_unsafe_request + and _is_conversation_locked_error(error) + and _should_preserve_conversation_locked_compatibility(retry_settings) + ): + if compatibility_retries_taken < COMPATIBILITY_CONVERSATION_LOCKED_RETRIES: + compatibility_retries_taken += 1 + delay = 1.0 * (2 ** (compatibility_retries_taken - 1)) + logger.debug( + ( + "Conversation locked during streamed request, retrying in %ss " + "(attempt %s/%s)." + ), + delay, + compatibility_retries_taken, + COMPATIBILITY_CONVERSATION_LOCKED_RETRIES, + ) + await rewind() + await _sleep_for_retry(delay) + request_attempt += 1 + continue + provider_advice = get_retry_advice( + ModelRetryAdviceRequest( + error=error, + attempt=policy_attempt, + stream=True, + previous_response_id=previous_response_id, + conversation_id=conversation_id, ) - await rewind() - await _sleep_for_retry(delay) - request_attempt += 1 - continue - provider_advice = get_retry_advice( - ModelRetryAdviceRequest( + ) + decision = await _evaluate_retry( error=error, attempt=policy_attempt, + max_retries=( + max(retry_settings.max_retries or 0, 0) if retry_settings is not None else 0 + ), + retry_policy=retry_settings.policy if retry_settings is not None else None, + retry_backoff=retry_settings.backoff if retry_settings is not None else None, stream=True, + replay_unsafe_request=replay_unsafe_request, + emitted_retry_unsafe_event=emitted_retry_unsafe_event, + provider_advice=provider_advice, previous_response_id=previous_response_id, conversation_id=conversation_id, ) - ) - decision = await _evaluate_retry( - error=error, - attempt=policy_attempt, - max_retries=( - max(retry_settings.max_retries or 0, 0) if retry_settings is not None else 0 - ), - retry_policy=retry_settings.policy if retry_settings is not None else None, - retry_backoff=retry_settings.backoff if retry_settings is not None else None, - stream=True, - replay_unsafe_request=replay_unsafe_request, - emitted_retry_unsafe_event=emitted_retry_unsafe_event, - provider_advice=provider_advice, - previous_response_id=previous_response_id, - conversation_id=conversation_id, - ) - if not decision.retry: - raise - - logger.debug( - "Retrying failed streamed model request in %ss (attempt %s/%s).", - decision.delay, - policy_attempt, - retry_settings.max_retries - if retry_settings is not None and retry_settings.max_retries is not None - else 0, - ) - await rewind() - await _sleep_for_retry(decision.delay or 0.0) - request_attempt += 1 - policy_attempt += 1 - failed_policy_attempts += 1 + if not decision.retry: + raise + + logger.debug( + "Retrying failed streamed model request in %ss (attempt %s/%s).", + decision.delay, + policy_attempt, + retry_settings.max_retries + if retry_settings is not None and retry_settings.max_retries is not None + else 0, + ) + await rewind() + await _sleep_for_retry(decision.delay or 0.0) + request_attempt += 1 + policy_attempt += 1 + failed_policy_attempts += 1 + finally: + model_spans.finish() diff --git a/src/agents/run_internal/tool_actions.py b/src/agents/run_internal/tool_actions.py index 18ae37b772..ec6cf6b2ae 100644 --- a/src/agents/run_internal/tool_actions.py +++ b/src/agents/run_internal/tool_actions.py @@ -55,13 +55,13 @@ normalize_max_output_length, normalize_shell_output, normalize_shell_output_entries, + pending_tool_span, render_shell_outputs, resolve_approval_rejection_message, resolve_approval_status, run_native_tool_post_invoke, serialize_shell_output, truncate_shell_outputs, - with_tool_function_span, ) if TYPE_CHECKING: @@ -121,8 +121,8 @@ async def execute( """Run a computer action, capturing a screenshot and notifying hooks.""" trace_tool_name = get_tool_trace_name_for_tool(action.computer_tool) or cls.TRACE_TOOL_NAME - async def _run_action(span: Any | None) -> RunItem: - if span is not None and config.trace_include_sensitive_data: + with pending_tool_span(config, trace_tool_name, action.tool_call.call_id) as span: + if config.trace_include_sensitive_data: span.span_data.input = _serialize_trace_payload( cls._get_trace_input_payload(action.tool_call) ) @@ -131,6 +131,7 @@ async def _run_action(span: Any | None) -> RunItem: tool=action.computer_tool, run_context=context_wrapper ) agent_hooks = agent.hooks + span.start(mark_as_current=True) context_wrapper._mark_tool_invocation_executed( action.tool_call, tool_name=action.computer_tool.name, @@ -152,16 +153,15 @@ async def _run_action(span: Any | None) -> RunItem: trace_include_sensitive_data=config.trace_include_sensitive_data, error_message=error_text, ) - if span is not None: - span.set_error( - SpanError( - message="Error running tool", - data={ - "tool_name": trace_tool_name, - "error": trace_error, - }, - ) + span.set_error( + SpanError( + message="Error running tool", + data={ + "tool_name": trace_tool_name, + "error": trace_error, + }, ) + ) log_tool_action_error("Failed to execute computer action", exc) raise @@ -211,17 +211,11 @@ async def finalize_output() -> None: await run_native_tool_post_invoke(finalize_output(), sibling_category_failure) - if span is not None and config.trace_include_sensitive_data: + if config.trace_include_sensitive_data: span.span_data.output = image_url return output_item - return await with_tool_function_span( - config=config, - tool_name=trace_tool_name, - fn=_run_action, - ) - @classmethod async def _execute_action_and_capture( cls, computer: Any, tool_call: ResponseComputerToolCall @@ -432,8 +426,14 @@ async def execute( ctx_wrapper=context_wrapper, data=call.tool_call, ) - output = call.local_shell_tool.executor(request) - result = await output if inspect.isawaitable(output) else output + with pending_tool_span(config, call.local_shell_tool.name, call.tool_call.call_id) as span: + span.start(mark_as_current=True) + if span.span_data.operation.detail == "content": + span.span_data.input = _serialize_trace_payload(call.tool_call.action) + output = call.local_shell_tool.executor(request) + result = await output if inspect.isawaitable(output) else output + if span.span_data.operation.detail == "content": + span.span_data.output = result raw_payload: dict[str, Any] = { "type": "local_shell_call_output", @@ -489,8 +489,8 @@ async def execute( tool_name=shell_tool.name, ) - async def _run_call(span: Any | None) -> RunItem: - if span is not None and config.trace_include_sensitive_data: + with pending_tool_span(config, shell_tool.name, shell_call.call_id) as span: + if config.trace_include_sensitive_data: span.span_data.input = _serialize_trace_payload( dataclasses.asdict(shell_call.action) ) @@ -526,6 +526,7 @@ async def _run_call(span: Any | None) -> RunItem: ) if approval_status is None: + span.span_data.operation.outcome = "paused" return approval_item if approval_status is False: @@ -549,6 +550,7 @@ async def _run_call(span: Any | None) -> RunItem: call.tool_call, tool_name=shell_tool.name, ) + span.start(mark_as_current=True) await gather_with_cancel( hooks.on_tool_start(context_wrapper, agent, shell_tool), ( @@ -608,16 +610,15 @@ async def _run_call(span: Any | None) -> RunItem: trace_include_sensitive_data=config.trace_include_sensitive_data, error_message=output_text, ) - if span is not None: - span.set_error( - SpanError( - message="Error running tool", - data={ - "tool_name": shell_tool.name, - "error": trace_error, - }, - ) + span.set_error( + SpanError( + message="Error running tool", + data={ + "tool_name": shell_tool.name, + "error": trace_error, + }, ) + ) if requested_max_output_length is not None: max_output_length = requested_max_output_length output_text = output_text[:max_output_length] @@ -675,17 +676,12 @@ async def finalize_output() -> None: await run_native_tool_post_invoke(finalize_output(), sibling_category_failure) - if span is not None and config.trace_include_sensitive_data: + span.span_data.operation.outcome = status + if config.trace_include_sensitive_data: span.span_data.output = output_text return output_item - return await with_tool_function_span( - config=config, - tool_name=shell_tool.name, - fn=_run_call, - ) - class CustomToolAction: """Execute Responses custom tool calls and return custom_tool_call_output items.""" @@ -725,8 +721,8 @@ async def execute( tool_name=custom_tool.name, ) - async def _run_call(span: Any | None) -> RunItem: - if span is not None and config.trace_include_sensitive_data: + with pending_tool_span(config, custom_tool.name, call_id) as span: + if config.trace_include_sensitive_data: span.span_data.input = tool_input approval_status = context_wrapper.get_approval_status( @@ -757,6 +753,7 @@ async def _run_call(span: Any | None) -> RunItem: ) if approval_status is None: + span.span_data.operation.outcome = "paused" return approval_item if approval_status is False: @@ -784,6 +781,7 @@ async def _run_call(span: Any | None) -> RunItem: call.tool_call, tool_name=custom_tool.name, ) + span.start(mark_as_current=True) await gather_with_cancel( hooks.on_tool_start(tool_context, agent, custom_tool), ( @@ -803,16 +801,15 @@ async def _run_call(span: Any | None) -> RunItem: trace_include_sensitive_data=config.trace_include_sensitive_data, error_message=output_text, ) - if span is not None: - span.set_error( - SpanError( - message="Error running tool", - data={ - "tool_name": custom_tool.name, - "error": trace_error, - }, - ) + span.set_error( + SpanError( + message="Error running tool", + data={ + "tool_name": custom_tool.name, + "error": trace_error, + }, ) + ) log_tool_action_error("Custom tool failed", exc) raw_item = cls._raw_tool_output_item( @@ -855,16 +852,10 @@ async def finalize_output() -> None: await run_native_tool_post_invoke(finalize_output(), sibling_category_failure) - if span is not None and config.trace_include_sensitive_data: + if config.trace_include_sensitive_data: span.span_data.output = output_text return output_item - return await with_tool_function_span( - config=config, - tool_name=custom_tool.name, - fn=_run_call, - ) - @staticmethod def _normalize_output(output: Any) -> str: return output if isinstance(output, str) else str(output) @@ -932,8 +923,8 @@ async def execute( tool_name=apply_patch_tool.name, ) - async def _run_call(span: Any | None) -> RunItem: - if span is not None and config.trace_include_sensitive_data: + with pending_tool_span(config, apply_patch_tool.name, call_id) as span: + if config.trace_include_sensitive_data: span.span_data.input = _serialize_trace_payload( [ { @@ -975,6 +966,7 @@ async def _run_call(span: Any | None) -> RunItem: ) if approval_status is None: + span.span_data.operation.outcome = "paused" return approval_item if approval_status is False: @@ -999,6 +991,7 @@ async def _run_call(span: Any | None) -> RunItem: call.tool_call, tool_name=apply_patch_tool.name, ) + span.start(mark_as_current=True) await gather_with_cancel( hooks.on_tool_start(context_wrapper, agent, apply_patch_tool), ( @@ -1043,16 +1036,15 @@ async def _run_call(span: Any | None) -> RunItem: trace_include_sensitive_data=config.trace_include_sensitive_data, error_message=output_text, ) - if span is not None: - span.set_error( - SpanError( - message="Error running tool", - data={ - "tool_name": apply_patch_tool.name, - "error": trace_error, - }, - ) + span.set_error( + SpanError( + message="Error running tool", + data={ + "tool_name": apply_patch_tool.name, + "error": trace_error, + }, ) + ) log_tool_action_error("Apply patch editor failed", exc) raw_item: dict[str, Any] = { @@ -1101,17 +1093,12 @@ async def finalize_output() -> None: await run_native_tool_post_invoke(finalize_output(), sibling_category_failure) - if span is not None and config.trace_include_sensitive_data: + span.span_data.operation.outcome = status + if config.trace_include_sensitive_data: span.span_data.output = output_text return output_item - return await with_tool_function_span( - config=config, - tool_name=apply_patch_tool.name, - fn=_run_call, - ) - __all__ = [ "ComputerAction", diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index d47e98db5d..a1c05b285e 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -12,8 +12,9 @@ import inspect import json from collections import deque -from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Literal, cast from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_input_item_param import ( @@ -98,13 +99,12 @@ ToolOutputGuardrailData, ToolOutputGuardrailResult, ) -from ..tracing import Span, SpanError, function_span, get_current_trace +from ..tracing import FunctionSpanData, Span, SpanError, custom_span, function_span from ..util import _coro, _error_tracing from ..util._approvals import evaluate_function_tool_approval from ..util._asyncio_tasks import _consume_future_exception, gather_with_cancel from ..util._custom_data import maybe_extract_custom_data, merge_custom_data from ..util._tool_errors import get_trace_tool_error -from ..util._types import MaybeAwaitable from ._asyncio_progress import get_function_tool_task_progress_deadline from .agent_bindings import AgentBindings, bind_public_agent from .approvals import append_approval_error_output @@ -151,7 +151,7 @@ "normalize_shell_output_entries", "format_shell_error", "get_trace_tool_error", - "with_tool_function_span", + "pending_tool_span", "build_litellm_json_tool_call", "collect_manual_mcp_approvals", "index_approval_items_by_call_id", @@ -169,7 +169,6 @@ "execute_approved_tools", ] -TToolSpanResult = TypeVar("TToolSpanResult") _FUNCTION_TOOL_CANCELLED_DRAIN_SECONDS = 0.25 _FUNCTION_TOOL_CANCELLED_IMMEDIATE_STEP_LIMIT = 64 _FUNCTION_TOOL_POST_INVOKE_WAIT_SECONDS = 0.1 @@ -1123,26 +1122,48 @@ def log_tool_action_error( ) -async def with_tool_function_span( - *, - config: RunConfig, - tool_name: str, - fn: Callable[[Span[Any] | None], MaybeAwaitable[TToolSpanResult]], -) -> TToolSpanResult: - """Execute a tool callback in a function span when tracing is active.""" - if config.tracing_disabled or get_current_trace() is None: - result = fn(None) - if inspect.isawaitable(result): - return await result - direct_result: object = result - return cast(TToolSpanResult, direct_result) - - with function_span(tool_name) as span: - result = fn(span) - if inspect.isawaitable(result): - return await result - span_result: object = result - return cast(TToolSpanResult, span_result) +@contextmanager +def pending_tool_span( + config: RunConfig, tool_name: str, call_id: str | None +) -> Iterator[Span[FunctionSpanData]]: + """Prepare a span that the executor starts after approval and input guardrails.""" + span = function_span(tool_name, disabled=config.tracing_disabled) + operation = span.span_data.operation + operation.tool_call_id = call_id + try: + yield span + except asyncio.CancelledError: + operation.outcome = "interrupted" + raise + except Exception: + operation.outcome = "failed" + if span.error is None: + span.set_error(SpanError(message="Error running tool", data=None)) + raise + finally: + if span.started_at is not None: + if operation.outcome is None: + operation.outcome = "failed" if span.error else "completed" + span.finish(reset_current=True) + elif span.trace_id == "no-op": + span.finish(reset_current=True) + else: + # An early return is a decision about a tool, before execution begins. + operation.outcome = operation.outcome or "rejected" + if operation.detail is None: + # Existing processors read approval inputs and results from function spans. + span.start() + span.finish() + else: + with custom_span( + "tool_approval", + data={ + "tool_name": tool_name, + "tool_call_id": call_id, + "outcome": operation.outcome, + }, + ): + pass def build_litellm_json_tool_call(output: ResponseFunctionToolCall) -> FunctionTool: @@ -1809,7 +1830,7 @@ async def _run_single_tool( or get_function_tool_trace_name(func_tool) or func_tool.name ) - with function_span(trace_tool_name) as span_fn: + with pending_tool_span(self.config, trace_tool_name, tool_call.call_id) as span_fn: tool_context_namespace = get_tool_call_namespace(raw_tool_call) if tool_context_namespace is None: tool_context_namespace = get_tool_call_namespace(tool_call) @@ -1836,8 +1857,11 @@ async def _run_single_tool( ) if approval_result is not None: result = approval_result + if isinstance(result.run_item, ToolApprovalItem): + span_fn.span_data.operation.outcome = "paused" else: result = await self._execute_single_tool_body( + span=span_fn, outer_task=outer_task, task_state=task_state, func_tool=func_tool, @@ -1850,7 +1874,7 @@ async def _run_single_tool( trace_include_sensitive_data=self.config.trace_include_sensitive_data, error_message=str(e), ) - _error_tracing.attach_error_to_current_span( + span_fn.set_error( SpanError( message="Error running tool", data={"tool_name": func_tool.name, "error": trace_error}, @@ -2021,6 +2045,7 @@ async def _maybe_execute_tool_approval( async def _execute_single_tool_body( self, *, + span: Span[FunctionSpanData], outer_task: asyncio.Task[Any] | None, task_state: _FunctionToolTaskState, func_tool: FunctionTool, @@ -2048,6 +2073,8 @@ async def _execute_single_tool_body( self.schema_bypassed_tool_runs.add(id(task_state.tool_run)) return rejected_message + span.start(mark_as_current=True) + if not is_nested_continuation: await gather_with_cancel( self.hooks.on_tool_start(tool_context, self.public_agent, func_tool), ( diff --git a/src/agents/tracing/_content.py b/src/agents/tracing/_content.py new file mode 100644 index 0000000000..5b79639a01 --- /dev/null +++ b/src/agents/tracing/_content.py @@ -0,0 +1,149 @@ +"""Bound the JSON values captured for an opted-in operation.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from typing import Any + +from pydantic import BaseModel + +CONTENT_LIMIT = 32 * 1024 +_TRUNCATED = "[truncated]" +_OMIT_FIELDS = { + "api_key", + "authorization", + "access_token", + "refresh_token", + "password", + "headers", + "encrypted_content", + "reasoning", + "reasoning_content", + "reasoning_details", + "thinking", + "image_url", + "file_data", + "image_data", + "audio_data", + "b64_json", + "client_metadata", + "x_api_key", +} +_OMIT_TYPES = { + "reasoning", + "reasoning_text", + "thinking", + "redacted_thinking", + "compaction", + "image", + "input_image", + "output_image", + "image_url", + "computer_screenshot", + "image_generation_call", + "audio", + "input_audio", + "output_audio", + "file", + "input_file", + "video", + "input_video", + "encrypted_content", +} +_OMIT_FIELDS = {key.replace("_", "") for key in _OMIT_FIELDS} +_DATA_URI = re.compile(r"data:[^\s;,]+(?:;[^\s,]*)?,[^\s\"'<>]+", re.IGNORECASE) + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + + +def _size(value: Any) -> int: + return len(json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8")) + + +def _text(value: str, budget: int) -> str: + if len(value) <= budget and _size(value) <= budget: + return value + low, high = 0, min(budget, len(value)) + while low < high: + pivot = (low + high + 1) // 2 + if _size(value[:pivot] + _TRUNCATED) <= budget: + low = pivot + else: + high = pivot - 1 + return value[:low] + _TRUNCATED + + +def _project(value: Any, budget: int, depth: int) -> Any: + if depth > 32: + return _TRUNCATED + if isinstance(value, str): + if value.lstrip().startswith(("{", "[")): + try: + embedded = json.loads(value) + except ValueError: + pass + else: + safe = _project(embedded, budget, depth + 1) + if safe == embedded and _size(value) <= budget: + return value + encoded = _json(safe) + if _size(encoded) > budget: + # Escaping can double the bytes. Rebudget the value rather than + # slicing encoded JSON into an invalid tool result. + encoded = _json(_project(embedded, (budget - 2) // 2, depth + 1)) + return encoded + value = _DATA_URI.sub("[omitted: media]", value) + return _text(value, budget) + if value is None or isinstance(value, (bool, float, int)): + return value if _size(value) <= budget else _TRUNCATED + if isinstance(value, BaseModel): + value = value.model_dump(mode="python") + if isinstance(value, Mapping): + kind = value.get("type") + if isinstance(kind, str) and kind in _OMIT_TYPES: + return {"type": kind, "omitted": True, "reason": "unsupported_content"} + resource = value.get("resource") + if kind == "resource" and isinstance(resource, Mapping) and "blob" in resource: + return {"type": kind, "omitted": True, "reason": "unsupported_content"} + result: dict[str, Any] = {} + used = 2 + identity = {"type", "role", "id", "call_id", "name"} + for key, item in sorted(value.items(), key=lambda entry: entry[0] not in identity): + if not isinstance(key, str): + continue + # Reserve space for a trailing marker before accepting another field. + available = budget - used - _size(key) - 4 - 32 + if available < 84: + result["truncated"] = True + break + if key.lower().replace("-", "").replace("_", "") in _OMIT_FIELDS: + item = "[omitted]" + result[key] = _project(item, available, depth + 1) + used += _size(key) + _size(result[key]) + 4 + return result + if isinstance(value, (list, tuple)): + items: list[Any] = [] + used = 2 + for item in value: + available = budget - used - 2 - 32 + if available < 84: + items.append({"truncated": True}) + break + items.append(_project(item, available, depth + 1)) + used += _size(items[-1]) + 2 + return items + return "[omitted: binary or unsupported]" + + +def bound_content(value: Any) -> Any: + """Keep readable text and structure within a 32 KiB serialized UTF-8 budget.""" + try: + captured = _project(value, CONTENT_LIMIT - 256, 0) + return captured if _size(captured) <= CONTENT_LIMIT else {"truncated": True} + except Exception: + # A tool result may have a custom serializer. Tracing must not fail its caller. + return "[omitted: serialization failed]" diff --git a/src/agents/tracing/_model.py b/src/agents/tracing/_model.py new file mode 100644 index 0000000000..bb38784814 --- /dev/null +++ b/src/agents/tracing/_model.py @@ -0,0 +1,208 @@ +"""Shared model operation lifecycle for provider adapters and Runner retries.""" + +from __future__ import annotations + +import asyncio +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from . import util +from .config import get_operation_detail +from .create import generation_span, response_span +from .span_data import GenerationSpanData, OperationMetadata, ResponseSpanData +from .spans import Span +from .traces import Trace + +if TYPE_CHECKING: + from ..models.interface import ModelTracing + +ModelSpan = Span[GenerationSpanData] | Span[ResponseSpanData] + + +@dataclass +class ModelRetrySpans: + attempt: int = 0 + operations: int = 0 + active: ModelSpan | None = None + pending: ModelSpan | None = None + ended_at: str | None = None + carried_usage: dict[str, Any] | None = None + + def begin_attempt(self, attempt: int) -> None: + if attempt != self.attempt: + self.attempt = attempt + self.operations = 0 + self.active = None + + def finish(self) -> None: + if self.pending is not None: + span, self.pending = self.pending, None + if isinstance(sys.exc_info()[1], asyncio.CancelledError): + span.span_data.operation.outcome = "interrupted" + span.span_data.operation.estimated_end = True + self.ended_at = None + if self.ended_at is not None: + span.finish_at(self.ended_at) + else: + span.finish() + + +_retry_spans: ContextVar[ModelRetrySpans | None] = ContextVar("model_retry_spans", default=None) + + +@contextmanager +def model_retry_scope(group: ModelRetrySpans, attempt: int) -> Iterator[None]: + group.begin_attempt(attempt) + token = _retry_spans.set(group) + try: + yield + finally: + _retry_spans.reset(token) + + +def acquire_model_span( + tracing: ModelTracing, + model: str | None = None, + *, + parent: Trace | Span[Any] | None = None, + started_at: str | None = None, +) -> ModelSpan: + detail = get_operation_detail(include_data=tracing.include_data()) + group = _retry_spans.get() if detail is not None else None + if group is not None: + if group.operations == 0 and group.pending is not None: + span, group.pending = group.pending, None + group.ended_at = None + group.carried_usage = span.span_data.usage + span.span_data.usage = None + span.span_data.operation = OperationMetadata( + detail=detail, api_type="responses", model=model + ) + if isinstance(span.span_data, GenerationSpanData): + span.span_data.input = None + span.span_data.output = None + span.span_data.model_config = None + span.span_data.tools = None + span.span_data.model = model + span.clear_error() + group.operations += 1 + group.active = span + return span + group.finish() + group.operations += 1 + span = ( + generation_span(model=model, parent=parent, disabled=tracing.is_disabled()) + if detail is not None + else response_span(parent=parent, disabled=tracing.is_disabled()) + ) + if group is not None: + group.active = span + span.span_data.operation.api_type = "responses" + span.span_data.operation.detail = detail + span.span_data.operation.model = model + if started_at is not None: + span.start_at(started_at) + else: + span.start() + return span + + +def finish_model_span(span: ModelSpan, *, ended_at: str | None = None) -> None: + operation = span.span_data.operation + group = _retry_spans.get() if operation.detail is not None else None + if span.ended_at is not None or (group is not None and group.pending is span): + return + if operation.outcome is None: + operation.outcome = "failed" if span.error is not None else "completed" + if group is not None and group.active is not span: + group = None + if group is not None and group.carried_usage is not None: + span.span_data.usage = _add_usage(group.carried_usage, span.span_data.usage or {}) + group.carried_usage = None + if group is not None and group.operations == 1 and operation.outcome == "failed": + group.finish() + group.pending = span + group.ended_at = ended_at or util.time_iso() + return + if ended_at is not None: + span.finish_at(ended_at) + else: + span.finish() + + +@contextmanager +def model_span(tracing: ModelTracing, model: str | None = None) -> Iterator[ModelSpan]: + span = acquire_model_span(tracing, model) + with span.as_current(): + try: + yield span + except BaseException as error: + if isinstance(error, Exception) or span.error is not None: + span.span_data.operation.outcome = "failed" + elif span.span_data.operation.outcome is None: + span.span_data.operation.outcome = "interrupted" + raise + finally: + finish_model_span(span) + + +def _add_usage(previous: dict[str, Any], current: dict[str, Any]) -> dict[str, Any]: + result = dict(previous) + for key, value in current.items(): + old = result.get(key) + if isinstance(old, dict) and isinstance(value, dict): + result[key] = _add_usage(old, value) + elif isinstance(old, int) and isinstance(value, int): + result[key] = old + value + else: + result[key] = value + return result + + +def record_model_input(data: GenerationSpanData, request: Any) -> None: + """Capture the approved request fields under one shared input-direction budget.""" + from ._content import bound_content + + if data.operation.detail != "content": + return + if not isinstance(request, dict): + data.input = [{"omitted": True, "reason": "unsupported input projection"}] + return + previous_response_id = request.get("previous_response_id") + if isinstance(previous_response_id, str): + data.operation.previous_response_id = previous_response_id + approved = { + key: request[key] for key in ("input", "truncated", "omitted", "reason") if key in request + } + if isinstance(approved.get("input"), str): + approved["input"] = [{"role": "user", "content": approved["input"]}] + instructions = request.get("instructions") + if isinstance(instructions, str): + approved["instructions"] = instructions + captured = bound_content(approved) + if not isinstance(captured, dict): + data.input = [{"omitted": True, "reason": "input serialization failed"}] + return + if "input" not in captured and captured.get("omitted"): + data.input = [captured] + return + marker = {key: captured[key] for key in ("truncated", "omitted", "reason") if key in captured} + if marker: + items = captured.get("input", []) + if isinstance(items, list): + # Rebudget after moving the outer marker so exporters cannot lose it on extraction. + with_marker = {"input": [*items, marker]} + if "instructions" in captured: + with_marker["instructions"] = captured["instructions"] + captured = bound_content(with_marker) + items = captured.get("input") + if isinstance(items, str): + items = [{"role": "user", "content": items}] + data.input = items + data.model_config = ( + {"instructions": captured["instructions"]} if "instructions" in captured else None + ) diff --git a/src/agents/tracing/config.py b/src/agents/tracing/config.py index fb6ee013d1..5516c46f86 100644 --- a/src/agents/tracing/config.py +++ b/src/agents/tracing/config.py @@ -1,7 +1,17 @@ from __future__ import annotations +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Literal + from typing_extensions import TypedDict +from .scope import Scope + +OperationDetail = Literal["metadata", "content"] +_operation_detail: ContextVar[OperationDetail | None] = ContextVar("operation_detail", default=None) + class TracingConfig(TypedDict, total=False): """Configuration for tracing behavior and export.""" @@ -12,7 +22,43 @@ class TracingConfig(TypedDict, total=False): include_task_and_turn_spans: bool """Whether the runner creates task and turn spans. Defaults to True when omitted.""" + operation_detail: OperationDetail + """Select metadata or bounded content for model calls and tools in this run. + + Omit to use existing SDK payload fields. Explicit content capture requires + trace_include_sensitive_data in the run configuration. + """ + def include_task_and_turn_spans(config: TracingConfig | None) -> bool: """Resolve whether automatic runner task and turn spans are enabled.""" return config is None or config.get("include_task_and_turn_spans", True) + + +def get_operation_detail(*, include_data: bool = True) -> OperationDetail | None: + """Return the current run's effective capture setting.""" + detail = _operation_detail.get() + if detail == "content": + span = Scope.get_current_span() + trace = Scope.get_current_trace() + if ( + not include_data + or (trace is not None and trace.trace_id == "no-op") + or (span is not None and span.trace_id == "no-op") + ): + return "metadata" + return detail + + +@contextmanager +def operation_tracing_scope( + config: TracingConfig | None, include_sensitive: bool +) -> Iterator[None]: + detail = config.get("operation_detail") if config else None + if detail is not None and not include_sensitive: + detail = "metadata" + token = _operation_detail.set(detail) + try: + yield + finally: + _operation_detail.reset(token) diff --git a/src/agents/tracing/processor_interface.py b/src/agents/tracing/processor_interface.py index d0f18bde38..1f8896797d 100644 --- a/src/agents/tracing/processor_interface.py +++ b/src/agents/tracing/processor_interface.py @@ -128,6 +128,18 @@ def force_flush(self) -> None: """ pass + def on_span_scope_enter(self, span: "Span[Any]") -> object | None: + """Activate optional execution context after a span becomes current. + + The return value belongs to the matching scope exit, independently of span end. + Existing processors do not need to implement these optional methods. + """ + return None + + def on_span_scope_exit(self, span: "Span[Any]", token: object) -> None: + """Restore the execution context associated with a matching scope entry.""" + return None + class TracingExporter(abc.ABC): """Exports traces and spans. For example, could log them or send them to a backend.""" diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index fcc807b778..d07e7230f6 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -174,6 +174,28 @@ def on_span_end(self, span: Span[Any]) -> None: diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) + def on_span_scope_enter(self, span: Span[Any]) -> object: + entered: list[tuple[TracingProcessor, object]] = [] + for processor in self._processors: + callback = getattr(processor, "on_span_scope_enter", None) + if callback is None: + continue + try: + token = callback(span) + if token is not None: + entered.append((processor, token)) + except Exception: + logger.warning("Error in trace processor during scope entry") + return entered + + def on_span_scope_exit(self, span: Span[Any], token: object) -> None: + # Restore the processors that entered, even if registration changed meanwhile. + for processor, scope_token in reversed(cast(list[tuple[TracingProcessor, object]], token)): + try: + processor.on_span_scope_exit(span, scope_token) + except Exception: + logger.warning("Error in trace processor during scope exit") + def shutdown(self, timeout: float | None = None) -> None: """ Called when the application stops. diff --git a/src/agents/tracing/span_data.py b/src/agents/tracing/span_data.py index 6cf38937c8..7f33eb805a 100644 --- a/src/agents/tracing/span_data.py +++ b/src/agents/tracing/span_data.py @@ -2,8 +2,12 @@ import abc from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from ._content import bound_content +from .config import OperationDetail, get_operation_detail + if TYPE_CHECKING: from openai.types.responses import Response, ResponseInputItemParam @@ -132,13 +136,45 @@ def export(self) -> dict[str, Any]: } +@dataclass(slots=True) +class OperationMetadata: + """Capture permission and source facts for one model call or tool execution.""" + + detail: OperationDetail | None = field(default_factory=get_operation_detail) + api_type: str | None = None + model: str | None = None + response_model: str | None = None + response_id: str | None = None + previous_response_id: str | None = None + tool_call_id: str | None = None + operation_id: str | None = None + outcome: str | None = None + estimated_end: bool = False + + def export(self) -> dict[str, Any]: + """Supply detailed operation facts alongside the existing hosted span schema.""" + values = { + "detail": self.detail, + "api_type": self.api_type, + "model": self.model, + "response_model": self.response_model, + "response_id": self.response_id, + "previous_response_id": self.previous_response_id, + "tool_call_id": self.tool_call_id, + "operation_id": self.operation_id, + "outcome": self.outcome, + "estimated_end": True if self.estimated_end else None, + } + return {key: value for key, value in values.items() if value is not None} + + class FunctionSpanData(SpanData): """ Represents a Function Span in the trace. Includes input, output and MCP data (if applicable). """ - __slots__ = ("name", "input", "output", "mcp_data") + __slots__ = ("name", "_input", "_output", "mcp_data", "operation") def __init__( self, @@ -148,10 +184,34 @@ def __init__( mcp_data: dict[str, Any] | None = None, ): self.name = name + self.operation = OperationMetadata() self.input = input self.output = output self.mcp_data = mcp_data + def _capture(self, value: Any) -> Any: + if self.operation.detail == "metadata": + return None + if self.operation.detail == "content" and value is not None: + return bound_content(value) + return value + + @property + def input(self) -> str | None: + return self._input + + @input.setter + def input(self, value: str | None) -> None: + self._input: str | None = self._capture(value) + + @property + def output(self) -> Any: + return self._output + + @output.setter + def output(self, value: Any) -> None: + self._output = self._capture(value) + @property def type(self) -> str: return "function" @@ -179,6 +239,7 @@ class GenerationSpanData(SpanData): "model_config", "usage", "tools", + "operation", ) def __init__( @@ -194,6 +255,7 @@ def __init__( self.output = output self.model = model self.model_config = model_config + self.operation = OperationMetadata(model=model) self.usage = usage self.tools = tools @@ -219,7 +281,7 @@ class ResponseSpanData(SpanData): Includes response and input. """ - __slots__ = ("response", "input", "usage", "_response_id") + __slots__ = ("response", "input", "usage", "_response_id", "operation") def __init__( self, @@ -233,15 +295,19 @@ def __init__( self.input = input self.usage = usage self._response_id: str | None = None + self.operation = OperationMetadata() @property def type(self) -> str: return "response" def export(self) -> dict[str, Any]: + response_id = self.response.id if self.response is not None else self._response_id + if self.operation.operation_id is not None: + response_id = self.operation.response_id return { "type": self.type, - "response_id": (self.response.id if self.response is not None else self._response_id), + "response_id": response_id, "usage": self.usage, } diff --git a/src/agents/tracing/spans.py b/src/agents/tracing/spans.py index 8f564beedb..cbbb35d97c 100644 --- a/src/agents/tracing/spans.py +++ b/src/agents/tracing/spans.py @@ -2,6 +2,8 @@ import abc import contextvars +from collections.abc import Iterator +from contextlib import contextmanager from typing import Any, Generic, TypeVar from typing_extensions import TypedDict @@ -10,7 +12,7 @@ from . import util from .processor_interface import TracingProcessor from .scope import Scope -from .span_data import SpanData +from .span_data import OperationMetadata, SpanData TSpanData = TypeVar("TSpanData", bound=SpanData) _SPAN_METADATA_ROUTING_KEYS = ("agent_harness_id",) @@ -46,6 +48,9 @@ def _finish_on_generator_exit(span: Span[Any]) -> None: """ span.finish(reset_current=False) + if isinstance(span, SpanImpl): + span._exit_processor_scope() + token: contextvars.Token[Span[Any] | None] | None = span._prev_span_token # type: ignore[attr-defined] if token is None: return @@ -145,6 +150,53 @@ def finish(self, reset_current: bool = False) -> None: """ pass + def start_at(self, started_at: str) -> None: + """Start an external operation at its source ISO 8601 timestamp.""" + self.start() + + def finish_at(self, ended_at: str) -> None: + """Finish an external operation at its source ISO 8601 timestamp.""" + self.finish() + + def clear_error(self) -> None: + """Clear a transient error when retrying the same operation.""" + + @contextmanager + def as_current(self) -> Iterator[Span[TSpanData]]: + """Activate an already-started span without changing its lifetime. + + Use this scope when one source-timed or retried operation spans several execution + intervals. Start and finish remain the caller's responsibility. + """ + token = Scope.set_current_span(self) + processor = getattr(self, "_processor", None) + callback = getattr(processor, "on_span_scope_enter", None) + scope_token: object | None = None + if callback is not None: + try: + scope_token = callback(self) + except Exception: + logger.warning("Error in trace processor during scope entry") + generator_exit = False + try: + yield self + except GeneratorExit: + generator_exit = True + raise + finally: + callback = getattr(processor, "on_span_scope_exit", None) + if callback is not None and scope_token is not None: + try: + callback(self, scope_token) + except Exception: + logger.warning("Error in trace processor during scope exit") + try: + Scope.reset_current_span(token) + except ValueError: + if not generator_exit: + raise + logger.debug("Skipping span context reset, token belongs to another context") + @abc.abstractmethod def __enter__(self) -> Span[TSpanData]: pass @@ -299,6 +351,7 @@ class SpanImpl(Span[TSpanData]): "_span_data", "_tracing_api_key", "_trace_metadata", + "_processor_scope_token", ) def __init__( @@ -322,6 +375,7 @@ def __init__( self._span_data = span_data self._tracing_api_key = tracing_api_key self._trace_metadata = trace_metadata + self._processor_scope_token: object | None = None @property def trace_id(self) -> str: @@ -340,26 +394,56 @@ def parent_id(self) -> str | None: return self._parent_id def start(self, mark_as_current: bool = False): + self._start(util.time_iso(), mark_as_current) + + def start_at(self, started_at: str) -> None: + self._start(started_at, mark_as_current=False) + + def _start(self, started_at: str, mark_as_current: bool) -> None: if self.started_at is not None: logger.warning("Span already started") return - self._started_at = util.time_iso() + self._started_at = started_at self._processor.on_span_start(self) if mark_as_current: self._prev_span_token = Scope.set_current_span(self) + callback = getattr(self._processor, "on_span_scope_enter", None) + if callback is not None: + try: + self._processor_scope_token = callback(self) + except Exception: + logger.warning("Error in trace processor during scope entry") def finish(self, reset_current: bool = False) -> None: + self._finish(util.time_iso(), reset_current) + + def finish_at(self, ended_at: str) -> None: + self._finish(ended_at, reset_current=False) + + def _finish(self, ended_at: str, reset_current: bool) -> None: if self.ended_at is not None: logger.warning("Span already finished") return - self._ended_at = util.time_iso() + self._ended_at = ended_at self._processor.on_span_end(self) if reset_current and self._prev_span_token is not None: + self._exit_processor_scope() Scope.reset_current_span(self._prev_span_token) self._prev_span_token = None + def _exit_processor_scope(self) -> None: + token = self._processor_scope_token + self._processor_scope_token = None + if token is not None: + callback = getattr(self._processor, "on_span_scope_exit", None) + if callback is not None: + try: + callback(self, token) + except Exception: + logger.warning("Error in trace processor during scope exit") + def __enter__(self) -> Span[TSpanData]: self.start(mark_as_current=True) return self @@ -373,6 +457,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): def set_error(self, error: SpanError) -> None: self._error = error + def clear_error(self) -> None: + self._error = None + @property def error(self) -> SpanError | None: return self._error @@ -418,6 +505,9 @@ def export(self) -> dict[str, Any] | None: metadata.update( {key: value for key, value in span_data_metadata.items() if key not in metadata} ) + operation = getattr(self.span_data, "operation", None) + if isinstance(operation, OperationMetadata) and operation.detail is not None: + metadata["operation"] = operation.export() if metadata: payload["metadata"] = metadata return payload diff --git a/src/agents/util/_error_tracing.py b/src/agents/util/_error_tracing.py index 230ef3acf2..71636256ee 100644 --- a/src/agents/util/_error_tracing.py +++ b/src/agents/util/_error_tracing.py @@ -7,6 +7,7 @@ from ..exceptions import ModelTimeoutError from ..logger import log_model_action_warning, logger from ..tracing import Span, SpanError, get_current_span +from ..tracing.config import get_operation_detail REDACTED_TRACE_ERROR_MESSAGE = "Error details are redacted." _MODEL_TIMEOUT_ERROR_ATTR = "_openai_agents_model_timeout_error" @@ -43,6 +44,12 @@ def record_current_task_model_timeout_on_span( return True +def include_error_text(include_sensitive: bool) -> bool: + # Detailed content covers submitted messages and results. An exception can + # contain transport headers reflected by the endpoint. + return include_sensitive and get_operation_detail() is None + + def get_trace_error( *, trace_include_sensitive_data: bool, @@ -50,7 +57,7 @@ def get_trace_error( redacted_message: str = REDACTED_TRACE_ERROR_MESSAGE, ) -> str: """Return a trace-safe error string based on the sensitive-data setting.""" - return error_message if trace_include_sensitive_data else redacted_message + return error_message if include_error_text(trace_include_sensitive_data) else redacted_message def attach_error_to_span(span: Span[Any], error: SpanError) -> None: @@ -74,7 +81,7 @@ def _model_error_text(error: Exception, *, trace_include_sensitive_data: bool) - provider exception with a side-effecting `__str__` is not invoked just to have its output thrown away by redaction. """ - if not trace_include_sensitive_data: + if not include_error_text(trace_include_sensitive_data): return REDACTED_TRACE_ERROR_MESSAGE try: return str(error) diff --git a/tests/models/test_response_operation_tracing.py b/tests/models/test_response_operation_tracing.py new file mode 100644 index 0000000000..ccaeec3440 --- /dev/null +++ b/tests/models/test_response_operation_tracing.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import asyncio +import json +from typing import Any, Literal + +import httpx2 +import pytest +from openai import AsyncOpenAI + +from agents import Agent, ModelSettings, RunConfig, Runner +from agents.models.openai_responses import OpenAIResponsesModel +from agents.retry import ModelRetrySettings +from agents.tracing.span_data import GenerationSpanData, ResponseSpanData +from tests.model_test_helpers import get_response_obj +from tests.test_responses import get_text_message +from tests.testing_processor import fetch_ordered_spans + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("detail", "exhausted", "retry_owner"), + [ + (None, False, "runner"), + ("metadata", False, "runner"), + ("content", False, "runner"), + ("metadata", True, "runner"), + (None, False, "transport"), + ], +) +async def test_runner_retries_preserve_default_attempts_and_coalesce_detailed_operations( + stream: bool, + detail: Literal["metadata", "content"] | None, + exhausted: bool, + retry_owner: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests: list[dict[str, Any]] = [] + + # A wire double preserves provider errors, stream parsing, and actual Runner retry ownership. + async def handler(request: httpx2.Request) -> httpx2.Response: + requests.append(json.loads(request.content)) + if len(requests) == 1 or exhausted: + return httpx2.Response( + 503, json={"error": {"message": "retry", "type": "server_error"}} + ) + payload = get_response_obj([get_text_message("done")]).model_dump(mode="json") + if stream: + event = {"type": "response.completed", "response": payload, "sequence_number": 0} + return httpx2.Response( + 200, + content=f"event: response.completed\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + return httpx2.Response(200, json=payload) + + async with AsyncOpenAI( + api_key="test", + base_url="https://provider.example/v1", + max_retries=1 if retry_owner == "transport" else 0, + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)), + ) as client: + monkeypatch.setattr(client, "_calculate_retry_timeout", lambda *args, **kwargs: 0) + agent = Agent( + name="retry", + model=OpenAIResponsesModel(model="fake", openai_client=client), + model_settings=ModelSettings( + retry=ModelRetrySettings( + max_retries=1, + backoff={"initial_delay": 0}, + policy=lambda _: True, + ) + if retry_owner == "runner" + else None + ), + ) + config = RunConfig(tracing={} if detail is None else {"operation_detail": detail}) + error: Exception | None = None + result = None + try: + if stream: + result = Runner.run_streamed(agent, "hello", run_config=config) + async for _ in result.stream_events(): + pass + else: + result = await Runner.run(agent, "hello", run_config=config) + except Exception as exc: + error = exc + + assert len(requests) == 2, repr(error) + assert (error is not None) is exhausted + if not exhausted: + assert result is not None + assert result.final_output == "done" + spans = [ + span + for span in fetch_ordered_spans() + if isinstance(span.span_data, (GenerationSpanData, ResponseSpanData)) + ] + assert len(spans) == (2 if detail is None and retry_owner == "runner" else 1) + assert all(span.started_at is not None and span.ended_at is not None for span in spans) + assert (spans[-1].error is not None) is exhausted + assert spans[-1].span_data.operation.outcome == ("failed" if exhausted else "completed") + if detail is not None: + assert isinstance(spans[0].span_data, GenerationSpanData) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["http", "websocket"]) +@pytest.mark.parametrize( + ("detail", "oversized"), [("content", False), ("content", True), ("metadata", False)] +) +async def test_content_uses_prepared_request_overrides( + transport: str, + detail: Literal["metadata", "content"], + oversized: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import AsyncMock + + from agents.models.openai_responses import OpenAIResponsesWSModel + from tests.models.test_openai_responses import DummyWSConnection + + requests: list[dict[str, Any]] = [] + response = get_response_obj([get_text_message("done")]).model_dump(mode="json") + + # These transport doubles preserve final request assembly, including extra_body overrides. + async def handler(request: httpx2.Request) -> httpx2.Response: + requests.append(json.loads(request.content)) + return httpx2.Response(200, json=response) + + frame = {"type": "response.completed", "response": response, "sequence_number": 0} + connection = DummyWSConnection([json.dumps(frame)]) + submitted = [{"role": "user", "content": "☃" * 20000 if oversized else "submitted input"}] + async with AsyncOpenAI( + api_key="test", + base_url="https://provider.example/v1", + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)), + ) as client: + model_class = OpenAIResponsesWSModel if transport == "websocket" else OpenAIResponsesModel + model = model_class(model="initial-model", openai_client=client) + if transport == "websocket": + monkeypatch.setattr( + model, "_open_websocket_connection", AsyncMock(return_value=connection) + ) + agent = Agent( + name="prepared", + instructions="initial instructions", + model=model, + model_settings=ModelSettings( + extra_headers={"Authorization": "transport-secret"}, + extra_body={ + "input": submitted, + "instructions": "submitted instructions", + "model": "submitted-model", + "previous_response_id": "previous-response", + "authorization": "body-secret", + }, + ), + ) + try: + result = await Runner.run( + agent, + "initial input", + run_config=RunConfig(tracing={"operation_detail": detail}), + ) + finally: + await model.close() + if transport == "websocket": + requests = connection.sent_messages + assert result.final_output == "done" + assert len(requests) == 1 + data = next( + span.span_data + for span in fetch_ordered_spans() + if isinstance(span.span_data, GenerationSpanData) + ) + assert requests[0]["input"] == submitted + if detail == "metadata": + assert data.input is None + assert data.output is None + assert data.model_config is None + elif oversized: + captured = json.dumps( + {"input": data.input, **(data.model_config or {})}, ensure_ascii=False + ).encode("utf-8") + assert len(captured) <= 32 * 1024 + assert b"truncated" in captured + else: + assert data.input == submitted + assert data.model_config == {"instructions": requests[0]["instructions"]} + assert data.model == "submitted-model" + assert data.operation.previous_response_id == ( + "previous-response" if detail == "content" else None + ) + assert "transport-secret" not in repr(data.export()) + assert "body-secret" not in repr(data.export()) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_cancel_during_retry_backoff_finishes_logical_model_span( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from agents.run_internal import model_retry + + in_backoff = asyncio.Event() + release = asyncio.Event() + + async def backoff(_delay: float) -> None: + in_backoff.set() + await release.wait() + + async def handler(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(503, json={"error": {"message": "retry", "type": "server_error"}}) + + monkeypatch.setattr(model_retry, "_sleep_for_retry", backoff) + async with AsyncOpenAI( + api_key="test", + max_retries=0, + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)), + ) as client: + agent = Agent( + name="cancel retry", + model=OpenAIResponsesModel(model="fake", openai_client=client), + model_settings=ModelSettings( + retry=ModelRetrySettings(max_retries=1, policy=lambda _: True) + ), + ) + task = asyncio.create_task( + Runner.run( + agent, "hello", run_config=RunConfig(tracing={"operation_detail": "metadata"}) + ) + ) + try: + await asyncio.wait_for(in_backoff.wait(), timeout=1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + release.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + spans = [ + span for span in fetch_ordered_spans() if isinstance(span.span_data, GenerationSpanData) + ] + assert len(spans) == 1 + assert spans[0].ended_at is not None + assert spans[0].span_data.operation.outcome == "interrupted" + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal", ["failed", "incomplete"]) +@pytest.mark.parametrize( + ("detail", "sensitive", "official"), + [ + (None, True, False), + ("content", True, False), + ("content", False, False), + ("metadata", False, True), + ], +) +async def test_failed_terminal_records_outcome_before_exposure_and_preserves_source_end( + terminal: str, + detail: Literal["metadata", "content"] | None, + sensitive: bool, + official: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from agents import ModelTracing, trace + from agents.models import openai_responses + from agents.tracing.config import operation_tracing_scope + from agents.tracing.scope import Scope + from agents.usage import Usage + + payload = get_response_obj( + [get_text_message("partial")], + response_id="terminal-response", + usage=Usage(requests=1, input_tokens=7, output_tokens=2, total_tokens=9), + ).model_dump(mode="json") + payload["status"] = terminal + if terminal == "incomplete": + payload["incomplete_details"] = {"reason": "max_output_tokens"} + else: + payload["error"] = {"code": "server_error", "message": "synthetic failure"} + event = {"type": f"response.{terminal}", "response": payload, "sequence_number": 0} + source_ends: list[str] = [] + original_clock = openai_responses.time_iso + + def terminal_clock() -> str: + source_ends.append(original_clock()) + return source_ends[-1] + + monkeypatch.setattr(openai_responses, "time_iso", terminal_clock) + async with AsyncOpenAI( + api_key="test", + base_url="https://api.openai.com/v1" if official else "https://provider.example/v1", + http_client=httpx2.AsyncClient( + transport=httpx2.MockTransport( + lambda _: httpx2.Response( + 200, + content=f"event: response.{terminal}\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) as client: + model = OpenAIResponsesModel(model="requested-model", openai_client=client) + with ( + trace("terminal"), + operation_tracing_scope( + {} if detail is None else {"operation_detail": detail}, sensitive + ), + ): + stream = model.stream_response( + system_instructions=None, + input="caller input", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.ENABLED if sensitive else ModelTracing.ENABLED_WITHOUT_DATA, + ) + try: + chunk = await stream.__anext__() + assert chunk.type == f"response.{terminal}" + live = Scope.get_current_span() + assert live is not None + assert live.span_data.operation.outcome == "failed" + assert live.span_data.operation.response_model == payload["model"] + assert live.span_data.usage["input_tokens"] == 7 + assert live.span_data.operation.response_id == ( + "terminal-response" if sensitive or official else None + ) + if isinstance(live.span_data, ResponseSpanData): + assert live.span_data.input is None + assert live.span_data.response is None + assert live.span_data.export()["response_id"] is None + elif not sensitive: + assert live.span_data.input is None + assert live.span_data.output is None + finally: + await stream.aclose() + spans = [ + span + for span in fetch_ordered_spans() + if isinstance(span.span_data, (GenerationSpanData, ResponseSpanData)) + ] + assert len(spans) == 1 + assert spans[0].span_data.operation.outcome == "failed" + assert spans[0].error is not None + assert spans[0].ended_at == source_ends[0] + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["http", "websocket"]) +@pytest.mark.parametrize("terminal", ["failed", "incomplete"]) +@pytest.mark.parametrize( + ("detail", "sensitive", "official"), + [ + (None, True, True), + ("metadata", True, True), + ("content", True, False), + ("content", False, False), + ], +) +async def test_nonstream_terminal_retains_permitted_metadata_and_partial_output( + transport: str, + terminal: str, + detail: Literal["metadata", "content"] | None, + sensitive: bool, + official: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import AsyncMock + + from agents import ModelBehaviorError + from agents.models.openai_responses import OpenAIResponsesWSModel + from agents.usage import Usage + from tests.models.test_openai_responses import DummyWSConnection + + response = get_response_obj( + [get_text_message("visible partial answer")], + response_id="terminal-response", + usage=Usage(requests=1, input_tokens=7, output_tokens=2, total_tokens=9), + ).model_dump(mode="json") + response["status"] = terminal + if terminal == "incomplete": + response["incomplete_details"] = {"reason": "max_output_tokens"} + else: + response["error"] = {"code": "server_error", "message": "synthetic failure"} + frame = {"type": f"response.{terminal}", "response": response, "sequence_number": 0} + connection = DummyWSConnection([json.dumps(frame)]) + async with AsyncOpenAI( + api_key="test", + base_url="https://api.openai.com/v1" if official else "https://provider.example/v1", + max_retries=0, + http_client=httpx2.AsyncClient( + transport=httpx2.MockTransport(lambda _: httpx2.Response(200, json=response)) + ), + ) as client: + model_class = OpenAIResponsesWSModel if transport == "websocket" else OpenAIResponsesModel + model = model_class(model="requested-model", openai_client=client) + if transport == "websocket": + monkeypatch.setattr( + model, "_open_websocket_connection", AsyncMock(return_value=connection) + ) + try: + with pytest.raises(ModelBehaviorError, match=f"response.{terminal}"): + await Runner.run( + Agent(name="failed response", model=model), + "caller input", + run_config=RunConfig( + tracing={} if detail is None else {"operation_detail": detail}, + trace_include_sensitive_data=sensitive, + ), + ) + finally: + await model.close() + span = next( + span + for span in fetch_ordered_spans() + if isinstance(span.span_data, (ResponseSpanData, GenerationSpanData)) + ) + data = span.span_data + assert span.error is not None + assert data.operation.outcome == "failed" + assert data.operation.response_model == response["model"] + assert data.usage["input_tokens"] == 7 + assert data.usage["output_tokens"] == 2 + if isinstance(data, ResponseSpanData): + assert data.response is None + assert data.input is None + assert data.export()["response_id"] is None + else: + assert data.operation.response_id == ( + "terminal-response" if sensitive or official else None + ) + assert data.output == (response["output"] if detail == "content" and sensitive else None) diff --git a/tests/test_local_shell_tool.py b/tests/test_local_shell_tool.py index ca385ace31..586cea0cd9 100644 --- a/tests/test_local_shell_tool.py +++ b/tests/test_local_shell_tool.py @@ -32,6 +32,7 @@ from tests.model_test_helpers import get_response_obj from .test_responses import get_text_message +from .testing_processor import fetch_ordered_spans class RecordingLocalShellExecutor: @@ -186,6 +187,10 @@ async def test_runner_executes_local_shell_calls() -> None: result = await Runner.run(agent, input="please run shell") + (span,) = [s for s in fetch_ordered_spans() if s.span_data.type == "function"] + assert span.span_data.name == tool.name + assert span.span_data.operation.tool_call_id == "call_local_shell" + assert span.span_data.operation.outcome == "completed" assert len(executor.calls) == 1 request = executor.calls[0] assert isinstance(request, LocalShellCommandRequest) diff --git a/tests/tracing/test_content_capture.py b/tests/tracing/test_content_capture.py new file mode 100644 index 0000000000..ad78a407c3 --- /dev/null +++ b/tests/tracing/test_content_capture.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator + +import httpx2 +import pytest +from openai import AsyncOpenAI, BadRequestError +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agents import Agent, RunConfig, Runner +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.models.openai_responses import OpenAIResponsesModel +from agents.tracing import ( + add_trace_processor, + function_span, + generation_span, + set_trace_processors, + trace, +) +from agents.tracing._content import CONTENT_LIMIT, bound_content +from tests.testing_processor import SPAN_PROCESSOR_TESTING, SpanProcessorForTests + + +@pytest.fixture +def exporter() -> Iterator[InMemorySpanExporter]: + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([processor]) + try: + yield exporter + finally: + processor.shutdown() + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() + + +def test_public_tool_export_redacts_embedded_credentials_and_binary(exporter) -> None: + arguments = { + "request": json.dumps( + { + "apiKey": "CREDENTIAL_CANARY", + "query": "visible query", + "media": json.dumps({"uri": "data:image/png;base64,NESTED_CANARY"}), + } + ), + "file": {"type": "input_file", "file_data": "FILE_CANARY"}, + } + result = json.dumps( + [ + {"type": "image", "data": "IMAGE_CANARY", "mimeType": "image/png"}, + {"type": "resource", "resource": {"blob": "BLOB_CANARY"}}, + {"type": "resource", "resource": {"text": "visible resource"}}, + {"type": "text", "text": "visible result"}, + ] + ) + with trace("content"), function_span("lookup", input=json.dumps(arguments)) as tool: + tool.span_data.operation.detail = "content" + tool.span_data.output = result + attributes = dict(exporter.get_finished_spans()[0].attributes) + serialized = json.dumps(attributes) + assert "CANARY" not in serialized + assert "visible query" in serialized + assert "visible resource" in serialized + assert "visible result" in serialized + assert "omitted" in serialized + # Function data remains a string; the exporter preserves embedded JSON validity. + assert isinstance(tool.span_data.input, str) + assert isinstance(json.loads(attributes["gen_ai.tool.call.result"]), list) + + +def test_public_model_export_omits_hidden_reasoning_and_embedded_media(exporter) -> None: + with trace("content"), generation_span(model="test-model") as model: + model.span_data.operation.detail = "content" + model.span_data.input = [ + { + "role": "user", + "content": "visible before data:image/png;base64,BINARY_CANARY visible after", + } + ] + model.span_data.output = [ + { + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": json.dumps( + {"Authorization": "AUTH_CANARY", "answer": "visible answer"} + ), + }, + {"type": "reasoning", "text": "REASONING_CANARY"}, + ], + } + ] + serialized = json.dumps(dict(exporter.get_finished_spans()[0].attributes)) + assert "CANARY" not in serialized + assert "visible before" in serialized + assert "visible after" in serialized + assert "visible answer" in serialized + assert "omitted" in serialized + + +def test_public_tool_export_retains_large_result_prefix_with_bounded_json(exporter) -> None: + with trace("content"), function_span("read_file") as tool: + tool.span_data.operation.detail = "content" + tool.span_data.output = json.dumps({"text": "file prefix: " + "é🙂" * 40_000}) + captured = exporter.get_finished_spans()[0].attributes["gen_ai.tool.call.result"] + assert len(json.dumps(captured, ensure_ascii=False).encode("utf-8")) <= CONTENT_LIMIT + decoded = json.loads(captured) + assert decoded["text"].startswith("file prefix: ") + assert decoded["text"].endswith("[truncated]") + + +@pytest.mark.parametrize( + "value", + [ + "é🙂" * 40_000, + [{"text": "x" * 180}] * 3_000, + {"input": [{"role": "user", "content": "x" * 80_000}], "instructions": "later"}, + ], +) +def test_projection_keeps_valid_structure_and_inline_budget_marker(value) -> None: + captured = bound_content(value) + serialized = json.dumps(captured, ensure_ascii=False) + assert len(serialized.encode("utf-8")) <= CONTENT_LIMIT + assert "truncated" in serialized + assert json.loads(serialized) == captured + + +def test_embedded_tool_json_remains_parseable_after_redaction_and_truncation() -> None: + raw = json.dumps({"accessToken": "TOKEN_CANARY", "result": "prefix " + "🙂" * 40_000}) + captured = bound_content(raw) + assert isinstance(captured, str) + decoded = json.loads(captured) + assert decoded["result"].startswith("prefix ") + assert decoded["result"].endswith("[truncated]") + assert decoded["accessToken"] == "[omitted]" + assert len(json.dumps(captured, ensure_ascii=False).encode("utf-8")) <= CONTENT_LIMIT + + +def test_embedded_json_that_fits_the_budget_remains_intact() -> None: + raw = json.dumps({"text": "x" * 27_000}) + assert bound_content(raw) == raw + + +def test_oversized_content_preserves_message_role_and_part_identity() -> None: + captured = bound_content( + [{"content": [{"text": "prefix " + "x" * 80_000, "type": "input_text"}], "role": "user"}] + ) + assert captured[0]["role"] == "user" + assert captured[0]["content"][0]["type"] == "input_text" + assert captured[0]["content"][0]["text"].startswith("prefix ") + assert captured[0]["content"][0]["text"].endswith("[truncated]") + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_provider_error_cannot_export_a_reflected_transport_credential( + exporter, stream +) -> None: + native = SpanProcessorForTests() + add_trace_processor(native) + authorization_headers: list[str] = [] + + # An actual provider request can receive an error message echoing its auth header. + # The wire double preserves the SDK's exception conversion and both Runner paths. + async def handler(request: httpx2.Request) -> httpx2.Response: + authorization = request.headers["authorization"] + authorization_headers.append(authorization) + return httpx2.Response( + 400, + json={"error": {"message": json.dumps({"headers": {"Authorization": authorization}})}}, + ) + + async with AsyncOpenAI( + api_key="TRANSPORT_CREDENTIAL_CANARY", + base_url="https://provider.example/v1", + max_retries=0, + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)), + ) as client: + agent = Agent( + name="worker", model=OpenAIResponsesModel(model="fixture", openai_client=client) + ) + config = RunConfig(tracing={"operation_detail": "content"}) + with pytest.raises(BadRequestError): + if stream: + result = Runner.run_streamed(agent, "hello", run_config=config) + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "hello", run_config=config) + + assert authorization_headers == ["Bearer TRANSPORT_CREDENTIAL_CANARY"] + spans = native.get_ordered_spans() + model_spans = [span for span in spans if span.span_data.type == "generation"] + assert len(model_spans) == 1 + assert model_spans[0].error is not None + assert model_spans[0].span_data.operation.outcome == "failed" + # Check every native span, including agent/task/turn error fields and hosted exports. + assert "TRANSPORT_CREDENTIAL_CANARY" not in json.dumps([span.export() for span in spans]) + for span in exporter.get_finished_spans(): + assert "TRANSPORT_CREDENTIAL_CANARY" not in str( + (dict(span.attributes), span.events, span.status.description) + ) diff --git a/tests/tracing/test_opentelemetry.py b/tests/tracing/test_opentelemetry.py new file mode 100644 index 0000000000..d55954e19d --- /dev/null +++ b/tests/tracing/test_opentelemetry.py @@ -0,0 +1,446 @@ +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +from collections.abc import Iterator + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.tracing import ( + agent_span, + custom_span, + function_span, + generation_span, + set_trace_processors, + task_span, + trace, + turn_span, +) +from tests.testing_processor import SPAN_PROCESSOR_TESTING + + +@pytest.fixture +def otel() -> Iterator[tuple[TracerProvider, InMemorySpanExporter]]: + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([processor]) + try: + yield provider, exporter + finally: + processor.shutdown() + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() + + +def test_mapping_and_legacy_content_privacy(otel) -> None: + provider, exporter = otel + with trace("group"), task_span("assistant"), agent_span("Worker"), turn_span(1, "Worker"): + with generation_span(model="test-model", input=[{"content": "private-input"}]) as model: + model.span_data.output = [{"content": "private-output"}] + model.span_data.usage = {"input_tokens": 13, "output_tokens": 5} + with function_span("lookup", input="private-arguments") as tool: + tool.span_data.output = "private-result" + spans = exporter.get_finished_spans() + by_operation = {span.attributes["gen_ai.operation.name"]: span for span in spans} + assert set(by_operation) == { + "invoke_runner", + "invoke_agent", + "openai.agents.turn", + "chat", + "execute_tool", + } + inference = by_operation["chat"] + assert by_operation["invoke_runner"].parent is None + assert inference.name == "chat test-model" + assert inference.attributes["gen_ai.usage.input_tokens"] == 13 + assert inference.attributes["gen_ai.usage.output_tokens"] == 5 + assert inference.parent.span_id == by_operation["openai.agents.turn"].context.span_id + assert by_operation["execute_tool"].parent.span_id == inference.parent.span_id + assert "private-" not in str([span.attributes for span in spans]) + + +def test_tool_scope_and_caller_context(otel) -> None: + provider, exporter = otel + from opentelemetry import trace as otel_trace + + tracer = provider.get_tracer("application") + with tracer.start_as_current_span("caller") as caller: + with trace("group"), task_span("assistant"): + with function_span("lookup"): + with tracer.start_as_current_span("database"): + pass + assert otel_trace.get_current_span() is not caller + assert otel_trace.get_current_span() is caller + by_name = {span.name: span for span in exporter.get_finished_spans()} + assert by_name["invoke_runner assistant"].parent.span_id == caller.get_span_context().span_id + assert by_name["database"].parent.span_id == by_name["execute_tool lookup"].context.span_id + + +def test_multiple_runs_inside_one_native_trace(otel) -> None: + provider, exporter = otel + tracer = provider.get_tracer("application") + with trace("conversation"): + for name in ("first", "second"): + with tracer.start_as_current_span(name): + with task_span("assistant"): + pass + spans = exporter.get_finished_spans() + callers = {span.name: span for span in spans if span.name in ("first", "second")} + runs = [span for span in spans if span.name == "invoke_runner assistant"] + assert len(runs) == 2 + assert {span.parent.span_id for span in runs} == { + span.context.span_id for span in callers.values() + } + + +def test_error_metadata_excludes_error_payload(otel) -> None: + _, exporter = otel + with trace("group"), function_span("lookup") as tool: + tool.set_error({"message": "private-arguments", "data": {"output": "private-result"}}) + span = exporter.get_finished_spans()[0] + assert span.status.is_ok is False + assert span.attributes["error.type"] == "_OTHER" + assert "private-" not in str(span.attributes) + assert span.status.description is None + assert not span.events + + +def test_sampling_and_provider_lifecycle() -> None: + provider = TracerProvider(sampler=ALWAYS_OFF) + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([processor]) + try: + with trace("group"), task_span("assistant"), function_span("lookup"): + pass + processor.force_flush() + processor.shutdown() + assert not exporter.get_finished_spans() + assert provider.force_flush() + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() + + +@pytest.mark.asyncio +async def test_concurrent_tool_context_and_content_permissions(otel) -> None: + provider, exporter = otel + from opentelemetry import trace as otel_trace + + tracer = provider.get_tracer("application") + started = [asyncio.Event(), asyncio.Event()] + release = [asyncio.Event(), asyncio.Event()] + + async def execute(index: int) -> None: + with function_span(f"lookup-{index}", input=f"private-input-{index}") as tool: + tool.span_data.operation.detail = "content" if index == 0 else "metadata" + tool.span_data.output = f"private-result-{index}" + started[index].set() + await release[index].wait() + with tracer.start_as_current_span(f"database-{index}"): + pass + + with tracer.start_as_current_span("caller") as caller: + with trace("group"), task_span("assistant"): + tasks = [asyncio.create_task(execute(index)) for index in range(2)] + await asyncio.gather(*(event.wait() for event in started)) + release[1].set() + await tasks[1] + release[0].set() + await tasks[0] + assert otel_trace.get_current_span() is caller + spans = {span.name: span for span in exporter.get_finished_spans()} + for index in range(2): + assert ( + spans[f"database-{index}"].parent.span_id + == spans[f"execute_tool lookup-{index}"].context.span_id + ) + assert "private-input-0" in str(spans["execute_tool lookup-0"].attributes) + assert "private-result-0" in str(spans["execute_tool lookup-0"].attributes) + assert "private-" not in str(spans["execute_tool lookup-1"].attributes) + + +@pytest.mark.asyncio +async def test_cancelled_tool_closes_scope(otel) -> None: + provider, exporter = otel + from opentelemetry import trace as otel_trace + + tracer = provider.get_tracer("application") + started = asyncio.Event() + + async def execute() -> None: + with function_span("lookup"): + started.set() + await asyncio.Future() + + with tracer.start_as_current_span("caller") as caller: + with trace("group"), task_span("assistant"): + task = asyncio.create_task(execute()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert otel_trace.get_current_span() is caller + spans = exporter.get_finished_spans() + assert len([span for span in spans if span.name == "execute_tool lookup"]) == 1 + + +def test_content_schema_and_budget(otel) -> None: + _, exporter = otel + with trace("group"), generation_span(model="test-model") as model: + model.span_data.operation.detail = "content" + model.span_data.input = [ + {"role": "user", "content": [{"type": "input_text", "text": "é" * 30_000}]} + ] + model.span_data.model_config = {"instructions": "instruction-canary"} + model.span_data.output = [ + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": '{"city":"Paris"}', + } + ] + span = exporter.get_finished_spans()[0] + inputs = span.attributes["gen_ai.input.messages"] + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"][0]["type"] == "text" + assert "[truncated]" in inputs[0]["parts"][0]["content"] + directions = { + key: value + for key, value in span.attributes.items() + if key in ("gen_ai.input.messages", "gen_ai.system_instructions") + } + assert len(json.dumps(directions, ensure_ascii=False).encode("utf-8")) <= 32 * 1024 + output = span.attributes["gen_ai.output.messages"] + assert output[0]["parts"][0] == { + "type": "tool_call", + "id": "call-1", + "name": "lookup", + "arguments": {"city": "Paris"}, + } + + +def test_failing_otel_export_does_not_change_sdk_execution(otel) -> None: + from opentelemetry.sdk.trace import SpanProcessor + + class FailingProcessor(SpanProcessor): + def on_start(self, span, parent_context=None): + raise RuntimeError("synthetic telemetry failure") + + def on_end(self, span): + raise RuntimeError("synthetic telemetry failure") + + provider, _ = otel + provider.add_span_processor(FailingProcessor()) + with trace("group"), task_span("assistant"), function_span("lookup"): + result = "application-result" + assert result == "application-result" + + +def test_core_import_without_opentelemetry() -> None: + script = """ +import importlib.abc +import sys + +class BlockOTel(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if fullname == 'opentelemetry' or fullname.startswith('opentelemetry.'): + raise ModuleNotFoundError('optional OTel dependency unavailable', name=fullname) + return None + +sys.meta_path.insert(0, BlockOTel()) +import agents +from agents.tracing import TracingProcessor +assert 'opentelemetry' not in sys.modules +""" + subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True) + + +def test_external_source_timestamps_preserve_nanoseconds(otel) -> None: + _, exporter = otel + with trace("group"): + model = generation_span(model="test-model") + model.start_at("2026-11-20T06:00:00.123456789Z") + model.finish_at("2026-11-20T06:00:01.987654321Z") + span = exporter.get_finished_spans()[0] + assert span.start_time == 1_795_154_400_123_456_789 + assert span.end_time == 1_795_154_401_987_654_321 + + +def test_reused_model_execution_scopes_preserve_live_parent(otel) -> None: + from opentelemetry import trace as otel_trace + + provider, exporter = otel + tracer = provider.get_tracer("application") + with tracer.start_as_current_span("caller") as caller: + with trace("group"): + model = generation_span(model="test-model") + model.start() + for attempt in range(2): + with model.as_current(): + with tracer.start_as_current_span(f"http-{attempt}"): + pass + assert otel_trace.get_current_span() is caller + model.finish() + by_name = {span.name: span for span in exporter.get_finished_spans()} + for attempt in range(2): + assert ( + by_name[f"http-{attempt}"].parent.span_id == by_name["chat test-model"].context.span_id + ) + + +def test_tool_approval_mapping_and_partial_coverage(otel) -> None: + provider, exporter = otel + tracer = provider.get_tracer("application") + with trace("group"): + with tracer.start_as_current_span("before-approval"): + approval = function_span("lookup") + approval.span_data.operation.outcome = "paused" + approval.start() + approval.finish() + with tracer.start_as_current_span("after-approval"): + with function_span("lookup"): + pass + with custom_span("tool_approval", {"tool_name": "denied", "outcome": "rejected"}): + pass + with custom_span( + "codex.tracing.coverage", + {"coverage": "partial", "provider": "codex", "reason": "unsupported"}, + ): + pass + spans = exporter.get_finished_spans() + executions = [ + span for span in spans if span.attributes.get("gen_ai.operation.name") == "execute_tool" + ] + approvals = [ + span + for span in spans + if span.attributes.get("gen_ai.operation.name") == "openai.agents.tool_approval" + ] + assert len(executions) == 1 + assert len(approvals) == 2 + by_name = {span.name: span for span in spans} + assert executions[0].parent.span_id == by_name["after-approval"].context.span_id + assert ( + by_name["openai.agents.tool_approval lookup"].parent.span_id + == by_name["before-approval"].context.span_id + ) + assert ( + by_name["codex.tracing.coverage"].attributes["openai.agents.operation.coverage"] + == "partial" + ) + + +def test_marker_only_content_preserves_omission(otel) -> None: + _, exporter = otel + with trace("group"), generation_span(model="test-model") as model: + model.span_data.operation.detail = "content" + model.span_data.input = [{"truncated": True}] + model.span_data.output = [{"truncated": True}] + span = exporter.get_finished_spans()[0] + for attribute in ("gen_ai.input.messages", "gen_ai.output.messages"): + assert list(span.attributes[attribute][0]["parts"]) == [ + {"type": "openai.omitted", "reason": "truncated"} + ] + + +@pytest.mark.asyncio +async def test_overlapping_runs_with_same_native_trace_id(otel) -> None: + provider, exporter = otel + tracer = provider.get_tracer("application") + first_done, second_started, release_second = (asyncio.Event() for _ in range(3)) + shared_id = "trace_" + "a" * 32 + + async def first() -> None: + with tracer.start_as_current_span("caller-first"): + with trace("first", trace_id=shared_id), task_span("first"): + await second_started.wait() + first_done.set() + + async def second() -> None: + with tracer.start_as_current_span("caller-second"): + with trace("second", trace_id=shared_id), task_span("second"): + second_started.set() + await release_second.wait() + with function_span("lookup"): + with tracer.start_as_current_span("database"): + pass + + first_task, second_task = asyncio.create_task(first()), asyncio.create_task(second()) + await first_done.wait() + assert "invoke_runner second" not in {span.name for span in exporter.get_finished_spans()} + release_second.set() + await asyncio.gather(first_task, second_task) + spans = {span.name: span for span in exporter.get_finished_spans()} + assert spans["invoke_runner second"].parent.span_id == spans["caller-second"].context.span_id + assert ( + spans["execute_tool lookup"].parent.span_id == spans["invoke_runner second"].context.span_id + ) + assert spans["database"].parent.span_id == spans["execute_tool lookup"].context.span_id + + +def test_trace_end_and_shutdown_close_missing_terminals_without_provider_shutdown(otel) -> None: + provider, exporter = otel + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([processor]) + with trace("group"): + pending = function_span("orphan") + pending.start() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["openai.agents.operation.outcome"] == "interrupted" + assert spans[0].attributes["openai.agents.operation.estimated_end"] is True + # A late completion cannot export the same operation twice. + pending.finish() + active_trace = trace("missing-trace-end") + active_trace.start(mark_as_current=True) + try: + missing_terminal = function_span("missing-terminal") + missing_terminal.start() + processor.shutdown() + processor.shutdown() + assert len(exporter.get_finished_spans()) == 2 + assert ( + exporter.get_finished_spans()[1].attributes["openai.agents.operation.outcome"] + == "interrupted" + ) + with provider.get_tracer("application").start_as_current_span("provider-still-active"): + pass + assert len(exporter.get_finished_spans()) == 3 + finally: + active_trace.finish(reset_current=True) + + +def test_trace_cleanup_continues_after_otel_end_failure(otel) -> None: + from opentelemetry.sdk.trace import SpanProcessor + + class FailingOnEnd(SpanProcessor): + def on_end(self, span): + raise RuntimeError("synthetic telemetry failure") + + provider, exporter = otel + provider.add_span_processor(FailingOnEnd()) + with trace("group"): + for name in ("first-orphan", "second-orphan"): + pending = function_span(name) + pending.start() + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert {span.name for span in spans} == { + "execute_tool first-orphan", + "execute_tool second-orphan", + } + assert all( + span.attributes["openai.agents.operation.outcome"] == "interrupted" for span in spans + ) diff --git a/tests/tracing/test_tool_operation_tracing.py b/tests/tracing/test_tool_operation_tracing.py new file mode 100644 index 0000000000..b258ae1c92 --- /dev/null +++ b/tests/tracing/test_tool_operation_tracing.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agents import Agent, RunConfig, Runner, RunState +from agents.decorators import tool +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.testing import ScriptedModel +from agents.tracing import get_current_span, set_trace_processors +from agents.tracing.config import OperationDetail +from tests.test_responses import get_function_tool_call +from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize("detail", [None, "metadata", "content"]) +@pytest.mark.parametrize("streamed", [False, True]) +async def test_approved_tool_runs_under_the_resuming_caller( + detail: OperationDetail | None, streamed: bool +) -> None: + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([SPAN_PROCESSOR_TESTING, processor]) + calls = [] + + @tool(needs_approval=True) + async def lookup() -> str: + span = get_current_span() + assert span + calls.append(span.span_id) + with tracer.start_as_current_span("query"): + return "success" + + agent = Agent( + name="worker", + tools=[lookup], + model=ScriptedModel(steps=[[get_function_tool_call("lookup", "{}", call_id="lookup-1")]]), + tool_use_behavior="stop_on_first_tool", + ) + config = RunConfig(tracing={"operation_detail": detail} if detail else {}) + + async def invoke(input: str | RunState[None]): + if streamed: + result = Runner.run_streamed(agent, input, run_config=config) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, input, run_config=config) + + try: + with tracer.start_as_current_span("original") as original: + paused = await invoke("lookup") + assert not calls + state = paused.to_state() + state.approve(paused.interruptions[0]) + with tracer.start_as_current_span("resume") as resume: + result = await invoke(state) + assert result.final_output == "success" + native = [span for span in fetch_ordered_spans() if span.span_id in calls] + assert len(native) == len(calls) == 1 + assert native[0].span_data.operation.outcome == "completed" + assert native[0].span_data.operation.tool_call_id == "lookup-1" + exported = exporter.get_finished_spans() + (decision,) = [s for s in exported if s.name == "openai.agents.tool_approval lookup"] + (execution,) = [s for s in exported if s.name == "execute_tool lookup"] + (query,) = [s for s in exported if s.name == "query"] + assert decision.context and execution.context and query.parent + assert decision.context.trace_id == original.get_span_context().trace_id + assert execution.context.trace_id == resume.get_span_context().trace_id + assert query.parent.span_id == execution.context.span_id + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + processor.shutdown() + provider.shutdown() diff --git a/uv.lock b/uv.lock index 636d6c80fa..8ca8cdb761 100644 --- a/uv.lock +++ b/uv.lock @@ -2852,6 +2852,9 @@ modal = [ mongodb = [ { name = "pymongo" }, ] +opentelemetry = [ + { name = "opentelemetry-api" }, +] realtime = [ { name = "websockets", version = "16.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "websockets", version = "17.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -2904,6 +2907,7 @@ dev = [ { name = "mkdocs-static-i18n" }, { name = "mkdocstrings", extra = ["python"] }, { name = "mypy" }, + { name = "opentelemetry-sdk" }, { name = "playwright" }, { name = "protobuf" }, { name = "pygments" }, @@ -2958,6 +2962,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'modal'", specifier = "==1.4.3" }, { name = "numpy", marker = "python_full_version >= '3.10' and extra == 'voice'", specifier = ">=2.2.0,<3" }, { name = "openai", specifier = ">=3.0.0,<4" }, + { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.45.0,<2" }, { name = "protobuf", marker = "extra == 'dapr'", specifier = ">=5.29.6,!=6.30.*,!=6.31.*,!=6.32.*,!=6.33.0.*,!=6.33.1.*,!=6.33.2.*,!=6.33.3.*,!=6.33.4.*" }, { name = "protobuf", marker = "extra == 'daytona'", specifier = ">=5.29.6,!=6.30.*,!=6.31.*,!=6.32.*,!=6.33.0.*,!=6.33.1.*,!=6.33.2.*,!=6.33.3.*,!=6.33.4.*" }, { name = "protobuf", marker = "extra == 'e2b'", specifier = ">=5.29.6,!=6.30.*,!=6.31.*,!=6.32.*,!=6.33.0.*,!=6.33.1.*,!=6.33.2.*,!=6.33.3.*,!=6.33.4.*" }, @@ -2984,7 +2989,7 @@ requires-dist = [ { name = "websockets", marker = "extra == 'realtime'", specifier = ">=15.0,<18" }, { name = "websockets", marker = "extra == 'voice'", specifier = ">=15.0,<18" }, ] -provides-extras = ["voice", "viz", "litellm", "any-llm", "realtime", "sqlalchemy", "encrypt", "redis", "dapr", "mongodb", "docker", "blaxel", "daytona", "cloudflare", "e2b", "modal", "vercel", "s3", "temporal"] +provides-extras = ["opentelemetry", "voice", "viz", "litellm", "any-llm", "realtime", "sqlalchemy", "encrypt", "redis", "dapr", "mongodb", "docker", "blaxel", "daytona", "cloudflare", "e2b", "modal", "vercel", "s3", "temporal"] [package.metadata.requires-dev] dev = [ @@ -3005,6 +3010,7 @@ dev = [ { name = "mkdocs-static-i18n", specifier = ">=1.3.0" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=0.28.0" }, { name = "mypy" }, + { name = "opentelemetry-sdk", specifier = ">=1.45.0,<2" }, { name = "playwright", specifier = "==1.63.0" }, { name = "protobuf", specifier = ">=5.29.6,!=6.30.*,!=6.31.*,!=6.32.*,!=6.33.0.*,!=6.33.1.*,!=6.33.2.*,!=6.33.3.*,!=6.33.4.*" }, { name = "pygments", specifier = ">=2.20.0" }, From 738e2fbccaeb28baaa1ad86e67338ea109de3b0b Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:26:59 -0800 Subject: [PATCH 03/33] feat(codex): trace model calls and native tools through the SDK --- patches/codex/0001-operation-tracing.patch | 3964 +++++++++++++++++ patches/codex/README.md | 139 + src/agents/models/_codex_tracing.py | 269 ++ src/agents/models/codex.py | 73 +- .../test_codex_operation_retry_boundary.py | 103 + tests/models/test_codex_provider.py | 734 +++ tests/models/test_operation_tracing_parity.py | 838 ++++ 7 files changed, 6093 insertions(+), 27 deletions(-) create mode 100644 patches/codex/0001-operation-tracing.patch create mode 100644 patches/codex/README.md create mode 100644 src/agents/models/_codex_tracing.py create mode 100644 tests/models/test_codex_operation_retry_boundary.py create mode 100644 tests/models/test_operation_tracing_parity.py diff --git a/patches/codex/0001-operation-tracing.patch b/patches/codex/0001-operation-tracing.patch new file mode 100644 index 0000000000..2ce3ab5093 --- /dev/null +++ b/patches/codex/0001-operation-tracing.patch @@ -0,0 +1,3964 @@ +diff --git a/codex-rs/analytics/src/client_tests.rs b/codex-rs/analytics/src/client_tests.rs +index d7d8ff6..4d04a6a 100644 +--- a/codex-rs/analytics/src/client_tests.rs ++++ b/codex-rs/analytics/src/client_tests.rs +@@ -746,6 +746,7 @@ fn sample_thread_fork_response() -> ClientResponsePayload { + + fn sample_turn_start_response() -> ClientResponsePayload { + ClientResponsePayload::TurnStart(TurnStartResponse { ++ operation_tracing: None, + turn: Turn { + id: "turn-1".to_string(), + root_turn_id: None, +diff --git a/codex-rs/analytics/src/tests/support.rs b/codex-rs/analytics/src/tests/support.rs +index c47593d..060571c 100644 +--- a/codex-rs/analytics/src/tests/support.rs ++++ b/codex-rs/analytics/src/tests/support.rs +@@ -263,6 +263,7 @@ pub(super) fn sample_turn_start_request(thread_id: &str, request_id: i64) -> Cli + + pub(super) fn sample_turn_start_response(turn_id: &str) -> ClientResponsePayload { + ClientResponsePayload::TurnStart(codex_app_server_protocol::TurnStartResponse { ++ operation_tracing: None, + turn: Turn { + id: turn_id.to_string(), + root_turn_id: None, +diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs +index 2936c01..fe3b0ad 100644 +--- a/codex-rs/app-server-protocol/src/protocol/common.rs ++++ b/codex-rs/app-server-protocol/src/protocol/common.rs +@@ -2029,6 +2029,10 @@ server_notification_definitions! { + ThreadSettingsUpdated => "thread/settings/updated" (v2::ThreadSettingsUpdatedNotification), + ThreadTokenUsageUpdated => "thread/tokenUsage/updated" (v2::ThreadTokenUsageUpdatedNotification), + TurnStarted => "turn/started" (v2::TurnStartedNotification), ++ #[experimental("turn/operation/started")] ++ TurnOperationStarted => "turn/operation/started" (v2::OperationTracingNotification), ++ #[experimental("turn/operation/completed")] ++ TurnOperationCompleted => "turn/operation/completed" (v2::OperationTracingNotification), + HookStarted => "hook/started" (v2::HookStartedNotification), + TurnCompleted => "turn/completed" (v2::TurnCompletedNotification), + HookCompleted => "hook/completed" (v2::HookCompletedNotification), +diff --git a/codex-rs/app-server-protocol/src/protocol/v2/mod.rs b/codex-rs/app-server-protocol/src/protocol/v2/mod.rs +index 0ecd549..e329b5d 100644 +--- a/codex-rs/app-server-protocol/src/protocol/v2/mod.rs ++++ b/codex-rs/app-server-protocol/src/protocol/v2/mod.rs +@@ -24,6 +24,7 @@ mod memory; + mod model; + mod model_provider_error; + mod notification; ++mod operation_tracing; + mod permissions; + mod plugin; + mod plugin_search; +@@ -67,6 +68,7 @@ pub use memory::*; + pub use model::*; + pub use model_provider_error::*; + pub use notification::*; ++pub use operation_tracing::*; + pub use permissions::*; + pub use plugin::*; + pub use plugin_search::*; +diff --git a/codex-rs/app-server-protocol/src/protocol/v2/operation_tracing.rs b/codex-rs/app-server-protocol/src/protocol/v2/operation_tracing.rs +new file mode 100644 +index 0000000..f9d1b44 +--- /dev/null ++++ b/codex-rs/app-server-protocol/src/protocol/v2/operation_tracing.rs +@@ -0,0 +1,136 @@ ++use crate::JsonSchema; ++use crate::TS; ++use serde::Deserialize; ++use serde::Serialize; ++use serde_json::Value; ++ ++/// Per-turn operation observation, delivered only to its initiating connection. ++#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] ++#[serde(rename_all = "camelCase")] ++#[ts(export_to = "v2/")] ++pub enum OperationTracingMode { ++ Metadata, ++ Content, ++} ++ ++#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] ++#[serde(rename_all = "camelCase")] ++#[ts(export_to = "v2/")] ++pub enum OperationTracingKind { ++ Model, ++ Tool, ++} ++ ++#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] ++#[serde(rename_all = "camelCase")] ++#[ts(export_to = "v2/")] ++pub enum OperationTracingOutcome { ++ Success, ++ Error, ++ Interrupted, ++} ++ ++/// Content projected before delivery, with explicit capture completeness. ++#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] ++#[serde(rename_all = "camelCase")] ++#[ts(export_to = "v2/")] ++pub struct OperationTracingContent { ++ pub value: Value, ++ pub truncated: bool, ++ pub omitted: bool, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub reason: Option, ++} ++ ++/// Source operation facts. Times are UTC RFC 3339, duration uses a monotonic clock. ++/// Input and output are absent in metadata mode and must never enter logs or history. ++#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] ++#[serde(rename_all = "camelCase")] ++#[ts(export_to = "v2/")] ++pub struct OperationTracingNotification { ++ pub operation_id: String, ++ pub thread_id: String, ++ pub turn_id: String, ++ pub kind: OperationTracingKind, ++ pub started_at: String, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub completed_at: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub duration_ms: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub outcome: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub parent_operation_id: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub requested_model: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub observed_model: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub response_id: Option, ++ #[serde(default)] ++ pub response_id_trusted: bool, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub usage: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub tool_name: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub call_id: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub input: Option, ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub output: Option, ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use crate::TurnStartParams; ++ use crate::experimental_api::ExperimentalApi; ++ use serde_json::json; ++ ++ #[test] ++ fn opt_in_requires_experimental_capability() { ++ let request: TurnStartParams = serde_json::from_value(json!({ ++ "threadId": "thread", "input": [], "operationTracing": "metadata" ++ })) ++ .unwrap(); ++ assert_eq!( ++ request.operation_tracing, ++ Some(OperationTracingMode::Metadata) ++ ); ++ assert_eq!( ++ request.experimental_reason(), ++ Some("turn/start.operationTracing") ++ ); ++ let request: TurnStartParams = ++ serde_json::from_value(json!({"threadId": "thread", "input": []})).unwrap(); ++ assert!(request.operation_tracing.is_none()); ++ } ++ ++ #[test] ++ fn metadata_notification_roundtrip_preserves_source_identity() { ++ let value = json!({ ++ "operationId": "op", "threadId": "thread", "turnId": "turn", ++ "kind": "model", "startedAt": "2026-11-20T00:00:00.000Z", ++ "requestedModel": "test-model", "responseIdTrusted": false ++ }); ++ let notification: OperationTracingNotification = ++ serde_json::from_value(value.clone()).unwrap(); ++ assert!(notification.input.is_none()); ++ assert!(notification.output.is_none()); ++ assert_eq!(serde_json::to_value(notification).unwrap(), value); ++ } ++} +diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +index 4580410..4ed8a48 100644 +--- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs ++++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +@@ -5108,6 +5108,7 @@ fn turn_start_params_preserve_explicit_null_service_tier() { + ); + + let without_override = TurnStartParams { ++ operation_tracing: None, + disabled_plugin_ids: None, + thread_id: "thread_123".to_string(), + client_user_message_id: None, +diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +index bf24445..ec7f2b6 100644 +--- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs ++++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +@@ -172,6 +172,11 @@ pub struct TurnToolOutput { + #[ts(export_to = "v2/")] + pub struct TurnStartParams { + pub thread_id: String, ++ /// Observe operations for this new turn on the initiating connection only. ++ #[experimental("turn/start.operationTracing")] ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub operation_tracing: Option, + /// Replace this thread's disabled plugin IDs. + /// Omitted/null preserves the list; [] clears it. + #[ts(optional = nullable)] +@@ -305,11 +310,16 @@ pub struct TurnStartParams { + pub cyber_access_program: Option, + } + +-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] ++#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS, ExperimentalApi)] + #[serde(rename_all = "camelCase")] + #[ts(export_to = "v2/")] + pub struct TurnStartResponse { + pub turn: Turn, ++ /// Effective observation mode. Absent when unrequested or steering an existing turn. ++ #[experimental("turn/start.operationTracing")] ++ #[serde(default, skip_serializing_if = "Option::is_none")] ++ #[ts(optional)] ++ pub operation_tracing: Option, + } + + #[derive( +diff --git a/codex-rs/app-server/src/message_processor_tracing_tests.rs b/codex-rs/app-server/src/message_processor_tracing_tests.rs +index aeff494..286fee6 100644 +--- a/codex-rs/app-server/src/message_processor_tracing_tests.rs ++++ b/codex-rs/app-server/src/message_processor_tracing_tests.rs +@@ -678,6 +678,7 @@ async fn turn_start_jsonrpc_span_parents_core_turn_spans() -> Result<()> { + ClientRequest::TurnStart { + request_id: RequestId::Integer(3), + params: TurnStartParams { ++ operation_tracing: None, + disabled_plugin_ids: None, + environments: None, + thread_id, +diff --git a/codex-rs/app-server/src/notification_media.rs b/codex-rs/app-server/src/notification_media.rs +index 99e4d1e..2ce64ea 100644 +--- a/codex-rs/app-server/src/notification_media.rs ++++ b/codex-rs/app-server/src/notification_media.rs +@@ -138,6 +138,8 @@ pub(crate) fn without_notification_media(notification: ServerNotification) -> Se + | ServerNotification::WindowsWorldWritableWarning(_) + | ServerNotification::WindowsSandboxSetupCompleted(_) + | ServerNotification::AccountLoginCompleted(_) ++ | ServerNotification::TurnOperationStarted(_) ++ | ServerNotification::TurnOperationCompleted(_) + | ServerNotification::GatewayOAuthChanged(_) => notification, + } + } +diff --git a/codex-rs/app-server/src/operation_observation.rs b/codex-rs/app-server/src/operation_observation.rs +new file mode 100644 +index 0000000..7e36f8a +--- /dev/null ++++ b/codex-rs/app-server/src/operation_observation.rs +@@ -0,0 +1,361 @@ ++//! Private operation delivery. Payloads bypass telemetry, history, and broadcasts. ++ ++use super::ConnectionId; ++use super::OutgoingEnvelope; ++use super::OutgoingMessageSender; ++use super::timestamped_server_notification; ++use codex_app_server_protocol::OperationTracingContent; ++use codex_app_server_protocol::OperationTracingKind; ++use codex_app_server_protocol::OperationTracingNotification; ++use codex_app_server_protocol::OperationTracingOutcome; ++use codex_app_server_protocol::ServerNotification; ++use codex_core::operation_observation::OperationContent; ++use codex_core::operation_observation::OperationEvent; ++use codex_core::operation_observation::OperationKind; ++use codex_core::operation_observation::OperationMetadata; ++use codex_core::operation_observation::OperationObserver; ++use codex_core::operation_observation::OperationOutcome; ++use codex_core::operation_observation::OperationPhase; ++use std::future::Future; ++use std::pin::Pin; ++use std::sync::Arc; ++use std::sync::Mutex; ++use tokio::sync::mpsc; ++use tokio::sync::watch; ++ ++struct PrivateOperationObserver { ++ thread_id: String, ++ turn_id: Mutex>, ++ sender: Mutex>>, ++ drained: watch::Receiver, ++} ++ ++impl PrivateOperationObserver { ++ fn owns_turn(&self, thread_id: &str, turn_id: &str) -> bool { ++ if thread_id != self.thread_id { ++ return false; ++ } ++ let mut bound = self ++ .turn_id ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if bound.as_ref().is_some_and(|id| id != turn_id) { ++ return false; ++ } ++ bound.get_or_insert_with(|| turn_id.to_owned()); ++ true ++ } ++} ++ ++impl OperationObserver for PrivateOperationObserver { ++ fn emit(&self, event: OperationEvent) { ++ if !self.owns_turn(&event.thread_id, &event.turn_id) { ++ return; ++ } ++ let phase = event.phase; ++ let payload = operation_notification(event); ++ let notification = match phase { ++ OperationPhase::Started => ServerNotification::TurnOperationStarted(payload), ++ OperationPhase::Completed => ServerNotification::TurnOperationCompleted(payload), ++ }; ++ let sender = self ++ .sender ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if let Some(sender) = sender.as_ref() { ++ let _ = sender.send(notification); ++ } ++ } ++ ++ fn turn_finished<'a>( ++ &'a self, ++ thread_id: &'a str, ++ turn_id: &'a str, ++ ) -> Pin + Send + 'a>> { ++ let owns_turn = self.owns_turn(thread_id, turn_id); ++ Box::pin(async move { ++ if !owns_turn { ++ return; ++ } ++ // Closing admission also fences empty turns and late finalizers. ++ drop( ++ self.sender ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner) ++ .take(), ++ ); ++ // Shared completion survives cancellation of an earlier terminal waiter. ++ let mut drained = self.drained.clone(); ++ let _ = drained.wait_for(|complete| *complete).await; ++ }) ++ } ++} ++ ++impl OutgoingMessageSender { ++ pub(crate) fn operation_observer( ++ self: &Arc, ++ connection_id: ConnectionId, ++ thread_id: String, ++ ) -> Arc { ++ let (sender, mut receiver) = mpsc::unbounded_channel(); ++ let (completed, drained) = watch::channel(false); ++ let outgoing = Arc::downgrade(self); ++ tokio::spawn(async move { ++ while let Some(notification) = receiver.recv().await { ++ let Some(outgoing) = outgoing.upgrade() else { ++ break; ++ }; ++ // Explicit owner delivery, without notification logging or analytics. ++ if outgoing ++ .sender ++ .send(OutgoingEnvelope::ToConnection { ++ connection_id, ++ message: timestamped_server_notification(notification), ++ write_complete_tx: None, ++ }) ++ .await ++ .is_err() ++ { ++ break; ++ } ++ } ++ let _ = completed.send(true); ++ }); ++ Arc::new(PrivateOperationObserver { ++ thread_id, ++ turn_id: Mutex::new(None), ++ sender: Mutex::new(Some(sender)), ++ drained, ++ }) ++ } ++} ++ ++fn operation_notification(event: OperationEvent) -> OperationTracingNotification { ++ let OperationEvent { ++ phase: _, ++ operation_id, ++ thread_id, ++ turn_id, ++ kind, ++ started_at, ++ completed_at, ++ duration_ms, ++ outcome, ++ metadata, ++ input, ++ output, ++ } = event; ++ let OperationMetadata { ++ parent_operation_id, ++ requested_model, ++ observed_model, ++ response_id, ++ response_id_trusted, ++ usage, ++ tool_name, ++ call_id, ++ } = metadata; ++ OperationTracingNotification { ++ operation_id, ++ thread_id, ++ turn_id, ++ started_at, ++ completed_at, ++ duration_ms, ++ kind: match kind { ++ OperationKind::Model => OperationTracingKind::Model, ++ OperationKind::Tool => OperationTracingKind::Tool, ++ }, ++ outcome: outcome.map(|outcome| match outcome { ++ OperationOutcome::Success => OperationTracingOutcome::Success, ++ OperationOutcome::Error => OperationTracingOutcome::Error, ++ OperationOutcome::Interrupted => OperationTracingOutcome::Interrupted, ++ }), ++ parent_operation_id, ++ requested_model, ++ observed_model, ++ response_id, ++ response_id_trusted, ++ usage, ++ tool_name, ++ call_id, ++ input: input.map(operation_content), ++ output: output.map(operation_content), ++ } ++} ++ ++fn operation_content(content: OperationContent) -> OperationTracingContent { ++ let OperationContent { ++ value, ++ truncated, ++ omitted, ++ reason, ++ } = content; ++ OperationTracingContent { ++ value, ++ truncated, ++ omitted, ++ reason, ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use codex_core::operation_observation::OperationCaptureMode; ++ use codex_core::operation_observation::OperationObservation; ++ ++ #[tokio::test] ++ async fn owner_only_delivery_and_source_terminal_fence() { ++ let (sender, mut receiver) = mpsc::channel(8); ++ let outgoing = Arc::new(OutgoingMessageSender::new( ++ sender, ++ codex_analytics::AnalyticsEventsClient::disabled(), ++ )); ++ let observer = outgoing.operation_observer(ConnectionId(7), "thread".into()); ++ let observation = ++ OperationObservation::new(OperationCaptureMode::Metadata, observer.clone()); ++ let operation = observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Model, ++ OperationMetadata::default(), ++ ); ++ operation.capture_input(&serde_json::json!({"value": "metadata-canary"})); ++ drop(operation); ++ observer.turn_finished("thread", "turn").await; ++ let mut phases = Vec::new(); ++ while let Ok(envelope) = receiver.try_recv() { ++ let OutgoingEnvelope::ToConnection { ++ connection_id, ++ message, ++ .. ++ } = envelope ++ else { ++ panic!("operation observations must never broadcast"); ++ }; ++ assert_eq!(connection_id, ConnectionId(7)); ++ let value = serde_json::to_value(message).unwrap(); ++ assert!(!value.to_string().contains("metadata-canary")); ++ phases.push(value["method"].as_str().unwrap().to_owned()); ++ } ++ assert_eq!( ++ phases, ++ ["turn/operation/started", "turn/operation/completed"] ++ ); ++ drop(observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Model, ++ OperationMetadata::default(), ++ )); ++ tokio::task::yield_now().await; ++ assert!(receiver.try_recv().is_err()); ++ } ++ ++ #[tokio::test] ++ async fn private_content_preserves_capture_fields() { ++ let (sender, mut receiver) = mpsc::channel(8); ++ let outgoing = Arc::new(OutgoingMessageSender::new( ++ sender, ++ codex_analytics::AnalyticsEventsClient::disabled(), ++ )); ++ let observer = outgoing.operation_observer(ConnectionId(9), "thread".into()); ++ let observation = ++ OperationObservation::new(OperationCaptureMode::Content, observer.clone()); ++ let operation = observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Tool, ++ OperationMetadata::default(), ++ ); ++ operation.capture_input(&serde_json::json!({"value": "private-canary"})); ++ operation.finish_with_output( ++ OperationOutcome::Success, ++ &serde_json::json!({"value": "private-output"}), ++ ); ++ observer.turn_finished("thread", "turn").await; ++ for phase in ["turn/operation/started", "turn/operation/completed"] { ++ let OutgoingEnvelope::ToConnection { ++ connection_id, ++ message, ++ .. ++ } = receiver.recv().await.unwrap() ++ else { ++ panic!("expected private delivery"); ++ }; ++ assert_eq!(connection_id, ConnectionId(9)); ++ let value = serde_json::to_value(message).unwrap(); ++ assert_eq!(value["method"], phase); ++ if phase.ends_with("completed") { ++ assert_eq!(value["params"]["input"]["value"]["value"], "private-canary"); ++ assert_eq!( ++ value["params"]["output"]["value"]["value"], ++ "private-output" ++ ); ++ } ++ } ++ } ++ ++ #[tokio::test] ++ async fn empty_source_terminal_excludes_all_late_observations() { ++ let (sender, mut receiver) = mpsc::channel(8); ++ let outgoing = Arc::new(OutgoingMessageSender::new( ++ sender, ++ codex_analytics::AnalyticsEventsClient::disabled(), ++ )); ++ let observer = outgoing.operation_observer(ConnectionId(7), "thread".into()); ++ let observation = ++ OperationObservation::new(OperationCaptureMode::Metadata, observer.clone()); ++ observer.turn_finished("thread", "empty-turn").await; ++ for (thread_id, turn_id) in [ ++ ("thread", "empty-turn"), ++ ("thread", "later-turn"), ++ ("child", "empty-turn"), ++ ] { ++ drop(observation.start( ++ thread_id.into(), ++ turn_id.into(), ++ OperationKind::Model, ++ OperationMetadata::default(), ++ )); ++ } ++ tokio::task::yield_now().await; ++ assert!(receiver.try_recv().is_err()); ++ } ++ #[tokio::test] ++ async fn cancelled_terminal_waiter_does_not_release_a_later_terminal_early() { ++ let (sender, mut receiver) = mpsc::channel(1); ++ let outgoing = Arc::new(OutgoingMessageSender::new( ++ sender, ++ codex_analytics::AnalyticsEventsClient::disabled(), ++ )); ++ let observer = outgoing.operation_observer(ConnectionId(7), "thread".into()); ++ let observation = ++ OperationObservation::new(OperationCaptureMode::Metadata, observer.clone()); ++ drop(observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Model, ++ OperationMetadata::default(), ++ )); ++ let mut first_waiter = observer.turn_finished("thread", "turn"); ++ assert!(futures::poll!(&mut first_waiter).is_pending()); ++ drop(first_waiter); ++ let second_waiter = observer.turn_finished("thread", "turn"); ++ let receive_operations = async { ++ for expected in ["turn/operation/started", "turn/operation/completed"] { ++ let OutgoingEnvelope::ToConnection { message, .. } = receiver.recv().await.unwrap() ++ else { ++ panic!("expected owner delivery"); ++ }; ++ assert_eq!(serde_json::to_value(message).unwrap()["method"], expected); ++ } ++ }; ++ tokio::time::timeout(std::time::Duration::from_secs(1), async { ++ tokio::join!(second_waiter, receive_operations); ++ }) ++ .await ++ .unwrap(); ++ } ++} +diff --git a/codex-rs/app-server/src/outgoing_message.rs b/codex-rs/app-server/src/outgoing_message.rs +index 35f5027..73a4662 100644 +--- a/codex-rs/app-server/src/outgoing_message.rs ++++ b/codex-rs/app-server/src/outgoing_message.rs +@@ -1706,3 +1706,6 @@ mod tests { + ); + } + } ++ ++#[path = "operation_observation.rs"] ++mod operation_observation; +diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs +index f310c4a..a9f727f 100644 +--- a/codex-rs/app-server/src/request_processors/turn_processor.rs ++++ b/codex-rs/app-server/src/request_processors/turn_processor.rs +@@ -2,6 +2,10 @@ use super::thread_input::ensure_direct_input_allowed; + use super::*; + use codex_app_server_protocol::ImageReference as V2ImageReference; + use codex_app_server_protocol::ThreadItem; ++use codex_core::WithTurnExtensionData; ++use codex_core::operation_observation::OperationCaptureMode; ++use codex_core::operation_observation::OperationObservation; ++use codex_extension_api::ExtensionDataInit; + use codex_protocol::error::CodexErrorDetails; + use codex_protocol::models::ContentItem; + use codex_protocol::models::FunctionCallOutputBody; +@@ -636,8 +640,27 @@ impl TurnRequestProcessor { + ) + .await?; + ++ // Always replace inherited turn data, including when this request opts out. ++ // Core applies this attachment only when the request starts a new turn. ++ let requested_operation_tracing = params.operation_tracing; ++ let mut turn_extension_init = ExtensionDataInit::new(); ++ if let Some(mode) = requested_operation_tracing { ++ let capture_mode = match mode { ++ codex_app_server_protocol::OperationTracingMode::Metadata => { ++ OperationCaptureMode::Metadata ++ } ++ codex_app_server_protocol::OperationTracingMode::Content => { ++ OperationCaptureMode::Content ++ } ++ }; ++ turn_extension_init.insert(OperationObservation::new( ++ capture_mode, ++ self.outgoing ++ .operation_observer(request_id.connection_id, thread_id.to_string()), ++ )); ++ } + let submission = thread +- .start_or_steer_turn( ++ .start_or_steer_turn(WithTurnExtensionData::new_turn_only( + TurnInputRequest::new(input) + .with_thread_settings(thread_settings) + .on_start(TurnStartOptions { +@@ -653,7 +676,8 @@ impl TurnRequestProcessor { + .with_responses_metadata(params.responsesapi_client_metadata) + .with_responses_metadata_patch(params.responsesapi_client_metadata_patch) + .with_trace(self.request_trace_context(&request_id).await), +- ) ++ turn_extension_init, ++ )) + .await + .map_err(|err| { + let error = match err.details() { +@@ -713,7 +737,10 @@ impl TurnRequestProcessor { + duration_ms: None, + }; + +- Ok(TurnStartResponse { turn }) ++ Ok(TurnStartResponse { ++ turn, ++ operation_tracing: requested_operation_tracing.filter(|_| started), ++ }) + } + + async fn build_environment_override( +diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs +index bec5909..1e7ebc1 100644 +--- a/codex-rs/app-server/tests/common/test_app_server.rs ++++ b/codex-rs/app-server/tests/common/test_app_server.rs +@@ -1091,7 +1091,7 @@ impl TestAppServer { + let response = self + .read_stream_until_response_message(RequestId::Integer(request_id)) + .await?; +- let TurnStartResponse { turn } = crate::to_response(response)?; ++ let TurnStartResponse { turn, .. } = crate::to_response(response)?; + let notification = self + .read_stream_until_matching_notification( + "turn/completed for started turn", +diff --git a/codex-rs/app-server/tests/suite/logging.rs b/codex-rs/app-server/tests/suite/logging.rs +index 348d6a8..c707b46 100644 +--- a/codex-rs/app-server/tests/suite/logging.rs ++++ b/codex-rs/app-server/tests/suite/logging.rs +@@ -772,7 +772,7 @@ async fn app_server_emits_structured_tool_call_timing_event() -> Result<()> { + .await? + .thread; + +- let TurnStartResponse { turn } = app_server ++ let TurnStartResponse { turn, .. } = app_server + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs +index 9a280c9..1e2c179 100644 +--- a/codex-rs/app-server/tests/suite/v2/analytics.rs ++++ b/codex-rs/app-server/tests/suite/v2/analytics.rs +@@ -220,7 +220,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> { + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(READ_TIMEOUT, app_server.read_response(request_id)).await??; + let reviews = timeout(READ_TIMEOUT, async { + loop { +diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs +index 76fe254..1292c6f 100644 +--- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs ++++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs +@@ -98,7 +98,7 @@ async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + + timeout( +@@ -197,7 +197,7 @@ async fn turn_start_sends_fork_lineage_in_turn_metadata_for_thread_fork_v2() -> + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + + timeout( +@@ -391,7 +391,7 @@ async fn turn_start_sends_nested_subagent_lineage_after_cold_thread_resume_v2() + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + + timeout( +@@ -517,7 +517,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + let turn_id = turn.id.clone(); + +@@ -672,7 +672,7 @@ async fn turn_start_forwards_client_metadata_to_responses_websocket_request_body + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + + timeout( +diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs +index 784e6f9..49c4cbd 100644 +--- a/codex-rs/app-server/tests/suite/v2/compaction.rs ++++ b/codex-rs/app-server/tests/suite/v2/compaction.rs +@@ -698,7 +698,7 @@ async fn send_turn_and_wait( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_id)).await??; + wait_for_turn_completed(mcp, &turn.id).await?; + Ok(turn.id) +diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket_unix.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket_unix.rs +index cb01a3b..e372272 100644 +--- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket_unix.rs ++++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket_unix.rs +@@ -264,7 +264,8 @@ async fn websocket_transport_allows_turn_interrupt_during_drain() -> Result<()> + to_response(read_response_for_id(&mut ws, /*id*/ 2).await?)?; + let thread_id = thread.id; + send_turn_start_request(&mut ws, /*id*/ 3, &thread_id).await?; +- let TurnStartResponse { turn } = to_response(read_response_for_id(&mut ws, /*id*/ 3).await?)?; ++ let TurnStartResponse { turn, .. } = ++ to_response(read_response_for_id(&mut ws, /*id*/ 3).await?)?; + let turn_id = turn.id; + timeout( + DEFAULT_READ_TIMEOUT, +@@ -427,7 +428,7 @@ async fn start_ctrl_c_restart_fixture(turn_delay: Duration) -> Result Res + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; +- let TurnStartResponse { turn } = to_response::(turn_resp)?; ++ let TurnStartResponse { turn, .. } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; +@@ -654,7 +654,7 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result(turn_resp)?; ++ let TurnStartResponse { turn, .. } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; +diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +index 0084570..876eb49 100644 +--- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs ++++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +@@ -1240,7 +1240,7 @@ async fn start_turn( + ..turn_start_params + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_req)).await??; + + Ok(turn.id) +diff --git a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +index cda1f49..b59b762 100644 +--- a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs ++++ b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +@@ -438,7 +438,7 @@ async fn openai_form_capability_follows_the_turn_starting_connection() -> Result + })?), + ) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + to_response(read_response_for_id(&mut supported_client, /*id*/ 6).await?)?; + + let (request_id, params) = loop { +@@ -789,7 +789,7 @@ impl ElicitationRoundTripFixture { + mcp.read_stream_until_response_message(RequestId::Integer(turn_start_id)), + ) + .await??; +- let TurnStartResponse { turn } = to_response(turn_start_resp)?; ++ let TurnStartResponse { turn, .. } = to_response(turn_start_resp)?; + + Ok(Self { + mcp, +diff --git a/codex-rs/app-server/tests/suite/v2/misalignment_policy.rs b/codex-rs/app-server/tests/suite/v2/misalignment_policy.rs +index 7c52208..16fae33 100644 +--- a/codex-rs/app-server/tests/suite/v2/misalignment_policy.rs ++++ b/codex-rs/app-server/tests/suite/v2/misalignment_policy.rs +@@ -196,7 +196,7 @@ async fn assert_policy_violation_completes_turn_with_typed_terminal_error( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = app_server ++ let TurnStartResponse { turn, .. } = app_server + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/request_permissions.rs b/codex-rs/app-server/tests/suite/v2/request_permissions.rs +index 3a061e9..374ed33 100644 +--- a/codex-rs/app-server/tests/suite/v2/request_permissions.rs ++++ b/codex-rs/app-server/tests/suite/v2/request_permissions.rs +@@ -107,7 +107,7 @@ text(await tools.exec_command({{cmd: "{command}"}}));"#, + } + events.push(create_final_assistant_message_sse_response("done")?); + let mock = responses::mount_sse_sequence(&server, events).await; +- let TurnStartResponse { turn } = app ++ let TurnStartResponse { turn, .. } = app + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -342,7 +342,7 @@ text(await tools.apply_patch("*** Begin Patch\n*** Add File: next.txt\n+next tur + ], + }; + let mock = responses::mount_sse_sequence(&server, events).await; +- let TurnStartResponse { turn } = app ++ let TurnStartResponse { turn, .. } = app + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -555,7 +555,9 @@ await tools.request_permissions({{permissions: {{file_system: {{write: ["."]}}}} + ], + ) + .await; +- let TurnStartResponse { turn: first_turn } = app ++ let TurnStartResponse { ++ turn: first_turn, .. ++ } = app + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -625,7 +627,7 @@ await tools.request_permissions({{permissions: {{file_system: {{write: ["."]}}}} + "next turn done", + )?); + let mock = responses::mount_sse_sequence(&server, events).await; +- let TurnStartResponse { turn } = app ++ let TurnStartResponse { turn, .. } = app + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs b/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs +index 13c983a..5976891 100644 +--- a/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs ++++ b/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs +@@ -154,7 +154,7 @@ async fn selected_plugin_mcp_startup_respects_explicit_mentions( + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; +- let TurnStartResponse { turn } = to_response(response)?; ++ let TurnStartResponse { turn, .. } = to_response(response)?; + wait_for_pid_file(&fixture.pid_file).await?; + if explicitly_mentioned { + // An explicit mention must outwait the optional one-second grace. +diff --git a/codex-rs/app-server/tests/suite/v2/sleep.rs b/codex-rs/app-server/tests/suite/v2/sleep.rs +index b515dbc..46e467f 100644 +--- a/codex-rs/app-server/tests/suite/v2/sleep.rs ++++ b/codex-rs/app-server/tests/suite/v2/sleep.rs +@@ -85,7 +85,7 @@ async fn clock_tools_emit_control_tool_analytics() -> Result<()> { + .start_thread(ThreadStartParams::default()) + .await? + .thread; +- let TurnStartResponse { turn } = app_server ++ let TurnStartResponse { turn, .. } = app_server + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs +index 6c902ce..613222b 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs +@@ -478,7 +478,7 @@ async fn assert_thread_fork_preserves_persisted_approvals_reviewer( + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; +- let TurnStartResponse { turn } = to_response(turn_resp)?; ++ let TurnStartResponse { turn, .. } = to_response(turn_resp)?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), +@@ -651,7 +651,7 @@ async fn assert_thread_fork_at_named_boundary_keeps_only_terminal_prefix( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_request_id)).await??; + turn_ids.push(turn.id); + timeout( +diff --git a/codex-rs/app-server/tests/suite/v2/thread_inject_items.rs b/codex-rs/app-server/tests/suite/v2/thread_inject_items.rs +index 931c45e..ab9ec94 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_inject_items.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_inject_items.rs +@@ -314,7 +314,7 @@ async fn thread_inject_items_adds_raw_response_items_to_thread_history( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(delegated_req)).await??; + let turn_started: TurnStartedNotification = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("turn/started")).await??; +diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs +index 8badd11..3b2a97d 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs +@@ -5282,8 +5282,9 @@ async fn thread_resume_rejects_history_when_thread_is_running() -> Result<()> { + primary.read_stream_until_response_message(RequestId::Integer(running_turn_request_id)), + ) + .await??; +- let TurnStartResponse { turn: running_turn } = +- to_response::(running_turn_resp)?; ++ let TurnStartResponse { ++ turn: running_turn, .. ++ } = to_response::(running_turn_resp)?; + assert_eq!(running_turn.items_view, TurnItemsView::NotLoaded); + timeout( + DEFAULT_READ_TIMEOUT, +@@ -5401,8 +5402,9 @@ async fn thread_resume_rejects_mismatched_path_for_running_thread_id() -> Result + primary.read_stream_until_response_message(RequestId::Integer(running_turn_request_id)), + ) + .await??; +- let TurnStartResponse { turn: running_turn } = +- to_response::(running_turn_resp)?; ++ let TurnStartResponse { ++ turn: running_turn, .. ++ } = to_response::(running_turn_resp)?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), +@@ -5556,8 +5558,9 @@ async fn thread_resume_rejoins_running_paginated_thread_with_initial_page() -> R + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn: seed_turn } = +- timeout(DEFAULT_READ_TIMEOUT, primary.read_response(seed_turn_id)).await??; ++ let TurnStartResponse { ++ turn: seed_turn, .. ++ } = timeout(DEFAULT_READ_TIMEOUT, primary.read_response(seed_turn_id)).await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), +@@ -5581,8 +5584,9 @@ async fn thread_resume_rejoins_running_paginated_thread_with_initial_page() -> R + primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)), + ) + .await??; +- let TurnStartResponse { turn: running_turn } = +- to_response::(running_turn_resp)?; ++ let TurnStartResponse { ++ turn: running_turn, .. ++ } = to_response::(running_turn_resp)?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), +diff --git a/codex-rs/app-server/tests/suite/v2/thread_revert.rs b/codex-rs/app-server/tests/suite/v2/thread_revert.rs +index ffbdbde..d762735 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_revert.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_revert.rs +@@ -641,7 +641,9 @@ async fn thread_revert_interrupts_active_turn_and_keeps_thread_loaded() -> Resul + }) + .await?; + +- let TurnStartResponse { turn: active_turn } = mcp ++ let TurnStartResponse { ++ turn: active_turn, .. ++ } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/thread_settings_update.rs b/codex-rs/app-server/tests/suite/v2/thread_settings_update.rs +index ab688b1..49b8eec 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_settings_update.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_settings_update.rs +@@ -612,7 +612,7 @@ async fn turn_start_settings_override_emits_thread_settings_updated() -> Result< + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_TIMEOUT, mcp.read_response(turn_request_id)).await??; + assert!(!turn.id.is_empty()); + +@@ -653,7 +653,7 @@ async fn start_text_turn(mcp: &mut TestAppServer, thread_id: String) -> Result<( + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn } = ++ let TurnStartResponse { turn, .. } = + timeout(DEFAULT_TIMEOUT, mcp.read_response(turn_request_id)).await??; + assert!(!turn.id.is_empty()); + Ok(()) +diff --git a/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs b/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs +index 561b890..639c279 100644 +--- a/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs ++++ b/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs +@@ -260,7 +260,7 @@ async fn check_thread_shell_command_in_active_turn(timeout_ms: Option) -> R + current_shell_output_command("active turn bang")? + }; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/turn_cost_otel.rs b/codex-rs/app-server/tests/suite/v2/turn_cost_otel.rs +index f9a06f2..016e364 100644 +--- a/codex-rs/app-server/tests/suite/v2/turn_cost_otel.rs ++++ b/codex-rs/app-server/tests/suite/v2/turn_cost_otel.rs +@@ -201,7 +201,7 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{}/metrics", protocol = "json" + }) + .await? + .expect("turn/start"); +- let TurnStartResponse { turn } = serde_json::from_value(response)?; ++ let TurnStartResponse { turn, .. } = serde_json::from_value(response)?; + timeout(Duration::from_secs(/*secs*/ 30), async { + while let Some(event) = client.next_event().await { + if let InProcessServerEvent::ServerNotification(event) = event +diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +index bdc1740..e5df44b 100644 +--- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs ++++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +@@ -84,7 +84,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { + .await?; + + // Start a turn that triggers a long-running command. +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -176,7 +176,7 @@ async fn turn_interrupt_rejects_completed_turn() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -269,7 +269,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs +index 7053107..99a34f8 100644 +--- a/codex-rs/app-server/tests/suite/v2/turn_start.rs ++++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs +@@ -171,7 +171,7 @@ async fn run_local_image_turn(detail: Option) -> Result> + let image_path = codex_home.path().join("image.png"); + std::fs::write(&image_path, TINY_PNG_BYTES)?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -571,7 +571,7 @@ async fn turn_start_with_empty_input_runs_model_request() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -698,7 +698,9 @@ async fn turn_start_steers_active_turn_and_returns_active_turn_id() -> Result<() + ..Default::default() + }) + .await?; +- let TurnStartResponse { turn: active_turn } = mcp ++ let TurnStartResponse { ++ turn: active_turn, .. ++ } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -742,7 +744,9 @@ async fn turn_start_steers_active_turn_and_returns_active_turn_id() -> Result<() + (thread.id.clone(), Some(false)) + ); + +- let TurnStartResponse { turn: steered_turn } = mcp ++ let TurnStartResponse { ++ turn: steered_turn, .. ++ } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -1361,7 +1365,7 @@ async fn turn_start_emits_raw_response_completed_with_upstream_usage( + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -1540,7 +1544,7 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -1796,7 +1800,9 @@ async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<() + }) + .await?; + +- let TurnStartResponse { turn: first_turn } = mcp ++ let TurnStartResponse { ++ turn: first_turn, .. ++ } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -1974,7 +1980,7 @@ async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -2211,7 +2217,7 @@ allow_local_binding = false + active_permission_profile.context("expected active permission profile")?; + assert_eq!(active_permission_profile.id, "managed-network"); + +- let TurnStartResponse { turn } = app_server ++ let TurnStartResponse { turn, .. } = app_server + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -2547,7 +2553,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( + .await?; + + // Start a turn with only input and thread_id set (no overrides). +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -2585,7 +2591,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( + assert_eq!(completed.turn.status, TurnStatus::Completed); + + // Send a second turn that exercises the overrides path: change the model. +- let TurnStartResponse { turn: turn2 } = mcp ++ let TurnStartResponse { turn: turn2, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -3335,7 +3341,7 @@ async fn run_turn_start_exec_approval_rejection_v2( + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -3510,6 +3516,7 @@ async fn turn_start_explicit_local_environment_updates_legacy_cwd_between_turns( + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { ++ operation_tracing: None, + disabled_plugin_ids: None, + environments: None, + thread_id: thread.id.clone(), +@@ -3562,6 +3569,7 @@ async fn turn_start_explicit_local_environment_updates_legacy_cwd_between_turns( + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { ++ operation_tracing: None, + disabled_plugin_ids: None, + environments: Some(vec![TurnEnvironmentParams { + environment_id: LOCAL_ENVIRONMENT_ID.to_string(), +@@ -3872,7 +3880,7 @@ async fn run_environment_selection_case( + let ThreadStartResponse { thread, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(thread_req)).await??; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -3969,7 +3977,7 @@ async fn turn_start_file_change_approval_v2() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -4300,7 +4308,7 @@ async fn turn_start_streams_apply_patch_change_updates_v2() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -5222,7 +5230,7 @@ async fn turn_start_file_change_approval_accept_for_session_persists_v2() -> Res + .await?; + + // First turn: expect FileChangeRequestApproval, respond with AcceptForSession, and verify the file exists. +- let TurnStartResponse { turn: turn_1 } = mcp ++ let TurnStartResponse { turn: turn_1, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -5314,7 +5322,7 @@ async fn turn_start_file_change_approval_accept_for_session_persists_v2() -> Res + assert_eq!(std::fs::read_to_string(&readme_path)?, "new line\n"); + + // Second turn: apply a patch to the same file. Approval should be skipped due to AcceptForSession. +- let TurnStartResponse { turn: turn_2 } = mcp ++ let TurnStartResponse { turn: turn_2, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -5450,7 +5458,7 @@ async fn run_turn_start_file_change_approval_rejection_v2( + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -5600,7 +5608,7 @@ async fn command_execution_notifications_include_process_id( + }) + .await?; + +- let TurnStartResponse { turn: _turn } = mcp ++ let TurnStartResponse { turn: _turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs +index 38f5cac..6565241 100644 +--- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs ++++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs +@@ -190,7 +190,7 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -311,7 +311,7 @@ async fn turn_steer_returns_active_turn_id(tool_outputs: bool) -> Result<()> { + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +@@ -491,7 +491,7 @@ async fn turn_steer_rejects_context_only_input_without_merging_context() -> Resu + }) + .await?; + +- let TurnStartResponse { turn } = mcp ++ let TurnStartResponse { turn, .. } = mcp + .request(|request_id| ClientRequest::TurnStart { + request_id, + params: TurnStartParams { +diff --git a/codex-rs/codex-api/src/common.rs b/codex-rs/codex-api/src/common.rs +index 2510f35..0a4af6c 100644 +--- a/codex-rs/codex-api/src/common.rs ++++ b/codex-rs/codex-api/src/common.rs +@@ -89,6 +89,8 @@ pub enum ResponseEvent { + /// Emitted when the server includes `OpenAI-Model` on the stream response. + /// This can differ from the requested model when backend safety routing applies. + ServerModel(String), ++ /// The response body's model name, for observation only, without routing authority. ++ ModelObservation(String), + /// Emitted when the server recommends additional account verification. + ModelVerifications(Vec), + /// Emitted when the server includes moderation metadata for first-party turn presentation. +diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs +index f43e285..d330da5 100644 +--- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs ++++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs +@@ -907,6 +907,11 @@ async fn run_websocket_response_stream( + } + continue; + } ++ if let Some(model) = event.observation_model() { ++ let _ = tx_event ++ .send(Ok(ResponseEvent::ModelObservation(model))) ++ .await; ++ } + let model = event.response_model(); + // Preserve terminal usage before any accompanying notification can block. + let event = match process_responses_event(event) { +diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs +index 67f627e..1e6c9cd 100644 +--- a/codex-rs/codex-api/src/sse/responses.rs ++++ b/codex-rs/codex-api/src/sse/responses.rs +@@ -190,6 +190,16 @@ impl ResponsesStreamEvent { + &self.kind + } + ++ /// Returns the body's model name for observation, without routing authority. ++ pub fn observation_model(&self) -> Option { ++ self.response ++ .as_ref()? ++ .get("model")? ++ .as_str() ++ .filter(|model| !model.is_empty()) ++ .map(str::to_owned) ++ } ++ + /// Returns the effective model reported by the server, if present. + /// + /// Precedence: +@@ -637,6 +647,15 @@ async fn process_sse_with_treatment( + let safety_buffering = event.safety_buffering(&safety_buffering_treatment); + let metadata_events = event.metadata_events(safety_buffering); + ++ if let Some(model) = event.observation_model() { ++ if tx_event ++ .send(Ok(ResponseEvent::ModelObservation(model))) ++ .await ++ .is_err() ++ { ++ return; ++ } ++ } + if let Some(model) = event.response_model() + && last_server_model.as_deref() != Some(model.as_str()) + { +@@ -1613,7 +1632,7 @@ mod tests { + } + + #[tokio::test] +- async fn process_sse_ignores_response_model_field_in_payload() { ++ async fn process_sse_reports_body_model_without_routing_authority() { + let events = run_sse(vec![ + json!({ + "type": "response.created", +@@ -1632,10 +1651,17 @@ mod tests { + ]) + .await; + +- assert_eq!(events.len(), 2); +- assert_matches!(&events[0], ResponseEvent::Created { .. }); ++ assert_eq!(events.len(), 4); ++ assert_matches!(&events[0], ResponseEvent::ModelObservation(model) if model == CYBER_RESTRICTED_MODEL_FOR_TESTS); ++ assert_matches!(&events[1], ResponseEvent::Created { .. }); ++ assert_matches!(&events[2], ResponseEvent::ModelObservation(model) if model == CYBER_RESTRICTED_MODEL_FOR_TESTS); ++ assert!( ++ !events ++ .iter() ++ .any(|event| matches!(event, ResponseEvent::ServerModel(_))) ++ ); + assert_matches!( +- &events[1], ++ &events[3], + ResponseEvent::Completed { + response_id, + token_usage: None, +diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs +index 2ac9b93..3f93332 100644 +--- a/codex-rs/core/src/client.rs ++++ b/codex-rs/core/src/client.rs +@@ -2033,6 +2033,11 @@ impl ResponsesHttpExecutor<'_> { + request.input = input; + } + inference_trace_attempt.record_started(&request); ++ crate::operation_observation::observe_model_request( ++ &request, ++ &request, ++ &client_setup.api_provider.base_url, ++ ); + let client = ApiResponsesClient::new( + transport, + client_setup.api_provider, +@@ -2185,6 +2190,7 @@ impl ModelClientSession { + } else { + session_telemetry_for_request(session_telemetry, &request) + }; ++ let operation_endpoint = client_setup.api_provider.base_url.clone(); + match self + .websocket_connection(WebsocketConnectParams { + session_telemetry, +@@ -2341,6 +2347,13 @@ impl ModelClientSession { + inference_trace_attempt.record_started(&ws_request); + } + ++ if !warmup { ++ crate::operation_observation::observe_model_request( ++ &ws_request, ++ &request, ++ &operation_endpoint, ++ ); ++ } + let websocket_connection = + self.websocket_session.connection.as_ref().ok_or_else(|| { + self.client.state.provider.map_api_error(ApiError::Stream( +@@ -2686,6 +2699,8 @@ where + mpsc::channel::>(RESPONSE_STREAM_CHANNEL_CAPACITY); + let consumer_dropped = CancellationToken::new(); + let consumer_dropped_for_stream = consumer_dropped.clone(); ++ let operation_observation = crate::operation_observation::current_model_operation() ++ .map(crate::operation_observation::ModelOperation::for_stream); + + tokio::spawn(async move { + let mut logged_error = false; +@@ -2702,6 +2717,7 @@ where + loop { + let event = tokio::select! { + _ = consumer_dropped.cancelled() => { ++ if let Some(operation) = &operation_observation { operation.source_ended(); } + inference_trace_attempt.record_cancelled( + STREAM_DROPPED_REASON, + upstream_request_id, +@@ -2712,8 +2728,21 @@ where + event = api_stream.next() => event, + }; + let Some(event) = event else { ++ if let Some(operation) = &operation_observation { ++ operation.source_ended(); ++ } + break; + }; ++ if let (Some(operation), Err(error)) = (&operation_observation, &event) { ++ operation.source_error(error); ++ } ++ if let (Some(operation), Ok(event)) = (&operation_observation, &event) { ++ operation.event(event); ++ } ++ // Body model names are observation facts, never session-routing updates. ++ if matches!(&event, Ok(ResponseEvent::ModelObservation(_))) { ++ continue; ++ } + match event { + Ok(ResponseEvent::OutputItemDone(mut item)) => { + // Assign IDs after interceptors, which may replace provider items. +diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs +index 61400a1..dcd01e4 100644 +--- a/codex-rs/core/src/lib.rs ++++ b/codex-rs/core/src/lib.rs +@@ -10,6 +10,7 @@ mod apps; + mod client; + mod client_common; + mod model_request; ++pub mod operation_observation; + mod realtime_context; + mod realtime_conversation; + mod realtime_history; +diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs +index 8c4ce1f..c6efa68 100644 +--- a/codex-rs/core/src/mcp_tool_call.rs ++++ b/codex-rs/core/src/mcp_tool_call.rs +@@ -559,6 +559,9 @@ async fn handle_approved_mcp_tool_call( + request_meta, + ) + .await?; ++ crate::tools::operation_observation::start_current_execution_with_input( ++ &tool_input, ++ ); + let mcp_call_trace = sess + .services + .rollout_thread_trace +diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs +index 9f4ef92..34a88e0 100644 +--- a/codex-rs/core/src/mcp_tool_call_tests.rs ++++ b/codex-rs/core/src/mcp_tool_call_tests.rs +@@ -1847,6 +1847,7 @@ async fn codex_apps_auth_elicitation_enabled_by_default_requests_elicitation( + #[test] + fn mcp_tool_call_ids_are_added_to_request_meta() { + let origin = crate::tools::context::ToolCallOrigin { ++ operation_id: None, + item_id: Some(ResponseItemId::from_server("fc-live".to_string())), + window_id: "thread-live:2".to_string(), + }; +diff --git a/codex-rs/core/src/operation_observation.rs b/codex-rs/core/src/operation_observation.rs +new file mode 100644 +index 0000000..1344548 +--- /dev/null ++++ b/codex-rs/core/src/operation_observation.rs +@@ -0,0 +1,1348 @@ ++//! Opt-in, in-memory operation facts for the host that owns a turn. ++//! ++//! These observations never enter the rollout or the telemetry logger. The host ++//! owns delivery and tracing policy; core owns operation boundaries and content. ++ ++use chrono::SecondsFormat; ++use chrono::Utc; ++use futures::FutureExt; ++use serde::Serialize; ++use serde_json::Value; ++use std::future::Future; ++use std::pin::Pin; ++use std::sync::Arc; ++use std::sync::Mutex; ++use std::time::Instant; ++ ++pub const CONTENT_LIMIT_BYTES: usize = 32 * 1024; ++ ++#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub enum OperationCaptureMode { ++ Metadata, ++ Content, ++} ++ ++#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub enum OperationKind { ++ Model, ++ Tool, ++} ++ ++#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub enum OperationOutcome { ++ Success, ++ Error, ++ Interrupted, ++} ++ ++#[derive(Clone, Copy, Debug, PartialEq, Eq)] ++pub enum OperationPhase { ++ Started, ++ Completed, ++} ++ ++#[derive(Clone, Debug, Default, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub struct OperationMetadata { ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub parent_operation_id: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub requested_model: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub observed_model: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub response_id: Option, ++ pub response_id_trusted: bool, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub usage: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub tool_name: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub call_id: Option, ++} ++ ++#[derive(Clone, Debug, PartialEq, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub struct OperationContent { ++ pub value: Value, ++ pub truncated: bool, ++ pub omitted: bool, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub reason: Option, ++} ++ ++#[derive(Clone, Debug, Serialize)] ++#[serde(rename_all = "camelCase")] ++pub struct OperationEvent { ++ #[serde(skip)] ++ pub phase: OperationPhase, ++ pub operation_id: String, ++ pub thread_id: String, ++ pub turn_id: String, ++ pub kind: OperationKind, ++ pub started_at: String, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub completed_at: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub duration_ms: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub outcome: Option, ++ #[serde(flatten)] ++ pub metadata: OperationMetadata, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub input: Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub output: Option, ++} ++ ++pub trait OperationObserver: Send + Sync { ++ /// Must return promptly. Delivery failures must not affect execution. ++ fn emit(&self, event: OperationEvent); ++ ++ /// Closes admission for this source turn before its terminal event. ++ fn turn_finished<'a>( ++ &'a self, ++ _thread_id: &'a str, ++ _turn_id: &'a str, ++ ) -> Pin + Send + 'a>> { ++ Box::pin(async {}) ++ } ++} ++ ++#[derive(Clone)] ++pub struct OperationObservation { ++ mode: OperationCaptureMode, ++ sink: Arc, ++} ++ ++impl std::fmt::Debug for OperationObservation { ++ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { ++ f.debug_struct("OperationObservation") ++ .field("mode", &self.mode) ++ .finish_non_exhaustive() ++ } ++} ++ ++impl OperationObservation { ++ pub fn new(mode: OperationCaptureMode, sink: Arc) -> Self { ++ Self { mode, sink } ++ } ++ ++ pub fn captures_content(&self) -> bool { ++ self.mode == OperationCaptureMode::Content ++ } ++ ++ pub fn start( ++ &self, ++ thread_id: String, ++ turn_id: String, ++ kind: OperationKind, ++ metadata: OperationMetadata, ++ ) -> OperationGuard { ++ let started = Instant::now(); ++ let event = OperationEvent { ++ phase: OperationPhase::Started, ++ operation_id: uuid::Uuid::new_v4().to_string(), ++ thread_id, ++ turn_id, ++ kind, ++ started_at: timestamp(), ++ completed_at: None, ++ duration_ms: None, ++ outcome: None, ++ metadata, ++ input: None, ++ output: None, ++ }; ++ self.emit(event.clone()); ++ OperationGuard { ++ operation_id: event.operation_id.clone(), ++ observation: self.clone(), ++ started, ++ state: Mutex::new(event), ++ } ++ } ++ ++ pub async fn turn_finished(&self, thread_id: &str, turn_id: &str) { ++ let future = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { ++ self.sink.turn_finished(thread_id, turn_id) ++ })); ++ if let Ok(future) = future { ++ let _ = std::panic::AssertUnwindSafe(future).catch_unwind().await; ++ } ++ } ++ ++ fn emit(&self, event: OperationEvent) { ++ // An optional host observer must never unwind an agent operation. ++ let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.sink.emit(event))); ++ } ++} ++ ++tokio::task_local! { ++ static CURRENT_MODEL_OPERATION: ModelOperation; ++} ++ ++pub struct OperationGuard { ++ operation_id: String, ++ observation: OperationObservation, ++ started: Instant, ++ state: Mutex, ++} ++ ++impl OperationGuard { ++ pub fn operation_id(&self) -> &str { ++ &self.operation_id ++ } ++ ++ pub fn capture_input(&self, value: &T) { ++ if self.observation.mode == OperationCaptureMode::Content { ++ self.update(|event| event.input = Some(project_content(value))); ++ } ++ } ++ ++ pub fn capture_output(&self, value: &T) { ++ if self.observation.mode == OperationCaptureMode::Content { ++ self.update(|event| event.output = Some(project_content(value))); ++ } ++ } ++ ++ fn update(&self, update: impl FnOnce(&mut OperationEvent)) { ++ let mut event = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if event.phase == OperationPhase::Started { ++ update(&mut event); ++ } ++ } ++ ++ pub fn finish_with_output_fn( ++ &self, ++ outcome: OperationOutcome, ++ output: impl FnOnce() -> T, ++ ) { ++ let completed_at = timestamp(); ++ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; ++ if self.observation.captures_content() { ++ self.capture_output(&output()); ++ } ++ self.finish_at(outcome, completed_at, duration_ms); ++ } ++ ++ pub fn finish_with_output(&self, outcome: OperationOutcome, output: &T) { ++ let completed_at = timestamp(); ++ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; ++ self.capture_output(output); ++ self.finish_at(outcome, completed_at, duration_ms); ++ } ++ ++ pub fn finish(&self, outcome: OperationOutcome) { ++ self.finish_at( ++ outcome, ++ timestamp(), ++ self.started.elapsed().as_secs_f64() * 1000.0, ++ ); ++ } ++ ++ fn finish_at(&self, outcome: OperationOutcome, completed_at: String, duration_ms: f64) { ++ let terminal = { ++ let mut event = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if event.phase == OperationPhase::Completed { ++ return; ++ } ++ event.phase = OperationPhase::Completed; ++ event.completed_at = Some(completed_at); ++ event.duration_ms = Some(duration_ms); ++ event.outcome = Some(outcome); ++ event.clone() ++ }; ++ self.observation.emit(terminal); ++ } ++} ++ ++impl Drop for OperationGuard { ++ fn drop(&mut self) { ++ self.finish(OperationOutcome::Interrupted); ++ } ++} ++ ++fn timestamp() -> String { ++ Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true) ++} ++ ++/// Projects JSON in memory, omitting binary/hidden reasoning and transport secrets. ++/// Inline markers survive consumers that extract values from the wire envelope. ++pub fn project_content(value: &T) -> OperationContent { ++ let value = match serde_json::to_value(value) { ++ Ok(value) => value, ++ Err(_) => return omitted_content("serialization_failed"), ++ }; ++ let mut projection = ContentProjection { ++ truncated: false, ++ omitted: false, ++ }; ++ let value = projection.project(value, CONTENT_LIMIT_BYTES - 256, 0); ++ let content = OperationContent { ++ value, ++ truncated: projection.truncated, ++ omitted: projection.omitted, ++ reason: if projection.truncated { ++ Some("capture_budget".into()) ++ } else if projection.omitted { ++ Some("unsupported_content".into()) ++ } else { ++ None ++ }, ++ }; ++ if json_size(&content) <= CONTENT_LIMIT_BYTES { ++ content ++ } else { ++ omitted_content("capture_budget") ++ } ++} ++ ++fn omitted_content(reason: &str) -> OperationContent { ++ OperationContent { ++ value: serde_json::json!({"omitted":true,"reason":reason}), ++ truncated: reason == "capture_budget", ++ omitted: true, ++ reason: Some(reason.into()), ++ } ++} ++ ++fn json_size(value: &(impl Serialize + ?Sized)) -> usize { ++ serde_json::to_vec(value).map_or(usize::MAX, |value| value.len()) ++} ++ ++struct ContentProjection { ++ truncated: bool, ++ omitted: bool, ++} ++ ++impl ContentProjection { ++ fn project(&mut self, value: Value, budget: usize, depth: usize) -> Value { ++ if depth > 32 { ++ self.truncated = true; ++ return Value::String("[truncated]".into()); ++ } ++ match value { ++ Value::String(text) => self.text(text, budget, depth), ++ Value::Array(items) => { ++ let mut result = Vec::new(); ++ let mut used: usize = 2; ++ for item in items { ++ let available = budget.saturating_sub(used + 34); ++ if available < 84 { ++ self.truncated = true; ++ result.push(serde_json::json!({"truncated":true})); ++ break; ++ } ++ let item = self.project(item, available, depth + 1); ++ used += json_size(&item) + 2; ++ result.push(item); ++ } ++ Value::Array(result) ++ } ++ Value::Object(items) => { ++ self.truncated |= items.get("truncated") == Some(&Value::Bool(true)); ++ self.omitted |= items.get("omitted") == Some(&Value::Bool(true)); ++ let kind = items.get("type").and_then(Value::as_str); ++ let binary_resource = kind == Some("resource") ++ && items ++ .get("resource") ++ .and_then(|value| value.get("blob")) ++ .is_some(); ++ if binary_resource ++ || kind.is_some_and(|kind| { ++ matches!( ++ kind, ++ "reasoning" ++ | "reasoning_text" ++ | "thinking" ++ | "redacted_thinking" ++ | "compaction" ++ | "encrypted_content" ++ | "image" ++ | "input_image" ++ | "output_image" ++ | "image_url" ++ | "computer_screenshot" ++ | "image_generation_call" ++ | "audio" ++ | "input_audio" ++ | "output_audio" ++ | "file" ++ | "input_file" ++ | "video" ++ | "input_video" ++ ) ++ }) ++ { ++ self.omitted = true; ++ return serde_json::json!({"type":kind,"omitted":true,"reason":"unsupported_content"}); ++ } ++ let mut result = serde_json::Map::new(); ++ let mut used: usize = 2; ++ let mut items: Vec<_> = items.into_iter().collect(); ++ items.sort_by_key(|(key, _)| { ++ !matches!(key.as_str(), "type" | "role" | "id" | "call_id" | "name") ++ }); ++ for (key, value) in items { ++ let available = budget.saturating_sub(used + json_size(&key) + 36); ++ if available < 84 { ++ self.truncated = true; ++ result.insert("truncated".into(), Value::Bool(true)); ++ break; ++ } ++ let normalized = key.to_ascii_lowercase().replace(['-', '_'], ""); ++ let value = if matches!( ++ normalized.as_str(), ++ "apikey" ++ | "xapikey" ++ | "authorization" ++ | "accesstoken" ++ | "refreshtoken" ++ | "password" ++ | "headers" ++ | "clientmetadata" ++ | "encryptedcontent" ++ | "reasoning" ++ | "reasoningcontent" ++ | "reasoningdetails" ++ | "thinking" ++ | "imageurl" ++ | "filedata" ++ | "imagedata" ++ | "audiodata" ++ | "b64json" ++ ) { ++ self.omitted = true; ++ Value::String("[omitted]".into()) ++ } else { ++ self.project(value, available, depth + 1) ++ }; ++ used += json_size(&key) + json_size(&value) + 4; ++ result.insert(key, value); ++ } ++ Value::Object(result) ++ } ++ value => value, ++ } ++ } ++ ++ fn text(&mut self, text: String, budget: usize, depth: usize) -> Value { ++ static DATA_URI: std::sync::LazyLock = std::sync::LazyLock::new(|| { ++ regex_lite::Regex::new(r#"(?i)data:[^\s;,]+(?:;[^\s,]*)?,[^\s\"'<>]+"#) ++ .expect("valid data URI pattern") ++ }); ++ self.truncated |= text.ends_with("[truncated]"); ++ self.omitted |= text.contains("[omitted]") || text.contains("[omitted: media]"); ++ if text.trim_start().starts_with(['{', '[']) { ++ if let Ok(embedded) = serde_json::from_str::(&text) { ++ let safe = self.project(embedded.clone(), budget, depth + 1); ++ if safe == embedded && json_size(&text) <= budget { ++ return Value::String(text); ++ } ++ let mut encoded = safe.to_string(); ++ if json_size(&encoded) > budget { ++ // Rebudget the value so escaped JSON remains valid. ++ encoded = self ++ .project(embedded, budget.saturating_sub(2) / 2, depth + 1) ++ .to_string(); ++ } ++ return Value::String(encoded); ++ } ++ } ++ let projected = DATA_URI.replace_all(&text, "[omitted: media]"); ++ self.omitted |= projected != text; ++ let text = projected.into_owned(); ++ if json_size(&text) <= budget { ++ return Value::String(text); ++ } ++ self.truncated = true; ++ let mut low = 0; ++ let mut high = text.len().min(budget); ++ while low < high { ++ let middle = low + (high - low + 1) / 2; ++ let mut end = middle; ++ while !text.is_char_boundary(end) { ++ end -= 1; ++ } ++ if json_size(&format!("{}[truncated]", &text[..end])) <= budget { ++ low = middle; ++ } else { ++ high = middle - 1; ++ } ++ } ++ while !text.is_char_boundary(low) { ++ low -= 1; ++ } ++ Value::String(format!("{}[truncated]", &text[..low])) ++ } ++} ++ ++#[derive(Clone)] ++pub(crate) struct ModelOperation { ++ observation: Arc, ++ thread_id: String, ++ turn_id: String, ++ requested_model: String, ++ stream_generation: Option, ++ state: Arc>, ++} ++ ++#[derive(Default)] ++struct ModelOperationState { ++ guard: Option, ++ output: Vec, ++ output_bytes: usize, ++ partial_text: String, ++ partial_truncated: bool, ++ fingerprint: Option, ++ generation: u64, ++ source_end: Option<(String, f64)>, ++ retry_waiting: bool, ++ authoritative_model: bool, ++} ++ ++impl ModelOperation { ++ pub(crate) fn new( ++ observation: Arc, ++ thread_id: String, ++ turn_id: String, ++ requested_model: String, ++ ) -> Self { ++ Self { ++ observation, ++ thread_id, ++ turn_id, ++ requested_model, ++ stream_generation: None, ++ state: Arc::new(Mutex::new(ModelOperationState::default())), ++ } ++ } ++ ++ pub(crate) async fn scope(&self, future: F) -> F::Output { ++ CURRENT_MODEL_OPERATION.scope(self.clone(), future).await ++ } ++ ++ pub(crate) fn finish(&self, outcome: OperationOutcome) { ++ // Only the source response.completed event establishes inference success. ++ // An outer successful turn handoff can still have preempted this stream. ++ let outcome = if outcome == OperationOutcome::Success { ++ OperationOutcome::Interrupted ++ } else { ++ outcome ++ }; ++ let state = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if let Some(guard) = &state.guard { ++ if let Some((completed_at, duration_ms)) = &state.source_end ++ && (outcome != OperationOutcome::Interrupted || !state.retry_waiting) ++ { ++ if self.observation.captures_content() { ++ guard.capture_output(&model_output(&state)); ++ } ++ guard.finish_at(outcome, completed_at.clone(), *duration_ms); ++ } else { ++ guard.finish_with_output_fn(outcome, || model_output(&state)); ++ } ++ } ++ } ++ ++ pub(crate) fn interruption_guard(&self) -> ModelInterruptionGuard { ++ ModelInterruptionGuard(self.clone()) ++ } ++ ++ pub(crate) fn retry_waiting(&self, waiting: bool) { ++ self.state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner) ++ .retry_waiting = waiting; ++ } ++ ++ pub(crate) fn source_ended(&self) { ++ let mut state = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if self ++ .stream_generation ++ .is_some_and(|generation| generation != state.generation) ++ { ++ return; ++ } ++ if state.source_end.is_none() { ++ let source_end = state ++ .guard ++ .as_ref() ++ .map(|guard| (timestamp(), guard.started.elapsed().as_secs_f64() * 1000.0)); ++ if source_end.is_some() { ++ state.source_end = source_end; ++ } ++ } ++ } ++ ++ pub(crate) fn source_error(&self, error: &codex_api::ApiError) { ++ self.source_ended(); ++ let state = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if self ++ .stream_generation ++ .is_some_and(|generation| generation != state.generation) ++ { ++ return; ++ } ++ if let codex_api::ApiError::ResponseIncomplete(incomplete) = error ++ && let Some(guard) = &state.guard ++ { ++ guard.update(|event| { ++ event.metadata.response_id = incomplete.response_id.clone(); ++ event.metadata.usage = incomplete ++ .token_usage ++ .as_ref() ++ .and_then(|usage| serde_json::to_value(usage).ok()); ++ }); ++ } ++ } ++ ++ fn request( ++ &self, ++ request: &T, ++ identity: &impl Serialize, ++ endpoint: &str, ++ ) { ++ let mut state = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ state.generation = state.generation.wrapping_add(1); ++ let previous_source_end = state.source_end.take(); ++ state.retry_waiting = false; ++ state.authoritative_model = false; ++ // Exclude routing metadata and changing transport fields from retry identity. ++ let fingerprint = request_fingerprint(identity); ++ if state.guard.is_some() && state.fingerprint == Some(fingerprint) { ++ if let Some(guard) = &state.guard { ++ guard.update(|event| { ++ event.metadata.response_id = None; ++ event.metadata.observed_model = None; ++ event.metadata.usage = None; ++ event.metadata.response_id_trusted = trusted_response_endpoint(endpoint); ++ }); ++ if self.observation.captures_content() { ++ guard.capture_input(&model_request_content(request)); ++ } ++ } ++ // A retried stream replaces any partial output from the failed attempt. ++ state.output.clear(); ++ state.output_bytes = 0; ++ state.partial_text.clear(); ++ state.partial_truncated = false; ++ return; ++ } ++ if let Some(previous) = state.guard.take() { ++ if self.observation.captures_content() { ++ previous.capture_output(&model_output(&state)); ++ } ++ if let Some((completed_at, duration_ms)) = previous_source_end { ++ previous.finish_at(OperationOutcome::Error, completed_at, duration_ms); ++ } else { ++ previous.finish(OperationOutcome::Error); ++ } ++ } ++ state.output.clear(); ++ state.output_bytes = 0; ++ state.partial_text.clear(); ++ state.partial_truncated = false; ++ state.fingerprint = Some(fingerprint); ++ let guard = self.observation.start( ++ self.thread_id.clone(), ++ self.turn_id.clone(), ++ OperationKind::Model, ++ OperationMetadata { ++ requested_model: Some(self.requested_model.clone()), ++ response_id_trusted: trusted_response_endpoint(endpoint), ++ ..Default::default() ++ }, ++ ); ++ // Capture the assembled wire body, rather than the caller's input. ++ if self.observation.captures_content() { ++ guard.capture_input(&model_request_content(request)); ++ } ++ state.guard = Some(guard); ++ } ++ ++ pub(crate) fn for_stream(mut self) -> Self { ++ self.stream_generation = Some( ++ self.state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner) ++ .generation, ++ ); ++ self ++ } ++ ++ pub(crate) fn event(&self, event: &codex_api::ResponseEvent) { ++ let mut state = self ++ .state ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ if self ++ .stream_generation ++ .is_some_and(|generation| generation != state.generation) ++ || state.source_end.is_some() ++ { ++ return; ++ } ++ match event { ++ codex_api::ResponseEvent::OutputTextDelta(text) ++ if self.observation.captures_content() => ++ { ++ let mut end = text ++ .len() ++ .min(CONTENT_LIMIT_BYTES.saturating_sub(state.partial_text.len())); ++ while !text.is_char_boundary(end) { ++ end -= 1; ++ } ++ state.partial_text.push_str(&text[..end]); ++ state.partial_truncated |= end < text.len(); ++ } ++ codex_api::ResponseEvent::OutputItemDone(item) ++ if self.observation.mode == OperationCaptureMode::Content => ++ { ++ if matches!(item, codex_protocol::models::ResponseItem::Message { .. }) { ++ state.partial_text.clear(); ++ state.partial_truncated = false; ++ } ++ if state.output_bytes < CONTENT_LIMIT_BYTES { ++ let content = project_content(item); ++ state.output_bytes += serde_json::to_vec(&content.value) ++ .map_or(CONTENT_LIMIT_BYTES, |value| value.len()); ++ state.output.push(content.value); ++ if content.truncated || content.omitted { ++ state.output.push(serde_json::json!({"omitted":content.omitted,"truncated":content.truncated,"reason":content.reason})); ++ } ++ } else if state ++ .output ++ .last() ++ .and_then(|item| item.get("reason")) ++ .and_then(Value::as_str) ++ != Some("capture_budget") ++ { ++ state ++ .output ++ .push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); ++ } ++ } ++ codex_api::ResponseEvent::ModelObservation(model) => { ++ if !state.authoritative_model ++ && let Some(guard) = &state.guard ++ { ++ guard.update(|event| event.metadata.observed_model = Some(model.clone())); ++ } ++ } ++ codex_api::ResponseEvent::ServerModel(model) => { ++ state.authoritative_model = true; ++ if let Some(guard) = &state.guard { ++ guard.update(|event| event.metadata.observed_model = Some(model.clone())); ++ } ++ } ++ codex_api::ResponseEvent::Created { response_id } => { ++ if let Some(guard) = &state.guard { ++ guard.update(|event| event.metadata.response_id = response_id.clone()); ++ } ++ } ++ codex_api::ResponseEvent::Completed { ++ response_id, ++ token_usage, ++ .. ++ } => { ++ if let Some(guard) = &state.guard { ++ guard.update(|event| { ++ event.metadata.response_id = Some(response_id.clone()); ++ event.metadata.usage = token_usage ++ .as_ref() ++ .and_then(|usage| serde_json::to_value(usage).ok()); ++ }); ++ guard.finish_with_output_fn(OperationOutcome::Success, || model_output(&state)); ++ } ++ } ++ _ => {} ++ } ++ } ++} ++ ++fn model_output(state: &ModelOperationState) -> Vec { ++ let mut output = state.output.clone(); ++ if !state.partial_text.is_empty() { ++ output.push(serde_json::json!({"type":"output_text","text":state.partial_text,"partial":true,"truncated":state.partial_truncated})); ++ } ++ output ++} ++ ++pub(crate) fn current_model_operation() -> Option { ++ CURRENT_MODEL_OPERATION.try_with(Clone::clone).ok() ++} ++ ++pub(crate) fn observe_model_request( ++ request: &T, ++ identity: &impl Serialize, ++ endpoint: &str, ++) { ++ if let Some(operation) = current_model_operation() { ++ operation.request(request, identity, endpoint); ++ } ++} ++ ++pub(crate) struct ModelInterruptionGuard(ModelOperation); ++ ++impl Drop for ModelInterruptionGuard { ++ fn drop(&mut self) { ++ self.0.finish(OperationOutcome::Interrupted); ++ } ++} ++ ++fn model_request_content(request: &(impl Serialize + ?Sized)) -> Value { ++ let mut value = model_request_identity(request); ++ if let Value::Object(body) = &mut value { ++ // The per-direction content budget belongs to messages and instructions. ++ // Tool definitions affect request identity but must not consume that budget. ++ body.retain(|key, _| { ++ matches!( ++ key.as_str(), ++ "instructions" | "input" | "previous_response_id" ++ ) ++ }); ++ } ++ value ++} ++ ++fn model_request_identity(request: &(impl Serialize + ?Sized)) -> Value { ++ let Ok(Value::Object(mut body)) = serde_json::to_value(request) else { ++ return Value::Null; ++ }; ++ // Preserve semantic request changes when grouping same-input transport retries. ++ body.retain(|key, _| { ++ matches!( ++ key.as_str(), ++ "model" ++ | "instructions" ++ | "input" ++ | "previous_response_id" ++ | "tools" ++ | "tool_choice" ++ | "text" ++ ) ++ }); ++ Value::Object(body) ++} ++ ++fn request_fingerprint(request: &impl Serialize) -> u64 { ++ use std::hash::Hash; ++ use std::hash::Hasher; ++ let value = model_request_identity(request); ++ let mut hasher = std::collections::hash_map::DefaultHasher::new(); ++ // The digest only lives with this operation; request content is not retained in metadata mode. ++ value.to_string().hash(&mut hasher); ++ hasher.finish() ++} ++ ++fn trusted_response_endpoint(endpoint: &str) -> bool { ++ url::Url::parse(endpoint) ++ .is_ok_and(|url| url.scheme() == "https" && url.host_str() == Some("api.openai.com")) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use pretty_assertions::assert_eq; ++ ++ #[derive(Default)] ++ struct RecordingObserver(Mutex>); ++ impl OperationObserver for RecordingObserver { ++ fn emit(&self, event: OperationEvent) { ++ self.0.lock().unwrap().push(event); ++ } ++ } ++ ++ fn observation(mode: OperationCaptureMode) -> (OperationObservation, Arc) { ++ let sink = Arc::new(RecordingObserver::default()); ++ (OperationObservation::new(mode, sink.clone()), sink) ++ } ++ ++ #[test] ++ fn metadata_never_captures_payload_and_finalizes_once() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ { ++ let operation = observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Tool, ++ OperationMetadata::default(), ++ ); ++ operation.capture_input(&serde_json::json!({"text":"INPUT_CANARY"})); ++ operation.finish_with_output(OperationOutcome::Success, &"OUTPUT_CANARY"); ++ operation.finish(OperationOutcome::Error); ++ } ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[0].phase, OperationPhase::Started); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); ++ assert_eq!(events[0].operation_id, events[1].operation_id); ++ assert_eq!(events[1].input, None); ++ assert_eq!(events[1].output, None); ++ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); ++ } ++ ++ #[test] ++ fn dropped_operation_reports_interruption() { ++ let (observation, sink) = observation(OperationCaptureMode::Content); ++ drop(observation.start( ++ "thread".into(), ++ "turn".into(), ++ OperationKind::Tool, ++ OperationMetadata::default(), ++ )); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); ++ assert!(events[1].duration_ms.unwrap() >= 0.0); ++ assert!(chrono::DateTime::parse_from_rfc3339(&events[1].started_at).is_ok()); ++ assert!( ++ chrono::DateTime::parse_from_rfc3339(events[1].completed_at.as_ref().unwrap()).is_ok() ++ ); ++ } ++ ++ #[test] ++ fn projection_bounds_utf8_and_excludes_hidden_or_binary_parts() { ++ let content = project_content(&serde_json::json!({ ++ "input":[{"role":"user","content":[{"type":"input_text","text":"hello"},{"type":"input_image","image_url":"BINARY_CANARY"}]},{"type":"reasoning","summary":[{"text":"REASONING_CANARY"}]}], ++ "authorization":"SECRET_CANARY", ++ "output":"é🙂".repeat(CONTENT_LIMIT_BYTES) ++ })); ++ let serialized = serde_json::to_string(&content).unwrap(); ++ assert!(serialized.len() <= CONTENT_LIMIT_BYTES); ++ assert!(content.truncated); ++ assert!(content.omitted); ++ assert!(!serialized.contains("CANARY")); ++ assert_eq!(content.value["input"][0]["role"], "user"); ++ assert_eq!(content.value["input"][0]["content"][0]["text"], "hello"); ++ } ++ ++ #[test] ++ fn mcp_binary_content_is_omitted_but_text_resources_remain_visible() { ++ let content = project_content(&serde_json::json!([ ++ {"type":"image","data":"IMAGE_CANARY","mimeType":"image/png"}, ++ {"type":"resource","resource":{"uri":"file:///image","blob":"BLOB_CANARY"}}, ++ {"type":"audio","data":"AUDIO_CANARY","mimeType":"audio/wav"}, ++ {"type":"resource","resource":{"uri":"file:///text","text":"visible text"}}, ++ {"type":"text","text":"visible output"} ++ ])); ++ let serialized = serde_json::to_string(&content).unwrap(); ++ assert!(content.omitted); ++ assert!(!serialized.contains("CANARY")); ++ assert_eq!(content.value[3]["resource"]["text"], "visible text"); ++ assert_eq!(content.value[4]["text"], "visible output"); ++ } ++ ++ #[test] ++ fn oversized_text_keeps_a_useful_utf8_prefix_and_inline_marker() { ++ let content = project_content(&format!( ++ "file prefix: {}", ++ "é🙂".repeat(CONTENT_LIMIT_BYTES) ++ )); ++ assert!(content.value.as_str().unwrap().starts_with("file prefix: ")); ++ assert!(content.value.as_str().unwrap().ends_with("[truncated]")); ++ assert!(content.truncated); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ ++ #[test] ++ fn native_tool_catalog_does_not_truncate_intact_model_input() { ++ let input = serde_json::json!([ ++ {"role":"user","content":[{"type":"input_text","text":"x".repeat(27_000)}]}, ++ ]); ++ let request = serde_json::json!({ ++ "model":"fixture-model", ++ "instructions":"worker instructions", ++ "input": input, ++ "tools":[{"type":"function","name":"native_tool","description":"catalog".repeat(20_000)}], ++ "text":{"format":{"type":"json_schema","schema":{"description":"schema".repeat(10_000)}}}, ++ "tool_choice":"auto", ++ }); ++ let content = project_content(&model_request_content(&request)); ++ assert!(!content.truncated); ++ assert!(!content.omitted); ++ assert_eq!(content.value["input"], input); ++ assert_eq!(content.value["instructions"], "worker instructions"); ++ assert!(content.value.get("tools").is_none()); ++ assert!(content.value.get("text").is_none()); ++ let mut changed = request.clone(); ++ changed["tools"][0]["name"] = Value::String("different_tool".into()); ++ assert_ne!(request_fingerprint(&request), request_fingerprint(&changed)); ++ assert_eq!( ++ model_request_content(&request), ++ model_request_content(&changed) ++ ); ++ } ++ ++ #[test] ++ fn embedded_json_that_fits_the_budget_remains_intact() { ++ let raw = serde_json::json!({"text":"x".repeat(27_000)}).to_string(); ++ let content = project_content(&raw); ++ assert_eq!(content.value, Value::String(raw)); ++ assert!(!content.truncated); ++ } ++ ++ #[test] ++ fn oversized_messages_preserve_roles_and_part_identity() { ++ let content = project_content(&serde_json::json!([ ++ {"content":[{"text":format!("prefix {}", "x".repeat(80_000)),"type":"input_text"}],"role":"user"}, ++ ])); ++ assert_eq!(content.value[0]["role"], "user"); ++ assert_eq!(content.value[0]["content"][0]["type"], "input_text"); ++ assert!( ++ content.value[0]["content"][0]["text"] ++ .as_str() ++ .unwrap() ++ .starts_with("prefix ") ++ ); ++ assert!( ++ content.value[0]["content"][0]["text"] ++ .as_str() ++ .unwrap() ++ .ends_with("[truncated]") ++ ); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ ++ #[test] ++ fn embedded_json_redacts_and_truncates_without_breaking_its_structure() { ++ let input = serde_json::json!({ ++ "accessToken":"TOKEN_CANARY", ++ "result":format!("prefix {}", "🙂".repeat(CONTENT_LIMIT_BYTES)), ++ }) ++ .to_string(); ++ let content = project_content(&input); ++ let embedded: Value = serde_json::from_str(content.value.as_str().unwrap()).unwrap(); ++ assert_eq!(embedded["accessToken"], "[omitted]"); ++ assert!(embedded["result"].as_str().unwrap().starts_with("prefix ")); ++ assert!( ++ embedded["result"] ++ .as_str() ++ .unwrap() ++ .ends_with("[truncated]") ++ ); ++ assert!(content.truncated); ++ assert!(content.omitted); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ assert!(!content.value.to_string().contains("CANARY")); ++ } ++ ++ #[test] ++ fn embedded_media_and_container_budget_have_visible_markers() { ++ let content = project_content(&serde_json::json!({ ++ "message":"before data:image/png;base64,BINARY_CANARY after", ++ "nested": [{"Authorization":"AUTH_CANARY","visible":"hello"}], ++ "nested_encoded": serde_json::json!({"request":serde_json::json!({"apiKey":"NESTED_CANARY","uri":"data:image/png;base64,MEDIA_CANARY","text":"visible nested"}).to_string()}).to_string(), ++ "results": vec![serde_json::json!({"text":"x".repeat(180)}); 3000], ++ })); ++ let serialized = serde_json::to_string(&content).unwrap(); ++ assert!(!serialized.contains("CANARY")); ++ assert!( ++ content.value["message"] ++ .as_str() ++ .unwrap() ++ .contains("[omitted: media]") ++ ); ++ assert_eq!(content.value["nested"][0]["visible"], "hello"); ++ assert!( ++ content.value["nested_encoded"] ++ .as_str() ++ .unwrap() ++ .contains("visible nested") ++ ); ++ assert!(content.value["results"].to_string().contains("truncated")); ++ assert!(serialized.len() <= CONTENT_LIMIT_BYTES); ++ } ++ ++ #[tokio::test] ++ async fn same_input_retry_has_one_live_start_and_changed_input_splits() { ++ let (observation, sink) = observation(OperationCaptureMode::Content); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ model.scope(async { ++ let request = serde_json::json!({"model":"model","input":[{"role":"user","content":"first"}],"client_metadata":{"secret":"META_CANARY"}}); ++ observe_model_request(&request, &request, "http://localhost:8080"); ++ assert_eq!(sink.0.lock().unwrap().len(), 1); ++ observe_model_request(&request, &request, "http://localhost:8080"); ++ assert_eq!(sink.0.lock().unwrap().len(), 1); ++ let changed = serde_json::json!({"model":"model","input":[{"role":"user","content":"second"}]}); ++ observe_model_request(&changed, &changed, "http://localhost:8080"); ++ model.event(&codex_api::ResponseEvent::Completed { response_id:"response".into(),token_usage:None,usage_metadata:None,end_turn:Some(true) }); ++ }).await; ++ model.finish(OperationOutcome::Error); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 4); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); ++ assert_eq!(events[3].outcome, Some(OperationOutcome::Success)); ++ assert_eq!(events[3].metadata.response_id.as_deref(), Some("response")); ++ assert!(!events[3].metadata.response_id_trusted); ++ assert!( ++ !serde_json::to_string(&*events) ++ .unwrap() ++ .contains("META_CANARY") ++ ); ++ } ++ ++ #[test] ++ fn body_model_fallback_never_overrides_authoritative_header_model() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "requested".into(), ++ ); ++ let request = serde_json::json!({"model":"requested","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ model.event(&codex_api::ResponseEvent::ModelObservation( ++ "body-first".into(), ++ )); ++ assert_eq!( ++ model ++ .state ++ .lock() ++ .unwrap() ++ .guard ++ .as_ref() ++ .unwrap() ++ .state ++ .lock() ++ .unwrap() ++ .metadata ++ .observed_model ++ .as_deref(), ++ Some("body-first") ++ ); ++ model.event(&codex_api::ResponseEvent::ServerModel("header".into())); ++ model.event(&codex_api::ResponseEvent::ModelObservation( ++ "body-last".into(), ++ )); ++ model.event(&codex_api::ResponseEvent::Completed { ++ response_id: "response".into(), ++ token_usage: None, ++ usage_metadata: None, ++ end_turn: Some(true), ++ }); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events[1].metadata.observed_model.as_deref(), Some("header")); ++ } ++ ++ #[test] ++ fn stale_stream_cannot_finish_a_new_attempt() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ let old = model.clone().for_stream(); ++ old.source_ended(); ++ model.request(&request, &request, "http://localhost"); ++ old.event(&codex_api::ResponseEvent::Completed { ++ response_id: "old".into(), ++ token_usage: None, ++ usage_metadata: None, ++ end_turn: Some(true), ++ }); ++ old.source_ended(); ++ assert_eq!(sink.0.lock().unwrap().len(), 1); ++ model.event(&codex_api::ResponseEvent::Completed { ++ response_id: "new".into(), ++ token_usage: None, ++ usage_metadata: None, ++ end_turn: Some(true), ++ }); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].metadata.response_id.as_deref(), Some("new")); ++ } ++ ++ #[test] ++ fn final_error_uses_stream_end_before_later_cleanup() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ model.source_ended(); ++ let expected = model.state.lock().unwrap().source_end.clone().unwrap(); ++ // A later EOF/drop must not move the source error boundary to tool cleanup. ++ model.source_ended(); ++ model.finish(OperationOutcome::Error); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events[1].completed_at.as_ref(), Some(&expected.0)); ++ assert_eq!(events[1].duration_ms, Some(expected.1)); ++ } ++ ++ #[test] ++ fn successful_handoff_without_response_completion_marks_inference_interrupted() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ model.finish(OperationOutcome::Success); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); ++ } ++ ++ #[test] ++ fn interruption_during_retry_wait_includes_backoff() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ model.source_ended(); ++ model.state.lock().unwrap().source_end = Some(("2000-01-01T00:00:00.000Z".into(), 0.0)); ++ model.retry_waiting(true); ++ model.finish(OperationOutcome::Interrupted); ++ let events = sink.0.lock().unwrap(); ++ assert_ne!( ++ events[1].completed_at.as_deref(), ++ Some("2000-01-01T00:00:00.000Z") ++ ); ++ assert!(events[1].duration_ms.unwrap() > 0.0); ++ } ++ ++ #[test] ++ fn changed_input_closes_previous_inference_at_saved_source_end() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ model.source_ended(); ++ let expected = model.state.lock().unwrap().source_end.clone().unwrap(); ++ let changed = serde_json::json!({"model":"model","input":["changed"]}); ++ model.request(&changed, &changed, "http://localhost"); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 3); ++ assert_eq!(events[1].completed_at.as_ref(), Some(&expected.0)); ++ assert_eq!(events[1].duration_ms, Some(expected.1)); ++ } ++ ++ #[test] ++ fn incomplete_response_preserves_usage_without_error_payload() { ++ let (observation, sink) = observation(OperationCaptureMode::Metadata); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost"); ++ let error = codex_api::ApiError::ResponseIncomplete(Box::new( ++ codex_protocol::error::ResponseIncompleteError { ++ reason: "ERROR_PAYLOAD_CANARY".into(), ++ response_id: Some("partial-response".into()), ++ token_usage: Some(codex_protocol::protocol::TokenUsage { ++ input_tokens: 7, ++ output_tokens: 3, ++ total_tokens: 10, ++ ..Default::default() ++ }), ++ }, ++ )); ++ model.source_error(&error); ++ model.finish(OperationOutcome::Error); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!( ++ events[1].metadata.response_id.as_deref(), ++ Some("partial-response") ++ ); ++ assert_eq!( ++ events[1].metadata.usage.as_ref().unwrap()["input_tokens"], ++ 7 ++ ); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); ++ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); ++ } ++ ++ #[test] ++ fn model_interruption_preserves_bounded_visible_partial_output() { ++ let (observation, sink) = observation(OperationCaptureMode::Content); ++ let model = ModelOperation::new( ++ Arc::new(observation), ++ "thread".into(), ++ "turn".into(), ++ "model".into(), ++ ); ++ let interrupted = model.interruption_guard(); ++ let request = serde_json::json!({"model":"model","input":[]}); ++ model.request(&request, &request, "http://localhost:8080"); ++ model.event(&codex_api::ResponseEvent::OutputTextDelta( ++ "partial answer".into(), ++ )); ++ drop(interrupted); ++ let events = sink.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); ++ assert_eq!( ++ events[1].output.as_ref().unwrap().value[0]["text"], ++ "partial answer" ++ ); ++ assert_eq!(events[1].output.as_ref().unwrap().value[0]["partial"], true); ++ } ++ ++ #[test] ++ fn response_id_trust_requires_exact_official_https_host() { ++ assert!(trusted_response_endpoint("https://api.openai.com/v1")); ++ assert!(!trusted_response_endpoint( ++ "https://chatgpt.com/backend-api/codex" ++ )); ++ assert!(!trusted_response_endpoint( ++ "https://api.openai.com.example.test/v1" ++ )); ++ assert!(!trusted_response_endpoint("http://api.openai.com/v1")); ++ assert!(!trusted_response_endpoint("http://127.0.0.1:9000/v1")); ++ } ++} +diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs +index cc418ca..66476ae 100644 +--- a/codex-rs/core/src/session/mod.rs ++++ b/codex-rs/core/src/session/mod.rs +@@ -2526,6 +2526,15 @@ impl Session { + + /// Persist the event to rollout and send it to clients. + pub(crate) async fn send_event(&self, turn_context: &TurnContext, msg: EventMsg) { ++ if matches!(&msg, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_)) ++ && let Some(observation) = turn_context ++ .extension_data ++ .get::() ++ { ++ observation ++ .turn_finished(&self.thread_id.to_string(), &turn_context.sub_id) ++ .await; ++ } + if let Some((item_id, started_at_ms)) = msg.command_approval_start() { + // The client can see the command before execution. Keep its first start + // for the canonical lifecycle events that will be saved in history. +diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs +index e4a092e..871cdc9 100644 +--- a/codex-rs/core/src/session/turn.rs ++++ b/codex-rs/core/src/session/turn.rs +@@ -1745,6 +1745,53 @@ async fn run_sampling_request( + client_session: &mut ModelClientSession, + input: Vec, + cancellation_token: CancellationToken, ++) -> CodexResult<(SamplingRequestResult, Vec)> { ++ let observation = step_context ++ .turn ++ .extension_data ++ .get::(); ++ let operation = observation.map(|observation| { ++ crate::operation_observation::ModelOperation::new( ++ observation, ++ sess.thread_id.to_string(), ++ step_context.turn.sub_id.clone(), ++ step_context.settings.model_info.slug.clone(), ++ ) ++ }); ++ let future = run_sampling_request_observed( ++ sess, ++ step_context, ++ turn_store, ++ turn_diff_tracker, ++ client_session, ++ input, ++ cancellation_token, ++ ); ++ if let Some(operation) = operation { ++ let _interrupted = operation.interruption_guard(); ++ let result = operation.scope(future).await; ++ operation.finish(match &result { ++ Ok(_) => crate::operation_observation::OperationOutcome::Success, ++ Err(error) if matches!(error.details(), CodexErrorDetails::TurnAborted) => { ++ crate::operation_observation::OperationOutcome::Interrupted ++ } ++ Err(_) => crate::operation_observation::OperationOutcome::Error, ++ }); ++ result ++ } else { ++ future.await ++ } ++} ++ ++#[allow(clippy::too_many_arguments)] ++async fn run_sampling_request_observed( ++ sess: Arc, ++ step_context: Arc, ++ turn_store: Arc, ++ turn_diff_tracker: SharedTurnDiffTracker, ++ client_session: &mut ModelClientSession, ++ input: Vec, ++ cancellation_token: CancellationToken, + ) -> CodexResult<(SamplingRequestResult, Vec)> { + let turn_context = Arc::clone(&step_context.turn); + let preempt = step_context.preempt.clone().unwrap_or_default(); +@@ -1854,6 +1901,9 @@ async fn run_sampling_request( + }, + }; + ++ if let Some(operation) = crate::operation_observation::current_model_operation() { ++ operation.retry_waiting(true); ++ } + let retry = handle_response_stream_error( + &mut retry_state, + max_retries, +@@ -1866,6 +1916,9 @@ async fn run_sampling_request( + .or_cancel(&preempt) + .or_cancel(&cancellation_token) + .await?; ++ if let Some(operation) = crate::operation_observation::current_model_operation() { ++ operation.retry_waiting(false); ++ } + if cancellation_token.is_cancelled() { + return Err(CodexErr::TurnAborted); + } +@@ -3170,6 +3223,8 @@ async fn try_run_sampling_request( + active_item_is_streaming_to_client = stream_item_to_client; + } + } ++ // Consumed by the client observer before session events are forwarded. ++ ResponseEvent::ModelObservation(_) => {} + ResponseEvent::ServerModel(server_model) => { + if !turn_context + .server_model_warning_emitted +diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs +index 4fa1c9d..28f4a00 100644 +--- a/codex-rs/core/src/session/turn_input.rs ++++ b/codex-rs/core/src/session/turn_input.rs +@@ -98,6 +98,7 @@ impl TurnStartKind { + struct PreparedTurnInputSettings { + thread_settings_update: Option, + start_options: TurnStartOptions, ++ new_turn_only_attachment: bool, + } + + impl PreparedTurnInputSettings { +@@ -108,7 +109,15 @@ impl PreparedTurnInputSettings { + thread_settings: impl Into>, + start_options: TurnStartOptions, + ) -> CodexResult { +- let thread_settings = thread_settings.into(); ++ let mut thread_settings = thread_settings.into(); ++ let new_turn_only = thread_settings ++ .turn_extension_init ++ .as_ref() ++ .and_then(|init| init.get::()); ++ let new_turn_only_attachment = new_turn_only.is_some(); ++ if let Some(init) = new_turn_only { ++ thread_settings.turn_extension_init = Some(init.0.clone()); ++ } + let thread_settings_update = if thread_settings.request + == ThreadSettingsOverrides::default() + && thread_settings.turn_extension_init.is_none() +@@ -128,6 +137,7 @@ impl PreparedTurnInputSettings { + Ok(Self { + thread_settings_update, + start_options, ++ new_turn_only_attachment, + }) + } + +@@ -215,9 +225,12 @@ impl PreparedTurnInputSettings { + /// Applies only persistent settings after steering succeeds. The active + /// turn keeps its existing context; subsequent turns see the update. + async fn apply_steered(self, session: &Session, submission_id: String) -> CodexResult<()> { +- let Some(thread_settings_update) = self.thread_settings_update else { ++ let Some(mut thread_settings_update) = self.thread_settings_update else { + return Ok(()); + }; ++ if self.new_turn_only_attachment { ++ thread_settings_update.turn_extension_init = None; ++ } + thread_settings::apply_update(session, submission_id, thread_settings_update) + .await + .map_err(|error| CodexErr::InvalidRequest(error.to_string())) +diff --git a/codex-rs/core/src/session/turn_input_tests.rs b/codex-rs/core/src/session/turn_input_tests.rs +index c8df7c1..5ccf2d4 100644 +--- a/codex-rs/core/src/session/turn_input_tests.rs ++++ b/codex-rs/core/src/session/turn_input_tests.rs +@@ -1238,3 +1238,128 @@ async fn scored_prompt_keeps_its_submission_item_id() { + assert_eq!(ui_ids.len(), 3); + session.abort_all_tasks(TurnAbortReason::Interrupted).await; + } ++ ++#[derive(Debug)] ++struct AttachmentOwner(&'static str); ++ ++#[test_case(false, false; "legacy_start_or_steer")] ++#[test_case(false, true; "legacy_steer_only")] ++#[test_case(true, false; "start_only_start_or_steer")] ++#[test_case(true, true; "start_only_steer_only")] ++#[tokio::test] ++async fn steering_respects_new_turn_only_attachment_policy(start_only: bool, steer_only: bool) { ++ let (session, turn_context, _rx) = make_session_and_context_with_rx().await; ++ session ++ .state ++ .lock() ++ .await ++ .session_configuration ++ .turn_extension_init ++ .insert(AttachmentOwner("original")); ++ turn_context ++ .extension_data ++ .insert(AttachmentOwner("original")); ++ session ++ .spawn_task( ++ Arc::clone(&turn_context), ++ vec![TurnInput::UserInput { ++ metadata: Default::default(), ++ content: vec![UserInput::Text { ++ text: "initial".into(), ++ text_elements: Vec::new(), ++ }], ++ client_id: None, ++ }], ++ NeverEndingTask { ++ kind: TaskKind::Regular, ++ listen_to_cancellation_token: true, ++ }, ++ ) ++ .await; ++ let mut init = codex_extension_api::ExtensionDataInit::new(); ++ init.insert(AttachmentOwner("steering")); ++ let request = TurnInputRequest::user_input(vec![UserInput::Text { ++ text: "steer".into(), ++ text_elements: Vec::new(), ++ }]); ++ let request = if start_only { ++ WithTurnExtensionData::new_turn_only(request, init) ++ } else { ++ WithTurnExtensionData::new(request, init) ++ }; ++ let mode = if steer_only { ++ TurnInputMode::Steer { ++ expected_turn_id: turn_context.sub_id.clone(), ++ } ++ } else { ++ TurnInputMode::StartOrSteer ++ }; ++ let result = handle(&session, request, mode, "steering-owner".into()) ++ .await ++ .unwrap(); ++ assert!(matches!(result, TurnInputSubmission::Steered { .. })); ++ assert_eq!( ++ turn_context ++ .extension_data ++ .get::() ++ .unwrap() ++ .0, ++ "original" ++ ); ++ session.abort_all_tasks(TurnAbortReason::Interrupted).await; ++ // Automatic/recovery contexts read this same saved configuration. An ++ // unacknowledged steering owner must not acquire those later turns. ++ let later = session ++ .new_turn_with_default_settings("later".into(), Default::default()) ++ .await; ++ assert_eq!( ++ later.extension_data.get::().unwrap().0, ++ if start_only { "original" } else { "steering" } ++ ); ++ assert!( ++ later ++ .extension_data ++ .get::() ++ .is_none() ++ ); ++} ++ ++#[tokio::test] ++async fn started_turn_unwraps_new_turn_only_attachment_before_persistence() { ++ let (session, _, _rx) = make_session_and_context_with_rx().await; ++ let mut init = codex_extension_api::ExtensionDataInit::new(); ++ init.insert(AttachmentOwner("started")); ++ let wrapped = ++ WithTurnExtensionData::new_turn_only(TurnInputRequest::user_input(Vec::new()), init); ++ let settings = PreparedTurnInputSettings::prepare( ++ &session, ++ WithTurnExtensionData { ++ request: ThreadSettingsOverrides::default(), ++ turn_extension_init: wrapped.turn_extension_init, ++ }, ++ TurnStartOptions::default(), ++ ) ++ .await ++ .unwrap(); ++ let turn = settings ++ .apply_started(&session, "started-owner".into(), TurnStartKind::User) ++ .await ++ .unwrap() ++ .unwrap(); ++ assert_eq!( ++ turn.extension_data.get::().unwrap().0, ++ "started" ++ ); ++ assert!( ++ turn.extension_data ++ .get::() ++ .is_none() ++ ); ++ let later = session ++ .new_turn_with_default_settings("later".into(), Default::default()) ++ .await; ++ assert_eq!( ++ later.extension_data.get::().unwrap().0, ++ "started" ++ ); ++} +diff --git a/codex-rs/core/src/tools/code_mode/execute_handler.rs b/codex-rs/core/src/tools/code_mode/execute_handler.rs +index 317e029..4354977 100644 +--- a/codex-rs/core/src/tools/code_mode/execute_handler.rs ++++ b/codex-rs/core/src/tools/code_mode/execute_handler.rs +@@ -208,7 +208,10 @@ impl CodeModeExecuteHandler { + invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + let handler_span = tracing::Span::current(); +- let originating_call = invocation.originating_call().await; ++ let mut originating_call = invocation.originating_call().await; ++ if let Some(origin) = originating_call.as_mut() { ++ origin.operation_id = crate::tools::operation_observation::current_tool_operation_id(); ++ } + let ToolInvocation { + session, + turn, +diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs +index fe7de2c..60aa2b8 100644 +--- a/codex-rs/core/src/tools/context.rs ++++ b/codex-rs/core/src/tools/context.rs +@@ -125,6 +125,8 @@ pub struct ToolInvocation { + /// Identity of the model call that started a tool invocation, retained across code-mode waits. + #[derive(Clone)] + pub(crate) struct ToolCallOrigin { ++ /// Execution observation retained across code-mode yields and detached callbacks. ++ pub(crate) operation_id: Option, + /// Best-effort lookup: invocations without a matching history item still + /// retain their known window ID. + pub(crate) item_id: Option, +@@ -158,6 +160,7 @@ impl ToolInvocation { + _ => None, + }); + Some(ToolCallOrigin { ++ operation_id: super::operation_observation::current_tool_operation_id(), + item_id, + window_id: self.session.current_window_id().await, + }) +diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs +index 64d6cc1..0c1c08a 100644 +--- a/codex-rs/core/src/tools/handlers/apply_patch.rs ++++ b/codex-rs/core/src/tools/handlers/apply_patch.rs +@@ -338,6 +338,10 @@ impl ApplyPatchHandler { + } + + impl CoreToolRuntime for ApplyPatchHandler { ++ fn approves_inside_handler(&self) -> bool { ++ true ++ } ++ + fn environments_for_input( + &self, + invocation: &ToolInvocation, +diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs +index 0eac846..59c158c 100644 +--- a/codex-rs/core/src/tools/handlers/dynamic.rs ++++ b/codex-rs/core/src/tools/handlers/dynamic.rs +@@ -174,6 +174,10 @@ impl DynamicToolHandler { + } + + impl CoreToolRuntime for DynamicToolHandler { ++ fn is_client_tool_handoff(&self) -> bool { ++ true ++ } ++ + fn finishes_on_cancellation(&self) -> bool { + true + } +diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs +index 8ac6845..a040228 100644 +--- a/codex-rs/core/src/tools/handlers/mcp.rs ++++ b/codex-rs/core/src/tools/handlers/mcp.rs +@@ -307,6 +307,10 @@ impl McpHandler { + } + + impl CoreToolRuntime for McpHandler { ++ fn approves_inside_handler(&self) -> bool { ++ true ++ } ++ + fn environments_for_input( + &self, + invocation: &ToolInvocation, +diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +index 6a81b62..00af650 100644 +--- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs ++++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +@@ -309,6 +309,10 @@ fn one_shot_exec_command_spec(spec: ToolSpec) -> ToolSpec { + } + + impl CoreToolRuntime for ExecCommandHandler { ++ fn approves_inside_handler(&self) -> bool { ++ true ++ } ++ + fn telemetry_tags(&self, invocation: &ToolInvocation) -> ToolTelemetryTags { + // Record the configured mode so fallback calls remain in their rollout cohort. + // This does not indicate whether the command actually used a snapshot. +diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs +index 3344ed8..ea173f5 100644 +--- a/codex-rs/core/src/tools/mod.rs ++++ b/codex-rs/core/src/tools/mod.rs +@@ -14,6 +14,7 @@ mod multi_agent_tool; + mod native_tool_review; + pub(crate) use native_tool_review::review as review_native_tool_call; + pub(crate) mod network_approval; ++pub(crate) mod operation_observation; + pub(crate) mod orchestrator; + pub(crate) mod parallel; + pub(crate) mod registry; +diff --git a/codex-rs/core/src/tools/operation_observation.rs b/codex-rs/core/src/tools/operation_observation.rs +new file mode 100644 +index 0000000..d07036a +--- /dev/null ++++ b/codex-rs/core/src/tools/operation_observation.rs +@@ -0,0 +1,174 @@ ++//! Execution observations, separate from admission, approval, and result hooks. ++ ++use std::sync::Arc; ++use std::sync::Mutex; ++ ++use serde::Serialize; ++use serde_json::Value; ++ ++use crate::function_tool::FunctionCallError; ++use crate::operation_observation::OperationGuard; ++use crate::operation_observation::OperationKind; ++use crate::operation_observation::OperationMetadata; ++use crate::operation_observation::OperationObservation; ++use crate::operation_observation::OperationOutcome; ++use crate::tools::context::ToolCallSource; ++use crate::tools::context::ToolInvocation; ++use crate::tools::context::ToolOutput; ++use crate::tools::context::ToolPayload; ++use crate::tools::registry::CoreToolRuntime; ++ ++tokio::task_local! { ++ static CURRENT_EXECUTION: Arc; ++} ++ ++struct ExecutionObservation { ++ observation: Arc, ++ thread_id: String, ++ turn_id: String, ++ metadata: OperationMetadata, ++ input: Option, ++ guard: Mutex>, ++} ++ ++impl ExecutionObservation { ++ fn start(&self, input: Option<&Value>) { ++ let mut guard = self ++ .guard ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ // Sandbox retries still belong to the same native tool execution. ++ if guard.is_some() { ++ return; ++ } ++ let started = self.observation.start( ++ self.thread_id.clone(), ++ self.turn_id.clone(), ++ OperationKind::Tool, ++ self.metadata.clone(), ++ ); ++ if let Some(input) = input.or(self.input.as_ref()) { ++ started.capture_input(input); ++ } ++ *guard = Some(started); ++ } ++ ++ fn operation_id(&self) -> Option { ++ self.guard ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner) ++ .as_ref() ++ .map(|guard| guard.operation_id().to_string()) ++ } ++ ++ fn finish( ++ &self, ++ invocation: &ToolInvocation, ++ result: &Result, FunctionCallError>, ++ ) { ++ let guard = self ++ .guard ++ .lock() ++ .unwrap_or_else(std::sync::PoisonError::into_inner); ++ let Some(guard) = guard.as_ref() else { ++ // A rejected approval has not executed a tool. ++ return; ++ }; ++ let outcome = if invocation.cancellation_token.is_cancelled() { ++ OperationOutcome::Interrupted ++ } else if result ++ .as_ref() ++ .is_ok_and(|output| output.success_for_logging()) ++ { ++ OperationOutcome::Success ++ } else { ++ OperationOutcome::Error ++ }; ++ guard.finish_with_output_fn(outcome, || match result { ++ Ok(output) => output.code_mode_result(&invocation.payload), ++ Err(error) => serde_json::json!({"error": error.to_string()}), ++ }); ++ } ++} ++ ++/// Called immediately before an approved sandbox or MCP execution starts. ++pub(crate) fn start_current_execution() { ++ let _ = CURRENT_EXECUTION.try_with(|execution| execution.start(None)); ++} ++ ++/// MCP preparation can rewrite arguments before sending them to the server. ++pub(crate) fn start_current_execution_with_input(input: &impl Serialize) { ++ let _ = CURRENT_EXECUTION.try_with(|execution| { ++ let input = execution ++ .observation ++ .captures_content() ++ .then(|| serde_json::to_value(input).ok()) ++ .flatten(); ++ execution.start(input.as_ref()); ++ }); ++} ++ ++pub(crate) fn current_tool_operation_id() -> Option { ++ CURRENT_EXECUTION ++ .try_with(|execution| execution.operation_id()) ++ .ok() ++ .flatten() ++} ++ ++pub(crate) async fn observe_execution( ++ tool: &dyn CoreToolRuntime, ++ invocation: &ToolInvocation, ++) -> Result, FunctionCallError> { ++ let Some(observation) = invocation.turn.extension_data.get::() else { ++ return tool.handle(invocation.clone()).await; ++ }; ++ if tool.is_client_tool_handoff() { ++ return tool.handle(invocation.clone()).await; ++ } ++ ++ let parent_operation_id = match &invocation.source { ++ ToolCallSource::CodeMode { cell_id, .. } => invocation ++ .session ++ .services ++ .code_mode_service ++ .cell_originating_call(&codex_code_mode::CellId::new(cell_id.clone())) ++ .and_then(|origin| origin.operation_id), ++ _ => current_tool_operation_id(), ++ }; ++ let input = observation ++ .captures_content() ++ .then(|| match &invocation.payload { ++ ToolPayload::Function { arguments } => { ++ serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.clone())) ++ } ++ ToolPayload::Custom { input } => Value::String(input.clone()), ++ ToolPayload::ToolSearch { arguments } => { ++ serde_json::to_value(arguments).unwrap_or(Value::Null) ++ } ++ }); ++ let execution = Arc::new(ExecutionObservation { ++ observation, ++ thread_id: invocation.session.thread_id.to_string(), ++ turn_id: invocation.turn.sub_id.clone(), ++ metadata: OperationMetadata { ++ tool_name: Some(crate::tools::flat_tool_name(&invocation.tool_name).into_owned()), ++ call_id: Some(invocation.call_id.clone()), ++ parent_operation_id, ++ ..Default::default() ++ }, ++ input, ++ guard: Mutex::new(None), ++ }); ++ if !tool.approves_inside_handler() { ++ execution.start(None); ++ } ++ let result = CURRENT_EXECUTION ++ .scope(Arc::clone(&execution), tool.handle(invocation.clone())) ++ .await; ++ execution.finish(invocation, &result); ++ result ++} ++ ++#[cfg(test)] ++#[path = "operation_observation_tests.rs"] ++mod tests; +diff --git a/codex-rs/core/src/tools/operation_observation_tests.rs b/codex-rs/core/src/tools/operation_observation_tests.rs +new file mode 100644 +index 0000000..7fda319 +--- /dev/null ++++ b/codex-rs/core/src/tools/operation_observation_tests.rs +@@ -0,0 +1,291 @@ ++use super::*; ++use crate::operation_observation::OperationCaptureMode; ++use crate::operation_observation::OperationEvent; ++use crate::operation_observation::OperationObserver; ++use crate::operation_observation::OperationPhase; ++use crate::session::step_context::StepContext; ++use crate::session::tests::make_session_and_context; ++use crate::tools::context::FunctionToolOutput; ++use crate::tools::context::ToolCallOrigin; ++use crate::tools::registry::ToolExecutor; ++use crate::turn_diff_tracker::TurnDiffTracker; ++use pretty_assertions::assert_eq; ++use tokio::sync::Notify; ++use tokio_util::sync::CancellationToken; ++ ++#[derive(Default)] ++struct Events(Mutex>); ++ ++impl OperationObserver for Events { ++ fn emit(&self, event: OperationEvent) { ++ self.0.lock().unwrap().push(event); ++ } ++} ++ ++struct Handler { ++ approved: Option>, ++ entered: Option>, ++ complete: Option>, ++ denied: bool, ++ handoff: bool, ++ success: bool, ++} ++ ++impl Default for Handler { ++ fn default() -> Self { ++ Self { ++ approved: None, ++ entered: None, ++ complete: None, ++ denied: false, ++ handoff: false, ++ success: true, ++ } ++ } ++} ++ ++impl ToolExecutor for Handler { ++ fn tool_name(&self) -> codex_tools::ToolName { ++ codex_tools::ToolName::plain("test_tool") ++ } ++ ++ fn spec(&self) -> codex_tools::ToolSpec { ++ codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool { ++ name: "test_tool".to_string(), ++ description: "Observation fixture.".to_string(), ++ strict: false, ++ defer_loading: None, ++ parameters: codex_tools::JsonSchema::default(), ++ output_schema: None, ++ }) ++ } ++ ++ fn handle<'a>(&'a self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'a> ++ where ++ ToolInvocation: 'a, ++ { ++ Box::pin(async move { ++ if let Some(approved) = &self.approved { ++ approved.notified().await; ++ } ++ if self.denied { ++ return Err(FunctionCallError::RespondToModel( ++ "Denied fixture.".to_string(), ++ )); ++ } ++ if self.approved.is_some() { ++ start_current_execution(); ++ } ++ if let Some(entered) = &self.entered { ++ entered.notify_one(); ++ } ++ if let Some(complete) = &self.complete { ++ complete.notified().await; ++ } ++ Ok(Box::new(FunctionToolOutput::from_text( ++ "private-tool-result".to_string(), ++ Some(self.success), ++ )) as Box) ++ }) ++ } ++} ++ ++impl CoreToolRuntime for Handler { ++ fn approves_inside_handler(&self) -> bool { ++ self.approved.is_some() || self.denied ++ } ++ ++ fn is_client_tool_handoff(&self) -> bool { ++ self.handoff ++ } ++} ++ ++async fn fixture(mode: OperationCaptureMode) -> (ToolInvocation, Arc) { ++ let (session, turn) = make_session_and_context().await; ++ let events = Arc::new(Events::default()); ++ turn.extension_data ++ .insert(OperationObservation::new(mode, events.clone())); ++ let turn = Arc::new(turn); ++ let invocation = ToolInvocation { ++ session: Arc::new(session), ++ step_context: StepContext::for_test(Arc::clone(&turn)), ++ turn, ++ cancellation_token: CancellationToken::new(), ++ tracker: Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())), ++ call_id: "call-1".to_string(), ++ tool_name: codex_tools::ToolName::plain("test_tool"), ++ source: ToolCallSource::Direct, ++ payload: ToolPayload::Function { ++ arguments: r#"{"query":"private-tool-argument"}"#.to_string(), ++ }, ++ environments: Default::default(), ++ }; ++ (invocation, events) ++} ++ ++#[tokio::test] ++async fn approval_precedes_live_execution_start() { ++ let (invocation, events) = fixture(OperationCaptureMode::Content).await; ++ let approved = Arc::new(Notify::new()); ++ let entered = Arc::new(Notify::new()); ++ let complete = Arc::new(Notify::new()); ++ let handler = Handler { ++ approved: Some(Arc::clone(&approved)), ++ entered: Some(Arc::clone(&entered)), ++ complete: Some(Arc::clone(&complete)), ++ ..Default::default() ++ }; ++ let task = tokio::spawn(async move { observe_execution(&handler, &invocation).await }); ++ tokio::task::yield_now().await; ++ assert!(events.0.lock().unwrap().is_empty()); ++ approved.notify_one(); ++ entered.notified().await; ++ { ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 1); ++ assert_eq!(events[0].phase, OperationPhase::Started); ++ assert_eq!(events[0].metadata.call_id.as_deref(), Some("call-1")); ++ } ++ complete.notify_one(); ++ assert!(task.await.unwrap().is_ok()); ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[0].operation_id, events[1].operation_id); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); ++ let content = serde_json::to_string(&events[1]).unwrap(); ++ assert!(content.contains("private-tool-argument")); ++ assert!(content.contains("private-tool-result")); ++} ++ ++#[tokio::test] ++async fn denied_execution_and_client_handoff_emit_no_native_operation() { ++ let (invocation, events) = fixture(OperationCaptureMode::Content).await; ++ assert!( ++ observe_execution( ++ &Handler { ++ denied: true, ++ ..Default::default() ++ }, ++ &invocation ++ ) ++ .await ++ .is_err() ++ ); ++ assert!( ++ observe_execution( ++ &Handler { ++ handoff: true, ++ ..Default::default() ++ }, ++ &invocation ++ ) ++ .await ++ .is_ok() ++ ); ++ assert!(events.0.lock().unwrap().is_empty()); ++} ++ ++#[tokio::test] ++async fn metadata_omits_payloads_and_retains_failed_execution() { ++ let (invocation, events) = fixture(OperationCaptureMode::Metadata).await; ++ let registry = crate::tools::registry::ToolRegistry::with_handler_for_test(Arc::new(Handler { ++ success: false, ++ ..Default::default() ++ })); ++ assert!( ++ registry ++ .dispatch_any_with_state(invocation, None) ++ .await ++ .is_ok() ++ ); ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); ++ assert!(events[1].input.is_none()); ++ assert!(events[1].output.is_none()); ++ let serialized = serde_json::to_string(&*events).unwrap(); ++ assert!(!serialized.contains("private-tool-argument")); ++ assert!(!serialized.contains("private-tool-result")); ++} ++ ++#[tokio::test] ++async fn abort_emits_one_interrupted_terminal() { ++ let (invocation, events) = fixture(OperationCaptureMode::Metadata).await; ++ let entered = Arc::new(Notify::new()); ++ let handler = Handler { ++ entered: Some(Arc::clone(&entered)), ++ complete: Some(Arc::new(Notify::new())), ++ ..Default::default() ++ }; ++ let task = tokio::spawn(async move { observe_execution(&handler, &invocation).await }); ++ entered.notified().await; ++ task.abort(); ++ assert!(task.await.err().unwrap().is_cancelled()); ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[0].operation_id, events[1].operation_id); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); ++} ++ ++#[tokio::test] ++async fn detached_parallel_code_cell_tools_retain_the_issuing_operation() { ++ let (mut first, events) = fixture(OperationCaptureMode::Metadata).await; ++ let cell_id = codex_code_mode::CellId::new("cell-1".to_string()); ++ first ++ .session ++ .services ++ .code_mode_service ++ .mark_cell_ready_for_dispatch( ++ &cell_id, ++ Some(ToolCallOrigin { ++ operation_id: Some("issuing-exec-operation".to_string()), ++ item_id: None, ++ window_id: "window-1".to_string(), ++ }), ++ ); ++ first.source = ToolCallSource::CodeMode { ++ cell_id: "cell-1".to_string(), ++ runtime_tool_call_id: "nested-1".to_string(), ++ }; ++ let mut second = first.clone(); ++ second.call_id = "call-2".to_string(); ++ second.source = ToolCallSource::CodeMode { ++ cell_id: "cell-1".to_string(), ++ runtime_tool_call_id: "nested-2".to_string(), ++ }; ++ let first_entered = Arc::new(Notify::new()); ++ let second_entered = Arc::new(Notify::new()); ++ let complete = Arc::new(Notify::new()); ++ let first_handler = Handler { ++ entered: Some(Arc::clone(&first_entered)), ++ complete: Some(Arc::clone(&complete)), ++ ..Default::default() ++ }; ++ let second_handler = Handler { ++ entered: Some(Arc::clone(&second_entered)), ++ complete: Some(Arc::clone(&complete)), ++ ..Default::default() ++ }; ++ let first_task = tokio::spawn(async move { observe_execution(&first_handler, &first).await }); ++ let second_task = ++ tokio::spawn(async move { observe_execution(&second_handler, &second).await }); ++ first_entered.notified().await; ++ second_entered.notified().await; ++ assert_eq!(events.0.lock().unwrap().len(), 2); ++ complete.notify_waiters(); ++ assert!(first_task.await.unwrap().is_ok()); ++ assert!(second_task.await.unwrap().is_ok()); ++ let events = events.0.lock().unwrap(); ++ let starts: Vec<_> = events ++ .iter() ++ .filter(|event| event.phase == OperationPhase::Started) ++ .collect(); ++ assert_eq!(starts.len(), 2); ++ assert_ne!(starts[0].operation_id, starts[1].operation_id); ++ for event in starts { ++ assert_eq!( ++ event.metadata.parent_operation_id.as_deref(), ++ Some("issuing-exec-operation") ++ ); ++ } ++} +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index a1a88e4..d9afeb6 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -102,6 +102,7 @@ impl ToolOrchestrator { + .as_ref() + .map(ActiveNetworkApproval::execution_proxy), + }; ++ super::operation_observation::start_current_execution(); + let run_result = tool + .run(req, &attempt_with_network_approval, &attempt_tool_ctx) + .await; +diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs +index 3dccc2e..bbbf568 100644 +--- a/codex-rs/core/src/tools/registry.rs ++++ b/codex-rs/core/src/tools/registry.rs +@@ -62,6 +62,16 @@ pub use codex_tools::ToolExposure; + /// Implementers provide the shared `ToolExecutor` behavior plus optional + /// core-owned metadata for hooks, telemetry, tool search, and argument diffs. + pub(crate) trait CoreToolRuntime: ToolExecutor { ++ /// Client callbacks execute outside this process and are observed by their owner. ++ fn is_client_tool_handoff(&self) -> bool { ++ false ++ } ++ ++ /// Defers the execution observation until the handler's approval has completed. ++ fn approves_inside_handler(&self) -> bool { ++ false ++ } ++ + /// Opts this invocation into synchronous native-call review before the handler runs. + fn requires_guardian_review(&self, _invocation: &ToolInvocation) -> bool { + false +@@ -1050,7 +1060,7 @@ async fn handle_any_tool( + ) -> Result { + let call_id = invocation.call_id.clone(); + let payload = invocation.payload.clone(); +- let output = tool.handle(invocation.clone()).await?; ++ let output = super::operation_observation::observe_execution(tool, &invocation).await?; + let post_tool_use_payload = + CoreToolRuntime::post_tool_use_payload(tool, &invocation, output.as_ref()); + let result = AnyToolResult { +diff --git a/codex-rs/core/src/turn_extension_data.rs b/codex-rs/core/src/turn_extension_data.rs +index 89f1127..e8990d2 100644 +--- a/codex-rs/core/src/turn_extension_data.rs ++++ b/codex-rs/core/src/turn_extension_data.rs +@@ -31,3 +31,21 @@ impl From for WithTurnExtensionData { + } + } + } ++ ++/// Carries start-only attachment policy through the existing submission envelope. ++/// PreparedTurnInputSettings unwraps this before storing the original initializer. ++#[derive(Debug)] ++pub(crate) struct NewTurnOnlyData(pub(crate) ExtensionDataInit); ++ ++impl WithTurnExtensionData { ++ /// Replaces saved host data only when this request starts a new turn. ++ /// Steering preserves the previously saved initializer. ++ pub fn new_turn_only( ++ request: codex_protocol::turn_input::TurnInputRequest, ++ turn_extension_init: ExtensionDataInit, ++ ) -> Self { ++ let mut init = ExtensionDataInit::new(); ++ init.insert(NewTurnOnlyData(turn_extension_init)); ++ Self::new(request, init) ++ } ++} +diff --git a/codex-rs/core/src/turn_timing.rs b/codex-rs/core/src/turn_timing.rs +index de45b74..9dd25e6 100644 +--- a/codex-rs/core/src/turn_timing.rs ++++ b/codex-rs/core/src/turn_timing.rs +@@ -492,6 +492,7 @@ fn response_event_records_turn_ttft(event: &ResponseEvent) -> bool { + | ResponseEvent::ReasoningContentDelta { .. } => true, + ResponseEvent::Created { .. } + | ResponseEvent::ServerModel(_) ++ | ResponseEvent::ModelObservation(_) + | ResponseEvent::ModelVerifications(_) + | ResponseEvent::TurnModerationMetadata(_) + | ResponseEvent::GuardianAssessment(_) +diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs +index 43cf70b..d692465 100644 +--- a/codex-rs/exec/src/lib.rs ++++ b/codex-rs/exec/src/lib.rs +@@ -1238,6 +1238,7 @@ async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> { + ClientRequest::TurnStart { + request_id: request_ids.next(), + params: TurnStartParams { ++ operation_tracing: None, + disabled_plugin_ids: None, + thread_id: primary_thread_id_for_span.clone(), + turn_trigger: Some("exec".to_string()), +diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs +index ced543a..193452e 100644 +--- a/codex-rs/otel/src/events/session_telemetry.rs ++++ b/codex-rs/otel/src/events/session_telemetry.rs +@@ -1406,7 +1406,9 @@ impl SessionTelemetry { + ResponseEvent::ReasoningSummaryPartAdded { .. } => { + "reasoning_summary_part_added".into() + } +- ResponseEvent::ServerModel(_) => "server_model".into(), ++ ResponseEvent::ServerModel(_) | ResponseEvent::ModelObservation(_) => { ++ "server_model".into() ++ } + ResponseEvent::ModelVerifications(_) => "model_verifications".into(), + ResponseEvent::TurnModerationMetadata(_) => "turn_moderation_metadata".into(), + ResponseEvent::GuardianAssessment(_) => "guardian_assessment".into(), +diff --git a/codex-rs/tui/src/app/app_server_event_targets.rs b/codex-rs/tui/src/app/app_server_event_targets.rs +index 13d30c2..ca841f5 100644 +--- a/codex-rs/tui/src/app/app_server_event_targets.rs ++++ b/codex-rs/tui/src/app/app_server_event_targets.rs +@@ -137,6 +137,10 @@ pub(super) fn server_notification_thread_target( + ServerNotification::ReasoningTextDelta(notification) => { + Some(notification.thread_id.as_str()) + } ++ ServerNotification::TurnOperationStarted(notification) ++ | ServerNotification::TurnOperationCompleted(notification) => { ++ Some(notification.thread_id.as_str()) ++ } + ServerNotification::ContextCompacted(notification) => Some(notification.thread_id.as_str()), + ServerNotification::ModelRerouted(notification) => Some(notification.thread_id.as_str()), + ServerNotification::ModelVerification(notification) => { +diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs +index be7da50..40e6c3f 100644 +--- a/codex-rs/tui/src/chatwidget/protocol.rs ++++ b/codex-rs/tui/src/chatwidget/protocol.rs +@@ -388,6 +388,8 @@ impl ChatWidget { + | ServerNotification::ThreadAttachmentUpdated(_) + | ServerNotification::RawResponseItemCompleted(_) + | ServerNotification::RawResponseCompleted(_) ++ | ServerNotification::TurnOperationStarted(_) ++ | ServerNotification::TurnOperationCompleted(_) + | ServerNotification::CommandExecOutputDelta(_) + | ServerNotification::ProcessOutputDelta(_) + | ServerNotification::ProcessExited(_) diff --git a/patches/codex/README.md b/patches/codex/README.md new file mode 100644 index 0000000000..c4adc2a751 --- /dev/null +++ b/patches/codex/README.md @@ -0,0 +1,139 @@ +# Codex operation tracing patch guide + +This patch adds private model and native tool observations to Codex 0.166.0. See the validation record for completed checks. + +Apply `patches/codex/0001-operation-tracing.patch` to `rust-v0.166.0` at `6ff1e1ca3b30fc8b914720681fcf5fe62e95f856`. Keep Python responsible for canonical Agents SDK spans and all OpenTelemetry export. Supply Rust observations for the active turn and nested native tools. Exclude independent agents and gateway propagation. Keep Rust exporter setup and PyO3 bindings outside this patch. + +## Wire contract + +Add an experimental `operationTracing` field to `turn/start` with `metadata` or `content`. Echo the effective mode in the reply. Ask for metadata by default when Python tracing is active. Python already holds the SDK-visible input and reply and can populate the existing SDK fields locally. Ask for content only with explicit per-run `"content"` and permission to trace sensitive data. With tracing disabled, omit the field and emit zero observations. Identify stock fallback as partial coverage. + +Emit `turn/operation/started` immediately, followed by `turn/operation/completed`. Include `operationId`, `threadId`, `turnId`, optional `parentOperationId`, and a `model` or `tool` kind. Identify each nested tool's parent. Include source start and end times, elapsed time from a monotonic clock, and a `success`, `error`, or `interrupted` outcome. Include the model name, source response ID subject to the SDK trust policy, and per-call usage when available. For a tool, include its name and call ID. + +Metadata example without captured content. + +```json +[ + { + "method": "turn/operation/started", + "params": { + "operationId": "op-1", + "threadId": "thread-1", + "turnId": "turn-1", + "kind": "model", + "requestedModel": "example-model", + "startedAt": "2026-11-20T00:00:00.000Z" + } + }, + { + "method": "turn/operation/completed", + "params": { + "operationId": "op-1", + "threadId": "thread-1", + "turnId": "turn-1", + "kind": "model", + "requestedModel": "example-model", + "startedAt": "2026-11-20T00:00:00.000Z", + "completedAt": "2026-11-20T00:00:01.000Z", + "durationMs": 1000, + "outcome": "success" + } + } +] +``` + +Extend `TurnStartParams`, `TurnStartResponse`, and the notification registry. Keep payload fields exclusive to effective content mode. [Turn protocol](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/app-server-protocol/src/protocol/v2/turn.rs) · [Notification registry](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/app-server-protocol/src/protocol/common.rs) + +## Source observations + +Capture HTTP and WebSocket requests after final assembly and input bounding in `core/src/client.rs`. Include exact submitted values within the capture budget and `previous_response_id` for continuation. Label wire deltas separately from expanded context. Group identical-input retries under one operation and split after changed assembled input. End model timing at stream termination before draining tools. Preserve requested and observed model identity when available. [Client](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/core/src/client.rs) · [Sampling](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/core/src/session/turn.rs) · [Retry guidance](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/core/src/responses_retry.rs) + +Observe native execution at the tool registry, with deferred start points immediately after authorization for sandboxed tools and MCP. Project actual results at the registry result boundary, including nested code-mode results. Keep dynamic requests as Python-owned callback interrupts and trace callback execution in Python. [Lifecycle types](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/ext/extension-api/src/contributors/tool_lifecycle.rs) · [Registry](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/core/src/tools/registry.rs) + +## Privacy and lifecycle + +Bind observations before sampling to the initiating opted-in connection. Deliver exclusively to that connection, including multiple-subscriber cases. Keep captured data in memory and exclude it from rollout persistence and logs. Existing outgoing helpers have been logging notifications and broadcasting to empty recipient lists. Add explicit targeting and redacted logging. [Outgoing delivery](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/app-server/src/outgoing_message.rs) + +Enforce a 32 KiB UTF-8 serialized capture budget per direction per operation before sending. Mark missing content with `omitted` and shortened content with `truncated`, including the reason. Describe completeness accurately. + +Use one finalizer per operation. Source turn completion closes private queue admission and awaits its drain before publishing `turn/completed`, including interrupted turns. Empty turns also close their observers. On child-process termination without a Rust terminal, close open IDs in Python with interruption and incomplete-observation metadata. + +## Build and checks + +Use the pinned Rust 1.95.0 toolchain and build prerequisites, including `just`. Replace the absolute paths below. Require a fresh checkout and the exact SHA before application. [Installation](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/docs/install.md) + +```sh +set -eu +export CARGO_PROFILE_DEV_DEBUG=0 +export CARGO_PROFILE_TEST_DEBUG=0 +export RUST_MIN_STACK=8388608 +SDK_ROOT=/path/to/sdk +CODEX_CHECKOUT=/path/to/fresh-codex +PATCH_FILE="$SDK_ROOT/patches/codex/0001-operation-tracing.patch" +git clone --branch rust-v0.166.0 --single-branch https://github.com/openai/codex.git "$CODEX_CHECKOUT" +cd "$CODEX_CHECKOUT" +test "$(git rev-parse HEAD)" = 6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 +test -z "$(git status --porcelain)" +git apply --check "$PATCH_FILE" +git apply "$PATCH_FILE" +just write-app-server-schema +just write-app-server-schema --experimental +cd "$CODEX_CHECKOUT/codex-rs" +cargo build -p codex-cli --bin codex +CODEX_TRACING_TEST_EXECUTABLE="$CODEX_CHECKOUT/codex-rs/target/debug/codex" +"$CODEX_TRACING_TEST_EXECUTABLE" --version +cd "$CODEX_CHECKOUT" +just fmt-check +git diff --check +cd "$CODEX_CHECKOUT/codex-rs" +cargo test -p codex-core -p codex-app-server --lib operation_observation +cargo test -p codex-core --lib new_turn_only +cargo test -p codex-app-server-protocol --lib +cargo test -p codex-api --lib process_sse_reports_body_model_without_routing_authority +cd "$SDK_ROOT" +CODEX_TRACING_TEST_EXECUTABLE="$CODEX_TRACING_TEST_EXECUTABLE" uv run --extra opentelemetry python -m pytest tests/models/test_operation_tracing_parity.py +``` + +For the optional real-stock gate on Linux x86_64, download the pinned official asset below. Choose the corresponding platform asset from the same release on other hosts. Keep the patched executable selected separately. + +```sh +STOCK_DIRECTORY=/path/to/codex-stock-0.166.0 +mkdir -p "$STOCK_DIRECTORY" +curl --fail --location --output "$STOCK_DIRECTORY/codex.tar.gz" \ + https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-x86_64-unknown-linux-musl.tar.gz +tar -xzf "$STOCK_DIRECTORY/codex.tar.gz" -C "$STOCK_DIRECTORY" +CODEX_TRACING_STOCK_EXECUTABLE="$STOCK_DIRECTORY/codex-x86_64-unknown-linux-musl" +"$CODEX_TRACING_STOCK_EXECUTABLE" --version +CODEX_TRACING_TEST_EXECUTABLE="$CODEX_TRACING_TEST_EXECUTABLE" CODEX_TRACING_STOCK_EXECUTABLE="$CODEX_TRACING_STOCK_EXECUTABLE" uv run --extra opentelemetry python -m pytest tests/models/test_operation_tracing_parity.py +``` + +Require the executable and mode acknowledgment in full integration tests. Cover default, disabled, metadata, and content modes with SDK tracing and OpenTelemetry separately and together. Verify starts, retries, nested tools, cancellation, source timing, and owner isolation. Use a synthetic canary to detect content in other clients, logs, or stored telemetry. Keep Rust regressions inside the patch. + +The executable embeds generated exports, so run both generators before building. The shipped patch contains hand-maintained source and tests only. The stable generator updates JSON/TypeScript fixtures, stable exports, and Python SDK types. The experimental generator updates `codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst`. Include generated outputs in the future upstream Codex PR. [Schema generator](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/app-server-protocol/scripts/write_schema_fixtures.py) + +## Validation + +Require the shared matrix to pass against the selected patched binary. With `CODEX_TRACING_STOCK_EXECUTABLE`, also verify explicit partial coverage on stock 0.166.0. Check live start/terminal ordering and closure after child-process interruption. The steering regressions must preserve the current turn's owner, reject future observation authority from an unacknowledged steer, and retain the existing `WithTurnExtensionData::new` behavior for other callers. + +Verify a fresh apply followed by both generators against the complete source tree used for the build. Keep hashes, timings, logs, and the resulting tree comparison in the PR validation record. + +## Refresh and upstream handoff + +Review source, tests, and generated outputs in the Codex checkout. Mark new Rust files with `git add -N` before refreshing the SDK-maintained patch. Keep generated files available for the future upstream PR and exclude them from this reviewable patch. Use the paths from the build commands above. + +```sh +set -eu +cd "$CODEX_CHECKOUT" +base=6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 +git diff --check "$base" +candidate=$(mktemp "${PATCH_FILE}.XXXXXX") +git -c diff.suppressBlankEmpty=true diff --binary "$base" -- . \ + ':!codex-rs/Cargo.lock' \ + ':!codex-rs/app-server-protocol/schema/**' \ + ':!sdk/python/src/openai_codex/generated/**' > "$candidate" +test -s "$candidate" +git apply --stat "$candidate" +mv "$candidate" "$PATCH_FILE" +``` + +Repeat the clone, apply, build, and checks with another unused checkout path. Prepare an upstream handoff containing the patch, base revision, wire examples, privacy decisions, and recorded validation. Remove the local patch after an upstream release with equivalent behavior passes integration. Retain capability negotiation and partial fallback during migration. diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py new file mode 100644 index 0000000000..ebe6d69965 --- /dev/null +++ b/src/agents/models/_codex_tracing.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from openai.types.responses import Response + +from ..tracing import ( + Span, + custom_span, + function_span, + get_current_span, + get_current_trace, + response_span, +) +from ..tracing._content import bound_content +from ..tracing._model import acquire_model_span, finish_model_span, record_model_input +from ..tracing.config import get_operation_detail +from ..tracing.provider import _is_noop_span, _is_noop_trace +from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData +from ..tracing.util import time_iso +from .interface import ModelTracing + + +@dataclass +class _Operation: + span: Span[Any] + kind: str + ended_at: str | None = None + + +class CodexOperationTracing: + """Translate connection-private operation facts into the current SDK trace.""" + + def __init__(self, tracing: ModelTracing, input: Any, model: str | None) -> None: + self._tracing = tracing + self._detail = get_operation_detail(include_data=tracing.include_data()) + self._input = input + self._model = model + self._parent = get_current_span() + active_trace = get_current_trace() + disabled = ( + tracing.is_disabled() + or active_trace is None + or _is_noop_trace(active_trace) + or _is_noop_span(self._parent) + ) + self.mode = None if disabled else self._detail or "metadata" + self._thread_id: str | None = None + self._turn_id: str | None = None + self._acknowledged = False + self._accepted_mode: str | None = None + self._operations: dict[str, _Operation] = {} + self._completed: dict[str, str] = {} + self._pending: _Operation | None = None + self._first_model = True + self._coverage_reason: str | None = None + self._coverage_emitted = False + + def request_mode(self) -> str | None: + # A hook may disable the provider after the active trace and parent were created. + if self.mode is not None and _is_noop_span(response_span(parent=self._parent)): + self.mode = None + return self.mode + + def bind(self, thread_id: str, turn_id: str, accepted_mode: Any) -> None: + self._thread_id, self._turn_id = thread_id, turn_id + self._acknowledged = self.mode is not None and ( + accepted_mode == self.mode or accepted_mode == "metadata" + ) + self._accepted_mode = accepted_mode if self._acknowledged else None + if self.mode is not None and not self._acknowledged: + self._coverage_reason = "unsupported" + elif self._acknowledged and accepted_mode != self.mode: + self._coverage_reason = "capture_downgraded" + + def observe(self, method: str, event: Any) -> None: + if self.mode is None: + return + try: + self._observe(method, event) + except Exception: + # Telemetry must never replay a model request or interrupt tool execution. + self._coverage_reason = "telemetry_error" + + def _observe(self, method: str, event: dict[str, Any]) -> None: + if not self._acknowledged: + self._coverage_reason = "telemetry_error" + return + if event.get("threadId") != self._thread_id or event.get("turnId") != self._turn_id: + self._coverage_reason = "telemetry_error" + return + operation_id = event["operationId"] + if not isinstance(operation_id, str) or not operation_id: + raise ValueError("Missing operation identity.") + if method == "turn/operation/started": + if operation_id in self._operations or operation_id in self._completed: + raise ValueError("Duplicate operation start.") + kind = event["kind"] + started_at = self._timestamp(event["startedAt"]) + parent_id = event.get("parentOperationId") + parent = self._parent + if parent_id is not None: + if parent_id in self._operations: + parent = self._operations[parent_id].span + else: + # Only identity is needed when a later child refers to a completed parent. + parent = function_span( + "codex.operation.parent", + span_id=self._completed[parent_id], + parent=self._parent, + ) + span: Span[Any] + if kind == "model": + self._finish_pending() + span = acquire_model_span( + self._tracing, + event.get("requestedModel") or self._model, + parent=parent, + started_at=started_at, + ) + if isinstance(span.span_data, ResponseSpanData) and self._first_model: + if self._tracing.include_data(): + span.span_data.input = self._input + self._first_model = False + elif kind == "tool": + span = function_span(event["toolName"], parent=parent) + span.span_data.operation.detail = self._detail + else: + raise ValueError("Unknown operation kind.") + operation = _Operation(span=span, kind=kind) + self._operations[operation_id] = operation + self._metadata(operation, event) + self._content(operation, event) + if kind == "tool": + span.start_at(started_at) + return + operation = self._operations[operation_id] + if operation.ended_at is not None or event.get("kind") != operation.kind: + raise ValueError("Invalid operation terminal.") + ended_at = self._timestamp(event["completedAt"]) + outcome = event["outcome"] + if outcome not in ("success", "error", "interrupted"): + raise ValueError("Unknown operation outcome.") + self._metadata(operation, event) + self._content(operation, event) + operation.ended_at = ended_at + operation.span.span_data.operation.outcome = { + "success": "completed", + "error": "failed", + }.get(outcome, outcome) + if outcome != "success": + operation.span.set_error({"message": f"Codex operation {outcome}.", "data": None}) + if operation.kind == "model": + self._finish_pending() + self._pending = operation + else: + operation.span.finish_at(operation.ended_at) + self._completed[operation_id] = operation.span.span_id + self._operations.pop(operation_id) + + @staticmethod + def _timestamp(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("Missing source timestamp.") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("Source timestamp must include timezone.") + return value + + def _metadata(self, operation: _Operation, event: dict[str, Any]) -> None: + data = operation.span.span_data + metadata = data.operation + metadata.operation_id = event["operationId"] + metadata.model = event.get("requestedModel") or metadata.model or self._model + metadata.response_model = event.get("observedModel") or metadata.response_model + metadata.tool_call_id = event.get("callId") or metadata.tool_call_id + if self._tracing.include_data() or event.get("responseIdTrusted") is True: + metadata.response_id = event.get("responseId") or metadata.response_id + else: + metadata.response_id = None + usage = event.get("usage") + if operation.kind == "model": + if isinstance(data, GenerationSpanData): + data.model = metadata.model + if isinstance(usage, dict): + data.usage = {key: value for key, value in usage.items() if isinstance(value, int)} + + def _content(self, operation: _Operation, event: dict[str, Any]) -> None: + if self._detail != "content" or self._accepted_mode != "content": + return + data = operation.span.span_data + for direction in ("input", "output"): + projection = event.get(direction) + if not isinstance(projection, dict): + continue + value = projection.get("value") + marker: dict[str, Any] = { + key: True for key in ("truncated", "omitted") if projection.get(key) is True + } + if marker and isinstance(projection.get("reason"), str): + marker["reason"] = projection["reason"] + if isinstance(data, GenerationSpanData) and direction == "input": + request = dict(value) if isinstance(value, dict) else {"input": value} + request.update(marker) + record_model_input(data, request) + continue + value = bound_content(value) + if value is None and marker: + value = marker + if isinstance(data, FunctionSpanData): + setattr(data, direction, value if isinstance(value, str) else json.dumps(value)) + elif isinstance(data, GenerationSpanData): + items = value if isinstance(value, list) else [{"content": value}] + if marker and marker not in items: + items = [*items, marker] + data.output = bound_content(items) + + def _finish_pending(self) -> None: + if self._pending is not None: + pending, self._pending = self._pending, None + finish_model_span(pending.span, ended_at=pending.ended_at) + + def close(self, *, interrupted: bool) -> None: + if self.mode is None: + return + estimated_end = time_iso() + for operation_id, operation in self._operations.items(): + if operation.ended_at is not None: + continue + operation.ended_at = estimated_end + operation.span.span_data.operation.outcome = "interrupted" + operation.span.span_data.operation.estimated_end = True + operation.span.set_error({"message": "Codex operation interrupted.", "data": None}) + if operation.kind == "model": + finish_model_span(operation.span, ended_at=operation.ended_at) + else: + operation.span.finish_at(operation.ended_at) + self._completed[operation_id] = operation.span.span_id + self._coverage_reason = "interrupted" + self._operations.clear() + if interrupted: + if not self._completed: + self._coverage_reason = self._coverage_reason or "interrupted" + self._finish_pending() + self._emit_coverage() + + def finish_response(self, response: Response) -> None: + if self.mode is not None and self._acknowledged and self._first_model: + self._coverage_reason = "missing_operations" + if self._pending is not None: + data = self._pending.span.span_data + if isinstance(data, ResponseSpanData) and self._tracing.include_data(): + data.response = response + self._finish_pending() + self.close(interrupted=False) + + def _emit_coverage(self) -> None: + if self._coverage_reason is None or self._coverage_emitted: + return + self._coverage_emitted = True + with custom_span( + "codex.tracing.coverage", + {"coverage": "partial", "provider": "codex", "reason": self._coverage_reason}, + parent=self._parent, + ): + pass diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 7d3c114961..8d059a2d05 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -41,6 +41,7 @@ from ..usage import Usage, _make_input_tokens_details, _response_usage_to_usage from ._codex_app_server import CodexAppServer, resolve_codex_executable from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call +from ._codex_tracing import CodexOperationTracing from .interface import Model, ModelProvider, ModelTracing @@ -311,6 +312,7 @@ async def stream_response( overrides, provider._env, ) + observations = CodexOperationTracing(tracing, input, self._name) output: dict[str, Any] = {} streamed: set[str] = set() indices: dict[str, int] = {} @@ -322,6 +324,7 @@ async def stream_response( raw_usage = None reasoning_summaries: list[str] = [] usage = None + sdk_done_event = None try: if provider._api_key is not None: await server.request( @@ -516,7 +519,12 @@ async def stream_response( turn_params["outputSchema"] = output_schema.json_schema() if sdk_sandbox: turn_params["environments"] = [] - turn_id = _id(await server.request("turn/start", turn_params), "turn") + operation_mode = observations.request_mode() + if operation_mode is not None: + turn_params["operationTracing"] = operation_mode + turn_result = await server.request("turn/start", turn_params) + turn_id = _id(turn_result, "turn") + observations.bind(thread_id, turn_id, turn_result.get("operationTracing")) response = Response( id=turn_id, created_at=time.time(), @@ -534,6 +542,12 @@ async def stream_response( sequence += 1 async for event in server.events(): params = event.get("params") or {} + if event.get("method") in ( + "turn/operation/started", + "turn/operation/completed", + ): + observations.observe(event["method"], params) + continue if "id" in event: if ( event["method"] != "item/tool/call" @@ -774,34 +788,39 @@ async def stream_response( type="response.output_item.done", ) sequence += 1 + if not completed: + raise RuntimeError("Codex finished without a completed response.") + if require_sdk_call and sdk_call is None: + raise UserError( + "CodexProvider tool_choice='required' did not return " + "an SDK tool or handoff call." + ) + if sdk_call is None and not output: + raise RuntimeError("Codex finished without a completed response.") + items = list(output.values()) + ([sdk_call] if sdk_call is not None else []) + if reasoning_summaries: + items[-1].codex_reasoning_summary = reasoning_summaries + if sdk_call is not None: + assert isinstance(sdk_call.id, str) + sdk_done_event = ResponseOutputItemDoneEvent( + item=sdk_call, + output_index=indices[sdk_call.id], + sequence_number=sequence, + type="response.output_item.done", + ) + sequence += 1 + items.sort(key=lambda item: indices[item.id]) + response = response.model_copy( + update={"output": items, "status": "completed", "usage": usage} + ) + if raw_usage is not None: + response.__dict__["codex_raw_usage"] = raw_usage + observations.finish_response(response) finally: + observations.close(interrupted=True) await server.close() - if not completed: - raise RuntimeError("Codex finished without a completed response.") - if require_sdk_call and sdk_call is None: - raise UserError( - "CodexProvider tool_choice='required' did not return an SDK tool or handoff call." - ) - if sdk_call is None and not output: - raise RuntimeError("Codex finished without a completed response.") - items = list(output.values()) + ([sdk_call] if sdk_call is not None else []) - if reasoning_summaries: - items[-1].codex_reasoning_summary = reasoning_summaries - if sdk_call is not None: - assert isinstance(sdk_call.id, str) - yield ResponseOutputItemDoneEvent( - item=sdk_call, - output_index=indices[sdk_call.id], - sequence_number=sequence, - type="response.output_item.done", - ) - sequence += 1 - items.sort(key=lambda item: indices[item.id]) - response = response.model_copy( - update={"output": items, "status": "completed", "usage": usage} - ) - if raw_usage is not None: - response.__dict__["codex_raw_usage"] = raw_usage + if sdk_done_event is not None: + yield sdk_done_event yield ResponseCompletedEvent( response=response, sequence_number=sequence, type="response.completed" ) diff --git a/tests/models/test_codex_operation_retry_boundary.py b/tests/models/test_codex_operation_retry_boundary.py new file mode 100644 index 0000000000..db03b22e93 --- /dev/null +++ b/tests/models/test_codex_operation_retry_boundary.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from agents import Agent, ModelSettings, RunConfig, Runner +from agents.models.codex import CodexProvider +from agents.retry import ModelRetrySettings, RetryDecision +from tests.models.test_codex_provider import FakeServer, OperationServer, servers as servers +from tests.testing_processor import fetch_ordered_spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "scenario", ["same_attempt", "multiple_before_retry", "single_before_retry"] +) +async def test_distinct_codex_inferences_do_not_merge_across_provider_or_runner_attempts( + tmp_path: Path, servers: Any, scenario: str +) -> None: + # App Server observations preserve internal inference boundaries that ScriptedModel cannot emit. + class InferenceServer(OperationServer): + def __init__(self, outcomes: list[str], complete: bool) -> None: + super().__init__(capture="content") + self.outcomes = outcomes + self.complete = complete + + async def events(self): + for index, outcome in enumerate(self.outcomes): + operation = { + "threadId": "thread_1", + "turnId": "turn_1", + "operationId": f"inference-{index}", + "kind": "model", + "requestedModel": "test-model", + "startedAt": f"2026-11-20T00:00:0{index}.000Z", + "input": {"value": {"input": [{"role": "user", "content": f"input-{index}"}]}}, + } + yield {"method": "turn/operation/started", "params": operation} + terminal = { + **operation, + "completedAt": f"2026-11-20T00:00:0{index}.250Z", + "outcome": outcome, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + if not self.complete: + terminal.update( + observedModel="stale-model", + responseId="stale-response", + output={"value": [{"role": "assistant", "content": "stale-output"}]}, + ) + yield {"method": "turn/operation/completed", "params": terminal} + if self.complete: + async for event in FakeServer.events(self): + yield event + + queue, _ = servers + if scenario == "single_before_retry": + queue.extend([InferenceServer(["error"], False), InferenceServer(["success"], True)]) + elif scenario == "multiple_before_retry": + queue.extend( + [InferenceServer(["success", "error"], False), InferenceServer(["success"], True)] + ) + else: + queue.append(InferenceServer(["error", "success"], True)) + agent = Agent( + name="boundaries", + model_settings=ModelSettings( + retry=ModelRetrySettings( + max_retries=1, + policy=lambda _: RetryDecision(retry=True, approve_unsafe_replay=True), + backoff={"initial_delay": 0}, + ) + ), + ) + result = await Runner.run( + agent, + "initial input", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "content"} + ), + ) + assert result.final_output == "A Codex response" + models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] + assert ( + len(models) + == {"same_attempt": 2, "multiple_before_retry": 3, "single_before_retry": 1}[scenario] + ) + assert len({span.span_id for span in models}) == len(models) + assert [span.span_data.operation.outcome for span in models].count("failed") == ( + 0 if scenario == "single_before_retry" else 1 + ) + assert all(span.ended_at is not None for span in models) + assert all( + span.span_data.usage["input_tokens"] == (20 if scenario == "single_before_retry" else 10) + for span in models + ) + if scenario == "single_before_retry": + assert models[0].error is None + assert models[0].span_data.output is None + assert models[0].span_data.operation.response_model is None + assert models[0].span_data.operation.response_id is None diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index c91c024683..f270ae3598 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1278,3 +1278,737 @@ async def events(self): else: await Runner.run(agent, "Use a tool.", run_config=config(tmp_path)) assert server.closed + + +class OperationServer(FakeServer): + # This double preserves the negotiated wire and source timing at the real Runner boundary. + def __init__(self, *, capture="metadata", malformed=False, active=False): + super().__init__() + self.capture = capture + self.malformed = malformed + self.active = active + + async def request(self, method, params): + result = await super().request(method, params) + if method == "turn/start": + result["operationTracing"] = self.capture + return result + + async def events(self): + common = {"threadId": "thread_1", "turnId": "turn_1"} + for index, kind in enumerate(["model", "tool", "model"], 1): + operation = { + **common, + "operationId": f"operation_{index}", + "kind": kind, + "requestedModel": "requested-model", + "startedAt": f"2026-11-20T00:00:0{index}.000Z", + } + if kind == "tool": + operation.update(toolName="read_file", callId="native_call") + if self.capture == "content": + operation["input"] = { + "value": {"input": [{"role": "user", "content": "native_input_secret"}]} + if kind == "model" + else {"path": "native_arg_secret"}, + "truncated": False, + "omitted": False, + } + yield {"method": "turn/operation/started", "params": operation} + if self.active: + async for event in super().events(): + yield event + return + terminal = { + **operation, + "completedAt": f"2026-11-20T00:00:0{index}.250Z", + "durationMs": 250, + "outcome": "success", + } + if kind == "model": + terminal.update( + responseId=f"resp_source_{index}", + responseIdTrusted=False, + observedModel="observed-model", + usage={"input_tokens": index * 10, "output_tokens": index * 5}, + ) + if self.capture == "content": + terminal["output"] = { + "value": [{"role": "assistant", "content": "native_result_secret"}] + if kind == "model" + else "native_result_secret", + "truncated": False, + "omitted": False, + } + yield {"method": "turn/operation/completed", "params": terminal} + if self.malformed: + yield {"method": "turn/operation/completed", "params": terminal} + yield { + "method": "turn/operation/started", + "params": {**operation, "threadId": "another_thread"}, + } + async for event in super().events(): + yield event + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("truncated", [False, True]) +async def test_codex_interrupted_visible_text_reaches_otel(tmp_path, servers, streamed, truncated): + import json + + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from agents.extensions.tracing import OpenTelemetryTracingProcessor + from agents.tracing import set_trace_processors + from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans + + # This wire double preserves the visible partial item produced before OutputItemDone. + partial = { + "type": "output_text", + "text": "visible partial answer", + "partial": True, + "truncated": truncated, + } + + class InterruptedServer(OperationServer): + async def events(self): + operation = { + "threadId": "thread_1", + "turnId": "turn_1", + "operationId": "model-1", + "kind": "model", + "requestedModel": "fixture", + "startedAt": "2026-11-20T00:00:00Z", + } + yield {"method": "turn/operation/started", "params": operation} + yield { + "method": "turn/operation/completed", + "params": { + **operation, + "completedAt": "2026-11-20T00:00:01Z", + "outcome": "interrupted", + "output": {"value": [partial], "truncated": False, "omitted": False}, + }, + } + raise RuntimeError("synthetic interrupted turn") + + queue, _ = servers + server = InterruptedServer(capture="content") + queue.append(server) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + set_trace_processors( + [SPAN_PROCESSOR_TESTING, OpenTelemetryTracingProcessor(tracer_provider=provider)] + ) + try: + config = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + ) + with pytest.raises(RuntimeError, match="synthetic interrupted turn"): + if streamed: + run = Runner.run_streamed(Agent(name="worker"), "hello", run_config=config) + async for _ in run.stream_events(): + pass + else: + await Runner.run(Agent(name="worker"), "hello", run_config=config) + native = next(span for span in fetch_ordered_spans() if span.span_data.type == "generation") + otel = next( + span + for span in exporter.get_finished_spans() + if span.attributes is not None + and span.attributes.get("gen_ai.operation.name") == "chat" + ) + assert native.span_data.output == [partial] + assert otel.attributes is not None + output = json.loads(json.dumps(otel.attributes["gen_ai.output.messages"])) + assert output[0]["role"] == "assistant" + assert output[0]["parts"][0] == { + "type": "text", + "content": partial["text"], + "partial": True, + "truncated": truncated, + } + assert native.span_data.operation.outcome == "interrupted" + assert otel.attributes["openai.agents.operation.outcome"] == "interrupted" + assert server.closed + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("detail", [None, "metadata", "content"]) +@pytest.mark.parametrize("sensitive", [False, True]) +async def test_codex_operations_preserve_source_spans_and_capture_policy( + tmp_path, servers, detail, sensitive +): + import json + + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + effective = "content" if detail == "content" and sensitive else "metadata" + server = OperationServer(capture=effective) + queue.append(server) + result = await Runner.run( + Agent(name="worker"), + "sdk_input_secret", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + trace_include_sensitive_data=sensitive, + tracing={} if detail is None else {"operation_detail": detail}, + ), + ) + assert result.final_output == "A Codex response" + assert dict(server.requests)["turn/start"]["operationTracing"] == effective + operations = [span for span in fetch_ordered_spans() if hasattr(span.span_data, "operation")] + models = [span for span in operations if span.span_data.type in {"response", "generation"}] + tools = [span for span in operations if span.span_data.type == "function"] + assert len(models) == 2 and len(tools) == 1 + assert {span.parent_id for span in operations} == {models[0].parent_id} + assert models[0].started_at == "2026-11-20T00:00:01.000Z" + assert models[0].ended_at == "2026-11-20T00:00:01.250Z" + assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" + assert models[0].span_data.usage == {"input_tokens": 10, "output_tokens": 5} + assert models[-1].span_data.usage == {"input_tokens": 30, "output_tokens": 15} + assert models[-1].span_data.operation.response_model == "observed-model" + assert models[-1].span_data.operation.outcome == "completed" + if detail is None: + assert all(span.span_data.type == "response" for span in models) + assert models[0].span_data.input == ( + [{"role": "user", "content": "sdk_input_secret"}] if sensitive else None + ) + assert models[-1].span_data.input is None + assert (models[-1].span_data.response is not None) == sensitive + if sensitive: + assert models[-1].span_data.response.id == "turn_1" + assert models[-1].span_data.response.usage.input_tokens == 10 + assert tools[0].span_data.input is None and tools[0].span_data.output is None + else: + assert all(span.span_data.type == "generation" for span in models) + assert (models[0].span_data.input is not None) == (effective == "content") + assert (tools[0].span_data.input is not None) == (effective == "content") + exported = json.dumps([span.export() for span in operations]) + assert ("native_result_secret" in exported) == (effective == "content") + if not sensitive: + assert all(span.span_data.operation.response_id is None for span in models) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("detail", [None, "metadata", "content"]) +async def test_stock_codex_has_one_partial_coverage_marker_without_synthetic_model_span( + tmp_path, servers, detail +): + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + queue.append(FakeServer()) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={} if detail is None else {"operation_detail": detail}, + ), + ) + assert result.final_output == "A Codex response" + spans = fetch_ordered_spans() + markers = [ + span for span in spans if getattr(span.span_data, "name", None) == "codex.tracing.coverage" + ] + assert len(markers) == 1 + assert markers[0].span_data.data["coverage"] == "partial" + assert not any(span.span_data.type in {"response", "generation"} for span in spans) + + +@pytest.mark.asyncio +async def test_codex_duplicate_and_foreign_telemetry_preserve_model_result(tmp_path, servers): + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(malformed=True) + queue.append(server) + run = Runner.run_streamed( + Agent(name="worker"), + "Hello", + run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), + ) + async for _ in run.stream_events(): + pass + assert run.final_output == "A Codex response" + spans = fetch_ordered_spans() + assert ( + sum(getattr(span.span_data, "name", None) == "codex.tracing.coverage" for span in spans) + == 1 + ) + assert sum(span.span_data.type == "response" for span in spans) == 2 + + +@pytest.mark.asyncio +async def test_codex_early_stream_close_finishes_observed_operation(tmp_path, servers): + from agents import ModelSettings + from agents.models.interface import ModelTracing + from agents.tracing import trace + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(active=True) + queue.append(server) + model = CodexProvider(cwd=tmp_path).get_model("test-model") + with trace("test"): + stream = model.stream_response( + None, + "Hello", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + assert (await anext(stream)).type == "response.created" + assert (await anext(stream)).type == "response.output_item.added" + await stream.aclose() + assert server.closed + models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] + assert len(models) == 1 + assert models[0].ended_at is not None + assert models[0].span_data.operation.outcome == "interrupted" + assert models[0].span_data.operation.estimated_end + + +@pytest.mark.asyncio +async def test_codex_nested_and_overlapping_tools_keep_explicit_parents(tmp_path, servers): + from tests.testing_processor import fetch_ordered_spans + + class NestedServer(OperationServer): + async def events(self): + common = {"threadId": "thread_1", "turnId": "turn_1"} + model = { + **common, + "operationId": "model", + "kind": "model", + "requestedModel": "test-model", + "startedAt": "2026-11-20T00:00:01Z", + } + yield {"method": "turn/operation/started", "params": model} + yield { + "method": "turn/operation/completed", + "params": {**model, "completedAt": "2026-11-20T00:00:02Z", "outcome": "success"}, + } + calls = [] + for name, parent in [("outer", None), ("nested", "outer"), ("parallel", None)]: + event = { + **common, + "operationId": name, + "kind": "tool", + "toolName": name, + "startedAt": "2026-11-20T00:00:02Z", + } + if parent: + event["parentOperationId"] = parent + calls.append(event) + yield {"method": "turn/operation/started", "params": event} + for event in reversed(calls): + yield { + "method": "turn/operation/completed", + "params": { + **event, + "completedAt": "2026-11-20T00:00:03Z", + "outcome": "success", + }, + } + async for event in FakeServer.events(self): + yield event + + queue, _ = servers + queue.append(NestedServer()) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), + ) + assert result.final_output == "A Codex response" + tools = { + span.span_data.name: span + for span in fetch_ordered_spans() + if span.span_data.type == "function" + } + assert tools["nested"].parent_id == tools["outer"].span_id + assert tools["parallel"].parent_id == tools["outer"].parent_id + assert tools["parallel"].started_at == tools["outer"].started_at + assert tools["parallel"].ended_at == tools["outer"].ended_at + + +@pytest.mark.asyncio +async def test_disabled_codex_tracing_omits_negotiation_and_operations(tmp_path, servers): + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(capture="content") + queue.append(server) + result = await Runner.run(Agent(name="worker"), "Hello", run_config=config(tmp_path)) + assert result.final_output == "A Codex response" + assert "operationTracing" not in dict(server.requests)["turn/start"] + assert fetch_ordered_spans() == [] + + +@pytest.mark.asyncio +async def test_codex_default_end_callback_receives_final_sdk_response_and_source_time( + tmp_path, servers, monkeypatch +): + from tests.testing_processor import SPAN_PROCESSOR_TESTING + + snapshots = [] + original = SPAN_PROCESSOR_TESTING.on_span_end + + def observe_end(span): + if span.span_data.type == "response": + snapshots.append((span.span_data.response, span.ended_at, span.span_data.usage)) + original(span) + + monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_end", observe_end) + queue, _ = servers + queue.append(OperationServer()) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), + ) + assert result.final_output == "A Codex response" + assert len(snapshots) == 2 + assert snapshots[0][0] is None + response, ended_at, usage = snapshots[-1] + assert response.id == "turn_1" + assert response.output[0].content[0].text == "A Codex response" + assert ended_at == "2026-11-20T00:00:03.250Z" + assert usage == {"input_tokens": 30, "output_tokens": 15} + + +@pytest.mark.asyncio +async def test_codex_content_retains_wire_omission_and_truncation_markers(tmp_path, servers): + import json + + from tests.testing_processor import fetch_ordered_spans + + class BoundedServer(OperationServer): + async def events(self): + async for event in super().events(): + params = event.get("params", {}) + if event["method"] == "turn/operation/started" and params["kind"] == "model": + params["input"]["truncated"] = True + params["input"]["reason"] = "capture_budget" + if event["method"] == "turn/operation/completed" and params["kind"] == "tool": + params["output"] = { + "value": None, + "omitted": True, + "truncated": False, + "reason": "unsupported_content", + } + yield event + + queue, _ = servers + queue.append(BoundedServer(capture="content")) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + ), + ) + assert result.final_output == "A Codex response" + models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] + assert all(any(item.get("truncated") for item in span.span_data.input) for span in models) + native = next(span for span in fetch_ordered_spans() if span.span_data.type == "function") + assert json.loads(native.span_data.output) == {"omitted": True, "reason": "unsupported_content"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disabled_parent", [False, True]) +async def test_codex_suppresses_capture_when_trace_provider_or_parent_is_disabled( + tmp_path, servers, disabled_parent +): + from contextlib import nullcontext + + from agents.tracing import set_tracing_disabled, trace + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(capture="content") + queue.append(server) + parent = trace("disabled", disabled=True) if disabled_parent else nullcontext() + if not disabled_parent: + set_tracing_disabled(True) + try: + with parent: + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + ), + ) + finally: + if not disabled_parent: + set_tracing_disabled(False) + assert result.final_output == "A Codex response" + assert "operationTracing" not in dict(server.requests)["turn/start"] + assert fetch_ordered_spans() == [] + + +@pytest.mark.asyncio +async def test_codex_capture_downgrade_marks_partial_coverage(tmp_path, servers): + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(capture="metadata") + queue.append(server) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + ), + ) + assert result.final_output == "A Codex response" + assert dict(server.requests)["turn/start"]["operationTracing"] == "content" + markers = [ + span + for span in fetch_ordered_spans() + if getattr(span.span_data, "name", None) == "codex.tracing.coverage" + ] + assert len(markers) == 1 + assert markers[0].span_data.data["reason"] == "capture_downgraded" + models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] + assert len(models) == 2 + assert all(span.span_data.input is None and span.span_data.output is None for span in models) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel", [False, True]) +async def test_codex_teardown_failure_finishes_pending_source_span(tmp_path, servers, cancel): + import asyncio + + from tests.testing_processor import fetch_ordered_spans + + closing = asyncio.Event() + + class TeardownServer(OperationServer): + async def close(self): + self.closed = True + closing.set() + if cancel: + await asyncio.Event().wait() + raise RuntimeError("teardown failed") + + queue, _ = servers + queue.append(TeardownServer()) + task = asyncio.create_task( + Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), + ) + ) + await closing.wait() + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError if cancel else RuntimeError): + await task + models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] + assert len(models) == 2 + assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" + + +@pytest.mark.asyncio +async def test_codex_releases_completed_tool_content_and_preserves_late_child_parent( + tmp_path, servers, monkeypatch +): + import gc + import weakref + + from tests.testing_processor import SPAN_PROCESSOR_TESTING + + completed_tools = {} + tool_edges = {} + original = SPAN_PROCESSOR_TESTING.on_span_end + + def release_completed(span): + if span.span_data.type == "function": + completed_tools[span.span_data.name] = weakref.ref(span) + tool_edges[span.span_data.name] = (span.span_id, span.parent_id) + else: + original(span) + + monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_end", release_completed) + + class LateChildServer(OperationServer): + async def events(self): + common = {"threadId": "thread_1", "turnId": "turn_1"} + for name, parent in [("outer", None), ("late_child", "outer")]: + if parent is not None: + gc.collect() + assert completed_tools["outer"]() is None + params = { + **common, + "operationId": name, + "kind": "tool", + "toolName": name, + "startedAt": "2026-11-20T00:00:01Z", + "input": {"value": "synthetic_content" * 1000}, + } + if parent is not None: + params["parentOperationId"] = parent + yield {"method": "turn/operation/started", "params": params} + yield { + "method": "turn/operation/completed", + "params": { + **params, + "completedAt": "2026-11-20T00:00:02Z", + "outcome": "success", + }, + } + async for event in FakeServer.events(self): + yield event + + queue, _ = servers + queue.append(LateChildServer(capture="content")) + result = await Runner.run( + Agent(name="worker"), + "Hello", + run_config=RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + ), + ) + assert result.final_output == "A Codex response" + assert tool_edges["late_child"][1] == tool_edges["outer"][0] + + +@pytest.mark.asyncio +async def test_codex_default_span_retains_exact_sdk_visible_response(tmp_path, servers): + from agents import ModelSettings + from agents.models.interface import ModelTracing + from agents.tracing import trace + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + queue.append(OperationServer()) + with trace("test"): + events = [ + event + async for event in CodexProvider(cwd=tmp_path) + .get_model(None) + .stream_response( + None, + "Hello", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ] + final = events[-1].response + models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] + assert models[-1].span_data.response is final + assert final.id == "turn_1" + assert final.usage is not None + assert final.usage.input_tokens == 10 + assert models[-1].span_data.usage["input_tokens"] == 30 + assert models[-1].span_data.operation.response_id == "resp_source_3" + + +@pytest.mark.asyncio +async def test_codex_direct_model_sensitive_veto_downgrades_child_capture(tmp_path, servers): + from agents import ModelSettings + from agents.models.interface import ModelTracing + from agents.tracing import trace + from agents.tracing.config import operation_tracing_scope + from tests.testing_processor import fetch_ordered_spans + + queue, _ = servers + server = OperationServer(capture="metadata") + queue.append(server) + with trace("test"), operation_tracing_scope({"operation_detail": "content"}, True): + async for _ in ( + CodexProvider(cwd=tmp_path) + .get_model(None) + .stream_response( + None, + "Hello", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED_WITHOUT_DATA, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ): + pass + assert dict(server.requests)["turn/start"]["operationTracing"] == "metadata" + models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] + assert len(models) == 2 + assert all(span.span_data.operation.detail == "metadata" for span in models) + assert all(span.span_data.input is None and span.span_data.output is None for span in models) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_codex_llm_hook_global_disable_prevents_child_capture(tmp_path, servers, streamed): + from agents import RunHooks + from agents.tracing import get_current_span, get_current_trace, set_tracing_disabled + from tests.testing_processor import fetch_ordered_spans + + class DisableCapture(RunHooks): + async def on_llm_start(self, context, agent, system_prompt, input_items): + active_trace, active_span = get_current_trace(), get_current_span() + assert active_trace is not None and active_trace.trace_id != "no-op" + assert active_span is not None and active_span.span_id != "no-op" + set_tracing_disabled(True) + + queue, _ = servers + server = OperationServer(capture="content") + queue.append(server) + run_config = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "content"} + ) + try: + if streamed: + result = Runner.run_streamed( + Agent(name="worker"), + "Hello", + hooks=DisableCapture(), + run_config=run_config, + ) + async for _ in result.stream_events(): + pass + else: + result = await Runner.run( + Agent(name="worker"), + "Hello", + hooks=DisableCapture(), + run_config=run_config, + ) + finally: + set_tracing_disabled(False) + assert result.final_output == "A Codex response" + assert "operationTracing" not in dict(server.requests)["turn/start"] + assert not any( + span.span_data.type in {"generation", "response"} for span in fetch_ordered_spans() + ) diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py new file mode 100644 index 0000000000..e0976e22b9 --- /dev/null +++ b/tests/models/test_operation_tracing_parity.py @@ -0,0 +1,838 @@ +"""Exercise provider-wire conversion and actual subprocess tool ownership over loopback.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shlex +import signal +import sys +import threading +import time +from collections.abc import Iterator +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +import pytest + +from agents import Agent, RunConfig, Runner +from agents.decorators import tool +from agents.models.codex import CodexProvider +from agents.models.interface import ModelProvider +from agents.models.openai_provider import OpenAIProvider +from agents.tracing import TracingConfig, set_trace_processors + +from ..testing_processor import SPAN_PROCESSOR_TESTING, SpanProcessorForTests + +pytestmark = pytest.mark.allow_call_model_methods + +MODEL = "fixture-model" +PROMPT = "parity_prompt_canary" +INSTRUCTIONS = "Answer using the supplied tools. parity_instructions_canary" +ARGUMENT = "parity_argument_canary" +TOOL_RESULT = "parity_tool_result_canary" +ANSWER = "parity_answer_canary" +FILE_CONTENT = "parity_native_file_canary" +PAYLOADS = (PROMPT, INSTRUCTIONS, ARGUMENT, TOOL_RESULT, ANSWER) + + +class _ResponsesFixture: + def __init__( + self, + *, + mixed: bool = False, + model_header: bool = True, + held: bool = False, + retry_once: bool = False, + observed_model: str = MODEL, + ) -> None: + self.mixed = mixed + self.model_header = model_header + self.held = held + self.retry_once = retry_once + self.observed_model = observed_model + self.request_started = threading.Event() + self.release = threading.Event() + self.requests: list[dict[str, Any]] = [] + self.intervals: list[tuple[int, int]] = [] + fixture = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: + pass + + def do_GET(self) -> None: + self._reply(b'{"data":[]}', "application/json") + + def _reply(self, body: bytes, content_type: str, status: int = 200) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + if status == 500: + self.send_header("Retry-After", "0") + if fixture.model_header: + self.send_header("openai-model", fixture.observed_model) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + def do_POST(self) -> None: + started = time.time_ns() + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + fixture.requests.append(request) + fixture.request_started.set() + if fixture.held: + fixture.release.wait(timeout=10) + number = len(fixture.requests) + if fixture.retry_once and number == 1: + fixture.intervals.append((started, time.time_ns())) + self._reply( + b'{"error":{"message":"Fixture retry","type":"server_error"}}', + "application/json", + 500, + ) + return + if fixture.retry_once: + number -= 1 + response = fixture.response(number) + if request.get("stream"): + item = response["output"][0] + events = [ + { + "type": "response.created", + "response": dict(response, output=[], status="in_progress"), + }, + { + "type": "response.output_item.added", + "output_index": 0, + "item": dict(item, status="in_progress"), + }, + {"type": "response.output_item.done", "output_index": 0, "item": item}, + {"type": "response.completed", "response": response}, + ] + body = "".join( + f"event: {event['type']}\ndata: " + + json.dumps(dict(event, sequence_number=index)) + + "\n\n" + for index, event in enumerate(events) + ).encode() + content_type = "text/event-stream" + else: + body = json.dumps(response).encode() + content_type = "application/json" + fixture.intervals.append((started, time.time_ns())) + self._reply(body, content_type) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread( + target=self.server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + self.thread.start() + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.server.server_port}" + + def response(self, number: int) -> dict[str, Any]: + if self.mixed and number == 1: + item: dict[str, Any] = { + "type": "function_call", + "name": "read_file", + "arguments": "{}", + "call_id": "native_read", + "id": "fc_native_read", + "status": "completed", + } + if any( + tool.get("type") == "namespace" and tool.get("name") == "mcp__parity" + for tool in self.requests[0].get("tools", []) + ): + item["namespace"] = "mcp__parity" + else: + item["name"] = "mcp__parity__read_file" + elif number == (2 if self.mixed else 1) and not self.retry_once: + item = { + "type": "function_call", + "name": "lookup", + "arguments": json.dumps({"key": ARGUMENT}), + "call_id": "python_lookup", + "id": "fc_python_lookup", + "status": "completed", + } + else: + item = { + "type": "message", + "id": "msg_answer", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": ANSWER, "annotations": []}], + } + return { + "id": f"resp_parity_{number}", + "object": "response", + "created_at": 1, + "model": self.observed_model, + "status": "completed", + "output": [item], + "usage": { + "input_tokens": 10 * number, + "output_tokens": 5 * number, + "total_tokens": 15 * number, + "input_tokens_details": {"cached_tokens": number}, + "output_tokens_details": {"reasoning_tokens": number}, + }, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + + def close(self) -> None: + self.release.set() + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + assert not self.thread.is_alive() + + +class _LiveProcessor(SpanProcessorForTests): + def __init__(self) -> None: + super().__init__() + self.started_models: list[Any] = [] + self.model_started = asyncio.Event() + + def on_span_start(self, span: Any) -> None: + super().on_span_start(span) + if span.span_data.type in {"response", "generation"}: + self.started_models.append(span) + self.model_started.set() + + +@pytest.fixture +def telemetry( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[tuple[Any, Any, SpanProcessorForTests, Any]]: + monkeypatch.delenv("OPENAI_AGENTS_TRACE_INCLUDE_SENSITIVE_DATA", raising=False) + pytest.importorskip("opentelemetry.sdk.trace") + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from agents.extensions.tracing import OpenTelemetryTracingProcessor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + native = _LiveProcessor() + otel_processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([native, otel_processor]) + try: + yield provider.get_tracer("operation-parity"), exporter, native, otel_processor + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() + + +def _codex_provider( + tmp_path: Path, server: _ResponsesFixture, *, pid_file: Path | None = None +) -> CodexProvider: + executable = os.environ.get("CODEX_TRACING_TEST_EXECUTABLE") + if not executable: + pytest.skip("Set CODEX_TRACING_TEST_EXECUTABLE to run patched Codex integration tests.") + assert Path(executable).is_file(), "CODEX_TRACING_TEST_EXECUTABLE must name a real binary." + home = tmp_path / "codex-home" + home.mkdir(mode=0o700) + config = f'''model = "{MODEL}" +model_provider = "fixture" +approval_policy = "never" +sandbox_mode = "read-only" +[analytics] +enabled = false +[features] +apps = false +hooks = false +plugins = false +[model_providers.fixture] +name = "Synthetic fixture" +base_url = "{server.base_url}" +wire_api = "responses" +requires_openai_auth = false +supports_websockets = false +[otel] +metrics_exporter = "none" +exporter = "none" +trace_exporter = "none" +''' + if server.mixed: + mcp_server = tmp_path / "parity_mcp.py" + mcp_server.write_text( + "import sys\n" + "from pathlib import Path\n" + "from mcp.server.fastmcp import FastMCP\n" + "from mcp.types import ToolAnnotations\n" + 'server = FastMCP("parity")\n' + "@server.tool(annotations=ToolAnnotations(readOnlyHint=True, destructiveHint=False))\n" + "def read_file() -> str:\n" + ' """Read the synthetic fixture file."""\n' + " return Path(sys.argv[1]).read_text()\n" + 'server.run(transport="stdio")\n' + ) + config += ( + "\n[mcp_servers.parity]\ncommand = " + + json.dumps(sys.executable) + + "\nargs = " + + json.dumps([str(mcp_server), str(tmp_path / "parity-native.txt")]) + + "\nstartup_timeout_sec = 10\ntool_timeout_sec = 5\n" + + 'default_tools_approval_mode = "approve"\n' + ) + (home / "config.toml").write_text(config) + environment = { + key: value + for key, value in os.environ.items() + if key in {"PATH", "HOME", "LANG", "LD_LIBRARY_PATH", "TMPDIR"} + } + environment["CODEX_HOME"] = str(home) + if pid_file is not None: + # The launcher records the real child PID and immediately executes the unmodified CLI. + launcher = tmp_path / "tracked-codex" + launcher.write_text( + "#!/bin/sh\nprintf '%s' \"$$\" > " + + shlex.quote(str(pid_file)) + + "\nexec " + + shlex.quote(str(Path(executable).resolve())) + + ' "$@"\n' + ) + launcher.chmod(0o700) + executable = str(launcher) + return CodexProvider(cwd=tmp_path, codex_executable=executable, env=environment) + + +def _ns(timestamp: str | None) -> int: + assert timestamp is not None + return int(datetime.fromisoformat(timestamp).timestamp() * 1_000_000_000) + + +def _model_provider(name: str, tmp_path: Path, server: _ResponsesFixture) -> ModelProvider: + if name == "codex": + return _codex_provider(tmp_path, server) + return OpenAIProvider( + api_key="loopback-test-placeholder", base_url=server.base_url, use_responses=True + ) + + +async def _exercise( + *, + tmp_path: Path, + telemetry: tuple[Any, Any, SpanProcessorForTests, Any], + provider_name: str, + streamed: bool, + capture: str, + mixed: bool = False, + consumer: str = "both", + omit_task_spans: bool = False, + model_header: bool = True, + observed_model: str = MODEL, +) -> None: + tracer, exporter, native, otel_processor = telemetry + set_trace_processors( + [native] + if consumer == "sdk" + else [otel_processor] + if consumer == "otel" + else [native, otel_processor] + ) + server = _ResponsesFixture( + mixed=mixed, + model_header=model_header, + observed_model=observed_model, + ) + callback_intervals: list[tuple[int, int]] = [] + model_provider = None + try: + (tmp_path / "parity-native.txt").write_text(FILE_CONTENT) + model_provider = _model_provider(provider_name, tmp_path, server) + + @tool + async def lookup(key: str) -> str: + assert key == ARGUMENT + started = time.time_ns() + with tracer.start_as_current_span("database.query"): + pass + callback_intervals.append((started, time.time_ns())) + return TOOL_RESULT + + agent = Agent(name="worker", model=MODEL, instructions=INSTRUCTIONS, tools=[lookup]) + tracing: TracingConfig = {} + if omit_task_spans: + tracing["include_task_and_turn_spans"] = False + if capture in {"metadata", "content", "veto"}: + tracing["operation_detail"] = "content" if capture == "veto" else capture + config = RunConfig( + model_provider=model_provider, + workflow_name="parity-workflow", + tracing=tracing, + tracing_disabled=capture == "disabled", + ) + if capture in {"veto", "default-veto"}: + config.trace_include_sensitive_data = False + with tracer.start_as_current_span("host_job") as host: + if streamed: + result = Runner.run_streamed(agent, PROMPT, run_config=config) + async for _ in result.stream_events(): + pass + else: + result = await Runner.run(agent, PROMPT, run_config=config) + assert result.final_output == ANSWER + assert len(server.requests) == (3 if mixed else 2) + assert all(request["model"] == MODEL for request in server.requests) + assert len(callback_intervals) == 1 + if mixed: + native_results = [ + item.get("output") + for item in server.requests[1]["input"] + if item.get("call_id") == "native_read" and "output" in item + ] + assert FILE_CONTENT in json.dumps(native_results), { + "result": native_results, + "mcp_tools": [ + tool + for tool in server.requests[0].get("tools", []) + if "parity" in json.dumps(tool) + ], + } + spans = native.get_ordered_spans() + otel = exporter.get_finished_spans() + database = next(span for span in otel if span.name == "database.query") + if capture == "disabled": + assert not spans + assert len(otel) == 2 + assert database.parent.span_id == host.get_span_context().span_id + return + + models = [span for span in spans if span.span_data.type in {"response", "generation"}] + tools = [span for span in spans if span.span_data.type == "function"] + turns = [span for span in spans if span.span_data.type == "turn"] + if consumer != "otel": + assert len(models) == len(server.requests) + assert len(tools) == (2 if mixed else 1) + assert len(turns) == (0 if omit_task_spans else 2) + assert {span.span_data.type for span in models} == { + "generation" if capture in {"metadata", "content", "veto"} else "response" + } + native_index = {span.span_id: span for span in spans} + assert all( + native_index[span.parent_id].span_data.type + == ("agent" if omit_task_spans else "turn") + for span in models + tools + ) + assert all(span.error is None for span in models + tools) + for index, span in enumerate(models, start=1): + assert span.span_data.operation.model == MODEL + assert span.span_data.operation.response_model == observed_model + assert span.span_data.operation.outcome == "completed" + if capture in {"metadata", "veto", "default-veto"}: + assert span.span_data.operation.response_id is None + else: + assert span.span_data.operation.response_id == f"resp_parity_{index}" + assert span.span_data.usage["input_tokens"] == 10 * index + assert span.span_data.usage["output_tokens"] == 5 * index + started, ended = server.intervals[index - 1] + assert _ns(span.started_at) <= started + 1_000_000 + assert _ns(span.ended_at) >= ended - 1_000_000 + if not omit_task_spans: + expected_turn_usage = [(30, 15), (30, 15)] if mixed else [(10, 5), (20, 10)] + assert [ + (span.span_data.usage["input_tokens"], span.span_data.usage["output_tokens"]) + for span in turns + ] == expected_turn_usage + if mixed: + native_tool = next(span for span in tools if span.span_data.name != "lookup") + assert _ns(models[0].started_at) <= _ns(native_tool.started_at) + assert _ns(native_tool.ended_at) <= _ns(models[1].started_at) + operations = [ + span for span in otel if span.attributes.get("gen_ai.operation.name") == "chat" + ] + executions = [ + span for span in otel if span.attributes.get("gen_ai.operation.name") == "execute_tool" + ] + if consumer == "sdk": + assert not operations and not executions + assert database.parent.span_id == host.get_span_context().span_id + return + assert len(operations) == len(server.requests) + assert len(executions) == (2 if mixed else 1) + otel_index = {span.context.span_id: span for span in otel} + for span in operations + executions: + assert span.context.trace_id == host.get_span_context().trace_id + assert otel_index[span.parent.span_id].attributes["gen_ai.operation.name"] == ( + "invoke_agent" if omit_task_spans else "openai.agents.turn" + ) + for index, span in enumerate( + sorted(operations, key=lambda value: value.start_time), start=1 + ): + assert span.attributes["gen_ai.request.model"] == MODEL + assert span.attributes["gen_ai.response.model"] == observed_model + assert span.attributes["gen_ai.usage.input_tokens"] == 10 * index + assert span.attributes["gen_ai.usage.output_tokens"] == 5 * index + assert span.attributes["openai.api.type"] == "responses" + assert span.kind.name == "CLIENT" + started, ended = server.intervals[index - 1] + assert span.start_time <= started + 1_000_000 + assert span.end_time >= ended - 1_000_000 + assert otel_index[database.parent.span_id] in executions + python_execution = otel_index[database.parent.span_id] + callback_started, callback_ended = callback_intervals[0] + assert python_execution.start_time <= callback_started + assert python_execution.end_time >= callback_ended + assert python_execution.start_time <= database.start_time <= database.end_time + assert database.end_time <= python_execution.end_time + if models: + assert _ns(models[0].ended_at) <= callback_started + 1_000_000 + for span in otel: + current = span + while current.parent is not None: + current = otel_index[current.parent.span_id] + assert current.context.span_id == host.get_span_context().span_id + + exported = json.dumps( + [ + { + "attributes": dict(span.attributes), + "events": [ + {"name": event.name, "attributes": dict(event.attributes or {})} + for event in span.events + ], + "status": span.status.description, + } + for span in otel + ] + ) + if capture == "content": + assert all(marker in exported for marker in PAYLOADS) + for span, request in zip(models, server.requests, strict=True): + submitted = {"input": request["input"], "instructions": request.get("instructions")} + assert len(json.dumps(submitted, ensure_ascii=False).encode()) < 32 * 1024 + assert span.span_data.input == request["input"] + if isinstance(request.get("instructions"), str): + assert span.span_data.model_config["instructions"] == request["instructions"] + else: + assert all(marker not in exported for marker in PAYLOADS) + if capture in {"metadata", "veto", "default-veto"}: + assert all(span.span_data.input is None for span in models + tools) + assert all(getattr(span.span_data, "output", None) is None for span in models + tools) + assert all(getattr(span.span_data, "response", None) is None for span in models) + if capture == "default" and consumer != "otel": + assert PROMPT in json.dumps(models[0].span_data.input) + assert ANSWER in models[-1].span_data.response.model_dump_json() + python_tool = next(span for span in tools if span.span_data.name == "lookup") + assert ARGUMENT in python_tool.span_data.input + assert TOOL_RESULT in str(python_tool.span_data.output) + if mixed: + native_payloads = json.dumps([span.export() for span in spans], default=str) + if capture == "content": + assert FILE_CONTENT in exported + else: + assert FILE_CONTENT not in exported + assert FILE_CONTENT not in native_payloads + finally: + if model_provider is not None: + await model_provider.aclose() + server.close() + + +@pytest.mark.parametrize("provider_name", ["default", "codex"]) +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) +@pytest.mark.parametrize( + "capture", ["default", "default-veto", "metadata", "content", "veto", "disabled"] +) +async def test_operation_tracing_parity( + tmp_path: Path, telemetry: Any, provider_name: str, streamed: bool, capture: str +) -> None: + await asyncio.wait_for( + _exercise( + tmp_path=tmp_path, + telemetry=telemetry, + provider_name=provider_name, + streamed=streamed, + capture=capture, + ), + timeout=30, + ) + + +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) +@pytest.mark.parametrize("capture", ["default", "metadata", "content", "veto"]) +async def test_codex_native_and_python_tool_parity( + tmp_path: Path, telemetry: Any, streamed: bool, capture: str +) -> None: + await asyncio.wait_for( + _exercise( + tmp_path=tmp_path, + telemetry=telemetry, + provider_name="codex", + streamed=streamed, + capture=capture, + mixed=True, + ), + timeout=30, + ) + + +@pytest.mark.parametrize("provider_name", ["default", "codex"]) +@pytest.mark.parametrize("consumer", ["sdk", "otel"]) +async def test_operation_tracing_individual_consumers( + tmp_path: Path, telemetry: Any, provider_name: str, consumer: str +) -> None: + await asyncio.wait_for( + _exercise( + tmp_path=tmp_path, + telemetry=telemetry, + provider_name=provider_name, + streamed=False, + capture="default", + consumer=consumer, + ), + timeout=30, + ) + + +async def test_operation_tracing_without_task_and_turn_spans( + tmp_path: Path, telemetry: Any +) -> None: + await asyncio.wait_for( + _exercise( + tmp_path=tmp_path, + telemetry=telemetry, + provider_name="default", + streamed=False, + capture="metadata", + omit_task_spans=True, + ), + timeout=30, + ) + + +@pytest.mark.parametrize("provider_name", ["default", "codex"]) +async def test_operation_tracing_observed_model_without_header( + tmp_path: Path, telemetry: Any, provider_name: str +) -> None: + await asyncio.wait_for( + _exercise( + tmp_path=tmp_path, + telemetry=telemetry, + provider_name=provider_name, + streamed=False, + capture="metadata", + model_header=False, + observed_model=f"{MODEL}-observed", + ), + timeout=30, + ) + + +@pytest.mark.parametrize( + ("provider_name", "termination"), + [("default", "cancel"), ("codex", "cancel"), ("codex", "child-exit")], +) +async def test_operation_tracing_live_start_and_cancel( + tmp_path: Path, telemetry: Any, provider_name: str, termination: str +) -> None: + if termination == "child-exit" and os.name != "posix": + pytest.skip("The transparent child-PID launcher requires a POSIX shell.") + tracer, exporter, native, _ = telemetry + server = _ResponsesFixture(held=True) + model_provider = None + task = None + try: + pid_file = tmp_path / "codex.pid" + model_provider = ( + _codex_provider(tmp_path, server, pid_file=pid_file) + if termination == "child-exit" + else _model_provider(provider_name, tmp_path, server) + ) + with tracer.start_as_current_span("host_job") as host: + task = asyncio.create_task( + Runner.run( + Agent(name="worker", model=MODEL), + PROMPT, + run_config=RunConfig( + model_provider=model_provider, tracing={"operation_detail": "metadata"} + ), + ) + ) + assert await asyncio.to_thread(server.request_started.wait, 5) + await asyncio.wait_for(native.model_started.wait(), 5) + assert len(native.started_models) == 1 + assert native.started_models[0].ended_at is None + if termination == "child-exit": + os.kill(int(pid_file.read_text()), signal.SIGKILL) + with pytest.raises(Exception, match="Codex App Server.*(?:exited|closed)"): + await asyncio.wait_for(task, 10) + else: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 10) + assert host.get_span_context().is_valid + models = [ + span for span in native.get_ordered_spans() if span.span_data.type == "generation" + ] + assert len(models) == 1 + assert models[0] is native.started_models[0] + assert models[0].ended_at is not None + assert models[0].span_data.operation.outcome == "interrupted" + chats = [ + span + for span in exporter.get_finished_spans() + if span.attributes.get("gen_ai.operation.name") == "chat" + ] + assert len(chats) == 1 + assert chats[0].end_time is not None + if termination == "child-exit": + assert models[0].span_data.operation.estimated_end is True + assert chats[0].attributes["openai.agents.operation.estimated_end"] is True + assert chats[0].attributes["openai.agents.operation.outcome"] == "interrupted" + coverage = [ + span + for span in native.get_ordered_spans() + if span.span_data.type == "custom" + and span.span_data.name == "codex.tracing.coverage" + ] + assert len(coverage) == 1 + assert coverage[0].span_data.data["coverage"] == "partial" + exported_coverage = [ + span + for span in exporter.get_finished_spans() + if span.name == "codex.tracing.coverage" + ] + assert len(exported_coverage) == 1 + assert exported_coverage[0].attributes["openai.agents.operation.coverage"] == "partial" + assert len(server.requests) == 1 + finally: + server.release.set() + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if model_provider is not None: + await model_provider.aclose() + server.close() + + +@pytest.mark.parametrize("provider_name", ["default", "codex"]) +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) +async def test_operation_tracing_transport_retry( + tmp_path: Path, telemetry: Any, provider_name: str, streamed: bool +) -> None: + tracer, exporter, native, _ = telemetry + server = _ResponsesFixture(retry_once=True) + model_provider = None + try: + model_provider = _model_provider(provider_name, tmp_path, server) + with tracer.start_as_current_span("host_job"): + config = RunConfig( + model_provider=model_provider, tracing={"operation_detail": "metadata"} + ) + agent = Agent(name="worker", model=MODEL) + if streamed: + result = Runner.run_streamed(agent, PROMPT, run_config=config) + async for _ in result.stream_events(): + pass + else: + result = await asyncio.wait_for(Runner.run(agent, PROMPT, run_config=config), 15) + assert result.final_output == ANSWER + assert len(server.requests) == 2 + assert server.requests[0]["input"] == server.requests[1]["input"] + models = [ + span for span in native.get_ordered_spans() if span.span_data.type == "generation" + ] + assert len(models) == 1 + assert models[0].error is None + assert models[0].span_data.operation.outcome == "completed" + assert models[0].span_data.usage["input_tokens"] == 10 + assert models[0].span_data.usage["output_tokens"] == 5 + assert _ns(models[0].started_at) <= server.intervals[0][0] + 1_000_000 + assert _ns(models[0].ended_at) >= server.intervals[-1][1] - 1_000_000 + chats = [ + span + for span in exporter.get_finished_spans() + if span.attributes.get("gen_ai.operation.name") == "chat" + ] + assert len(chats) == 1 + assert chats[0].attributes["gen_ai.usage.input_tokens"] == 10 + assert chats[0].attributes["gen_ai.usage.output_tokens"] == 5 + finally: + if model_provider is not None: + await model_provider.aclose() + server.close() + + +async def test_stock_codex_reports_partial_coverage( + tmp_path: Path, telemetry: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + stock = os.environ.get("CODEX_TRACING_STOCK_EXECUTABLE") + if not stock: + pytest.skip("Set CODEX_TRACING_STOCK_EXECUTABLE to run the stock binary coverage gate.") + monkeypatch.setenv("CODEX_TRACING_TEST_EXECUTABLE", stock) + tracer, exporter, native, _ = telemetry + server = _ResponsesFixture() + model_provider = None + calls: list[str] = [] + try: + model_provider = _codex_provider(tmp_path, server) + + @tool + async def lookup(key: str) -> str: + calls.append(key) + return TOOL_RESULT + + with tracer.start_as_current_span("host_job") as host: + result = await asyncio.wait_for( + Runner.run( + Agent(name="worker", model=MODEL, tools=[lookup]), + PROMPT, + run_config=RunConfig( + model_provider=model_provider, tracing={"operation_detail": "metadata"} + ), + ), + 30, + ) + assert result.final_output == ANSWER + assert calls == [ARGUMENT] + assert len(server.requests) == 2 + spans = native.get_ordered_spans() + assert not [span for span in spans if span.span_data.type in {"response", "generation"}] + assert len([span for span in spans if span.span_data.type == "function"]) == 1 + coverage = [ + span + for span in spans + if span.span_data.type == "custom" and span.span_data.name == "codex.tracing.coverage" + ] + assert len(coverage) == 2 + assert all( + span.span_data.data["coverage"] == "partial" + and span.span_data.data["reason"] == "unsupported" + for span in coverage + ) + exported = exporter.get_finished_spans() + assert not [ + span for span in exported if span.attributes.get("gen_ai.operation.name") == "chat" + ] + assert ( + len( + [ + span + for span in exported + if span.attributes.get("gen_ai.operation.name") == "execute_tool" + ] + ) + == 1 + ) + assert len([span for span in exported if span.name == "codex.tracing.coverage"]) == 2 + assert all(span.context.trace_id == host.get_span_context().trace_id for span in exported) + finally: + if model_provider is not None: + await model_provider.aclose() + server.close() From 73348336fdcce05ef3586a71c5f31ab53abc0e7f Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:54:37 -0800 Subject: [PATCH 04/33] test: make sandbox collection and App Server startup portable --- tests/models/test_codex_app_server.py | 15 +++++++-------- tests/sandbox/test_manifest_file_content.py | 3 ++- tests/sandbox/test_provider_capabilities.py | 8 +++++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/models/test_codex_app_server.py b/tests/models/test_codex_app_server.py index 02fb487d10..bdda3fa70e 100644 --- a/tests/models/test_codex_app_server.py +++ b/tests/models/test_codex_app_server.py @@ -101,12 +101,10 @@ async def test_initialization_retries_only_child_exit_and_reaps_failed_attempt( monkeypatch.setattr("agents.models._codex_app_server._INITIALIZE_RETRY_DELAYS", (0, 0, 0)) - script = tmp_path / "codex-app-server" + # The real launcher supplies this filename as Python's script argument. + script = tmp_path / "app-server" script.write_text( - "#!" - + sys.executable - + "\n" - + r""" + r""" import json, os, pathlib, sys attempt_file = pathlib.Path(os.environ["ATTEMPT_FILE"]) try: @@ -126,7 +124,6 @@ async def test_initialization_retries_only_child_exit_and_reaps_failed_attempt( print(json.dumps({"id": message["id"], "result": {"ok": True}}), flush=True) """ ) - script.chmod(0o700) opened = [] original = CodexAppServer.open @@ -142,7 +139,9 @@ async def open_server(*args): "FAILURE_MODE": failure, } if failure == "once": - server = await CodexAppServer.open_initialized(str(script), str(tmp_path), [], environment) + server = await CodexAppServer.open_initialized( + sys.executable, str(tmp_path), [], environment + ) assert len(opened) == 2 assert opened[0]._process.returncode is not None assert await server.request("thread/start", {}) == {"ok": True} @@ -150,7 +149,7 @@ async def open_server(*args): else: error = _CodexProcessExitedError if failure == "always" else RuntimeError with pytest.raises(error): - await CodexAppServer.open_initialized(str(script), str(tmp_path), [], environment) + await CodexAppServer.open_initialized(sys.executable, str(tmp_path), [], environment) assert len(opened) == (4 if failure == "always" else 1) assert all(server._process.returncode is not None for server in opened) diff --git a/tests/sandbox/test_manifest_file_content.py b/tests/sandbox/test_manifest_file_content.py index eab72ee419..56036e8771 100644 --- a/tests/sandbox/test_manifest_file_content.py +++ b/tests/sandbox/test_manifest_file_content.py @@ -10,7 +10,6 @@ from agents.sandbox import Manifest from agents.sandbox.entries import Dir, File -from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient def test_inline_binary_file_round_trips_through_nested_manifest_json() -> None: @@ -76,6 +75,8 @@ def test_inline_file_json_schemas_declare_the_encoded_object() -> None: async def test_inline_binary_file_survives_real_session_serialization_and_resume( tmp_path: Path, ) -> None: + from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient + client = UnixLocalSandboxClient(inherit_host_environment=False) content = b"SQLite format 3\x00\x80\xff\x00" manifest = Manifest( diff --git a/tests/sandbox/test_provider_capabilities.py b/tests/sandbox/test_provider_capabilities.py index 3eb43cb91b..f452b08bc5 100644 --- a/tests/sandbox/test_provider_capabilities.py +++ b/tests/sandbox/test_provider_capabilities.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from pathlib import Path from typing import ClassVar @@ -13,7 +14,6 @@ from agents.sandbox.errors import WorkspaceReadNotFoundError from agents.testing import ScriptedModel, scripted_sandbox_session from agents.tool import FunctionTool -from tests.sandbox._filesystem_test_session import FilesystemTestSandboxClient from tests.test_responses import get_final_output_message, get_function_tool_call @@ -29,7 +29,10 @@ class _FunctionSandboxModel(ScriptedModel): @pytest.mark.asyncio @pytest.mark.parametrize("approve", [True, False]) +@pytest.mark.skipif(sys.platform == "win32", reason="Filesystem test session requires Unix-local") async def test_function_sandbox_patch_uses_runner_approval_and_session_files(approve: bool) -> None: + from tests.sandbox._filesystem_test_session import FilesystemTestSandboxClient + client = FilesystemTestSandboxClient() session = await client.create(manifest=Manifest()) model = _FunctionSandboxModel( @@ -225,7 +228,10 @@ def get_model(self, model_name: str | None) -> Model: @pytest.mark.asyncio +@pytest.mark.skipif(sys.platform == "win32", reason="Filesystem test session requires Unix-local") async def test_memory_generation_uses_bound_provider_for_both_omitted_phases() -> None: + from tests.sandbox._filesystem_test_session import FilesystemTestSandboxClient + client = FilesystemTestSandboxClient() session = await client.create(manifest=Manifest()) model = _FunctionSandboxModel( From 83962e9357d87f4f4cf81371ccf8294faad689f1 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:54:45 -0800 Subject: [PATCH 05/33] fix(tracing): preserve source nanoseconds on Python 3.10 --- src/agents/extensions/tracing/opentelemetry.py | 9 +++------ src/agents/tracing/util.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 93dba6b328..55ac858e37 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -11,11 +11,9 @@ from __future__ import annotations import json -import re import threading from collections.abc import Mapping, Sequence from contextvars import ContextVar -from datetime import datetime from typing import Any from opentelemetry import context as otel_context, trace as otel_trace @@ -37,6 +35,7 @@ ) from ...tracing.spans import Span from ...tracing.traces import Trace +from ...tracing.util import _parse_iso_timestamp from ...version import __version__ _scope_owner: ContextVar[object | None] = ContextVar("agents_otel_scope_owner", default=None) @@ -45,10 +44,8 @@ def _timestamp(value: str | None) -> int | None: if value is None: return None - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - seconds = int(parsed.replace(microsecond=0).timestamp()) - fraction = re.search(r"T\d{2}:\d{2}:\d{2}\.(\d+)", value) - nanos = int(fraction.group(1).ljust(9, "0")[:9]) if fraction else 0 + parsed, nanos = _parse_iso_timestamp(value) + seconds = int(parsed.timestamp()) return seconds * 1_000_000_000 + nanos diff --git a/src/agents/tracing/util.py b/src/agents/tracing/util.py index 7f436d0192..8b2a5269d5 100644 --- a/src/agents/tracing/util.py +++ b/src/agents/tracing/util.py @@ -1,6 +1,23 @@ +import re +from datetime import datetime + from .setup import get_trace_provider +def _parse_iso_timestamp(value: str) -> tuple[datetime, int]: + """Parse timezone-aware source time without losing its nanosecond fraction.""" + fraction = re.search( + r"^\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}\.(\d+)(?=Z$|[+-]\d{2}:\d{2}$|$)", value + ) + # Python 3.10 cannot parse arbitrary fractional precision; retain it separately. + whole_seconds = value[: fraction.start(1) - 1] + value[fraction.end(1) :] if fraction else value + parsed = datetime.fromisoformat(whole_seconds.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("Source timestamp must include timezone.") + nanos = int(fraction.group(1).ljust(9, "0")[:9]) if fraction else parsed.microsecond * 1000 + return parsed.replace(microsecond=0), nanos + + def time_iso() -> str: """Return the current time in ISO 8601 format.""" return get_trace_provider().time_iso() From 62d933cd6894a2973b237967ae11199fafdce6fa Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 00:54:53 -0800 Subject: [PATCH 06/33] fix(codex): preserve source timestamps on Python 3.10 --- src/agents/models/_codex_tracing.py | 7 ++----- tests/models/test_codex_provider.py | 4 ++-- tests/models/test_operation_tracing_parity.py | 9 ++++++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py index ebe6d69965..bcc7966756 100644 --- a/src/agents/models/_codex_tracing.py +++ b/src/agents/models/_codex_tracing.py @@ -2,7 +2,6 @@ import json from dataclasses import dataclass -from datetime import datetime from typing import Any from openai.types.responses import Response @@ -20,7 +19,7 @@ from ..tracing.config import get_operation_detail from ..tracing.provider import _is_noop_span, _is_noop_trace from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData -from ..tracing.util import time_iso +from ..tracing.util import _parse_iso_timestamp, time_iso from .interface import ModelTracing @@ -165,9 +164,7 @@ def _observe(self, method: str, event: dict[str, Any]) -> None: def _timestamp(value: Any) -> str: if not isinstance(value, str): raise ValueError("Missing source timestamp.") - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - if parsed.tzinfo is None: - raise ValueError("Source timestamp must include timezone.") + _parse_iso_timestamp(value) return value def _metadata(self, operation: _Operation, event: dict[str, Any]) -> None: diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index f270ae3598..5622382851 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1302,7 +1302,7 @@ async def events(self): "operationId": f"operation_{index}", "kind": kind, "requestedModel": "requested-model", - "startedAt": f"2026-11-20T00:00:0{index}.000Z", + "startedAt": f"2026-11-20T00:00:0{index}.000000000Z", } if kind == "tool": operation.update(toolName="read_file", callId="native_call") @@ -1471,7 +1471,7 @@ async def test_codex_operations_preserve_source_spans_and_capture_policy( tools = [span for span in operations if span.span_data.type == "function"] assert len(models) == 2 and len(tools) == 1 assert {span.parent_id for span in operations} == {models[0].parent_id} - assert models[0].started_at == "2026-11-20T00:00:01.000Z" + assert models[0].started_at == "2026-11-20T00:00:01.000000000Z" assert models[0].ended_at == "2026-11-20T00:00:01.250Z" assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" assert models[0].span_data.usage == {"input_tokens": 10, "output_tokens": 5} diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py index e0976e22b9..64f6745c6e 100644 --- a/tests/models/test_operation_tracing_parity.py +++ b/tests/models/test_operation_tracing_parity.py @@ -272,9 +272,12 @@ def _codex_provider( mcp_server.write_text( "import sys\n" "from pathlib import Path\n" - "from mcp.server.fastmcp import FastMCP\n" + "try:\n" + " from mcp.server.mcpserver import MCPServer\n" + "except ImportError:\n" + " from mcp.server.fastmcp import FastMCP as MCPServer\n" "from mcp.types import ToolAnnotations\n" - 'server = FastMCP("parity")\n' + 'server = MCPServer("parity")\n' "@server.tool(annotations=ToolAnnotations(readOnlyHint=True, destructiveHint=False))\n" "def read_file() -> str:\n" ' """Read the synthetic fixture file."""\n' @@ -313,7 +316,7 @@ def _codex_provider( def _ns(timestamp: str | None) -> int: assert timestamp is not None - return int(datetime.fromisoformat(timestamp).timestamp() * 1_000_000_000) + return int(datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp() * 1_000_000_000) def _model_provider(name: str, tmp_path: Path, server: _ResponsesFixture) -> ModelProvider: From bcd1b2cfaf2ccd8aa20ad0a99a9888ae02cb67fa Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:22:19 -0800 Subject: [PATCH 07/33] fix(codex): resolve Windows image paths and file URLs --- src/agents/models/_codex_inputs.py | 17 +++++++++-------- tests/models/test_codex_inputs.py | 14 +++++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/agents/models/_codex_inputs.py b/src/agents/models/_codex_inputs.py index 471cb9d6ce..be1762066d 100644 --- a/src/agents/models/_codex_inputs.py +++ b/src/agents/models/_codex_inputs.py @@ -6,7 +6,8 @@ import re from pathlib import Path from typing import Any, cast -from urllib.parse import unquote, urlparse +from urllib.parse import urlparse +from urllib.request import url2pathname from openai.types.responses import ResponseFunctionToolCall from pydantic import TypeAdapter @@ -24,13 +25,13 @@ def _image_content(part: dict[str, Any]) -> dict[str, Any]: image = part.get("image_url") if isinstance(image, str) and not image.startswith("data:"): - parsed = urlparse(image) - if parsed.scheme == "file" and parsed.netloc in ("", "localhost"): - path = Path(unquote(parsed.path)) - elif not parsed.scheme and Path(image).is_absolute(): - path = Path(image) - else: - raise UserError("CodexProvider accepts images as local paths or image data URLs.") + path = Path(image) + if not path.is_absolute(): + parsed = urlparse(image) + if parsed.scheme == "file" and parsed.netloc in ("", "localhost"): + path = Path(url2pathname(parsed.path)) + else: + raise UserError("CodexProvider accepts images as local paths or image data URLs.") media_type = mimetypes.guess_type(path.name)[0] if media_type not in ("image/png", "image/jpeg", "image/webp", "image/gif"): raise UserError("CodexProvider accepts PNG, JPEG, WebP, and GIF images.") diff --git a/tests/models/test_codex_inputs.py b/tests/models/test_codex_inputs.py index 07b300dbc3..3915057f6c 100644 --- a/tests/models/test_codex_inputs.py +++ b/tests/models/test_codex_inputs.py @@ -65,9 +65,11 @@ def test_tool_result_continuation_uses_history_without_fabricated_user_input(): assert turn == [] -def test_data_and_local_images_preserve_content_in_turn_and_tool_output(tmp_path): - image = tmp_path / "image.png" +@pytest.mark.parametrize("local_form", ["url", "path"]) +def test_data_and_local_images_preserve_content_in_turn_and_tool_output(tmp_path, local_form): + image = tmp_path / "image % space.png" image.write_bytes(b"image bytes") + local_image = image.as_uri() if local_form == "url" else str(image) data = "data:image/png;base64,aW1hZ2UgYnl0ZXM=" history, turn = prepare_codex_history( cast( @@ -77,7 +79,7 @@ def test_data_and_local_images_preserve_content_in_turn_and_tool_output(tmp_path { "type": "function_call_output", "call_id": "c", - "output": [{"type": "input_image", "image_url": image.as_uri()}], + "output": [{"type": "input_image", "image_url": local_image}], }, { "role": "user", @@ -105,6 +107,12 @@ def test_data_and_local_images_preserve_content_in_turn_and_tool_output(tmp_path [{"role": "user", "content": [{"type": "input_file", "file_id": "file_1"}]}], [{"type": "function_call", "name": "f", "call_id": "c", "arguments": "{}"}], [{"type": "function_call_output", "call_id": "c", "output": "orphan"}], + [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "file://remote/image.png"}], + } + ], ], ) def test_unsupported_history_is_rejected(items: Any): From 1afab3ff1d8993d5d3b634fc78d75e0137d6570f Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:22:21 -0800 Subject: [PATCH 08/33] test(models): use the Windows subprocess event loop --- tests/models/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/models/conftest.py diff --git a/tests/models/conftest.py b/tests/models/conftest.py new file mode 100644 index 0000000000..bd05d89c68 --- /dev/null +++ b/tests/models/conftest.py @@ -0,0 +1,9 @@ +import asyncio +import sys +from collections.abc import Callable + +if sys.platform == "win32": + # Importing Modal during collection changes the Windows loop policy. + # The platform's default Proactor loop supports subprocess pipes. + def pytest_asyncio_loop_factories() -> dict[str, Callable[[], asyncio.AbstractEventLoop]]: + return {"proactor": asyncio.ProactorEventLoop} From 52b89a3ecb0646b96f5f937bfa4534d2e0684ac4 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:22:28 -0800 Subject: [PATCH 09/33] fix(tracing): enforce capture budgets before publishing generation payloads --- src/agents/models/_trace.py | 5 +- src/agents/tracing/_content.py | 7 +- src/agents/tracing/span_data.py | 92 +++++++++++++++-------- tests/tracing/test_content_capture.py | 104 +++++++++++++++++++++++--- 4 files changed, 163 insertions(+), 45 deletions(-) diff --git a/src/agents/models/_trace.py b/src/agents/models/_trace.py index 4f8991e29e..2173099b69 100644 --- a/src/agents/models/_trace.py +++ b/src/agents/models/_trace.py @@ -59,7 +59,10 @@ def populate_generation_span( ) -> None: """Populate generation trace data from a Chat Completions adapter response.""" if tracing.include_data(): - span_generation.span_data.output = [final_response.model_dump()] + if span_generation.span_data.operation.detail == "content": + span_generation.span_data.output = [item.model_dump() for item in final_response.output] + else: + span_generation.span_data.output = [final_response.model_dump()] if final_response.usage is not None: span_generation.span_data.usage = { diff --git a/src/agents/tracing/_content.py b/src/agents/tracing/_content.py index 5b79639a01..0f567863df 100644 --- a/src/agents/tracing/_content.py +++ b/src/agents/tracing/_content.py @@ -54,6 +54,7 @@ } _OMIT_FIELDS = {key.replace("_", "") for key in _OMIT_FIELDS} _DATA_URI = re.compile(r"data:[^\s;,]+(?:;[^\s,]*)?,[^\s\"'<>]+", re.IGNORECASE) +_JSON_PREFIX = re.compile(r"\s*[\[{]") def _json(value: Any) -> str: @@ -81,7 +82,9 @@ def _project(value: Any, budget: int, depth: int) -> Any: if depth > 32: return _TRUNCATED if isinstance(value, str): - if value.lstrip().startswith(("{", "[")): + if _JSON_PREFIX.match(value): + if len(value) > budget or len(value.encode("utf-8")) > budget: + return _json({"omitted": True, "truncated": True, "reason": "capture_budget"}) try: embedded = json.loads(value) except ValueError: @@ -96,7 +99,7 @@ def _project(value: Any, budget: int, depth: int) -> Any: # slicing encoded JSON into an invalid tool result. encoded = _json(_project(embedded, (budget - 2) // 2, depth + 1)) return encoded - value = _DATA_URI.sub("[omitted: media]", value) + value = _DATA_URI.sub("[omitted: media]", _text(value, budget)) return _text(value, budget) if value is None or isinstance(value, (bool, float, int)): return value if _size(value) <= budget else _TRUNCATED diff --git a/src/agents/tracing/span_data.py b/src/agents/tracing/span_data.py index 7f33eb805a..a4bb0e23bd 100644 --- a/src/agents/tracing/span_data.py +++ b/src/agents/tracing/span_data.py @@ -3,7 +3,7 @@ import abc from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast from ._content import bound_content from .config import OperationDetail, get_operation_detail @@ -168,13 +168,47 @@ def export(self) -> dict[str, Any]: return {key: value for key, value in values.items() if value is not None} -class FunctionSpanData(SpanData): +_Payload = TypeVar("_Payload") +_Input = TypeVar("_Input") +_Output = TypeVar("_Output") + + +def _capture_payload(operation: OperationMetadata, value: _Payload) -> _Payload | None: + if operation.detail == "metadata": + return None + if operation.detail == "content" and value is not None: + return cast(_Payload, bound_content(value)) + return value + + +class _CapturedSpanData(SpanData, Generic[_Input, _Output]): + __slots__ = ("_input", "_output", "operation") + operation: OperationMetadata + + @property + def input(self) -> _Input | None: + return self._input + + @input.setter + def input(self, value: _Input | None) -> None: + self._input = _capture_payload(self.operation, value) + + @property + def output(self) -> _Output | None: + return self._output + + @output.setter + def output(self, value: _Output | None) -> None: + self._output = _capture_payload(self.operation, value) + + +class FunctionSpanData(_CapturedSpanData[str, Any]): """ Represents a Function Span in the trace. Includes input, output and MCP data (if applicable). """ - __slots__ = ("name", "_input", "_output", "mcp_data", "operation") + __slots__ = ("name", "mcp_data") def __init__( self, @@ -189,29 +223,6 @@ def __init__( self.output = output self.mcp_data = mcp_data - def _capture(self, value: Any) -> Any: - if self.operation.detail == "metadata": - return None - if self.operation.detail == "content" and value is not None: - return bound_content(value) - return value - - @property - def input(self) -> str | None: - return self._input - - @input.setter - def input(self, value: str | None) -> None: - self._input: str | None = self._capture(value) - - @property - def output(self) -> Any: - return self._output - - @output.setter - def output(self, value: Any) -> None: - self._output = self._capture(value) - @property def type(self) -> str: return "function" @@ -226,20 +237,19 @@ def export(self) -> dict[str, Any]: } -class GenerationSpanData(SpanData): +class GenerationSpanData( + _CapturedSpanData[Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]] +): """ Represents a Generation Span in the trace. Includes input, output, model, model configuration, usage, and optional tool definitions. """ __slots__ = ( - "input", - "output", "model", - "model_config", + "_model_config", "usage", - "tools", - "operation", + "_tools", ) def __init__( @@ -251,14 +261,30 @@ def __init__( usage: dict[str, Any] | None = None, tools: Sequence[Mapping[str, Any]] | None = None, ): + self.operation = OperationMetadata(model=model) self.input = input self.output = output self.model = model self.model_config = model_config - self.operation = OperationMetadata(model=model) self.usage = usage self.tools = tools + @property + def model_config(self) -> Mapping[str, Any] | None: + return self._model_config + + @model_config.setter + def model_config(self, value: Mapping[str, Any] | None) -> None: + self._model_config = _capture_payload(self.operation, value) + + @property + def tools(self) -> Sequence[Mapping[str, Any]] | None: + return self._tools + + @tools.setter + def tools(self, value: Sequence[Mapping[str, Any]] | None) -> None: + self._tools = _capture_payload(self.operation, value) + @property def type(self) -> str: return "generation" diff --git a/tests/tracing/test_content_capture.py b/tests/tracing/test_content_capture.py index ad78a407c3..e0f3e75814 100644 --- a/tests/tracing/test_content_capture.py +++ b/tests/tracing/test_content_capture.py @@ -21,6 +21,8 @@ trace, ) from agents.tracing._content import CONTENT_LIMIT, bound_content +from agents.tracing.config import OperationDetail, operation_tracing_scope +from agents.tracing.span_data import GenerationSpanData from tests.testing_processor import SPAN_PROCESSOR_TESTING, SpanProcessorForTests @@ -104,21 +106,21 @@ def test_public_model_export_omits_hidden_reasoning_and_embedded_media(exporter) assert "omitted" in serialized -def test_public_tool_export_retains_large_result_prefix_with_bounded_json(exporter) -> None: +def test_public_tool_export_omits_oversized_structured_result_with_bounded_json(exporter) -> None: with trace("content"), function_span("read_file") as tool: tool.span_data.operation.detail = "content" tool.span_data.output = json.dumps({"text": "file prefix: " + "é🙂" * 40_000}) captured = exporter.get_finished_spans()[0].attributes["gen_ai.tool.call.result"] assert len(json.dumps(captured, ensure_ascii=False).encode("utf-8")) <= CONTENT_LIMIT decoded = json.loads(captured) - assert decoded["text"].startswith("file prefix: ") - assert decoded["text"].endswith("[truncated]") + assert decoded == {"omitted": True, "truncated": True, "reason": "capture_budget"} @pytest.mark.parametrize( "value", [ "é🙂" * 40_000, + " " * (CONTENT_LIMIT * 8) + "ordinary text", [{"text": "x" * 180}] * 3_000, {"input": [{"role": "user", "content": "x" * 80_000}], "instructions": "later"}, ], @@ -129,16 +131,35 @@ def test_projection_keeps_valid_structure_and_inline_budget_marker(value) -> Non assert len(serialized.encode("utf-8")) <= CONTENT_LIMIT assert "truncated" in serialized assert json.loads(serialized) == captured + if isinstance(value, str): + assert captured.endswith("[truncated]") -def test_embedded_tool_json_remains_parseable_after_redaction_and_truncation() -> None: - raw = json.dumps({"accessToken": "TOKEN_CANARY", "result": "prefix " + "🙂" * 40_000}) +@pytest.mark.parametrize( + "raw", + [ + json.dumps({"accessToken": "TOKEN_CANARY", "result": "prefix " + "🙂" * 40_000}), + "[" + ",".join(['{"apiKey":"TOKEN_CANARY"}'] * 60_000) + "]", + json.dumps({"text": "🙂" * 10_000}, ensure_ascii=False), + " " * (CONTENT_LIMIT * 8) + '{"apiKey":"TOKEN_CANARY"}', + ], + ids=["object", "array", "utf8-byte-limit", "whitespace-prefix"], +) +def test_oversized_embedded_tool_json_is_omitted_without_parsing( + raw: str, monkeypatch: pytest.MonkeyPatch +) -> None: + original_loads = json.loads + monkeypatch.setattr( + "agents.tracing._content.json.loads", + lambda _: pytest.fail("Oversized structured content must not be parsed."), + ) captured = bound_content(raw) assert isinstance(captured, str) - decoded = json.loads(captured) - assert decoded["result"].startswith("prefix ") - assert decoded["result"].endswith("[truncated]") - assert decoded["accessToken"] == "[omitted]" + assert original_loads(captured) == { + "omitted": True, + "truncated": True, + "reason": "capture_budget", + } assert len(json.dumps(captured, ensure_ascii=False).encode("utf-8")) <= CONTENT_LIMIT @@ -207,3 +228,68 @@ async def handler(request: httpx2.Request) -> httpx2.Response: assert "TRANSPORT_CREDENTIAL_CANARY" not in str( (dict(span.attributes), span.events, span.status.description) ) + + +@pytest.mark.parametrize("detail", [None, "metadata", "content"]) +@pytest.mark.parametrize("assign", [False, True], ids=["constructor", "assignment"]) +def test_generation_payload_capture_policy(detail: OperationDetail | None, assign: bool) -> None: + messages = [{"role": "user", "content": "visible prompt " + "é🙂" * 40_000}] + output = [ + { + "role": "assistant", + "reasoning_content": "HIDDEN_CANARY", + "content": "visible answer " + "é🙂" * 40_000, + } + ] + config = { + "instructions": "visible instructions " + "é🙂" * 40_000, + "metadata": {"Authorization": "AUTH_CANARY"}, + } + tools = [ + { + "type": "function", + "function": {"name": "lookup", "description": "visible tool " + "é🙂" * 40_000}, + } + ] + usage = {"input_tokens": 10, "output_tokens": 5} + policy = {"operation_detail": detail} if detail is not None else None + with operation_tracing_scope(policy, include_sensitive=True): + if assign: + data = GenerationSpanData(model="fixture", usage=usage) + data.input, data.output, data.model_config, data.tools = messages, output, config, tools + else: + data = GenerationSpanData(messages, output, "fixture", config, usage, tools) + captured = (data.input, data.output, data.model_config, data.tools) + assert all( + value is repeated + for value, repeated in zip( + captured, (data.input, data.output, data.model_config, data.tools), strict=True + ) + ) + originals = (messages, output, config, tools) + assert data.model == "fixture" and data.usage is usage + if detail is None: + assert all(value is original for value, original in zip(captured, originals, strict=True)) + elif detail == "metadata": + assert captured == (None, None, None, None) + assert "CANARY" not in json.dumps(data.export()) + else: + assert all( + len(json.dumps(value, ensure_ascii=False).encode()) <= CONTENT_LIMIT + for value in captured + ) + exported = json.dumps(data.export(), ensure_ascii=False) + assert "CANARY" not in exported + assert "visible prompt" in exported and "visible answer" in exported + assert "visible instructions" in exported and "visible tool" in exported + + +def test_generation_payload_assignments_honor_content_veto() -> None: + with operation_tracing_scope({"operation_detail": "content"}, include_sensitive=False): + data = GenerationSpanData() + data.input = [{"role": "user", "content": "INPUT_CANARY"}] + data.output = [{"role": "assistant", "content": "OUTPUT_CANARY"}] + data.model_config = {"instructions": "INSTRUCTIONS_CANARY"} + data.tools = [{"description": "TOOL_CANARY"}] + assert (data.input, data.output, data.model_config, data.tools) == (None, None, None, None) + assert "CANARY" not in json.dumps(data.export()) From eff7774383cf29765ede26c67df4142fc3df08a5 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:22:31 -0800 Subject: [PATCH 10/33] fix(tracing): preserve Chat Completions messages and tool calls in OpenTelemetry --- .../extensions/tracing/opentelemetry.py | 23 +++++++- tests/tracing/test_opentelemetry.py | 56 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 55ac858e37..5b1c7e7dc8 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -82,7 +82,7 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: if isinstance(item.get(flag), bool): part[flag] = item[flag] parts.append(part) - elif item.get("truncated") is True: + elif item.get("truncated") is True and "content" not in item and "tool_calls" not in item: parts.append({"type": "openai.omitted", "reason": "truncated"}) elif kind in ("function_call", "custom_tool_call"): role = "assistant" @@ -105,7 +105,11 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: ) elif kind == "message": content = item.get("content") - if isinstance(content, str): + if role == "tool" and "tool_call_id" in item: + parts.append( + {"type": "tool_call_response", "id": item["tool_call_id"], "response": content} + ) + elif isinstance(content, str): parts.append({"type": "text", "content": content}) elif isinstance(content, Sequence): for part in content: @@ -116,6 +120,18 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: else: # Preserve part order without exporting media or hidden reasoning. parts.append({"type": "openai.omitted", "reason": "unsupported_content"}) + for call in item.get("tool_calls") or (): + if not isinstance(call, Mapping) or call.get("type") != "function": + continue + function = call.get("function") + if isinstance(function, Mapping): + canonical = { + "type": "function_call", + "call_id": call.get("id"), + "name": function.get("name", ""), + "arguments": function.get("arguments"), + } + parts.extend(_messages([canonical], output=True)[0]["parts"]) else: parts.append({"type": "openai.omitted", "reason": "unsupported_content"}) if output and messages: @@ -372,6 +388,9 @@ def _content_attributes( else None ) ) + if isinstance(data, ResponseSpanData): + inputs = bound_content(inputs) if inputs is not None else None + output = bound_content(output) if output is not None else None request: dict[str, Any] = {} if inputs is not None: request["messages"] = _messages(inputs) diff --git a/tests/tracing/test_opentelemetry.py b/tests/tracing/test_opentelemetry.py index d55954e19d..1c65745494 100644 --- a/tests/tracing/test_opentelemetry.py +++ b/tests/tracing/test_opentelemetry.py @@ -444,3 +444,59 @@ def on_end(self, span): assert all( span.attributes["openai.agents.operation.outcome"] == "interrupted" for span in spans ) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_chat_completions_native_payloads_reach_otel(otel, monkeypatch, streamed) -> None: + from unittest.mock import AsyncMock + + from openai import AsyncOpenAI + from openai.types.chat import ChatCompletionChunk + + from agents import ModelSettings, ModelTracing, OpenAIChatCompletionsModel + from agents.tracing.config import operation_tracing_scope + from tests.models.test_openai_chatcompletions import _minimal_chat_completion + + _, exporter = otel + completion = _minimal_chat_completion("answer") + call = { + "id": "call-lookup", + "type": "function", + "function": {"name": "lookup", "arguments": '{"key":"value"}'}, + } + completion.choices[0].message = type(completion.choices[0].message).model_validate( + {**completion.choices[0].message.model_dump(), "tool_calls": [call]} + ) + + async def chunks(): + payload = completion.model_dump() + delta = {**payload["choices"][0]["message"], "tool_calls": [{**call, "index": 0}]} + payload.update( + object="chat.completion.chunk", + choices=[{"index": 0, "delta": delta, "finish_reason": "tool_calls"}], + ) + yield ChatCompletionChunk.model_validate(payload) + + async with AsyncOpenAI(api_key="test-placeholder") as client: + monkeypatch.setattr( + client.chat.completions, + "create", + AsyncMock(return_value=chunks() if streamed else completion), + ) + model = OpenAIChatCompletionsModel("fixture", client) + with trace("group"), operation_tracing_scope({"operation_detail": "content"}, True): + args = (None, "prompt", ModelSettings(), [], None, [], ModelTracing.ENABLED) + if streamed: + async for _ in model.stream_response(*args): + pass + else: + await model.get_response(*args) + span = next(s for s in exporter.get_finished_spans() if s.name == "chat fixture") + messages = json.loads(json.dumps(span.attributes["gen_ai.output.messages"])) + assert messages[0]["role"] == "assistant" + assert messages[0]["parts"] == [ + {"type": "text", "content": "answer"}, + {"type": "tool_call", "id": "call-lookup", "name": "lookup", "arguments": {"key": "value"}}, + ] From 459729bb18616a1f8adb1ffa26b468854334e714 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:40:47 -0800 Subject: [PATCH 11/33] fix(codex): bound allocations when capturing operation payloads --- patches/codex/0001-operation-tracing.patch | 946 ++++++++++++++++++--- patches/codex/README.md | 2 +- 2 files changed, 822 insertions(+), 126 deletions(-) diff --git a/patches/codex/0001-operation-tracing.patch b/patches/codex/0001-operation-tracing.patch index 2ce3ab5093..8fd585be19 100644 --- a/patches/codex/0001-operation-tracing.patch +++ b/patches/codex/0001-operation-tracing.patch @@ -1482,22 +1482,25 @@ index 67f627e..1e6c9cd 100644 response_id, token_usage: None, diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs -index 2ac9b93..3f93332 100644 +index 2ac9b93..8a01fc3 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs -@@ -2033,6 +2033,11 @@ impl ResponsesHttpExecutor<'_> { +@@ -2033,6 +2033,14 @@ impl ResponsesHttpExecutor<'_> { request.input = input; } inference_trace_attempt.record_started(&request); + crate::operation_observation::observe_model_request( -+ &request, ++ &crate::operation_observation::ModelRequestContent { ++ input: &request.input, ++ previous_response_id: None, ++ }, + &request, + &client_setup.api_provider.base_url, + ); let client = ApiResponsesClient::new( transport, client_setup.api_provider, -@@ -2185,6 +2190,7 @@ impl ModelClientSession { +@@ -2185,6 +2193,7 @@ impl ModelClientSession { } else { session_telemetry_for_request(session_telemetry, &request) }; @@ -1505,13 +1508,17 @@ index 2ac9b93..3f93332 100644 match self .websocket_connection(WebsocketConnectParams { session_telemetry, -@@ -2341,6 +2347,13 @@ impl ModelClientSession { +@@ -2341,6 +2350,17 @@ impl ModelClientSession { inference_trace_attempt.record_started(&ws_request); } + if !warmup { ++ let ResponsesWsRequest::ResponseCreate(payload) = &ws_request; + crate::operation_observation::observe_model_request( -+ &ws_request, ++ &crate::operation_observation::ModelRequestContent { ++ input: payload.input, ++ previous_response_id: payload.previous_response_id.as_deref(), ++ }, + &request, + &operation_endpoint, + ); @@ -1519,7 +1526,7 @@ index 2ac9b93..3f93332 100644 let websocket_connection = self.websocket_session.connection.as_ref().ok_or_else(|| { self.client.state.provider.map_api_error(ApiError::Stream( -@@ -2686,6 +2699,8 @@ where +@@ -2686,6 +2706,8 @@ where mpsc::channel::>(RESPONSE_STREAM_CHANNEL_CAPACITY); let consumer_dropped = CancellationToken::new(); let consumer_dropped_for_stream = consumer_dropped.clone(); @@ -1528,7 +1535,7 @@ index 2ac9b93..3f93332 100644 tokio::spawn(async move { let mut logged_error = false; -@@ -2702,6 +2717,7 @@ where +@@ -2702,6 +2724,7 @@ where loop { let event = tokio::select! { _ = consumer_dropped.cancelled() => { @@ -1536,7 +1543,7 @@ index 2ac9b93..3f93332 100644 inference_trace_attempt.record_cancelled( STREAM_DROPPED_REASON, upstream_request_id, -@@ -2712,8 +2728,21 @@ where +@@ -2712,8 +2735,21 @@ where event = api_stream.next() => event, }; let Some(event) = event else { @@ -1598,10 +1605,10 @@ index 9f4ef92..34a88e0 100644 }; diff --git a/codex-rs/core/src/operation_observation.rs b/codex-rs/core/src/operation_observation.rs new file mode 100644 -index 0000000..1344548 +index 0000000..cea2a98 --- /dev/null +++ b/codex-rs/core/src/operation_observation.rs -@@ -0,0 +1,1348 @@ +@@ -0,0 +1,1970 @@ +//! Opt-in, in-memory operation facts for the host that owns a turn. +//! +//! These observations never enter the rollout or the telemetry logger. The host @@ -1619,6 +1626,7 @@ index 0000000..1344548 +use std::time::Instant; + +pub const CONTENT_LIMIT_BYTES: usize = 32 * 1024; ++pub(crate) const CONTENT_VALUE_LIMIT_BYTES: usize = CONTENT_LIMIT_BYTES - 256; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] @@ -1801,13 +1809,19 @@ index 0000000..1344548 + &self.operation_id + } + -+ pub fn capture_input(&self, value: &T) { ++ pub fn capture_input(&self, value: &Value) { + if self.observation.mode == OperationCaptureMode::Content { + self.update(|event| event.input = Some(project_content(value))); + } + } + -+ pub fn capture_output(&self, value: &T) { ++ pub(crate) fn capture_input_content(&self, content: OperationContent) { ++ if self.observation.captures_content() { ++ self.update(|event| event.input = Some(content)); ++ } ++ } ++ ++ pub fn capture_output(&self, value: &Value) { + if self.observation.mode == OperationCaptureMode::Content { + self.update(|event| event.output = Some(project_content(value))); + } @@ -1823,11 +1837,7 @@ index 0000000..1344548 + } + } + -+ pub fn finish_with_output_fn( -+ &self, -+ outcome: OperationOutcome, -+ output: impl FnOnce() -> T, -+ ) { ++ pub fn finish_with_output_fn(&self, outcome: OperationOutcome, output: impl FnOnce() -> Value) { + let completed_at = timestamp(); + let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; + if self.observation.captures_content() { @@ -1836,7 +1846,7 @@ index 0000000..1344548 + self.finish_at(outcome, completed_at, duration_ms); + } + -+ pub fn finish_with_output(&self, outcome: OperationOutcome, output: &T) { ++ pub fn finish_with_output(&self, outcome: OperationOutcome, output: &Value) { + let completed_at = timestamp(); + let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; + self.capture_output(output); @@ -1882,16 +1892,53 @@ index 0000000..1344548 + +/// Projects JSON in memory, omitting binary/hidden reasoning and transport secrets. +/// Inline markers survive consumers that extract values from the wire envelope. -+pub fn project_content(value: &T) -> OperationContent { -+ let value = match serde_json::to_value(value) { -+ Ok(value) => value, -+ Err(_) => return omitted_content("serialization_failed"), -+ }; ++pub fn project_content(value: &Value) -> OperationContent { ++ project_with(|projection| projection.project(value, CONTENT_VALUE_LIMIT_BYTES, 0)) ++} ++ ++pub(crate) fn project_text(text: &str) -> OperationContent { ++ project_with(|projection| projection.text(text, CONTENT_VALUE_LIMIT_BYTES, 0)) ++} ++ ++/// Typed adapters stop serialization before constructing an oversized owned JSON value. ++fn bounded_value(value: &(impl Serialize + ?Sized)) -> Result { ++ struct BoundedWriter(Vec); ++ impl std::io::Write for BoundedWriter { ++ fn write(&mut self, bytes: &[u8]) -> std::io::Result { ++ if bytes.len() > CONTENT_VALUE_LIMIT_BYTES.saturating_sub(self.0.len()) { ++ return Err(std::io::Error::other("capture_budget")); ++ } ++ self.0.extend_from_slice(bytes); ++ Ok(bytes.len()) ++ } ++ fn flush(&mut self) -> std::io::Result<()> { ++ Ok(()) ++ } ++ } ++ let mut writer = BoundedWriter(Vec::new()); ++ serde_json::to_writer(&mut writer, value).map_err(|error| { ++ if error.is_io() { ++ "capture_budget" ++ } else { ++ "serialization_failed" ++ } ++ })?; ++ serde_json::from_slice(&writer.0).map_err(|_| "serialization_failed") ++} ++ ++pub(crate) fn project_serializable_content(value: &(impl Serialize + ?Sized)) -> OperationContent { ++ match bounded_value(value) { ++ Ok(value) => project_content(&value), ++ Err(reason) => omitted_content(reason), ++ } ++} ++ ++fn project_with(project: impl FnOnce(&mut ContentProjection) -> Value) -> OperationContent { + let mut projection = ContentProjection { + truncated: false, + omitted: false, + }; -+ let value = projection.project(value, CONTENT_LIMIT_BYTES - 256, 0); ++ let value = project(&mut projection); + let content = OperationContent { + value, + truncated: projection.truncated, @@ -1913,7 +1960,11 @@ index 0000000..1344548 + +fn omitted_content(reason: &str) -> OperationContent { + OperationContent { -+ value: serde_json::json!({"omitted":true,"reason":reason}), ++ value: if reason == "capture_budget" { ++ serde_json::json!({"omitted":true,"truncated":true,"reason":reason}) ++ } else { ++ serde_json::json!({"omitted":true,"reason":reason}) ++ }, + truncated: reason == "capture_budget", + omitted: true, + reason: Some(reason.into()), @@ -1921,7 +1972,18 @@ index 0000000..1344548 +} + +fn json_size(value: &(impl Serialize + ?Sized)) -> usize { -+ serde_json::to_vec(value).map_or(usize::MAX, |value| value.len()) ++ struct Counter(usize); ++ impl std::io::Write for Counter { ++ fn write(&mut self, bytes: &[u8]) -> std::io::Result { ++ self.0 = self.0.saturating_add(bytes.len()); ++ Ok(bytes.len()) ++ } ++ fn flush(&mut self) -> std::io::Result<()> { ++ Ok(()) ++ } ++ } ++ let mut counter = Counter(0); ++ serde_json::to_writer(&mut counter, value).map_or(usize::MAX, |_| counter.0) +} + +struct ContentProjection { @@ -1930,7 +1992,7 @@ index 0000000..1344548 +} + +impl ContentProjection { -+ fn project(&mut self, value: Value, budget: usize, depth: usize) -> Value { ++ fn project(&mut self, value: &Value, budget: usize, depth: usize) -> Value { + if depth > 32 { + self.truncated = true; + return Value::String("[truncated]".into()); @@ -1993,11 +2055,14 @@ index 0000000..1344548 + } + let mut result = serde_json::Map::new(); + let mut used: usize = 2; -+ let mut items: Vec<_> = items.into_iter().collect(); -+ items.sort_by_key(|(key, _)| { -+ !matches!(key.as_str(), "type" | "role" | "id" | "call_id" | "name") -+ }); -+ for (key, value) in items { ++ // Preserve structural keys without collecting an unbounded list of references. ++ let structural = ++ |key: &str| matches!(key, "type" | "role" | "id" | "call_id" | "name"); ++ let ordered = items ++ .iter() ++ .filter(|(key, _)| structural(key)) ++ .chain(items.iter().filter(|(key, _)| !structural(key))); ++ for (key, value) in ordered { + let available = budget.saturating_sub(used + json_size(&key) + 36); + if available < 84 { + self.truncated = true; @@ -2032,41 +2097,230 @@ index 0000000..1344548 + self.project(value, available, depth + 1) + }; + used += json_size(&key) + json_size(&value) + 4; -+ result.insert(key, value); ++ result.insert(key.clone(), value); + } + Value::Object(result) + } -+ value => value, ++ value => value.clone(), ++ } ++ } ++ ++ fn response_item( ++ &mut self, ++ item: &codex_protocol::models::ResponseItem, ++ budget: usize, ++ ) -> Value { ++ use codex_protocol::models::ContentItem; ++ use codex_protocol::models::ResponseItem; ++ match bounded_value(item) { ++ Ok(value) => return self.project(&value, budget, 0), ++ Err("capture_budget") => {} ++ Err(reason) => { ++ self.omitted = true; ++ return omitted_content(reason).value; ++ } ++ } ++ if let Some(value) = self.oversized_tool_result(item, budget) { ++ return value; ++ } ++ let ResponseItem::Message { ++ role, ++ content, ++ id, ++ status, ++ phase, ++ encrypted_content, ++ internal_chat_message_metadata_passthrough, ++ .. ++ } = item ++ else { ++ self.truncated = true; ++ self.omitted = true; ++ return omitted_content("capture_budget").value; ++ }; ++ // Large messages retain borrowed visible text without cloning media or the full item. ++ let mut result = serde_json::Map::new(); ++ result.insert("type".into(), Value::String("message".into())); ++ result.insert("role".into(), self.text(role, budget.min(256), 0)); ++ for (key, value) in [ ++ ("id", bounded_value(id)), ++ ("status", bounded_value(status)), ++ ("phase", bounded_value(phase)), ++ ] { ++ if let Ok(value) = value { ++ if !value.is_null() { ++ result.insert(key.into(), self.project(&value, 256, 0)); ++ } ++ } else { ++ self.omitted = true; ++ result.insert("omitted".into(), Value::Bool(true)); ++ } ++ } ++ if encrypted_content.is_some() || internal_chat_message_metadata_passthrough.is_some() { ++ self.omitted = true; ++ result.insert("omitted".into(), Value::Bool(true)); ++ } ++ let mut used = json_size(&result) + 24; ++ let mut parts = Vec::new(); ++ for part in content { ++ let available = budget.saturating_sub(used + 96); ++ if available < 84 { ++ self.truncated = true; ++ parts.push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); ++ break; ++ } ++ let value = match part { ++ ContentItem::InputText { text } => { ++ serde_json::json!({"type":"input_text","text":self.text(text, available, 0)}) ++ } ++ ContentItem::OutputText { ++ text, ++ annotations, ++ logprobs, ++ } => { ++ let mut value = serde_json::json!({"type":"output_text","text":self.text(text, available, 0)}); ++ if annotations.is_some() || logprobs.is_some() { ++ self.omitted = true; ++ value["omitted"] = Value::Bool(true); ++ } ++ value ++ } ++ ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => { ++ self.omitted = true; ++ serde_json::json!({"type":if matches!(part, ContentItem::InputImage { .. }) {"input_image"} else {"input_audio"},"omitted":true,"reason":"unsupported_content"}) ++ } ++ }; ++ used += json_size(&value) + 2; ++ parts.push(value); ++ } ++ result.insert("content".into(), Value::Array(parts)); ++ Value::Object(result) ++ } ++ ++ fn oversized_tool_result( ++ &mut self, ++ item: &codex_protocol::models::ResponseItem, ++ budget: usize, ++ ) -> Option { ++ use codex_protocol::models::{ ++ FunctionCallOutputBody, FunctionCallOutputContentItem, ResponseItem, ++ }; ++ let (kind, call_id, name, output) = match item { ++ ResponseItem::FunctionCallOutput { ++ call_id, ++ name, ++ output, ++ .. ++ } => ( ++ "function_call_output", ++ call_id.as_deref(), ++ name.as_deref(), ++ output, ++ ), ++ ResponseItem::CustomToolCallOutput { ++ call_id, ++ name, ++ output, ++ .. ++ } => ( ++ "custom_tool_call_output", ++ Some(call_id.as_str()), ++ name.as_deref(), ++ output, ++ ), ++ _ => return None, ++ }; ++ self.omitted = true; ++ let mut result = serde_json::json!({"type":kind,"omitted":true,"reason":"capture_budget"}); ++ for (key, value) in [("call_id", call_id), ("name", name)] { ++ if let Some(value) = value { ++ result[key] = self.text(value, budget.min(256), 0); ++ } + } ++ let mut used = json_size(&result) + 24; ++ result["output"] = match &output.body { ++ FunctionCallOutputBody::Text(text) => { ++ self.text(text, budget.saturating_sub(used + 64), 0) ++ } ++ FunctionCallOutputBody::ContentItems(items) => { ++ let mut parts = Vec::new(); ++ for part in items { ++ let available = budget.saturating_sub(used + 96); ++ if available < 84 { ++ self.truncated = true; ++ parts.push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); ++ break; ++ } ++ let value = match part { ++ FunctionCallOutputContentItem::InputText { text } => { ++ serde_json::json!({"type":"input_text","text":self.text(text, available, 0)}) ++ } ++ _ => { ++ let kind = match part { ++ FunctionCallOutputContentItem::InputImage { .. } => "input_image", ++ FunctionCallOutputContentItem::InputAudio { .. } => "input_audio", ++ FunctionCallOutputContentItem::EncryptedContent { .. } => { ++ "encrypted_content" ++ } ++ FunctionCallOutputContentItem::InputText { .. } => unreachable!(), ++ }; ++ serde_json::json!({"type":kind,"omitted":true,"reason":"unsupported_content"}) ++ } ++ }; ++ used += json_size(&value) + 2; ++ parts.push(value); ++ } ++ Value::Array(parts) ++ } ++ }; ++ Some(self.project(&result, budget, 0)) + } + -+ fn text(&mut self, text: String, budget: usize, depth: usize) -> Value { ++ fn text(&mut self, text: &str, budget: usize, depth: usize) -> Value { + static DATA_URI: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex_lite::Regex::new(r#"(?i)data:[^\s;,]+(?:;[^\s,]*)?,[^\s\"'<>]+"#) + .expect("valid data URI pattern") + }); + self.truncated |= text.ends_with("[truncated]"); + self.omitted |= text.contains("[omitted]") || text.contains("[omitted: media]"); ++ // Classification borrows the text; bound raw bytes before allocating a JSON tree. ++ let parse_limit = budget.min(CONTENT_LIMIT_BYTES); + if text.trim_start().starts_with(['{', '[']) { ++ if text.len() > parse_limit { ++ self.truncated = true; ++ self.omitted = true; ++ return Value::String( ++ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) ++ .to_string(), ++ ); ++ } + if let Ok(embedded) = serde_json::from_str::(&text) { -+ let safe = self.project(embedded.clone(), budget, depth + 1); ++ let safe = self.project(&embedded, budget, depth + 1); + if safe == embedded && json_size(&text) <= budget { -+ return Value::String(text); ++ return Value::String(text.to_owned()); + } + let mut encoded = safe.to_string(); + if json_size(&encoded) > budget { + // Rebudget the value so escaped JSON remains valid. + encoded = self -+ .project(embedded, budget.saturating_sub(2) / 2, depth + 1) ++ .project(&embedded, budget.saturating_sub(2) / 2, depth + 1) + .to_string(); + } + return Value::String(encoded); + } + } -+ let projected = DATA_URI.replace_all(&text, "[omitted: media]"); ++ // Regex replacement and JSON sizing only see a bounded UTF-8 prefix. ++ let mut end = text.len().min(budget); ++ while !text.is_char_boundary(end) { ++ end -= 1; ++ } ++ let source_truncated = end < text.len(); ++ self.truncated |= source_truncated; ++ let text = &text[..end]; ++ let projected = DATA_URI.replace_all(text, "[omitted: media]"); + self.omitted |= projected != text; + let text = projected.into_owned(); -+ if json_size(&text) <= budget { ++ if !source_truncated && json_size(&text) <= budget { + return Value::String(text); + } + self.truncated = true; @@ -2078,7 +2332,7 @@ index 0000000..1344548 + while !text.is_char_boundary(end) { + end -= 1; + } -+ if json_size(&format!("{}[truncated]", &text[..end])) <= budget { ++ if json_size(&&text[..end]).saturating_add("[truncated]".len()) <= budget { + low = middle; + } else { + high = middle - 1; @@ -2223,7 +2477,7 @@ index 0000000..1344548 + fn request( + &self, + request: &T, -+ identity: &impl Serialize, ++ identity: &codex_api::ResponsesApiRequest, + endpoint: &str, + ) { + let mut state = self @@ -2330,7 +2584,7 @@ index 0000000..1344548 + state.partial_truncated = false; + } + if state.output_bytes < CONTENT_LIMIT_BYTES { -+ let content = project_content(item); ++ let content = project_response_item(item); + state.output_bytes += serde_json::to_vec(&content.value) + .map_or(CONTENT_LIMIT_BYTES, |value| value.len()); + state.output.push(content.value); @@ -2387,21 +2641,60 @@ index 0000000..1344548 + } +} + -+fn model_output(state: &ModelOperationState) -> Vec { ++fn model_output(state: &ModelOperationState) -> Value { + let mut output = state.output.clone(); + if !state.partial_text.is_empty() { + output.push(serde_json::json!({"type":"output_text","text":state.partial_text,"partial":true,"truncated":state.partial_truncated})); + } -+ output ++ Value::Array(output) +} + +pub(crate) fn current_model_operation() -> Option { + CURRENT_MODEL_OPERATION.try_with(Clone::clone).ok() +} + ++/// Only wire messages and continuation identity consume the content budget. ++#[derive(Serialize)] ++pub(crate) struct ModelRequestContent<'a> { ++ #[serde(serialize_with = "serialize_model_inputs")] ++ pub input: &'a [codex_protocol::models::ResponseItem], ++ #[serde(skip_serializing_if = "Option::is_none")] ++ pub previous_response_id: Option<&'a str>, ++} ++ ++fn serialize_model_inputs( ++ input: &[codex_protocol::models::ResponseItem], ++ serializer: S, ++) -> Result { ++ use serde::ser::SerializeSeq; ++ let mut sequence = serializer.serialize_seq(None)?; ++ let mut projection = ContentProjection { ++ truncated: false, ++ omitted: false, ++ }; ++ let mut used = 0; ++ for item in input { ++ let available = CONTENT_VALUE_LIMIT_BYTES.saturating_sub(used + 512); ++ if available < 84 { ++ sequence.serialize_element( ++ &serde_json::json!({"truncated":true,"reason":"capture_budget"}), ++ )?; ++ break; ++ } ++ let value = projection.response_item(item, available); ++ used += json_size(&value) + 2; ++ sequence.serialize_element(&value)?; ++ } ++ sequence.end() ++} ++ ++fn project_response_item(item: &codex_protocol::models::ResponseItem) -> OperationContent { ++ project_with(|projection| projection.response_item(item, CONTENT_VALUE_LIMIT_BYTES)) ++} ++ +pub(crate) fn observe_model_request( + request: &T, -+ identity: &impl Serialize, ++ identity: &codex_api::ResponsesApiRequest, + endpoint: &str, +) { + if let Some(operation) = current_model_operation() { @@ -2418,7 +2711,10 @@ index 0000000..1344548 +} + +fn model_request_content(request: &(impl Serialize + ?Sized)) -> Value { -+ let mut value = model_request_identity(request); ++ let mut value = match bounded_value(request) { ++ Ok(value) => value, ++ Err(reason) => return omitted_content(reason).value, ++ }; + if let Value::Object(body) = &mut value { + // The per-direction content budget belongs to messages and instructions. + // Tool definitions affect request identity but must not consume that budget. @@ -2432,34 +2728,43 @@ index 0000000..1344548 + value +} + -+fn model_request_identity(request: &(impl Serialize + ?Sized)) -> Value { -+ let Ok(Value::Object(mut body)) = serde_json::to_value(request) else { -+ return Value::Null; -+ }; -+ // Preserve semantic request changes when grouping same-input transport retries. -+ body.retain(|key, _| { -+ matches!( -+ key.as_str(), -+ "model" -+ | "instructions" -+ | "input" -+ | "previous_response_id" -+ | "tools" -+ | "tool_choice" -+ | "text" -+ ) -+ }); -+ Value::Object(body) ++fn model_request_identity(request: &codex_api::ResponsesApiRequest) -> impl Serialize + '_ { ++ #[derive(Serialize)] ++ struct Identity<'a> { ++ input: &'a [codex_protocol::models::ResponseItem], ++ model: &'a str, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ text: Option<&'a codex_api::TextControls>, ++ tool_choice: &'a str, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ tools: Option<&'a codex_api::ResponsesApiTools>, ++ } ++ Identity { ++ input: &request.input, ++ model: &request.model, ++ text: request.text.as_ref(), ++ tool_choice: &request.tool_choice, ++ tools: request.tools.as_ref(), ++ } +} + -+fn request_fingerprint(request: &impl Serialize) -> u64 { -+ use std::hash::Hash; ++fn request_fingerprint(request: &codex_api::ResponsesApiRequest) -> u64 { + use std::hash::Hasher; -+ let value = model_request_identity(request); -+ let mut hasher = std::collections::hash_map::DefaultHasher::new(); -+ // The digest only lives with this operation; request content is not retained in metadata mode. -+ value.to_string().hash(&mut hasher); -+ hasher.finish() ++ struct Digest(std::collections::hash_map::DefaultHasher); ++ impl std::io::Write for Digest { ++ fn write(&mut self, bytes: &[u8]) -> std::io::Result { ++ self.0.write(bytes); ++ Ok(bytes.len()) ++ } ++ fn flush(&mut self) -> std::io::Result<()> { ++ Ok(()) ++ } ++ } ++ let mut digest = Digest(std::collections::hash_map::DefaultHasher::new()); ++ // Typed semantic fields borrow the prepared body. Transport metadata is excluded; ++ // request retries clone history and retain its map ordering and raw tool JSON. ++ let _ = serde_json::to_writer(&mut digest, &model_request_identity(request)); ++ digest.0.finish() +} + +fn trusted_response_endpoint(endpoint: &str) -> bool { @@ -2485,6 +2790,86 @@ index 0000000..1344548 + (OperationObservation::new(mode, sink.clone()), sink) + } + ++ // Capture fixtures remain independent from the prepared transport identity. ++ fn test_request(value: &Value) -> codex_api::ResponsesApiRequest { ++ codex_api::ResponsesApiRequest { ++ model: value["model"].as_str().unwrap_or("model").to_owned(), ++ input: vec![codex_protocol::models::ResponseItem::Message { ++ role: "user".into(), ++ content: vec![codex_protocol::models::ContentItem::InputText { ++ text: value["input"].to_string(), ++ }], ++ encrypted_content: None, ++ status: None, ++ id: None, ++ phase: None, ++ internal_chat_message_metadata_passthrough: None, ++ }], ++ tools: value.get("tools").map(|tools| { ++ Arc::::from( ++ serde_json::value::to_raw_value(tools).unwrap(), ++ ) ++ .into() ++ }), ++ tool_choice: value["tool_choice"].as_str().unwrap_or("auto").to_owned(), ++ text: None, ++ stream: true, ++ service_tier: None, ++ parallel_tool_calls: true, ++ reasoning: None, ++ store: false, ++ stream_options: None, ++ include: vec![], ++ prompt_cache_key: None, ++ client_metadata: None, ++ access_programs: None, ++ } ++ } ++ ++ #[test] ++ fn fingerprint_ignores_transport_metadata_and_tracks_semantic_fields() { ++ let old_identity = |request: &codex_api::ResponsesApiRequest| { ++ let mut value = serde_json::to_value(request).unwrap(); ++ value.as_object_mut().unwrap().retain(|key, _| { ++ matches!( ++ key.as_str(), ++ "model" | "input" | "tools" | "tool_choice" | "text" ++ ) ++ }); ++ value ++ }; ++ let request = test_request(&serde_json::json!({"input":["same"]})); ++ let original = request_fingerprint(&request); ++ let mut retry = request.clone(); ++ retry.client_metadata = Some(std::collections::HashMap::from([ ++ ("routing".into(), "new route".into()), ++ ("trace".into(), "new trace".into()), ++ ])); ++ retry.service_tier = Some("priority".into()); ++ retry.prompt_cache_key = Some("new cache key".into()); ++ assert_eq!(old_identity(&request), old_identity(&retry)); ++ assert_eq!(original, request_fingerprint(&retry)); ++ let mut changed_model = request.clone(); ++ changed_model.model = "different".into(); ++ let mut changed_input = request.clone(); ++ changed_input.input.clear(); ++ let mut changed_choice = request.clone(); ++ changed_choice.tool_choice = "none".into(); ++ let mut changed_text = request.clone(); ++ changed_text.text = Some(codex_api::TextControls::default()); ++ let changed_tools = test_request(&serde_json::json!({"input":["same"],"tools":[]})); ++ for changed in [ ++ changed_model, ++ changed_input, ++ changed_choice, ++ changed_text, ++ changed_tools, ++ ] { ++ assert_ne!(old_identity(&request), old_identity(&changed)); ++ assert_ne!(original, request_fingerprint(&changed)); ++ } ++ } ++ + #[test] + fn metadata_never_captures_payload_and_finalizes_once() { + let (observation, sink) = observation(OperationCaptureMode::Metadata); @@ -2496,7 +2881,10 @@ index 0000000..1344548 + OperationMetadata::default(), + ); + operation.capture_input(&serde_json::json!({"text":"INPUT_CANARY"})); -+ operation.finish_with_output(OperationOutcome::Success, &"OUTPUT_CANARY"); ++ operation.finish_with_output( ++ OperationOutcome::Success, ++ &Value::String("OUTPUT_CANARY".into()), ++ ); + operation.finish(OperationOutcome::Error); + } + let events = sink.0.lock().unwrap(); @@ -2562,7 +2950,7 @@ index 0000000..1344548 + + #[test] + fn oversized_text_keeps_a_useful_utf8_prefix_and_inline_marker() { -+ let content = project_content(&format!( ++ let content = project_text(&format!( + "file prefix: {}", + "é🙂".repeat(CONTENT_LIMIT_BYTES) + )); @@ -2585,7 +2973,9 @@ index 0000000..1344548 + "text":{"format":{"type":"json_schema","schema":{"description":"schema".repeat(10_000)}}}, + "tool_choice":"auto", + }); -+ let content = project_content(&model_request_content(&request)); ++ let selected = ++ serde_json::json!({"input": request["input"], "instructions":request["instructions"]}); ++ let content = project_content(&model_request_content(&selected)); + assert!(!content.truncated); + assert!(!content.omitted); + assert_eq!(content.value["input"], input); @@ -2594,17 +2984,16 @@ index 0000000..1344548 + assert!(content.value.get("text").is_none()); + let mut changed = request.clone(); + changed["tools"][0]["name"] = Value::String("different_tool".into()); -+ assert_ne!(request_fingerprint(&request), request_fingerprint(&changed)); -+ assert_eq!( -+ model_request_content(&request), -+ model_request_content(&changed) ++ assert_ne!( ++ request_fingerprint(&test_request(&request)), ++ request_fingerprint(&test_request(&changed)) + ); + } + + #[test] + fn embedded_json_that_fits_the_budget_remains_intact() { + let raw = serde_json::json!({"text":"x".repeat(27_000)}).to_string(); -+ let content = project_content(&raw); ++ let content = project_text(&raw); + assert_eq!(content.value, Value::String(raw)); + assert!(!content.truncated); + } @@ -2632,25 +3021,55 @@ index 0000000..1344548 + } + + #[test] -+ fn embedded_json_redacts_and_truncates_without_breaking_its_structure() { -+ let input = serde_json::json!({ -+ "accessToken":"TOKEN_CANARY", -+ "result":format!("prefix {}", "🙂".repeat(CONTENT_LIMIT_BYTES)), -+ }) -+ .to_string(); -+ let content = project_content(&input); ++ fn embedded_json_redacts_in_budget_sensitive_fields() { ++ let input = ++ serde_json::json!({"accessToken":"TOKEN_CANARY","result":"visible"}).to_string(); ++ let content = project_text(&input); + let embedded: Value = serde_json::from_str(content.value.as_str().unwrap()).unwrap(); + assert_eq!(embedded["accessToken"], "[omitted]"); -+ assert!(embedded["result"].as_str().unwrap().starts_with("prefix ")); -+ assert!( -+ embedded["result"] -+ .as_str() -+ .unwrap() -+ .ends_with("[truncated]") ++ assert_eq!(embedded["result"], "visible"); ++ assert!(!content.truncated); ++ assert!(content.omitted); ++ assert!(!content.value.to_string().contains("CANARY")); ++ } ++ ++ #[test] ++ fn oversized_json_text_is_omitted_before_structural_projection() { ++ let inputs = [ ++ serde_json::json!({"authorization":"LARGE_JSON_CANARY","items":vec![0; CONTENT_LIMIT_BYTES * 8]}).to_string(), ++ format!("{}{}", " ".repeat(CONTENT_LIMIT_BYTES * 2), r#"{"authorization":"WHITESPACE_CANARY","result":"visible"}"#), ++ ]; ++ for raw in inputs { ++ let content = project_text(&raw); ++ let marker: Value = serde_json::from_str(content.value.as_str().unwrap()).unwrap(); ++ assert_eq!( ++ marker, ++ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) ++ ); ++ assert!(content.omitted); ++ assert!(content.truncated); ++ assert_eq!(content.reason.as_deref(), Some("capture_budget")); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ assert!(!content.value.to_string().contains("CANARY")); ++ } ++ } ++ ++ #[test] ++ fn embedded_json_parse_limit_follows_remaining_direction_budget() { ++ let raw = ++ serde_json::json!({"authorization":"NESTED_JSON_CANARY","text":"y".repeat(8_000)}) ++ .to_string(); ++ assert!(raw.len() < CONTENT_LIMIT_BYTES); ++ let content = project_content( ++ &serde_json::json!({"a":"x".repeat(CONTENT_VALUE_LIMIT_BYTES - 1_200),"z":raw}), ++ ); ++ let marker: Value = serde_json::from_str(content.value["z"].as_str().unwrap()).unwrap(); ++ assert_eq!( ++ marker, ++ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) + ); -+ assert!(content.truncated); + assert!(content.omitted); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ assert!(content.truncated); + assert!(!content.value.to_string().contains("CANARY")); + } + @@ -2681,6 +3100,216 @@ index 0000000..1344548 + assert!(serialized.len() <= CONTENT_LIMIT_BYTES); + } + ++ #[test] ++ fn borrowed_large_plain_text_retains_prefix_and_redacts_media() { ++ let value = serde_json::json!({ ++ "role":"tool", ++ "text":format!("visible data:image/png;base64,MEDIA_CANARY {}", "é🙂".repeat(CONTENT_LIMIT_BYTES * 8)), ++ "authorization":"TOKEN_CANARY", ++ }); ++ let content = project_content(&value); ++ assert_eq!(content.value["role"], "tool"); ++ assert!( ++ content.value["text"] ++ .as_str() ++ .unwrap() ++ .starts_with("visible [omitted: media] ") ++ ); ++ assert!( ++ content.value["text"] ++ .as_str() ++ .unwrap() ++ .ends_with("[truncated]") ++ ); ++ assert!(content.truncated); ++ assert!(content.omitted); ++ assert!(!content.value.to_string().contains("CANARY")); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ ++ fn tool_result_item( ++ output: codex_protocol::models::FunctionCallOutputPayload, ++ custom: bool, ++ ) -> codex_protocol::models::ResponseItem { ++ use codex_protocol::models::ResponseItem; ++ if custom { ++ ResponseItem::CustomToolCallOutput { ++ id: None, ++ call_id: "call-file".into(), ++ name: Some("read_file".into()), ++ output, ++ internal_chat_message_metadata_passthrough: None, ++ } ++ } else { ++ ResponseItem::FunctionCallOutput { ++ id: None, ++ call_id: Some("call-file".into()), ++ name: Some("read_file".into()), ++ namespace: None, ++ output, ++ internal_chat_message_metadata_passthrough: None, ++ } ++ } ++ } ++ ++ #[test] ++ fn oversized_tool_results_preserve_call_identity_and_readable_utf8_prefix() { ++ for custom in [false, true] { ++ let item = tool_result_item( ++ codex_protocol::models::FunctionCallOutputPayload::from_text(format!( ++ "file prefix: {}", ++ "é🙂".repeat(CONTENT_LIMIT_BYTES) ++ )), ++ custom, ++ ); ++ let content = project_with(|projection| { ++ projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES) ++ }); ++ assert_eq!(content.value["call_id"], "call-file"); ++ assert_eq!( ++ content.value["type"], ++ if custom { ++ "custom_tool_call_output" ++ } else { ++ "function_call_output" ++ } ++ ); ++ let output = content.value["output"].as_str().unwrap(); ++ assert!(output.starts_with("file prefix: ")); ++ assert!(output.ends_with("[truncated]")); ++ assert!(content.truncated); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ } ++ ++ #[test] ++ fn oversized_multimodal_tool_result_keeps_text_and_omits_binary_parts() { ++ use codex_protocol::models::{ ++ FunctionCallOutputContentItem, FunctionCallOutputPayload, ImageReference, ++ }; ++ let item = tool_result_item( ++ FunctionCallOutputPayload::from_content_items(vec![ ++ FunctionCallOutputContentItem::InputImage { ++ image: ImageReference::Inline { ++ image_url: "IMAGE_CANARY".repeat(CONTENT_LIMIT_BYTES), ++ }, ++ detail: None, ++ }, ++ FunctionCallOutputContentItem::InputAudio { ++ audio_url: "AUDIO_CANARY".repeat(CONTENT_LIMIT_BYTES), ++ }, ++ FunctionCallOutputContentItem::InputText { ++ text: format!("file prefix: {}", "x".repeat(CONTENT_LIMIT_BYTES * 4)), ++ }, ++ ]), ++ false, ++ ); ++ let content = ++ project_with(|projection| projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES)); ++ assert_eq!(content.value["call_id"], "call-file"); ++ assert_eq!(content.value["output"][0]["omitted"], true); ++ assert_eq!(content.value["output"][1]["omitted"], true); ++ assert!( ++ content.value["output"][2]["text"] ++ .as_str() ++ .unwrap() ++ .starts_with("file prefix: ") ++ ); ++ assert!(!content.value.to_string().contains("CANARY")); ++ assert!(content.omitted); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ ++ #[test] ++ fn oversized_json_tool_result_uses_the_existing_raw_json_guard() { ++ let raw = serde_json::json!({"authorization":"TOOL_JSON_CANARY","text":"x".repeat(CONTENT_LIMIT_BYTES * 4)}).to_string(); ++ let item = tool_result_item( ++ codex_protocol::models::FunctionCallOutputPayload::from_text(raw), ++ true, ++ ); ++ let content = ++ project_with(|projection| projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES)); ++ let marker: Value = ++ serde_json::from_str(content.value["output"].as_str().unwrap()).unwrap(); ++ assert_eq!(marker["reason"], "capture_budget"); ++ assert_eq!(marker["omitted"], true); ++ assert_eq!(content.value["call_id"], "call-file"); ++ assert!(!content.value.to_string().contains("CANARY")); ++ assert!(content.omitted); ++ assert!(content.truncated); ++ } ++ ++ #[test] ++ fn typed_oversized_items_stop_serialization_with_explicit_omission() { ++ #[derive(Serialize)] ++ struct TypedItem<'a> { ++ text: &'a str, ++ } ++ let text = "x".repeat(CONTENT_LIMIT_BYTES * 128); ++ let content = project_serializable_content(&TypedItem { text: &text }); ++ assert!(content.omitted); ++ assert!(content.truncated); ++ assert_eq!(content.value["truncated"], true); ++ assert_eq!(content.reason.as_deref(), Some("capture_budget")); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ ++ #[test] ++ fn oversized_typed_messages_keep_plain_prefix_and_omit_structured_text() { ++ for kind in ["input_text", "output_text"] { ++ for structured in [false, true] { ++ let text = if structured { ++ serde_json::json!({"authorization":"MESSAGE_CANARY","text":"x".repeat(96 * 1024)}).to_string() ++ } else { ++ format!("TEXT_PREFIX {}", "é".repeat(96 * 1024)) ++ }; ++ let item: codex_protocol::models::ResponseItem = serde_json::from_value(serde_json::json!({ ++ "type":"message","id":"msg_test","role":"user","content":[{"type":kind,"text":text}] ++ })).unwrap(); ++ let output = project_response_item(&item); ++ let request = ModelRequestContent { ++ input: std::slice::from_ref(&item), ++ previous_response_id: Some("previous"), ++ }; ++ let input = project_content(&model_request_content(&request)); ++ for (content, value) in ++ [(&output, &output.value), (&input, &input.value["input"][0])] ++ { ++ assert_eq!(value["role"], "user"); ++ assert_eq!(value["id"], "msg_test"); ++ let text = value["content"][0]["text"].as_str().unwrap(); ++ if structured { ++ let marker: Value = serde_json::from_str(text).unwrap(); ++ assert_eq!(marker["reason"], "capture_budget"); ++ assert!(content.omitted); ++ assert!(!text.contains("CANARY")); ++ } else { ++ assert!(text.starts_with("TEXT_PREFIX ")); ++ assert!(text.ends_with("[truncated]")); ++ } ++ assert!(content.truncated); ++ assert!(json_size(content) <= CONTENT_LIMIT_BYTES); ++ } ++ assert_eq!(input.value["previous_response_id"], "previous"); ++ } ++ } ++ } ++ ++ #[test] ++ fn oversized_media_does_not_consume_visible_message_text_budget() { ++ let item: codex_protocol::models::ResponseItem = serde_json::from_value(serde_json::json!({ ++ "type":"message","role":"user","content":[ ++ {"type":"input_image","image_url":format!("data:image/png;base64,{}", "x".repeat(2_000_000))}, ++ {"type":"input_text","text":"VISIBLE_AFTER_MEDIA"} ++ ] ++ })).unwrap(); ++ let content = project_response_item(&item); ++ assert_eq!(content.value["content"][0]["omitted"], true); ++ assert_eq!(content.value["content"][1]["text"], "VISIBLE_AFTER_MEDIA"); ++ assert!(content.omitted); ++ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); ++ } ++ + #[tokio::test] + async fn same_input_retry_has_one_live_start_and_changed_input_splits() { + let (observation, sink) = observation(OperationCaptureMode::Content); @@ -2692,12 +3321,12 @@ index 0000000..1344548 + ); + model.scope(async { + let request = serde_json::json!({"model":"model","input":[{"role":"user","content":"first"}],"client_metadata":{"secret":"META_CANARY"}}); -+ observe_model_request(&request, &request, "http://localhost:8080"); ++ observe_model_request(&request, &test_request(&request), "http://localhost:8080"); + assert_eq!(sink.0.lock().unwrap().len(), 1); -+ observe_model_request(&request, &request, "http://localhost:8080"); ++ observe_model_request(&request, &test_request(&request), "http://localhost:8080"); + assert_eq!(sink.0.lock().unwrap().len(), 1); + let changed = serde_json::json!({"model":"model","input":[{"role":"user","content":"second"}]}); -+ observe_model_request(&changed, &changed, "http://localhost:8080"); ++ observe_model_request(&changed, &test_request(&changed), "http://localhost:8080"); + model.event(&codex_api::ResponseEvent::Completed { response_id:"response".into(),token_usage:None,usage_metadata:None,end_turn:Some(true) }); + }).await; + model.finish(OperationOutcome::Error); @@ -2724,7 +3353,7 @@ index 0000000..1344548 + "requested".into(), + ); + let request = serde_json::json!({"model":"requested","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + model.event(&codex_api::ResponseEvent::ModelObservation( + "body-first".into(), + )); @@ -2768,10 +3397,10 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + let old = model.clone().for_stream(); + old.source_ended(); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + old.event(&codex_api::ResponseEvent::Completed { + response_id: "old".into(), + token_usage: None, @@ -2801,7 +3430,7 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + model.source_ended(); + let expected = model.state.lock().unwrap().source_end.clone().unwrap(); + // A later EOF/drop must not move the source error boundary to tool cleanup. @@ -2822,7 +3451,7 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + model.finish(OperationOutcome::Success); + let events = sink.0.lock().unwrap(); + assert_eq!(events.len(), 2); @@ -2839,7 +3468,7 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + model.source_ended(); + model.state.lock().unwrap().source_end = Some(("2000-01-01T00:00:00.000Z".into(), 0.0)); + model.retry_waiting(true); @@ -2862,11 +3491,11 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + model.source_ended(); + let expected = model.state.lock().unwrap().source_end.clone().unwrap(); + let changed = serde_json::json!({"model":"model","input":["changed"]}); -+ model.request(&changed, &changed, "http://localhost"); ++ model.request(&changed, &test_request(&changed), "http://localhost"); + let events = sink.0.lock().unwrap(); + assert_eq!(events.len(), 3); + assert_eq!(events[1].completed_at.as_ref(), Some(&expected.0)); @@ -2883,7 +3512,7 @@ index 0000000..1344548 + "model".into(), + ); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost"); ++ model.request(&request, &test_request(&request), "http://localhost"); + let error = codex_api::ApiError::ResponseIncomplete(Box::new( + codex_protocol::error::ResponseIncompleteError { + reason: "ERROR_PAYLOAD_CANARY".into(), @@ -2922,7 +3551,7 @@ index 0000000..1344548 + ); + let interrupted = model.interruption_guard(); + let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &request, "http://localhost:8080"); ++ model.request(&request, &test_request(&request), "http://localhost:8080"); + model.event(&codex_api::ResponseEvent::OutputTextDelta( + "partial answer".into(), + )); @@ -3352,24 +3981,28 @@ index 3344ed8..ea173f5 100644 pub(crate) mod registry; diff --git a/codex-rs/core/src/tools/operation_observation.rs b/codex-rs/core/src/tools/operation_observation.rs new file mode 100644 -index 0000000..d07036a +index 0000000..2c84e35 --- /dev/null +++ b/codex-rs/core/src/tools/operation_observation.rs -@@ -0,0 +1,174 @@ +@@ -0,0 +1,184 @@ +//! Execution observations, separate from admission, approval, and result hooks. + +use std::sync::Arc; +use std::sync::Mutex; + -+use serde::Serialize; +use serde_json::Value; + +use crate::function_tool::FunctionCallError; ++use crate::operation_observation::CONTENT_VALUE_LIMIT_BYTES; ++use crate::operation_observation::OperationContent; +use crate::operation_observation::OperationGuard; +use crate::operation_observation::OperationKind; +use crate::operation_observation::OperationMetadata; +use crate::operation_observation::OperationObservation; +use crate::operation_observation::OperationOutcome; ++use crate::operation_observation::project_content; ++use crate::operation_observation::project_serializable_content; ++use crate::operation_observation::project_text; +use crate::tools::context::ToolCallSource; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolOutput; @@ -3385,12 +4018,12 @@ index 0000000..d07036a + thread_id: String, + turn_id: String, + metadata: OperationMetadata, -+ input: Option, ++ input: Option, + guard: Mutex>, +} + +impl ExecutionObservation { -+ fn start(&self, input: Option<&Value>) { ++ fn start(&self, input: Option) { + let mut guard = self + .guard + .lock() @@ -3405,8 +4038,8 @@ index 0000000..d07036a + OperationKind::Tool, + self.metadata.clone(), + ); -+ if let Some(input) = input.or(self.input.as_ref()) { -+ started.capture_input(input); ++ if let Some(input) = input.or_else(|| self.input.clone()) { ++ started.capture_input_content(input); + } + *guard = Some(started); + } @@ -3455,14 +4088,13 @@ index 0000000..d07036a +} + +/// MCP preparation can rewrite arguments before sending them to the server. -+pub(crate) fn start_current_execution_with_input(input: &impl Serialize) { ++pub(crate) fn start_current_execution_with_input(input: &Value) { + let _ = CURRENT_EXECUTION.try_with(|execution| { + let input = execution + .observation + .captures_content() -+ .then(|| serde_json::to_value(input).ok()) -+ .flatten(); -+ execution.start(input.as_ref()); ++ .then(|| project_content(input)); ++ execution.start(input); + }); +} + @@ -3497,11 +4129,18 @@ index 0000000..d07036a + .captures_content() + .then(|| match &invocation.payload { + ToolPayload::Function { arguments } => { -+ serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.clone())) ++ // Trace-only parsing must not allocate a JSON tree beyond the capture budget. ++ if arguments.len() > CONTENT_VALUE_LIMIT_BYTES { ++ project_content(&serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"})) ++ } else if let Ok(value) = serde_json::from_str::(arguments) { ++ project_content(&value) ++ } else { ++ project_text(arguments) ++ } + } -+ ToolPayload::Custom { input } => Value::String(input.clone()), ++ ToolPayload::Custom { input } => project_text(input), + ToolPayload::ToolSearch { arguments } => { -+ serde_json::to_value(arguments).unwrap_or(Value::Null) ++ project_serializable_content(arguments) + } + }); + let execution = Arc::new(ExecutionObservation { @@ -3532,10 +4171,10 @@ index 0000000..d07036a +mod tests; diff --git a/codex-rs/core/src/tools/operation_observation_tests.rs b/codex-rs/core/src/tools/operation_observation_tests.rs new file mode 100644 -index 0000000..7fda319 +index 0000000..b94ecd9 --- /dev/null +++ b/codex-rs/core/src/tools/operation_observation_tests.rs -@@ -0,0 +1,291 @@ +@@ -0,0 +1,348 @@ +use super::*; +use crate::operation_observation::OperationCaptureMode; +use crate::operation_observation::OperationEvent; @@ -3827,6 +4466,63 @@ index 0000000..7fda319 + ); + } +} ++ ++#[tokio::test] ++async fn oversized_function_arguments_are_omitted_without_changing_execution() { ++ let (mut invocation, events) = fixture(OperationCaptureMode::Content).await; ++ invocation.payload = ToolPayload::Function { ++ arguments: serde_json::json!({"authorization":"TOOL_ARGUMENT_CANARY","items":vec![0; CONTENT_VALUE_LIMIT_BYTES * 8]}).to_string(), ++ }; ++ assert!( ++ observe_execution(&Handler::default(), &invocation) ++ .await ++ .is_ok() ++ ); ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); ++ let content = events[1].input.as_ref().unwrap(); ++ assert_eq!( ++ content.value, ++ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) ++ ); ++ assert!(content.omitted); ++ assert!(content.truncated); ++ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); ++} ++ ++#[tokio::test] ++async fn oversized_custom_input_keeps_prefix_without_changing_execution() { ++ let (mut invocation, events) = fixture(OperationCaptureMode::Content).await; ++ invocation.payload = ToolPayload::Custom { ++ input: format!( ++ "visible custom input {}", ++ "é🙂".repeat(CONTENT_VALUE_LIMIT_BYTES * 8) ++ ), ++ }; ++ assert!( ++ observe_execution(&Handler::default(), &invocation) ++ .await ++ .is_ok() ++ ); ++ let events = events.0.lock().unwrap(); ++ assert_eq!(events.len(), 2); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); ++ let content = events[1].input.as_ref().unwrap(); ++ assert!( ++ content ++ .value ++ .as_str() ++ .unwrap() ++ .starts_with("visible custom input ") ++ ); ++ assert!(content.value.as_str().unwrap().ends_with("[truncated]")); ++ assert!(content.truncated); ++ assert!( ++ serde_json::to_vec(content).unwrap().len() ++ <= crate::operation_observation::CONTENT_LIMIT_BYTES ++ ); ++} diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index a1a88e4..d9afeb6 100644 --- a/codex-rs/core/src/tools/orchestrator.rs diff --git a/patches/codex/README.md b/patches/codex/README.md index c4adc2a751..aeb9a4d318 100644 --- a/patches/codex/README.md +++ b/patches/codex/README.md @@ -54,7 +54,7 @@ Observe native execution at the tool registry, with deferred start points immedi Bind observations before sampling to the initiating opted-in connection. Deliver exclusively to that connection, including multiple-subscriber cases. Keep captured data in memory and exclude it from rollout persistence and logs. Existing outgoing helpers have been logging notifications and broadcasting to empty recipient lists. Add explicit targeting and redacted logging. [Outgoing delivery](https://github.com/openai/codex/blob/6ff1e1ca3b30fc8b914720681fcf5fe62e95f856/codex-rs/app-server/src/outgoing_message.rs) -Enforce a 32 KiB UTF-8 serialized capture budget per direction per operation before sending. Mark missing content with `omitted` and shortened content with `truncated`, including the reason. Describe completeness accurately. +Enforce a 32 KiB UTF-8 serialized capture budget per direction per operation before sending. Bound raw UTF-8 bytes before parsing JSON-looking strings or native function arguments. Oversized JSON becomes an explicit `omitted` and `truncated` marker with reason `capture_budget`, without retaining a raw prefix. Keep embedded markers valid JSON strings. Preserve in-budget structured content and sensitive-field omission. Borrow normalized JSON values and raw custom-tool text during projection. Bound ordinary text to a UTF-8 prefix before media replacement and JSON sizing. Serialize typed adapters through a capped writer before creating owned JSON trees. Preserve bounded visible text and explicit media omissions for oversized messages and tool results. Omit other oversized typed items. Hash borrowed request identity fields without retaining serialized request copies. The existing tool-result conversion supplies an owned value before projection. Mark missing content with `omitted` and shortened content with `truncated`, including the reason. Describe completeness accurately. Use one finalizer per operation. Source turn completion closes private queue admission and awaits its drain before publishing `turn/completed`, including interrupted turns. Empty turns also close their observers. On child-process termination without a Rust terminal, close open IDs in Python with interruption and incomplete-observation metadata. From f4334c83a94300cb41d48142aeb92468b2a72978 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:40:51 -0800 Subject: [PATCH 12/33] fix(codex): preserve UTF-8 in captured native tool results --- src/agents/models/_codex_tracing.py | 5 ++--- tests/models/test_codex_provider.py | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py index bcc7966756..9461208d11 100644 --- a/src/agents/models/_codex_tracing.py +++ b/src/agents/models/_codex_tracing.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from dataclasses import dataclass from typing import Any @@ -14,7 +13,7 @@ get_current_trace, response_span, ) -from ..tracing._content import bound_content +from ..tracing._content import _json, bound_content from ..tracing._model import acquire_model_span, finish_model_span, record_model_input from ..tracing.config import get_operation_detail from ..tracing.provider import _is_noop_span, _is_noop_trace @@ -208,7 +207,7 @@ def _content(self, operation: _Operation, event: dict[str, Any]) -> None: if value is None and marker: value = marker if isinstance(data, FunctionSpanData): - setattr(data, direction, value if isinstance(value, str) else json.dumps(value)) + setattr(data, direction, value if isinstance(value, str) else _json(value)) elif isinstance(data, GenerationSpanData): items = value if isinstance(value, list) else [{"content": value}] if marker and marker not in items: diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 5622382851..a6529c6833 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1281,6 +1281,9 @@ async def events(self): class OperationServer(FakeServer): + # ASCII escapes would push this UTF-8 file over the capture budget. + file_text = "native_result_secret" + "🙂" * 3000 + # This double preserves the negotiated wire and source timing at the real Runner boundary. def __init__(self, *, capture="metadata", malformed=False, active=False): super().__init__() @@ -1336,7 +1339,7 @@ async def events(self): terminal["output"] = { "value": [{"role": "assistant", "content": "native_result_secret"}] if kind == "model" - else "native_result_secret", + else {"text": self.file_text}, "truncated": False, "omitted": False, } @@ -1495,6 +1498,8 @@ async def test_codex_operations_preserve_source_spans_and_capture_policy( assert (tools[0].span_data.input is not None) == (effective == "content") exported = json.dumps([span.export() for span in operations]) assert ("native_result_secret" in exported) == (effective == "content") + if effective == "content": + assert json.loads(tools[0].span_data.output) == {"text": OperationServer.file_text} if not sensitive: assert all(span.span_data.operation.response_id is None for span in models) From a2a5c886fd339d7cb4c69f78c740081c35e05a52 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 01:40:59 -0800 Subject: [PATCH 13/33] test(tracing): compare callback and tool times at SDK clock precision --- tests/models/test_operation_tracing_parity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py index 64f6745c6e..1ea7bbe37e 100644 --- a/tests/models/test_operation_tracing_parity.py +++ b/tests/models/test_operation_tracing_parity.py @@ -489,10 +489,10 @@ async def lookup(key: str) -> str: assert otel_index[database.parent.span_id] in executions python_execution = otel_index[database.parent.span_id] callback_started, callback_ended = callback_intervals[0] - assert python_execution.start_time <= callback_started - assert python_execution.end_time >= callback_ended - assert python_execution.start_time <= database.start_time <= database.end_time - assert database.end_time <= python_execution.end_time + # The SDK records the tool interval with datetime's microsecond precision. + assert python_execution.start_time // 1_000 <= callback_started // 1_000 + assert callback_started <= database.start_time <= database.end_time <= callback_ended + assert callback_ended // 1_000 <= python_execution.end_time // 1_000 if models: assert _ns(models[0].ended_at) <= callback_started + 1_000_000 for span in otel: From 7472be41cc96cdd154c2f0423ac71ffd91e48256 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 18:17:26 -0800 Subject: [PATCH 14/33] Bound AnyLLM Responses capture using the submitted request --- src/agents/extensions/models/any_llm_model.py | 25 ++-- tests/models/test_any_llm_model.py | 8 +- tests/tracing/test_content_capture.py | 122 ++++++++++++++++++ 3 files changed, 143 insertions(+), 12 deletions(-) diff --git a/src/agents/extensions/models/any_llm_model.py b/src/agents/extensions/models/any_llm_model.py index 80b5759ad5..b6f07520af 100644 --- a/src/agents/extensions/models/any_llm_model.py +++ b/src/agents/extensions/models/any_llm_model.py @@ -51,10 +51,13 @@ Converter as OpenAIResponsesConverter, _coerce_response_includables, _materialize_responses_tool_params, + _record_prepared_request, + _record_response_operation, ) from ...retry import ModelRetryAdvice, ModelRetryAdviceRequest from ...tool import Tool -from ...tracing import generation_span, response_span +from ...tracing import generation_span +from ...tracing._model import model_span from ...tracing.span_data import GenerationSpanData from ...tracing.spans import Span from ...usage import ( @@ -414,7 +417,7 @@ async def _get_response_via_responses( prompt: ResponsePromptParam | None, ) -> ModelResponse: with ( - response_span(disabled=tracing.is_disabled()) as span_response, + model_span(tracing, self.model) as span_response, model_span_errors( span_response, message="Error getting response", @@ -465,9 +468,7 @@ async def _get_response_via_responses( span_response.span_data.usage = model_usage_to_span_usage(usage) - if tracing.include_data(): - span_response.span_data.response = response - span_response.span_data.input = input + _record_response_operation(span_response, response, input, tracing) return ModelResponse( output=response.output, @@ -496,7 +497,7 @@ async def _stream_response_via_responses( prompt: ResponsePromptParam | None, ) -> AsyncGenerator[ResponseStreamEvent, None]: with ( - response_span(disabled=tracing.is_disabled()) as span_response, + model_span(tracing, self.model) as span_response, model_span_errors( span_response, message="Error streaming response", @@ -561,9 +562,10 @@ async def _stream_response_via_responses( requests=_requests_for_response_without_usage(final_response) ) ) - if tracing.include_data() and final_response is not None: - span_response.span_data.response = final_response - span_response.span_data.input = input + if final_response is not None: + _record_response_operation( + span_response, final_response, input, tracing + ) if terminal_failure_error is not None: # The failure is already known here. A consumer that stops at # this event closes the generator, which raises GeneratorExit @@ -1123,6 +1125,11 @@ async def _fetch_responses_response( } transport_kwargs = self._build_responses_transport_kwargs(model_settings) + prepared_request = dict(request_kwargs) + extra_body = prepared_request.get("extra_body") + if isinstance(extra_body, Mapping): + prepared_request.update(extra_body) + _record_prepared_request(prepared_request) response = await self._call_any_llm_responses( request_kwargs=request_kwargs, transport_kwargs=transport_kwargs, diff --git a/tests/models/test_any_llm_model.py b/tests/models/test_any_llm_model.py index e0c20d0b1e..7f4775f2ac 100644 --- a/tests/models/test_any_llm_model.py +++ b/tests/models/test_any_llm_model.py @@ -2376,14 +2376,16 @@ async def test_any_llm_responses_stream_populates_span_before_yielding_completed ) -> None: """The response span must carry the response and input when the completed event arrives.""" stream = _ClosableStream([_completed_event()]) - module, model = _responses_module_and_model_with_stream(monkeypatch, stream) - spans = _capture_spans(monkeypatch, module, "response_span") + from agents.tracing import get_current_span + + _module, model = _responses_module_and_model_with_stream(monkeypatch, stream) with trace(workflow_name="any-llm-responses-span"): stream_agen = cast(Any, _stream_events(model, ModelTracing.ENABLED)) async for event in stream_agen: if event.type == "response.completed": - [span] = spans + span = get_current_span() + assert span is not None assert span.span_data.response is not None assert span.span_data.response.id == "resp_123" assert span.span_data.input == "hi" diff --git a/tests/tracing/test_content_capture.py b/tests/tracing/test_content_capture.py index ac32d7166d..1c9797adf5 100644 --- a/tests/tracing/test_content_capture.py +++ b/tests/tracing/test_content_capture.py @@ -397,3 +397,125 @@ async def handler(request: httpx2.Request) -> httpx2.Response: assert "user-message" in json.dumps(captured) assert "truncated" in json.dumps(captured) assert data.tools is None + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("mode", ["default", "content", "metadata", "veto"]) +async def test_any_llm_responses_capture_bounds_sdk_and_otel_without_changing_provider_data( + exporter, monkeypatch, stream: bool, mode: str +) -> None: + pytest.importorskip("any_llm") + from any_llm.providers.openai import base as any_llm_openai + + from agents import ModelSettings + from agents.extensions.models.any_llm_model import AnyLLMModel + from agents.tracing.span_data import ResponseSpanData + from tests.models.test_any_llm_model import _response, _responses_output + + native = SpanProcessorForTests() + add_trace_processor(native) + input_secret, output_secret = "INPUT_CREDENTIAL_CANARY", "OUTPUT_CREDENTIAL_CANARY" + prompt = "UNSENT_INPUT_CANARY" + wire_input = [ + {"role": "user", "content": json.dumps({"api_key": input_secret, "query": "input-visible"})} + ] + wire_instructions = "wire-instructions-visible " + "i" * 40_000 + long_output = "answer-visible " + "o" * 40_000 + response = _response(json.dumps({"password": output_secret, "result": "output-visible"})) + content = response.output[0].content + assert content is not None + content.extend(_responses_output(long_output)[0].content) + payload = response.model_dump(mode="json") + requests = [] + + async def handler(request: httpx2.Request) -> httpx2.Response: + requests.append(json.loads(request.content)) + if stream: + event = {"type": "response.completed", "response": payload, "sequence_number": 0} + return httpx2.Response( + 200, + content=f"event: response.completed\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + return httpx2.Response(200, json=payload) + + async with AsyncOpenAI( + api_key="test", + base_url="https://provider.example/v1", + max_retries=0, + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler), trust_env=False), + ) as client: + # Keep the real AnyLLM provider and replace only its HTTP client. + monkeypatch.setattr(any_llm_openai, "AsyncOpenAI", lambda **kwargs: client) + agent = Agent( + name="worker", + model=AnyLLMModel("openai/fixture", api="responses", api_key="test"), + instructions="UNSENT_INSTRUCTIONS_CANARY", + model_settings=ModelSettings( + extra_body={"input": wire_input, "instructions": wire_instructions} + ), + ) + config = RunConfig( + tracing={} + if mode == "default" + else {"operation_detail": "metadata" if mode == "metadata" else "content"}, + trace_include_sensitive_data=mode != "veto", + ) + if stream: + result = Runner.run_streamed(agent, prompt, run_config=config) + async for _ in result.stream_events(): + pass + else: + result = await Runner.run(agent, prompt, run_config=config) + assert len(requests) == 1 + assert requests[0]["input"] == wire_input + assert requests[0]["instructions"] == wire_instructions + assert long_output in result.final_output and output_secret in result.final_output + data = next( + span.span_data + for span in native.get_ordered_spans() + if span.span_data.type in {"response", "generation"} + ) + if isinstance(data, ResponseSpanData): + captured_input = data.input + captured_output = ( + [item.model_dump(mode="json") for item in data.response.output] + if data.response + else None + ) + else: + captured_input = {"input": data.input, "model_config": data.model_config} + captured_output = data.output + attributes = dict( + next( + span + for span in exporter.get_finished_spans() + if span.attributes.get("gen_ai.operation.name") == "chat" + ).attributes + ) + exported_input = { + key: attributes[key] + for key in ("gen_ai.input.messages", "gen_ai.system_instructions") + if key in attributes + } + exported_output = attributes.get("gen_ai.output.messages") + if mode == "default": + assert isinstance(data, ResponseSpanData) + assert data.input == [{"role": "user", "content": prompt}] + assert long_output in json.dumps(captured_output) + assert not exported_input and exported_output is None + elif mode in {"metadata", "veto"}: + assert data.input is None and captured_output is None + assert not exported_input and exported_output is None + else: + for value in (captured_input, captured_output, exported_input, exported_output): + encoded = json.dumps(value, ensure_ascii=False) + assert len(encoded.encode()) <= CONTENT_LIMIT + assert input_secret not in encoded and output_secret not in encoded + assert "UNSENT_" not in encoded and "truncated" in encoded + for value in (captured_input, exported_input): + assert "input-visible" in json.dumps(value) + assert "wire-instructions-visible" in json.dumps(value) + assert "output-visible" in json.dumps(captured_output) From 4ab1102365c139cfb1953db945776d82ace9f9ed Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 19:04:19 -0800 Subject: [PATCH 15/33] Preserve visible refusal text in OpenTelemetry messages --- .../extensions/tracing/opentelemetry.py | 5 +++++ tests/tracing/test_content_capture.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 8048eb0aa9..2883788459 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -117,9 +117,14 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: continue if part.get("type") in ("input_text", "output_text", "text"): parts.append({"type": "text", "content": part.get("text", "")}) + elif part.get("type") == "refusal": + parts.append({"type": "text", "content": part.get("refusal", "")}) else: # Preserve part order without exporting media or hidden reasoning. parts.append({"type": "openai.omitted", "reason": "unsupported_content"}) + refusal = item.get("refusal") + if isinstance(refusal, str): + parts.append({"type": "text", "content": refusal}) for call in item.get("tool_calls") or (): if not isinstance(call, Mapping) or call.get("type") != "function": continue diff --git a/tests/tracing/test_content_capture.py b/tests/tracing/test_content_capture.py index bea90f5a89..d99d6a5ecc 100644 --- a/tests/tracing/test_content_capture.py +++ b/tests/tracing/test_content_capture.py @@ -112,6 +112,25 @@ def test_public_model_export_omits_hidden_reasoning_and_embedded_media(exporter) assert "omitted" in serialized +@pytest.mark.parametrize( + "message", + [ + {"content": [{"type": "refusal", "refusal": "Visible refusal."}]}, + {"content": None, "refusal": "Visible refusal."}, + ], + ids=["responses", "chat-completions"], +) +def test_public_model_export_preserves_visible_refusal_text(exporter, message) -> None: + with trace("refusal"), generation_span(model="test-model") as model: + model.span_data.operation.detail = "content" + model.span_data.output = [{"role": "assistant", **message}] + attributes = exporter.get_finished_spans()[0].attributes + messages = json.loads(json.dumps(attributes["gen_ai.output.messages"])) + assert messages == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Visible refusal."}]} + ] + + def test_public_tool_export_omits_oversized_structured_result_with_bounded_json(exporter) -> None: with trace("content"), function_span("read_file") as tool: tool.span_data.operation.detail = "content" From 690de4c7048ae75a7944bcd69beea24971ada9e4 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 19:47:36 -0800 Subject: [PATCH 16/33] Time the first macOS parity case using the existing wheel artifact --- .../codex-macos-parity-diagnosis.yml | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 .github/workflows/codex-macos-parity-diagnosis.yml diff --git a/.github/workflows/codex-macos-parity-diagnosis.yml b/.github/workflows/codex-macos-parity-diagnosis.yml new file mode 100644 index 0000000000..0a59b26f26 --- /dev/null +++ b/.github/workflows/codex-macos-parity-diagnosis.yml @@ -0,0 +1,128 @@ +name: Temporary Codex macOS parity diagnosis + +on: + push: + branches: [feature/codex-operation-tracing] + +permissions: + contents: read + actions: read + +jobs: + diagnose: + runs-on: macos-latest + timeout-minutes: 15 + defaults: + run: + shell: python + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: 61d97f8f1b0519c7312996c827a46eb5e4b7a1f1 + persist-credentials: false + sparse-checkout: | + native/codex/verify_installed.py + pyproject.toml + tests/__init__.py + tests/models/__init__.py + tests/testing_processor.py + tests/conftest.py + tests/models/test_operation_tracing_parity.py + tests/models/test_codex_mutable_tool_hooks.py + tests/mcp/__init__.py + tests/mcp/helpers.py + tests/mcp/model_compat.py + sparse-checkout-cone-mode: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 + with: + python-version: '3.14' + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: codex-wheel-macosx_11_0_arm64 + path: ${{ runner.temp }}/codex-wheel + run-id: 44859239869 + github-token: ${{ github.token }} + - name: Exercise a fresh installation outside the checkout + working-directory: ${{ runner.temp }} + env: + VERIFIER: ${{ github.workspace }}/native/codex/verify_installed.py + run: | + import os, shutil, subprocess, venv + from pathlib import Path + wheel, = Path('codex-wheel').glob('*.whl') + venv.create('codex-venv', with_pip=True) + python = Path('codex-venv') / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') + subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', str(wheel)], check=True) + shutil.copy2(os.environ['VERIFIER'], 'verify_installed.py') + subprocess.run([str(python), '-I', 'verify_installed.py', '--output', 'installed.json'], check=True) + - name: Time the unchanged first parity case and the no-DNS fixture + working-directory: ${{ runner.temp }} + env: + CODEX_TRACING_NATIVE_TEST: '1' + PARITY_TESTS: ${{ github.workspace }}/tests/models/test_operation_tracing_parity.py + run: | + import hashlib, json, os, shutil, subprocess, sys, time + from pathlib import Path + python = Path('codex-venv') / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') + subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', + 'pytest>=9.0.3', 'pytest-asyncio', 'opentelemetry-sdk>=1.45.0,<2'], check=True) + tests = Path(os.environ['PARITY_TESTS']) + probe = Path('timing_probe.py').resolve() + probe.write_text('"""Run an installed-wheel parity case with synchronous startup timing only."""\nimport faulthandler\nimport socket\nimport sys\nimport time\nimport pytest\n\n\nclass TimingProbe:\n @pytest.hookimpl(hookwrapper=True)\n def pytest_runtest_call(self, item):\n if \'test_operation_tracing_parity[default-run-default]\' not in item.nodeid:\n yield\n return\n import agents.run_internal.run_loop as run_loop\n started = time.monotonic()\n restored = []\n\n def observe(owner, name):\n original = getattr(owner, name)\n\n def timed(*args, **kwargs):\n before = time.monotonic()\n print(f\'PROBE {name} entering +{before-started:.3f}s\', flush=True)\n try:\n return original(*args, **kwargs)\n finally:\n print(f\'PROBE {name} leaving duration={time.monotonic()-before:.3f}s\', flush=True)\n setattr(owner, name, timed)\n restored.append((owner, name, original))\n\n observe(socket, \'getfqdn\')\n observe(item.module._ResponsesFixture, \'__init__\')\n observe(item.module, \'_model_provider\')\n observe(item.module, \'tool\')\n observe(item.module, \'Agent\')\n observe(run_loop, \'gather_with_cancel\')\n faulthandler.dump_traceback_later(10, repeat=True)\n try:\n yield\n finally:\n faulthandler.cancel_dump_traceback_later()\n for owner, name, original in reversed(restored):\n setattr(owner, name, original)\n print(f\'PROBE total={time.monotonic()-started:.3f}s\', flush=True)\n\n\nraise SystemExit(pytest.main([*sys.argv[1:], \'--import-mode=importlib\', \'-q\', \'-s\'], plugins=[TimingProbe()]))\n') + wheel, = Path('codex-wheel').glob('*.whl') + receipt = {'source_head': '61d97f8f1b0519c7312996c827a46eb5e4b7a1f1', + 'artifact_run': 44859239869, 'wheel': wheel.name, + 'wheel_sha256': hashlib.sha256(wheel.read_bytes()).hexdigest(), 'cases': []} + def run_case(label, path): + command = [str(python), '-I', str(probe), + str(path) + '::test_operation_tracing_parity[default-run-default]'] + started = time.monotonic() + try: + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=120) + code, output = result.returncode, result.stdout + except subprocess.TimeoutExpired as error: + code = 124 + output = error.stdout or b'' + if isinstance(output, bytes): + output = output.decode(errors='replace') + Path(label + '.log').write_text(output) + print('CASE ' + label, flush=True) + print(output, flush=True) + receipt['cases'].append({'label': label, 'exit_code': code, + 'seconds': time.monotonic() - started, + 'command': command}) + run_case('unchanged', tests) + candidate = Path('no-dns-candidate').resolve() + candidate.mkdir() + shutil.copytree(tests.parents[1], candidate / 'tests') + shutil.copy2(tests.parents[2] / 'pyproject.toml', candidate / 'pyproject.toml') + patched = candidate / 'tests/models/test_operation_tracing_parity.py' + source = patched.read_text() + assert source.count('class _ResponsesFixture:') == 1 + assert source.count('self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)') == 1 + source = source.replace('from pathlib import Path\n', + 'from pathlib import Path\nfrom socketserver import TCPServer\n') + source = source.replace('class _ResponsesFixture:', + 'class _LoopbackHTTPServer(ThreadingHTTPServer):\n' + ' def server_bind(self) -> None:\n' + ' TCPServer.server_bind(self)\n' + ' self.server_name, self.server_port = self.server_address[:2]\n\n\n' + 'class _ResponsesFixture:') + source = source.replace('self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)', + 'self.server = _LoopbackHTTPServer(("127.0.0.1", 0), Handler)') + patched.write_text(source) + run_case('no-dns', patched) + Path('diagnosis.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(receipt, indent=2)) + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: codex-macos-parity-diagnosis-${{ github.run_id }} + path: | + ${{ runner.temp }}/installed.json + ${{ runner.temp }}/diagnosis.json + ${{ runner.temp }}/unchanged.log + ${{ runner.temp }}/no-dns.log + if-no-files-found: warn + retention-days: 7 From 0692e224a5b729c58b1f55d7635e213b3271008a Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 19:50:54 -0800 Subject: [PATCH 17/33] Bind the parity test server without reverse DNS on loopback --- .../codex-macos-parity-diagnosis.yml | 128 ------------------ tests/models/test_operation_tracing_parity.py | 10 +- 2 files changed, 9 insertions(+), 129 deletions(-) delete mode 100644 .github/workflows/codex-macos-parity-diagnosis.yml diff --git a/.github/workflows/codex-macos-parity-diagnosis.yml b/.github/workflows/codex-macos-parity-diagnosis.yml deleted file mode 100644 index 0a59b26f26..0000000000 --- a/.github/workflows/codex-macos-parity-diagnosis.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Temporary Codex macOS parity diagnosis - -on: - push: - branches: [feature/codex-operation-tracing] - -permissions: - contents: read - actions: read - -jobs: - diagnose: - runs-on: macos-latest - timeout-minutes: 15 - defaults: - run: - shell: python - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - ref: 61d97f8f1b0519c7312996c827a46eb5e4b7a1f1 - persist-credentials: false - sparse-checkout: | - native/codex/verify_installed.py - pyproject.toml - tests/__init__.py - tests/models/__init__.py - tests/testing_processor.py - tests/conftest.py - tests/models/test_operation_tracing_parity.py - tests/models/test_codex_mutable_tool_hooks.py - tests/mcp/__init__.py - tests/mcp/helpers.py - tests/mcp/model_compat.py - sparse-checkout-cone-mode: false - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 - with: - python-version: '3.14' - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 - with: - name: codex-wheel-macosx_11_0_arm64 - path: ${{ runner.temp }}/codex-wheel - run-id: 44859239869 - github-token: ${{ github.token }} - - name: Exercise a fresh installation outside the checkout - working-directory: ${{ runner.temp }} - env: - VERIFIER: ${{ github.workspace }}/native/codex/verify_installed.py - run: | - import os, shutil, subprocess, venv - from pathlib import Path - wheel, = Path('codex-wheel').glob('*.whl') - venv.create('codex-venv', with_pip=True) - python = Path('codex-venv') / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') - subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', str(wheel)], check=True) - shutil.copy2(os.environ['VERIFIER'], 'verify_installed.py') - subprocess.run([str(python), '-I', 'verify_installed.py', '--output', 'installed.json'], check=True) - - name: Time the unchanged first parity case and the no-DNS fixture - working-directory: ${{ runner.temp }} - env: - CODEX_TRACING_NATIVE_TEST: '1' - PARITY_TESTS: ${{ github.workspace }}/tests/models/test_operation_tracing_parity.py - run: | - import hashlib, json, os, shutil, subprocess, sys, time - from pathlib import Path - python = Path('codex-venv') / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') - subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', - 'pytest>=9.0.3', 'pytest-asyncio', 'opentelemetry-sdk>=1.45.0,<2'], check=True) - tests = Path(os.environ['PARITY_TESTS']) - probe = Path('timing_probe.py').resolve() - probe.write_text('"""Run an installed-wheel parity case with synchronous startup timing only."""\nimport faulthandler\nimport socket\nimport sys\nimport time\nimport pytest\n\n\nclass TimingProbe:\n @pytest.hookimpl(hookwrapper=True)\n def pytest_runtest_call(self, item):\n if \'test_operation_tracing_parity[default-run-default]\' not in item.nodeid:\n yield\n return\n import agents.run_internal.run_loop as run_loop\n started = time.monotonic()\n restored = []\n\n def observe(owner, name):\n original = getattr(owner, name)\n\n def timed(*args, **kwargs):\n before = time.monotonic()\n print(f\'PROBE {name} entering +{before-started:.3f}s\', flush=True)\n try:\n return original(*args, **kwargs)\n finally:\n print(f\'PROBE {name} leaving duration={time.monotonic()-before:.3f}s\', flush=True)\n setattr(owner, name, timed)\n restored.append((owner, name, original))\n\n observe(socket, \'getfqdn\')\n observe(item.module._ResponsesFixture, \'__init__\')\n observe(item.module, \'_model_provider\')\n observe(item.module, \'tool\')\n observe(item.module, \'Agent\')\n observe(run_loop, \'gather_with_cancel\')\n faulthandler.dump_traceback_later(10, repeat=True)\n try:\n yield\n finally:\n faulthandler.cancel_dump_traceback_later()\n for owner, name, original in reversed(restored):\n setattr(owner, name, original)\n print(f\'PROBE total={time.monotonic()-started:.3f}s\', flush=True)\n\n\nraise SystemExit(pytest.main([*sys.argv[1:], \'--import-mode=importlib\', \'-q\', \'-s\'], plugins=[TimingProbe()]))\n') - wheel, = Path('codex-wheel').glob('*.whl') - receipt = {'source_head': '61d97f8f1b0519c7312996c827a46eb5e4b7a1f1', - 'artifact_run': 44859239869, 'wheel': wheel.name, - 'wheel_sha256': hashlib.sha256(wheel.read_bytes()).hexdigest(), 'cases': []} - def run_case(label, path): - command = [str(python), '-I', str(probe), - str(path) + '::test_operation_tracing_parity[default-run-default]'] - started = time.monotonic() - try: - result = subprocess.run(command, text=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, timeout=120) - code, output = result.returncode, result.stdout - except subprocess.TimeoutExpired as error: - code = 124 - output = error.stdout or b'' - if isinstance(output, bytes): - output = output.decode(errors='replace') - Path(label + '.log').write_text(output) - print('CASE ' + label, flush=True) - print(output, flush=True) - receipt['cases'].append({'label': label, 'exit_code': code, - 'seconds': time.monotonic() - started, - 'command': command}) - run_case('unchanged', tests) - candidate = Path('no-dns-candidate').resolve() - candidate.mkdir() - shutil.copytree(tests.parents[1], candidate / 'tests') - shutil.copy2(tests.parents[2] / 'pyproject.toml', candidate / 'pyproject.toml') - patched = candidate / 'tests/models/test_operation_tracing_parity.py' - source = patched.read_text() - assert source.count('class _ResponsesFixture:') == 1 - assert source.count('self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)') == 1 - source = source.replace('from pathlib import Path\n', - 'from pathlib import Path\nfrom socketserver import TCPServer\n') - source = source.replace('class _ResponsesFixture:', - 'class _LoopbackHTTPServer(ThreadingHTTPServer):\n' - ' def server_bind(self) -> None:\n' - ' TCPServer.server_bind(self)\n' - ' self.server_name, self.server_port = self.server_address[:2]\n\n\n' - 'class _ResponsesFixture:') - source = source.replace('self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)', - 'self.server = _LoopbackHTTPServer(("127.0.0.1", 0), Handler)') - patched.write_text(source) - run_case('no-dns', patched) - Path('diagnosis.json').write_text(json.dumps(receipt, indent=2) + '\n') - print(json.dumps(receipt, indent=2)) - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - if: always() - with: - name: codex-macos-parity-diagnosis-${{ github.run_id }} - path: | - ${{ runner.temp }}/installed.json - ${{ runner.temp }}/diagnosis.json - ${{ runner.temp }}/unchanged.log - ${{ runner.temp }}/no-dns.log - if-no-files-found: warn - retention-days: 7 diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py index 44b3081191..05291cf533 100644 --- a/tests/models/test_operation_tracing_parity.py +++ b/tests/models/test_operation_tracing_parity.py @@ -12,6 +12,7 @@ from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from socketserver import TCPServer from typing import Any import pytest @@ -37,6 +38,13 @@ PAYLOADS = (PROMPT, INSTRUCTIONS, ARGUMENT, TOOL_RESULT, ANSWER) +class _LoopbackHTTPServer(ThreadingHTTPServer): + def server_bind(self) -> None: + # This fixture has been using numeric loopback addresses without reverse DNS. + TCPServer.server_bind(self) + self.server_name, self.server_port = self.server_address[:2] + + class _ResponsesFixture: def __init__( self, @@ -126,7 +134,7 @@ def do_POST(self) -> None: fixture.intervals.append((started, time.time_ns())) self._reply(body, content_type) - self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server = _LoopbackHTTPServer(("127.0.0.1", 0), Handler) self.thread = threading.Thread( target=self.server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True ) From 4d1072616163632088209681b895efe53b6dcbd3 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 20:23:28 -0800 Subject: [PATCH 18/33] Resume pending Codex command and file edit approvals after a worker restart (#6115) * Resume pending Codex commands and file edits after a worker restart * Reject repeat execution under an already completed native approval --- native/codex/src/lib.rs | 36 +- native/codex/src/native_approval.rs | 157 +++ native/codex/src/operation_events.rs | 46 +- .../codex/0002-native-approval-recovery.patch | 1242 +++++++++++++++++ src/agents/memory/resumable_session.py | 17 + src/agents/models/_codex_items.py | 76 +- src/agents/models/codex.py | 140 +- src/agents/run.py | 4 + src/agents/run_context.py | 4 + src/agents/run_internal/native_approvals.py | 213 +++ src/agents/run_internal/resumable_session.py | 7 + src/agents/run_internal/run_steps.py | 2 + src/agents/run_internal/turn_resolution.py | 43 +- src/agents/run_state.py | 65 +- tests/memory/test_native_approval_recovery.py | 336 +++++ tests/models/test_codex_pending_approval.py | 166 +++ 16 files changed, 2525 insertions(+), 29 deletions(-) create mode 100644 native/codex/src/native_approval.rs create mode 100644 patches/codex/0002-native-approval-recovery.patch create mode 100644 src/agents/run_internal/native_approvals.py create mode 100644 tests/memory/test_native_approval_recovery.py create mode 100644 tests/models/test_codex_pending_approval.py diff --git a/native/codex/src/lib.rs b/native/codex/src/lib.rs index 141c2d49a8..acafa6ea45 100644 --- a/native/codex/src/lib.rs +++ b/native/codex/src/lib.rs @@ -1,5 +1,6 @@ //! Private, asynchronous Core bindings for the opt-in Codex model provider. mod config; +mod native_approval; mod operation_events; mod tool_hooks; @@ -449,7 +450,8 @@ impl Thread { final_output_json_schema: schema, ..Default::default() }); - let request = WithTurnExtensionData::new(request, capture); + let request = + WithTurnExtensionData::new(request, capture).capture_native_approvals(); match Box::pin(lease.thread.start_turn_if_idle(request)) .await .map_err(|_| "Unable to start the Codex turn.".to_owned())? @@ -497,6 +499,38 @@ impl Thread { }) } + #[pyo3(signature = (call_json, request_json, preconditions_json, operation_capture=None))] + fn resolve_approval<'py>( + &self, + py: Python<'py>, + call_json: String, + request_json: String, + preconditions_json: String, + operation_capture: Option, + ) -> PyResult> { + let call = serde_json::from_str(&call_json) + .map_err(|_| py_error("Invalid saved Codex tool call."))?; + let request = serde_json::from_str(&request_json) + .map_err(|_| py_error("Invalid saved Codex approval request."))?; + let preconditions = serde_json::from_str(&preconditions_json) + .map_err(|_| py_error("Invalid saved Codex file preconditions."))?; + let lease = self.lease.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + lease.ensure_open().map_err(py_error)?; + // Restoration owns a separate capture queue, leaving normal next_event delivery intact. + let operations = operation_events::OperationEvents::new(); + let capture = operations + .initializer(operation_capture.as_deref()) + .map_err(py_error)?; + let (output, turn_id, failure) = + native_approval::resolve(&lease.thread, call, request, preconditions, capture) + .await + .map_err(py_error)?; + let notifications = operations.drain_notifications(&lease.id.to_string()).await; + Ok((output, turn_id, notifications, failure)) + }) + } + fn close<'py>(&self, py: Python<'py>) -> PyResult> { let cleanup = self.lease.begin_close(); pyo3_async_runtimes::tokio::future_into_py( diff --git a/native/codex/src/native_approval.rs b/native/codex/src/native_approval.rs new file mode 100644 index 0000000000..d192567b11 --- /dev/null +++ b/native/codex/src/native_approval.rs @@ -0,0 +1,157 @@ +//! Resume one native call after the SDK has atomically claimed its saved approval. + +use codex_core_api::{CodexThread, StartIfIdleSubmission}; +use codex_extension_api::ExtensionDataInit; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::{EventMsg, Op, ReviewDecision}; + +use crate::NativeResult; + +pub(super) async fn resolve( + thread: &CodexThread, + call: ResponseItem, + expected: EventMsg, + preconditions: Option, + capture: ExtensionDataInit, +) -> NativeResult<(Option, String, Option)> { + let call_id = match (&call, &expected) { + (ResponseItem::FunctionCall { call_id, .. }, EventMsg::ExecApprovalRequest(request)) + if call_id == &request.call_id => + { + call_id.clone() + } + ( + ResponseItem::CustomToolCall { call_id, .. }, + EventMsg::ApplyPatchApprovalRequest(request), + ) if call_id == &request.call_id => call_id.clone(), + _ => return Err("The saved Codex approval does not match its tool call.".into()), + }; + let patch = matches!(expected, EventMsg::ApplyPatchApprovalRequest(_)); + // Restoration dispatches this call through Core. Core checks the saved request + // against its freshly prepared action before it can ask us for the decision. + let StartIfIdleSubmission::Started { turn_id, .. } = + Box::pin(thread.restore_tool_call_if_idle(call, expected, preconditions, Some(capture))) + .await + .map_err(|_| "Could not restore the pending Codex approval.")? + else { + return Err("The Codex thread is busy.".into()); + }; + + let outcome: NativeResult = async { + let mut answered = false; + let mut finished = false; + let mut result = None; + loop { + let event = thread + .next_event() + .await + .map_err(|_| "The restored Codex turn ended unexpectedly.")?; + if event.id != turn_id { + continue; + } + let decision = match event.msg { + EventMsg::ExecApprovalRequest(request) + if !patch + && !answered + && request.call_id == call_id + && request.approval_id.is_none() + && request.turn_id == turn_id + && request + .effective_available_decisions() + .contains(&ReviewDecision::Approved) => + { + Op::ExecApproval { + id: request.call_id, + turn_id: Some(turn_id.clone()), + decision: ReviewDecision::Approved, + } + } + EventMsg::ApplyPatchApprovalRequest(request) + if patch + && !answered + && request.call_id == call_id + && request.turn_id == turn_id => + { + Op::PatchApproval { + id: request.call_id, + decision: ReviewDecision::Approved, + } + } + EventMsg::ExecApprovalRequest(_) + | EventMsg::ApplyPatchApprovalRequest(_) + | EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) + | EventMsg::DynamicToolCallRequest(_) => { + return Err("Codex requested another decision while restoring the call.".into()); + } + EventMsg::ExecCommandEnd(end) + if answered && !patch && end.call_id == call_id && end.turn_id == turn_id => + { + finished = true; + continue; + } + EventMsg::PatchApplyEnd(end) + if answered && patch && end.call_id == call_id && end.turn_id == turn_id => + { + finished = true; + continue; + } + EventMsg::RawResponseItem(item) => { + let output = match item.item { + ResponseItem::FunctionCallOutput { + call_id: Some(id), + output, + .. + } if !patch && id == call_id => output, + ResponseItem::CustomToolCallOutput { + call_id: id, + output, + .. + } if patch && id == call_id => output, + _ => continue, + }; + if !answered || result.is_some() { + return Err( + "Codex could not confirm the saved approval before dispatch.".into(), + ); + } + result = output.body.to_text(); + continue; + } + EventMsg::TurnComplete(_) => { + if !finished { + // An exec can yield a handle to a still-running process. That handle + // dies with this thread. Don't checkpoint its uncertain effect as done. + return Err( + "Codex could not confirm that the approved call finished.".into() + ); + } + return result + .ok_or_else(|| "Codex did not complete the approved call.".into()); + } + EventMsg::TurnAborted(_) | EventMsg::Error(_) | EventMsg::ShutdownComplete => { + return Err( + "Codex could not complete the restored approval under current policy." + .into(), + ); + } + _ => continue, + }; + thread + .submit(decision) + .await + .map_err(|_| "Could not submit the saved approval to Codex.")?; + answered = true; + } + } + .await; + match outcome { + Ok(output) => Ok((Some(output), turn_id, None)), + Err(failure) => { + // Close real Core work before the caller drains its cancelled/failed observations. + let _ = thread.shutdown_and_wait().await; + Ok((None, turn_id, Some(failure))) + } + } +} diff --git a/native/codex/src/operation_events.rs b/native/codex/src/operation_events.rs index d996d14640..a3589860ee 100644 --- a/native/codex/src/operation_events.rs +++ b/native/codex/src/operation_events.rs @@ -101,6 +101,25 @@ impl OperationEvents { Ok(init) } + pub(crate) async fn drain_notifications(&self, thread_id: &str) -> Vec { + let mut state = self.state.lock().await; + let mut events = Vec::new(); + while let Ok(event) = state.receiver.try_recv() { + events.push(event); + } + let dropped = self.dropped.swap(0, Ordering::Relaxed); + if dropped != 0 { + events.push( + json!({ + "method": "turn/operation/dropped", + "params": {"threadId": thread_id, "droppedCount": dropped}, + }) + .to_string(), + ); + } + events + } + pub(crate) async fn next_event( &self, thread: &CodexThread, @@ -128,12 +147,22 @@ impl OperationEvents { event = thread.next_event() => { let event = event.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; match &event.msg { - EventMsg::ExecApprovalRequest(_) - | EventMsg::ApplyPatchApprovalRequest(_) - | EventMsg::RequestPermissions(_) + EventMsg::ExecApprovalRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) => { + let preconditions = thread + .capture_tool_approval_preconditions(event.msg.clone()) + .await + .map_err(|_| "This Codex approval cannot be safely resumed.".to_owned())?; + state.pending = Some(json!({ + "id": event.id, + "msg": event.msg, + "preconditions": preconditions, + }).to_string()); + continue; + } + EventMsg::RequestPermissions(_) | EventMsg::RequestUserInput(_) | EventMsg::ElicitationRequest(_) => { - return Err("Codex requested interactive approval. This provider supports non-interactive execution only.".to_owned()); + return Err("Codex requested an unsupported kind of interactive approval.".to_owned()); } _ => {} } @@ -168,11 +197,10 @@ mod tests { }, ); guard.finish_with_output(OperationOutcome::Success, &json!({"text": "visible"})); - let mut state = events.state.lock().await; - let start: serde_json::Value = - serde_json::from_str(&state.receiver.recv().await.unwrap()).unwrap(); - let end: serde_json::Value = - serde_json::from_str(&state.receiver.recv().await.unwrap()).unwrap(); + let notifications = events.drain_notifications("thread_1").await; + assert_eq!(notifications.len(), 2); + let start: serde_json::Value = serde_json::from_str(¬ifications[0]).unwrap(); + let end: serde_json::Value = serde_json::from_str(¬ifications[1]).unwrap(); assert_eq!(start["method"], "turn/operation/started"); assert_eq!(end["method"], "turn/operation/completed"); assert_eq!(start["params"]["operationId"], end["params"]["operationId"]); diff --git a/patches/codex/0002-native-approval-recovery.patch b/patches/codex/0002-native-approval-recovery.patch new file mode 100644 index 0000000000..683d169073 --- /dev/null +++ b/patches/codex/0002-native-approval-recovery.patch @@ -0,0 +1,1242 @@ +diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock +index 1d27abb..f8972a1 100644 +--- a/codex-rs/Cargo.lock ++++ b/codex-rs/Cargo.lock +@@ -3051,6 +3051,7 @@ dependencies = [ + "serde_json", + "serial_test", + "sha1 0.10.6", ++ "sha2 0.10.9", + "shlex", + "similar", + "tempfile", +diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml +index 542abf9..5ffc5cf 100644 +--- a/codex-rs/core/Cargo.toml ++++ b/codex-rs/core/Cargo.toml +@@ -114,6 +114,7 @@ rmcp = { workspace = true, default-features = false, features = [ + serde = { workspace = true, features = ["derive"] } + serde_json = { workspace = true } + sha1 = { workspace = true } ++sha2 = { workspace = true } + shlex = { workspace = true } + similar = { workspace = true } + tempfile = { workspace = true } +diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs +index 968d331..9d0bd2e 100644 +--- a/codex-rs/core/src/codex_thread.rs ++++ b/codex-rs/core/src/codex_thread.rs +@@ -45,6 +45,7 @@ use codex_protocol::protocol::AskForApproval; + use codex_protocol::protocol::EnvironmentConfig; + use codex_protocol::protocol::EnvironmentConfigState; + use codex_protocol::protocol::Event; ++use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::MultiAgentVersion; + use codex_protocol::protocol::Op; + use codex_protocol::protocol::SandboxPolicy; +@@ -474,10 +475,39 @@ impl CodexThread { + .await + } + ++ /// Replays one native call without model sampling, requiring the original human approval. ++ pub async fn restore_tool_call_if_idle( ++ &self, ++ call: ResponseItem, ++ expected_approval: EventMsg, ++ preconditions: Option, ++ turn_extension_init: Option, ++ ) -> CodexResult { ++ let restore = ++ crate::restore_tool_call::RestoreToolCall::new(call, expected_approval, preconditions)?; ++ let mut init = match turn_extension_init { ++ Some(init) => init, ++ None => self.config_snapshot().await.turn_extension_init, ++ }; ++ init.insert(restore); ++ self.start_turn_if_idle(crate::WithTurnExtensionData::new( ++ TurnInputRequest::user_input(Vec::new()), ++ init, ++ )) ++ .await ++ } ++ ++ /// Captures live preconditions before retiring a worker waiting for its first native approval. ++ pub async fn capture_tool_approval_preconditions( ++ &self, ++ approval: EventMsg, ++ ) -> CodexResult { ++ crate::restore_tool_call::capture(&self.session, approval).await ++ } ++ + /// Starts a regular turn only when the thread is idle. + /// +- /// Core declines the input without recording or enqueueing it when idle +- /// work cannot start. ++ /// Core declines the input without recording or enqueueing it when idle work cannot start. + pub async fn start_turn_if_idle( + &self, + request: impl Into>, +diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs +index dcd01e4..dc14807 100644 +--- a/codex-rs/core/src/lib.rs ++++ b/codex-rs/core/src/lib.rs +@@ -32,6 +32,8 @@ pub use codex_protocol::turn_input::TurnStartOptions; + pub use responses_metadata::CodexResponsesMetadata; + pub use turn_metadata::detached_memory_responses_metadata; + mod codex_thread; ++mod restore_tool_call; ++mod patch_preconditions; + mod turn_extension_data; + pub use turn_extension_data::WithTurnExtensionData; + mod compact_model_fallback; +diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs +new file mode 100644 +index 0000000..2e9781e +--- /dev/null ++++ b/codex-rs/core/src/patch_preconditions.rs +@@ -0,0 +1,306 @@ ++//! Filesystem preconditions for restoring an interrupted native patch approval. ++ ++use crate::session::step_context::StepContext; ++use crate::tools::ApprovalContext; ++use codex_file_system::EnvironmentAccess; ++use codex_file_system::GetMetadataOptions; ++use codex_file_system::ReadFileOptions; ++use codex_protocol::error::CodexErr; ++use codex_protocol::error::Result as CodexResult; ++use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; ++use codex_protocol::protocol::FileChange; ++use codex_utils_path_uri::PathUri; ++use serde::Deserialize; ++use serde::Serialize; ++use serde_json::Value; ++use sha2::Digest; ++use sha2::Sha256; ++use std::collections::BTreeMap; ++use std::io::ErrorKind; ++use std::path::Component; ++use std::path::Path; ++use std::path::PathBuf; ++ ++#[derive(Debug, Deserialize, Serialize, PartialEq)] ++#[serde(deny_unknown_fields)] ++struct PatchPreconditions { ++ version: u32, ++ files: BTreeMap>, ++} ++ ++fn invalid(message: impl Into) -> CodexErr { ++ CodexErr::InvalidRequest(message.into()) ++} ++ ++pub(crate) async fn capture_patch_approval_preconditions( ++ context: &ApprovalContext, ++ approval: &ApplyPatchApprovalRequestEvent, ++) -> CodexResult { ++ let review = &context.review_context; ++ let environment = review ++ .environments() ++ .single_local_environment() ++ .ok_or_else(|| { ++ invalid("Restoring patch approvals requires a single selected local environment") ++ })?; ++ let grants = review ++ .turn() ++ .granted_permissions(&environment.selection.environment_id); ++ serde_json::to_value(capture(&environment.fs_accessor(grants), approval).await?) ++ .map_err(|error| invalid(error.to_string())) ++} ++ ++async fn capture_step( ++ step: &StepContext, ++ approval: &ApplyPatchApprovalRequestEvent, ++) -> CodexResult { ++ let environment = step ++ .environments ++ .single_local_environment() ++ .ok_or_else(|| { ++ invalid("Restoring patch approvals requires a single selected local environment") ++ })?; ++ capture(&step.fs_accessor(environment), approval).await ++} ++ ++pub(crate) async fn validate_patch_preconditions( ++ step: &StepContext, ++ approval: &ApplyPatchApprovalRequestEvent, ++ checkpoint: &Value, ++) -> CodexResult<()> { ++ let expected: PatchPreconditions = serde_json::from_value(checkpoint.clone()) ++ .map_err(|error| invalid(format!("Invalid patch approval preconditions: {error}")))?; ++ let observed = capture_step(step, approval).await?; ++ if expected != observed { ++ return Err(invalid( ++ "The pending patch or its files have changed since the original approval request", ++ )); ++ } ++ Ok(()) ++} ++ ++async fn capture( ++ fs: &impl EnvironmentAccess, ++ approval: &ApplyPatchApprovalRequestEvent, ++) -> CodexResult { ++ if approval.changes.is_empty() { ++ return Err(invalid("A pending patch must include file changes")); ++ } ++ let mut files = BTreeMap::new(); ++ for (path, change) in &approval.changes { ++ let is_add = matches!(change, FileChange::Add { .. }); ++ let original = observe_file(fs, path, is_add).await?; ++ if let FileChange::Delete { content } = change ++ && original.as_deref() != Some(&digest(content.as_bytes())) ++ { ++ return Err(invalid( ++ "The pending deletion no longer matches its original file", ++ )); ++ } ++ files.insert(path.clone(), original); ++ if let FileChange::Update { ++ move_path: Some(destination), ++ .. ++ } = change ++ { ++ if approval.changes.contains_key(destination) || files.contains_key(destination) { ++ return Err(invalid( ++ "Pending patch move destinations must be distinct from changed files", ++ )); ++ } ++ files.insert( ++ destination.clone(), ++ observe_file(fs, destination, true).await?, ++ ); ++ } ++ } ++ Ok(PatchPreconditions { version: 1, files }) ++} ++ ++fn digest(bytes: &[u8]) -> String { ++ format!("{:x}", Sha256::digest(bytes)) ++} ++ ++async fn observe_file( ++ fs: &impl EnvironmentAccess, ++ path: &Path, ++ must_be_absent: bool, ++) -> CodexResult> { ++ if !path.is_absolute() ++ || path ++ .components() ++ .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) ++ { ++ return Err(invalid( ++ "Pending patch paths must be normalized absolute local paths", ++ )); ++ } ++ // The selected local filesystem's no-follow operations reject symlinks in ++ // every component, including directory ancestors, without opening targets. ++ let uri = PathUri::from_host_native_path(path).map_err(|error| invalid(error.to_string()))?; ++ match fs ++ .get_metadata( ++ &uri, ++ GetMetadataOptions { ++ follow_symlinks: false, ++ }, ++ ) ++ .await ++ { ++ Err(error) if must_be_absent && error.kind() == ErrorKind::NotFound => Ok(None), ++ Err(error) => Err(invalid(format!( ++ "Cannot inspect pending patch file: {error}" ++ ))), ++ Ok(_) if must_be_absent => Err(invalid("A pending patch destination already exists")), ++ Ok(metadata) if !metadata.is_file || metadata.is_symlink => { ++ Err(invalid("Pending patch sources must be regular files")) ++ } ++ Ok(_) => { ++ let bytes = fs ++ .read_file( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false, ++ }, ++ ) ++ .await ++ .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; ++ Ok(Some(digest(&bytes))) ++ } ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use codex_exec_server::LOCAL_FS; ++ use codex_file_system::FileSystemEnvironmentAccessor; ++ ++ fn approval(path: PathBuf, change: FileChange) -> ApplyPatchApprovalRequestEvent { ++ ApplyPatchApprovalRequestEvent { ++ call_id: "original-call".into(), ++ turn_id: "original-turn".into(), ++ started_at_ms: 0, ++ changes: [(path, change)].into_iter().collect(), ++ reason: None, ++ grant_root: None, ++ } ++ } ++ ++ #[tokio::test] ++ async fn existing_update_binds_full_original_contents() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().join("existing.txt"); ++ std::fs::write(&path, "old\nuntouched\n").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ path.clone(), ++ FileChange::Update { ++ unified_diff: "-old\n+new\n".into(), ++ move_path: None, ++ }, ++ ); ++ let original = capture(&fs, &request).await.unwrap(); ++ assert_eq!(original, capture(&fs, &request).await.unwrap()); ++ std::fs::write(&path, "old\nchanged outside patch context\n").unwrap(); ++ assert_ne!(original, capture(&fs, &request).await.unwrap()); ++ } ++ ++ #[tokio::test] ++ async fn add_requires_absent_destination_and_blocks_completed_replay() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().join("new.txt"); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ path.clone(), ++ FileChange::Add { ++ content: "new\n".into(), ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_ok()); ++ std::fs::write(path, "new\n").unwrap(); ++ assert!(capture(&fs, &request).await.is_err()); ++ } ++ ++ #[tokio::test] ++ async fn move_requires_absent_destination() { ++ let dir = tempfile::tempdir().unwrap(); ++ let source = dir.path().join("old.txt"); ++ let destination = dir.path().join("new.txt"); ++ std::fs::write(&source, "old\n").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ source, ++ FileChange::Update { ++ unified_diff: "-old\n+new\n".into(), ++ move_path: Some(destination.clone()), ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_ok()); ++ std::fs::write(destination, "pre-existing work\n").unwrap(); ++ assert!(capture(&fs, &request).await.is_err()); ++ } ++ ++ #[tokio::test] ++ async fn delete_requires_original_contents() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().join("old.txt"); ++ std::fs::write(&path, "original\n").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ path.clone(), ++ FileChange::Delete { ++ content: "original\n".into(), ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_ok()); ++ std::fs::write(path, "changed\n").unwrap(); ++ assert!(capture(&fs, &request).await.is_err()); ++ } ++ ++ #[cfg(unix)] ++ #[tokio::test] ++ async fn rejects_symlink_sources_and_ancestors() { ++ let dir = tempfile::tempdir().unwrap(); ++ let original = dir.path().join("original"); ++ std::fs::create_dir(&original).unwrap(); ++ std::fs::write(original.join("file"), "original\n").unwrap(); ++ let link = dir.path().join("link"); ++ std::os::unix::fs::symlink(&original, &link).unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ link.join("file"), ++ FileChange::Update { ++ unified_diff: "-original\n+new\n".into(), ++ move_path: None, ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_err()); ++ let request = approval( ++ link.join("new"), ++ FileChange::Add { ++ content: "new".into(), ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_err()); ++ let file_link = dir.path().join("file-link"); ++ std::os::unix::fs::symlink(original.join("file"), &file_link).unwrap(); ++ let request = approval( ++ file_link, ++ FileChange::Update { ++ unified_diff: "-original\n+new\n".into(), ++ move_path: None, ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_err()); ++ let dangling_link = dir.path().join("dangling-link"); ++ std::os::unix::fs::symlink(dir.path().join("absent"), &dangling_link).unwrap(); ++ let request = approval( ++ dangling_link, ++ FileChange::Add { ++ content: "new".into(), ++ }, ++ ); ++ assert!(capture(&fs, &request).await.is_err()); ++ } ++} +diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs +new file mode 100644 +index 0000000..984cbce +--- /dev/null ++++ b/codex-rs/core/src/restore_tool_call.rs +@@ -0,0 +1,642 @@ ++//! Model-free replay of one native call, requiring the same fresh human approval. ++ ++use std::collections::HashMap; ++use std::sync::atomic::{AtomicBool, Ordering}; ++use std::sync::{Arc, Mutex, OnceLock}; ++ ++use codex_protocol::approvals::ExecApprovalKind; ++use codex_protocol::config_types::ApprovalsReviewer; ++use codex_protocol::error::{CodexErr, Result as CodexResult}; ++use codex_protocol::models::ResponseItem; ++use codex_protocol::openai_models::ToolMode; ++use codex_protocol::protocol::{EventMsg, ReviewDecision}; ++use serde_json::{Value, json}; ++use sha2::{Digest, Sha256}; ++use tokio_util::sync::CancellationToken; ++ ++use crate::session::session::Session; ++use crate::session::step_context::StepContext; ++use crate::session::turn_context::TurnContext; ++use crate::state::TaskKind; ++use crate::tools::ApprovalContext; ++use crate::tools::parallel::ToolCallRuntime; ++use crate::tools::router::ToolRouter; ++use crate::tools::sandboxing::ApprovalAction; ++ ++fn refusal(message: &str) -> CodexErr { ++ CodexErr::InvalidRequest(format!("Native tool restoration refused: {message}")) ++} ++ ++#[derive(Debug, Clone)] ++struct CapturableApproval { ++ event: Option, ++ permissions: Value, ++ patch: Option, ++} ++ ++#[derive(Debug, Default)] ++struct CapturableApprovals(Mutex>); ++ ++impl crate::WithTurnExtensionData { ++ /// Enables pre-execution native approval checkpoint capture for SDK-owned turns. ++ pub fn capture_native_approvals(mut self) -> Self { ++ self.turn_extension_init ++ .get_or_insert_with(Default::default) ++ .insert(CapturableApprovals::default()); ++ self ++ } ++} ++ ++fn permission_snapshot( ++ turn: &TurnContext, ++ environments: &crate::environment_selection::TurnEnvironmentSnapshot, ++ policy: codex_protocol::protocol::AskForApproval, ++ reviewer: ApprovalsReviewer, ++) -> Value { ++ let environments: Vec<_> = environments ++ .turn_environments() ++ .map(|environment| { ++ json!({ ++ "id": environment.selection.environment_id, ++ "profile": environment.permission_profile_with_workspace_roots(), ++ "symbolic_profile": environment.permission_profile(), ++ "network": environment.config().network_policy, ++ "grants": turn.granted_permissions(&environment.selection.environment_id), ++ }) ++ }) ++ .collect(); ++ let mut snapshot = json!({ ++ "approval_policy": policy, "reviewer": reviewer, "environments": environments, ++ "managed_network": turn.config.permissions.network.as_ref().map(|network| network.environment_policy()), ++ }); ++ snapshot.sort_all_objects(); ++ json!(format!( ++ "{:x}", ++ Sha256::digest(snapshot.to_string().as_bytes()) ++ )) ++} ++ ++pub(crate) struct CaptureLease { ++ state: Arc, ++ call_id: String, ++} ++ ++impl Drop for CaptureLease { ++ fn drop(&mut self) { ++ self.state.0.lock().unwrap().remove(&self.call_id); ++ } ++} ++ ++pub(crate) async fn capture_lease( ++ action: &ApprovalAction, ++ ctx: &ApprovalContext, ++) -> Option { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || !matches!( ++ action, ++ ApprovalAction::ExecCommand { .. } | ApprovalAction::ApplyPatch { .. } ++ ) ++ { ++ return None; ++ } ++ let state = ctx ++ .review_context ++ .turn() ++ .extension_data ++ .get::()?; ++ let patch = if let ApprovalAction::ApplyPatch { changes, .. } = action { ++ let event = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { ++ call_id: ctx.call_id.clone(), ++ turn_id: ctx.review_context.turn().sub_id.clone(), ++ started_at_ms: 0, ++ changes: changes.as_ref().clone(), ++ reason: ctx.approval_reason.clone(), ++ grant_root: None, ++ }; ++ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, &event) ++ .await ++ .ok() ++ } else { ++ None ++ }; ++ state.0.lock().unwrap().insert( ++ ctx.call_id.clone(), ++ CapturableApproval { ++ event: None, ++ patch, ++ permissions: permission_snapshot( ++ ctx.review_context.turn(), ++ ctx.review_context.environments(), ++ ctx.review_context.approval_policy, ++ ctx.review_context.approvals_reviewer, ++ ), ++ }, ++ ); ++ Some(CaptureLease { ++ state, ++ call_id: ctx.call_id.clone(), ++ }) ++} ++ ++fn approval_id(event: &EventMsg) -> Option<&str> { ++ match event { ++ EventMsg::ExecApprovalRequest(event) ++ if event.kind == ExecApprovalKind::Command ++ && event.approval_id.is_none() ++ && event.network_approval_context.is_none() => ++ { ++ Some(&event.call_id) ++ } ++ EventMsg::ApplyPatchApprovalRequest(event) if event.grant_root.is_none() => { ++ Some(&event.call_id) ++ } ++ _ => None, ++ } ++} ++ ++fn comparable(event: &EventMsg) -> CodexResult { ++ let mut value = serde_json::to_value(event).map_err(|_| refusal("invalid approval event"))?; ++ let object = value ++ .as_object_mut() ++ .ok_or_else(|| refusal("invalid approval event"))?; ++ object.remove("turn_id"); ++ object.remove("started_at_ms"); ++ Ok(value) ++} ++ ++pub(crate) fn record_capture_event(turn: &TurnContext, event: &EventMsg) { ++ let Some(id) = approval_id(event) else { return }; ++ if let Some(state) = turn.extension_data.get::() { ++ if let Some(pending) = state.0.lock().unwrap().get_mut(id) { ++ pending.event = Some(event.clone()); ++ } ++ } ++} ++ ++#[derive(Debug)] ++pub(crate) struct RestoreToolCall { ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Value, ++ owner: OnceLock, ++ requested: AtomicBool, ++ approved: AtomicBool, ++ executed: AtomicBool, ++} ++ ++impl RestoreToolCall { ++ pub(crate) fn new( ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Option, ++ ) -> CodexResult { ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let supported = match (&call, &expected) { ++ ( ++ ResponseItem::FunctionCall { ++ name, ++ call_id, ++ namespace, ++ .. ++ }, ++ EventMsg::ExecApprovalRequest(_), ++ ) => { ++ name == "exec_command" ++ && call_id == id ++ && namespace.as_deref().is_none_or(|ns| ns == "functions") ++ } ++ ( ++ ResponseItem::CustomToolCall { ++ name, ++ call_id, ++ namespace, ++ .. ++ }, ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ) => { ++ name == "apply_patch" ++ && call_id == id ++ && namespace.as_deref().is_none_or(|ns| ns == "functions") ++ } ++ _ => false, ++ }; ++ if !supported { ++ return Err(refusal( ++ "only one direct exec_command or apply_patch call is supported", ++ )); ++ } ++ let checkpoint = ++ checkpoint.ok_or_else(|| refusal("missing original pre-execution checkpoint"))?; ++ if checkpoint.get("version") != Some(&json!(1)) ++ || checkpoint.get("first_approval") != Some(&json!(true)) ++ { ++ return Err(refusal("checkpoint does not match the original approval")); ++ } ++ Ok(Self { ++ call, ++ expected, ++ checkpoint, ++ owner: OnceLock::new(), ++ requested: AtomicBool::new(false), ++ approved: AtomicBool::new(false), ++ executed: AtomicBool::new(false), ++ }) ++ } ++ ++ pub(crate) fn accepts_approval(&self, action: &ApprovalAction, ctx: &ApprovalContext) -> bool { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || ctx.strict_auto_review ++ || self.requested.load(Ordering::SeqCst) ++ || approval_id(&self.expected) != Some(ctx.call_id.as_str()) ++ { ++ return false; ++ } ++ matches!( ++ (&self.expected, action), ++ ( ++ EventMsg::ExecApprovalRequest(_), ++ ApprovalAction::ExecCommand { .. } ++ ) | ( ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ApprovalAction::ApplyPatch { .. } ++ ) ++ ) ++ } ++ ++ pub(crate) fn is_patch(&self) -> bool { ++ matches!(self.expected, EventMsg::ApplyPatchApprovalRequest(_)) ++ } ++ ++ pub(crate) fn claim(&self, turn: &TurnContext) -> bool { ++ self.owner.set(turn.sub_id.clone()).is_ok() ++ } ++} ++ ++pub(crate) fn active(turn: &TurnContext) -> Option> { ++ turn.extension_data ++ .get::() ++ .filter(|restore| restore.owner.get() == Some(&turn.sub_id)) ++} ++ ++async fn validate_step(session: &Arc, step: &StepContext) -> CodexResult<()> { ++ let config = session.get_config().await; ++ let requirements = config.config_layer_stack.requirements(); ++ if requirements.auto_review_required_for_model(&step.settings.model_info.slug) ++ || requirements ++ .approvals_reviewer ++ .can_set(&ApprovalsReviewer::User) ++ .is_err() ++ { ++ return Err(refusal("managed automatic review is unsupported")); ++ } ++ if step.tool_router.tool_mode() != ToolMode::Direct ++ || step.environments.single_local_environment().is_none() ++ { ++ return Err(refusal( ++ "only direct tools in one local environment are supported", ++ )); ++ } ++ if step.settings.approvals_reviewer() != ApprovalsReviewer::User ++ || step.turn.strict_auto_review_enabled() ++ { ++ return Err(refusal("automatic approval review is unsupported")); ++ } ++ let hooks = session.hooks().list_with_environment_hooks( ++ crate::hook_environment_executor::environment_hook_sources(&step.environments, &[]), ++ ); ++ if !hooks.hooks.is_empty() ++ || !hooks.warnings.is_empty() ++ || !step.executor_hook_sources.is_empty() ++ { ++ return Err(refusal("configured hooks are unsupported")); ++ } ++ if let Some(restore) = active(&step.turn) { ++ if restore.checkpoint.get("permissions") ++ != Some(&permission_snapshot( ++ &step.turn, ++ &step.environments, ++ step.settings.approval_policy(), ++ step.settings.approvals_reviewer(), ++ )) ++ { ++ return Err(refusal("permissions or approval policy changed")); ++ } ++ } ++ Ok(()) ++} ++ ++pub(crate) async fn capture(session: &Arc, expected: EventMsg) -> CodexResult { ++ let (turn, cancellation) = session ++ .active_turn_context_and_cancellation_token() ++ .await ++ .ok_or_else(|| refusal("no active approval"))?; ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let state = turn ++ .extension_data ++ .get::() ++ .ok_or_else(|| refusal("not a first pre-execution approval"))?; ++ let recorded = state ++ .0 ++ .lock() ++ .unwrap() ++ .get(id) ++ .cloned() ++ .ok_or_else(|| refusal("not a first pre-execution approval"))?; ++ if recorded ++ .event ++ .as_ref() ++ .and_then(|event| serde_json::to_value(event).ok()) ++ != serde_json::to_value(&expected).ok() ++ { ++ return Err(refusal("approval does not match the live request")); ++ } ++ let step = session.capture_step_context(turn, &cancellation).await?; ++ validate_step(session, &step).await?; ++ let patch = match &expected { ++ EventMsg::ApplyPatchApprovalRequest(event) => { ++ let patch = recorded ++ .patch ++ .clone() ++ .ok_or_else(|| refusal("patch preimages could not be captured before approval"))?; ++ crate::patch_preconditions::validate_patch_preconditions(&step, event, &patch).await?; ++ Some(patch) ++ } ++ _ => None, ++ }; ++ if !state.0.lock().unwrap().contains_key(id) { ++ return Err(refusal("approval is no longer pending")); ++ } ++ Ok(json!({"version":1,"first_approval":true,"permissions":recorded.permissions,"patch":patch})) ++} ++ ++pub(crate) fn compare_request(turn: &TurnContext, event: &EventMsg) -> bool { ++ record_capture_event(turn, event); ++ let Some(restore) = active(turn) else { ++ return true; ++ }; ++ !restore.requested.swap(true, Ordering::SeqCst) ++ && approval_id(event) == approval_id(&restore.expected) ++ && matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected) ++} ++ ++pub(crate) fn record_decision(turn: &TurnContext, decision: ReviewDecision) -> ReviewDecision { ++ if let Some(restore) = active(turn) { ++ if decision == ReviewDecision::Approved { ++ restore.approved.store(true, Ordering::SeqCst); ++ } else if !matches!( ++ decision, ++ ReviewDecision::Denied { .. } | ReviewDecision::Abort ++ ) { ++ return ReviewDecision::Abort; ++ } ++ } ++ decision ++} ++ ++pub(crate) async fn validate_before_execution( ++ session: &Arc, ++ step: &StepContext, ++) -> CodexResult<()> { ++ let Some(restore) = active(&step.turn) else { ++ return Ok(()); ++ }; ++ validate_step(session, step).await?; ++ if !restore.approved.load(Ordering::SeqCst) { ++ return Err(refusal("a matching fresh human approval is required")); ++ } ++ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { ++ let checkpoint = restore ++ .checkpoint ++ .get("patch") ++ .ok_or_else(|| refusal("missing patch preimages"))?; ++ crate::patch_preconditions::validate_patch_preconditions(step, event, checkpoint).await?; ++ } ++ if restore.executed.swap(true, Ordering::SeqCst) { ++ return Err(refusal("a restored native call cannot retry execution")); ++ } ++ Ok(()) ++} ++ ++pub(crate) async fn run( ++ session: Arc, ++ turn: Arc, ++ restore: Arc, ++ cancellation: CancellationToken, ++) -> crate::tasks::SessionTaskResult { ++ session.emit_turn_started(&turn, TaskKind::Regular).await; ++ let step = session ++ .capture_step_context(Arc::clone(&turn), &cancellation) ++ .await?; ++ validate_step(&session, &step).await?; ++ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { ++ let checkpoint = restore ++ .checkpoint ++ .get("patch") ++ .ok_or_else(|| refusal("missing patch preimages"))?; ++ crate::patch_preconditions::validate_patch_preconditions(&step, event, checkpoint).await?; ++ } ++ let call = ToolRouter::build_tool_call(restore.call.clone()) ++ .map_err(|_| refusal("invalid native call"))? ++ .ok_or_else(|| refusal("unsupported native call"))?; ++ if !step.tool_router.exposes_tool(&call.tool_name) { ++ return Err(refusal("tool is unavailable in the replacement worker")); ++ } ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &restore.call) ++ .await; ++ let result = ToolCallRuntime::new(Arc::clone(&session), Arc::clone(&step), Default::default()) ++ .handle_tool_call(call, cancellation) ++ .await?; ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &result.item).await; ++ Ok(None) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use codex_protocol::protocol::AskForApproval; ++ ++ fn command() -> ResponseItem { ++ serde_json::from_value(json!({ ++ "type":"function_call", "call_id":"original", "name":"exec_command", ++ "arguments":"{\"cmd\":\"printf original\"}", ++ })) ++ .unwrap() ++ } ++ ++ fn approval() -> EventMsg { ++ serde_json::from_value(json!({ ++ "type":"exec_approval_request", "kind":"command", "call_id":"original", ++ "turn_id":"first-turn", "started_at_ms":1, "command":["sh","-c","printf original"], ++ "cwd":"/workspace", "environmentId":"local", "parsed_cmd":[], ++ })) ++ .unwrap() ++ } ++ ++ fn checkpoint() -> Value { ++ json!({"version":1,"first_approval":true,"permissions":"test-fingerprint","patch":null}) ++ } ++ ++ #[test] ++ fn rejects_unrelated_calls_and_noninitial_approvals() { ++ let mut call = command(); ++ if let ResponseItem::FunctionCall { call_id, .. } = &mut call { ++ *call_id = "sibling".into(); ++ } ++ assert!(RestoreToolCall::new(call, approval(), Some(checkpoint())).is_err()); ++ let mut request = approval(); ++ if let EventMsg::ExecApprovalRequest(event) = &mut request { ++ event.approval_id = Some("subcommand".into()); ++ } ++ assert!(RestoreToolCall::new(command(), request, Some(checkpoint())).is_err()); ++ assert!(RestoreToolCall::new(command(), approval(), None).is_err()); ++ } ++ ++ #[tokio::test] ++ async fn requires_exact_fresh_approval_and_consumes_execution_once() { ++ let (session, turn) = crate::session::tests::make_session_and_context().await; ++ let session = Arc::new(session); ++ let turn = Arc::new(turn); ++ let step = session ++ .capture_step_context(Arc::clone(&turn), &CancellationToken::new()) ++ .await ++ .unwrap(); ++ let mut preconditions = checkpoint(); ++ preconditions["permissions"] = permission_snapshot( ++ &turn, ++ &step.environments, ++ step.settings.approval_policy(), ++ step.settings.approvals_reviewer(), ++ ); ++ let restore = RestoreToolCall::new(command(), approval(), Some(preconditions)).unwrap(); ++ turn.extension_data.insert(restore); ++ let restore = turn.extension_data.get::().unwrap(); ++ assert!(restore.claim(&turn)); ++ assert!(!restore.claim(&turn)); ++ assert!(validate_before_execution(&session, &step).await.is_err()); ++ assert!(compare_request(&turn, &approval())); ++ assert_eq!( ++ record_decision(&turn, ReviewDecision::Approved), ++ ReviewDecision::Approved ++ ); ++ assert!(validate_before_execution(&session, &step).await.is_ok()); ++ assert!(validate_before_execution(&session, &step).await.is_err()); ++ assert!(!compare_request(&turn, &approval())); ++ } ++ ++ #[tokio::test] ++ async fn refuses_changed_command_before_approval() { ++ let (_, turn) = crate::session::tests::make_session_and_context().await; ++ turn.extension_data ++ .insert(RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap()); ++ assert!( ++ turn.extension_data ++ .get::() ++ .unwrap() ++ .claim(&turn) ++ ); ++ let mut changed = approval(); ++ if let EventMsg::ExecApprovalRequest(event) = &mut changed { ++ event.command.push("unexpected".into()); ++ } ++ assert!(!compare_request(&turn, &changed)); ++ assert!(!compare_request(&turn, &approval())); ++ assert!(!active(&turn).unwrap().approved.load(Ordering::SeqCst)); ++ } ++ ++ #[tokio::test] ++ async fn refuses_changed_authority_even_with_matching_approval() { ++ let (session, turn) = crate::session::tests::make_session_and_context().await; ++ let session = Arc::new(session); ++ let turn = Arc::new(turn); ++ let step = session ++ .capture_step_context(Arc::clone(&turn), &CancellationToken::new()) ++ .await ++ .unwrap(); ++ assert_ne!( ++ permission_snapshot( ++ &turn, ++ &step.environments, ++ AskForApproval::OnRequest, ++ ApprovalsReviewer::User ++ ), ++ permission_snapshot( ++ &turn, ++ &step.environments, ++ AskForApproval::Never, ++ ApprovalsReviewer::User ++ ), ++ ); ++ turn.extension_data ++ .insert(RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap()); ++ assert!( ++ turn.extension_data ++ .get::() ++ .unwrap() ++ .claim(&turn) ++ ); ++ assert!(compare_request(&turn, &approval())); ++ record_decision(&turn, ReviewDecision::Approved); ++ assert!(validate_before_execution(&session, &step).await.is_err()); ++ } ++ #[tokio::test] ++ async fn capture_is_opt_in_and_excludes_execution_retries() { ++ let (_, turn) = crate::session::tests::make_session_and_context().await; ++ let turn = Arc::new(turn); ++ let action = ApprovalAction::ExecCommand { ++ id: "original".into(), ++ environment_id: "local".into(), ++ command: vec!["printf".into(), "original".into()], ++ hook_command: "printf original".into(), ++ cwd: turn.cwd.clone().into(), ++ sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, ++ additional_permissions: None, ++ justification: None, ++ tty: false, ++ proposed_execpolicy_amendment: None, ++ }; ++ let mut ctx = ApprovalContext { ++ review_context: crate::guardian::GuardianReviewContext::from(&turn), ++ cancellation_token: None, ++ call_id: "original".into(), ++ tool_name: codex_tools::ToolName { ++ name: "exec_command".into(), ++ namespace: None, ++ }, ++ strict_auto_review: false, ++ approval_reason: None, ++ retry_reason: None, ++ network_approval_context: None, ++ }; ++ assert!(capture_lease(&action, &ctx).await.is_none()); ++ let restore = RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap(); ++ assert!(restore.accepts_approval(&action, &ctx)); ++ ctx.network_approval_context = Some(codex_protocol::approvals::NetworkApprovalContext { ++ host: "unexpected.invalid".into(), ++ protocol: codex_protocol::approvals::NetworkApprovalProtocol::Http, ++ }); ++ assert!(!restore.accepts_approval(&action, &ctx)); ++ ctx.network_approval_context = None; ++ turn.extension_data.insert(CapturableApprovals::default()); ++ ctx.retry_reason = Some("sandbox retry".into()); ++ assert!(!restore.accepts_approval(&action, &ctx)); ++ assert!(capture_lease(&action, &ctx).await.is_none()); ++ ctx.retry_reason = None; ++ let lease = capture_lease(&action, &ctx).await.unwrap(); ++ record_capture_event(&turn, &approval()); ++ let state = turn.extension_data.get::().unwrap(); ++ assert!( ++ state ++ .0 ++ .lock() ++ .unwrap() ++ .get("original") ++ .unwrap() ++ .event ++ .is_some() ++ ); ++ drop(lease); ++ assert!(state.0.lock().unwrap().is_empty()); ++ } ++} +diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs +index 66476ae..c092ec5 100644 +--- a/codex-rs/core/src/session/mod.rs ++++ b/codex-rs/core/src/session/mod.rs +@@ -2930,7 +2930,7 @@ impl Session { + .map(|task| Arc::clone(&task.turn_context)) + } + +- async fn active_turn_context_and_cancellation_token( ++ pub(crate) async fn active_turn_context_and_cancellation_token( + &self, + ) -> Option<(Arc, CancellationToken)> { + let active = self.active_turn.lock().await; +@@ -3175,9 +3175,15 @@ impl Session { + available_decisions: Some(available_decisions), + parsed_cmd, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +@@ -3219,9 +3225,15 @@ impl Session { + reason, + grant_root, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs +index 6c6f3d1..1309ac1 100644 +--- a/codex-rs/core/src/tasks/regular.rs ++++ b/codex-rs/core/src/tasks/regular.rs +@@ -44,6 +44,14 @@ impl SessionTask for RegularTask { + input: Vec, + cancellation_token: CancellationToken, + ) -> SessionTaskResult { ++ if let Some(restore) = ctx ++ .extension_data ++ .get::() ++ { ++ if restore.claim(&ctx) { ++ return crate::restore_tool_call::run(sess, ctx, restore, cancellation_token).await; ++ } ++ } + if ctx + .config + .features +diff --git a/codex-rs/core/src/tools/approvals.rs b/codex-rs/core/src/tools/approvals.rs +index 74082aa..ebaa126 100644 +--- a/codex-rs/core/src/tools/approvals.rs ++++ b/codex-rs/core/src/tools/approvals.rs +@@ -499,6 +499,18 @@ impl Session { + action: ApprovalAction, + ctx: ApprovalContext, + ) -> ApprovalResult { ++ if let Some(restore) = crate::restore_tool_call::active(ctx.review_context.turn()) { ++ if !restore.accepts_approval(&action, &ctx) { ++ return ApprovalResult { ++ result: Err(ToolError::Rejected( ++ "native restoration cannot request another approval or escalation".into(), ++ )), ++ source: None, ++ cancellation_reason: None, ++ }; ++ } ++ } ++ let _capture_lease = crate::restore_tool_call::capture_lease(&action, &ctx).await; + // Retained authority outside current grants needs a new sandbox approval. + // Current grants permit review of the input; they never approve the input itself. + let policy = ctx.review_context.turn().approval_policy(); +@@ -723,7 +735,7 @@ impl Session { + .into_iter() + .map(|key| (key, &policy_fingerprint)) + .collect(); +- with_cached_approval(&self.services, tool_name, cache_keys, || async { ++ let request = || async { + self.request_command_approval( + ctx.review_context.turn(), + ExecApprovalKind::Command, +@@ -742,8 +754,12 @@ impl Session { + /*plugin_attribution_override*/ None, + ) + .await +- }) +- .await ++ }; ++ if crate::restore_tool_call::active(ctx.review_context.turn()).is_some() { ++ request().await ++ } else { ++ with_cached_approval(&self.services, tool_name, cache_keys, request).await ++ } + } + ApprovalAction::WriteStdin { + id, +@@ -788,9 +804,11 @@ impl Session { + .retry_reason + .clone() + .or_else(|| ctx.approval_reason.clone()); +- if *permissions_preapproved && reason.is_none() { ++ let restoring = ++ crate::restore_tool_call::active(ctx.review_context.turn()).is_some(); ++ if *permissions_preapproved && reason.is_none() && !restoring { + ReviewDecision::Approved +- } else if reason.is_some() { ++ } else if reason.is_some() || restoring { + self.request_patch_approval( + ctx.review_context.turn(), + ctx.call_id.clone(), +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index d9afeb6..32a5820 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -61,6 +61,14 @@ impl ToolOrchestrator { + where + T: ToolRuntime, + { ++ if let Err(error) = crate::restore_tool_call::validate_before_execution( ++ &tool_ctx.session, ++ &tool_ctx.step_context, ++ ) ++ .await ++ { ++ return (Err(ToolError::Codex(error)), None); ++ } + let network_approval = match begin_network_approval( + &tool_ctx.session, + &tool_ctx.step_context.turn, +@@ -165,6 +173,13 @@ impl ToolOrchestrator { + let requirement = tool.exec_approval_requirement(req).unwrap_or_else(|| { + default_exec_approval_requirement(approval_policy, &file_system_sandbox_policy) + }); ++ if crate::restore_tool_call::active(turn_ctx).is_some() ++ && !matches!(requirement, ExecApprovalRequirement::NeedsApproval { .. }) ++ { ++ return Err(ToolError::Rejected( ++ "native restoration requires a fresh human approval".into(), ++ )); ++ } + match &requirement { + ExecApprovalRequirement::Skip { .. } => { + if strict_auto_review { +@@ -228,12 +243,24 @@ impl ToolOrchestrator { + + // 2) First attempt under the selected sandbox. + let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); +- let sandbox_override = sandbox_override_for_first_attempt( +- tool.sandbox_permissions(req), +- &requirement, +- &file_system_sandbox_policy, +- /*already_approved*/ initial_approval_source.is_some(), +- ); ++ // Restored patches get one attempt. Use only authority the normal approved retry already grants. ++ let approved_restored_patch = crate::restore_tool_call::active(turn_ctx) ++ .is_some_and(|restore| restore.is_patch()) ++ && initial_approval_source == Some(ApprovalResolutionSource::User) ++ && !strict_auto_review ++ && unsandboxed_allowed ++ && tool.wants_no_sandbox_approval(approval_policy) ++ && tool.should_bypass_approval(approval_policy, true); ++ let sandbox_override = if approved_restored_patch { ++ SandboxOverride::BypassSandboxFirstAttempt ++ } else { ++ sandbox_override_for_first_attempt( ++ tool.sandbox_permissions(req), ++ &requirement, ++ &file_system_sandbox_policy, ++ /*already_approved*/ initial_approval_source.is_some(), ++ ) ++ }; + let (escalated_profile, writable_root_metadata) = if sandbox_override + == SandboxOverride::EscalatedSandboxWithRestrictions + { diff --git a/src/agents/memory/resumable_session.py b/src/agents/memory/resumable_session.py index a9f1314edd..abb58b9b85 100644 --- a/src/agents/memory/resumable_session.py +++ b/src/agents/memory/resumable_session.py @@ -283,6 +283,7 @@ async def load_state( return state def _validate_run(self, input: Any) -> None: + from ..run_internal.native_approvals import native_receipts_from_snapshot from ..run_state import RunState if self._checkpoint.status == "active": @@ -300,6 +301,22 @@ def _validate_run(self, input: Any) -> None: or binding.revision != self._checkpoint.revision ): raise ResumableSessionError("invalid_state") + invalid_native_state = False + try: + assert self._checkpoint.run_state is not None + original = json.loads(self._checkpoint.run_state) + original_receipts = native_receipts_from_snapshot(original) + processed = input._last_processed_response + if original_receipts is not None or ( + processed is not None and processed.native_approvals + ): + invalid_native_state = ( + native_receipts_from_snapshot(input.to_json()) != original_receipts + ) + except Exception: + invalid_native_state = True + if invalid_native_state: + raise ResumableSessionError("invalid_state") def _get_history(self) -> list[TResponseInputItem]: return copy.deepcopy(self._checkpoint.history) diff --git a/src/agents/models/_codex_items.py b/src/agents/models/_codex_items.py index 66ee061da9..f555164166 100644 --- a/src/agents/models/_codex_items.py +++ b/src/agents/models/_codex_items.py @@ -1,9 +1,18 @@ -"""Internal output types attesting to work already completed by the Codex adapter.""" +"""Internal output types attesting to work observed directly by the Codex adapter.""" + +import json +from pathlib import Path +from typing import Any, cast from openai.types.responses import ( + ResponseCustomToolCall, ResponseCustomToolCallOutputItem, + ResponseFunctionToolCall, ResponseFunctionToolCallOutputItem, ) +from pydantic import Field + +from ..exceptions import UserError class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): @@ -12,3 +21,68 @@ class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): class CodexCustomToolCallOutput(ResponseCustomToolCallOutputItem): """A matching native custom call has already completed in Codex.""" + + +class CodexPendingFunctionCall(ResponseFunctionToolCall): + """A native function call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +class CodexPendingCustomCall(ResponseCustomToolCall): + """A native custom call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +def native_approval_checkpoint(raw_item: Any) -> dict[str, Any] | None: + """Validate an adapter-owned receipt against the call shown for approval.""" + # An unrelated approval may define __getattribute__("__class__"). Don't run it + # while taking a detached copy of a pending call or a session checkpoint. + raw_type = type(raw_item) + if raw_type is not CodexPendingFunctionCall and raw_type is not CodexPendingCustomCall: + return None + checkpoint = raw_item.checkpoint + required = {"version", "cwd", "call", "request", "model"} + if ( + not isinstance(checkpoint, dict) + or not required <= checkpoint.keys() + or set(checkpoint) - required - {"preconditions"} + or type(checkpoint["version"]) is not int + or checkpoint["version"] != 1 + or not isinstance(checkpoint["cwd"], str) + or not Path(checkpoint["cwd"]).is_absolute() + or not isinstance(checkpoint["model"], str) + or not checkpoint["model"] + or not isinstance(checkpoint["request"], dict) + or not isinstance(checkpoint["call"], dict) + or ("preconditions" in checkpoint and not isinstance(checkpoint["preconditions"], dict)) + ): + raise UserError("Invalid Codex native approval checkpoint.") + model = ( + ResponseFunctionToolCall + if isinstance(raw_item, CodexPendingFunctionCall) + else ResponseCustomToolCall + ) + display_metadata = {"codex_reasoning_summary"} + displayed = model.model_validate(raw_item.model_dump()).model_dump(exclude=display_metadata) + stored = model.model_validate(checkpoint["call"]).model_dump(exclude=display_metadata) + if displayed != stored or raw_item.name not in {"exec_command", "apply_patch"}: + raise UserError("Codex native approval no longer matches its original call.") + return cast(dict[str, Any], json.loads(json.dumps(checkpoint, allow_nan=False))) + + +def restore_native_approval_call(raw_item: Any, checkpoint: Any) -> Any: + """Restore only explicit SDK checkpoint metadata, never provider output fields.""" + if not isinstance(raw_item, dict) or not isinstance(checkpoint, dict): + raise UserError("Invalid Codex native approval checkpoint.") + item: CodexPendingFunctionCall | CodexPendingCustomCall + item_type = raw_item.get("type") + if item_type == "function_call": + item = CodexPendingFunctionCall(**raw_item, checkpoint=checkpoint) + elif item_type == "custom_tool_call": + item = CodexPendingCustomCall(**raw_item, checkpoint=checkpoint) + else: + raise UserError("Invalid Codex native approval call type.") + native_approval_checkpoint(item) + return item diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 5ad77ed914..4589219f5d 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -52,10 +52,15 @@ _response_usage_to_usage, ) from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call -from ._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, +) from ._codex_tool_hooks import CodexToolHooks from ._codex_tracing import CodexOperationTracing -from ._run_context import get_model_tool_hooks +from ._run_context import ModelToolHooks, get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing @@ -112,6 +117,86 @@ async def aclose(self) -> None: def get_model(self, model_name: str | None) -> Model: return _CodexModel(model_name, self) + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], tracing: ModelTracing + ) -> str: + """Run only the saved invocation, without a model turn or its old Codex home.""" + if checkpoint.get("version") != 1 or checkpoint.get("cwd") != self._cwd: + raise UserError("This Codex approval belongs to a different workspace or SDK version.") + + settings = {"model": checkpoint["model"]} + observations = CodexOperationTracing(tracing, None, checkpoint["model"]) + thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) + interrupted = True + try: + output, turn_id, notifications, failure = await thread.resolve_approval( + json.dumps(checkpoint["call"]), + json.dumps(checkpoint["request"]), + json.dumps(checkpoint.get("preconditions")), + observations.request_mode(), + ) + observations.bind(thread.thread_id, turn_id) + for encoded in notifications: + event = json.loads(encoded) + observations.observe(event["method"], event.get("params")) + if failure is not None: + raise RuntimeError(failure) + interrupted = False + return str(output) + finally: + try: + observations.close(interrupted=interrupted) + finally: + await _join_close(thread.close()) + + +def _pending_native_call( + call: Any, event: dict[str, Any], cwd: str, model: str +) -> CodexPendingFunctionCall | CodexPendingCustomCall: + request = event["msg"] + wire = call.model_dump(exclude_none=True) + if event.get("preconditions") is None: + raise UserError("Codex could not confirm that this tool is still waiting to run.") + checkpoint = { + "version": 1, + "cwd": cwd, + "model": model, + "call": wire, + "request": request, + "preconditions": event["preconditions"], + } + if getattr(call, "namespace", None) not in (None, "functions"): + raise UserError("Only built-in Codex tools support native approval recovery.") + if ( + isinstance(call, ResponseFunctionToolCall) + and call.name == "exec_command" + and request["type"] == "exec_approval_request" + and request.get("kind", "command") == "command" + and request.get("approval_id") is None + and request.get("network_approval_context") is None + and request.get("environmentId") in (None, "local") + ): + return CodexPendingFunctionCall(**wire, checkpoint=checkpoint) + if ( + isinstance(call, ResponseCustomToolCall) + and call.name == "apply_patch" + and request["type"] == "apply_patch_approval_request" + ): + return CodexPendingCustomCall(**wire, checkpoint=checkpoint) + raise UserError("This Codex approval cannot safely be resumed on another worker.") + + +def _uses_sdk_workspace(tools: list[Tool], bindings: ModelToolHooks | None) -> bool: + settings = bindings.settings if bindings else None + return bool( + settings + and settings.sdk_only_tool_hooks + and (settings.prepare_tool_input or settings.prepare_tool_output) + ) or any( + getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) + for tool in tools + ) + def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: supported = { @@ -177,6 +262,16 @@ def __init__(self, name: str | None, provider: CodexProvider) -> None: def get_default_model_settings(self) -> ModelSettings: return ModelSettings() + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], *, tools: list[Tool], tracing: ModelTracing + ) -> str: + bindings = get_model_tool_hooks() + if _uses_sdk_workspace(tools, bindings) or CodexToolHooks.for_run(bindings, set()): + raise UserError( + "The current workspace or native tool hooks do not allow restoring this call." + ) + return await self._provider._run_approved_native_call(checkpoint, tracing) + def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice: # Native Codex tools may have changed the workspace before a failed response. return ModelRetryAdvice(replay_safety="unsafe") @@ -275,19 +370,7 @@ async def stream_response( ) provider = self._provider bindings = get_model_tool_hooks() - tool_execution = bindings.settings if bindings is not None else None - classic_preparation = ( - tool_execution is not None - and tool_execution.sdk_only_tool_hooks - and ( - tool_execution.prepare_tool_input is not None - or tool_execution.prepare_tool_output is not None - ) - ) - sdk_sandbox = classic_preparation or any( - getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) - for tool in tools - ) + sdk_sandbox = _uses_sdk_workspace(tools, bindings) native_hooks = None if sdk_sandbox else CodexToolHooks.for_run(bindings, identities) if sdk_sandbox: system_instructions = "\n\n".join( @@ -366,6 +449,7 @@ async def stream_response( sdk_calls: dict[str, Any] = {} native_pending: dict[str, tuple[str, str]] = {} native_completed: set[str] = set() + pending_approval: dict[str, Any] | None = None sampling_pending = False interrupt_requested = False sequence = 0 @@ -411,6 +495,18 @@ async def interrupt_when_ready() -> None: ) sequence += 1 while True: + if pending_approval is not None and not sampling_pending: + # All calls from this sampling response are now known. A sibling + # still in flight has an unknown outcome and cannot be dropped. + call_id = pending_approval["msg"]["call_id"] + if set(native_pending) != {call_id} or sdk_calls: + raise UserError("Codex can only pause one outstanding native call.") + item_id, _ = native_pending.pop(call_id) + output[item_id] = _pending_native_call( + output[item_id], pending_approval, provider._cwd, thread.model + ) + # Close the waiting thread before handing the pause to the Runner. + break encoded = await thread.next_event() if encoded is None: raise RuntimeError("Codex finished without a completed response.") @@ -437,6 +533,20 @@ async def interrupt_when_ready() -> None: ) if event.get("id") not in ("", turn_id): continue + if kind in ("exec_approval_request", "apply_patch_approval_request"): + if ( + message.get("turn_id") != turn_id + or pending_approval is not None + or native_hooks is not None + or sdk_sandbox + or require_sdk_call + ): + raise UserError( + "Codex can only pause a single built-in tool call before execution, " + "with native workspace tools enabled and without custom tool hooks." + ) + pending_approval = event + continue if kind == "dynamic_tool_call_request": if message.get("turnId") != turn_id: continue diff --git a/src/agents/run.py b/src/agents/run.py index 57ca88bbe2..715b5b4f24 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -871,6 +871,9 @@ async def _run_impl( run_state._reasoning_item_id_policy = resolved_reasoning_item_id_policy run_state.set_trace(get_current_trace()) + if isinstance(session, ResumableRun): + session.bind_state(run_state) + use_task_and_turn_spans = include_task_and_turn_spans(run_config.tracing) current_task_span: Span[TaskSpanData] | None = ( task_span(name=trace_workflow_name) if use_task_and_turn_spans else None @@ -2912,6 +2915,7 @@ async def run_loop() -> None: if isinstance(session, ResumableRun): try: await session.begin(input) + session.bind_state(run_state) except BaseException: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 77ae747a8e..6ddaa62170 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -1189,6 +1189,10 @@ def _apply_approval_decision( rejection_message: str | None = None, ) -> None: """Record an approval or rejection decision.""" + from .models._codex_items import native_approval_checkpoint + + if native_approval_checkpoint(approval_item.raw_item) is not None and always: + raise UserError("Native approvals require a decision for the original call only.") hosted_request = get_hosted_mcp_approval_request_identity(approval_item) if hosted_request is not None: call_id = hosted_request.request_id diff --git a/src/agents/run_internal/native_approvals.py b/src/agents/run_internal/native_approvals.py new file mode 100644 index 0000000000..2709737ac5 --- /dev/null +++ b/src/agents/run_internal/native_approvals.py @@ -0,0 +1,213 @@ +"""Keep native approval state and execution separate from SDK tool dispatch.""" + +from __future__ import annotations + +from typing import Any, cast + +from ..exceptions import UserError +from ..items import ToolCallOutputItem, ToolCallOutputTypes +from ..memory.resumable_session import _ResumableSessionBinding +from ..models._codex_items import native_approval_checkpoint +from ..models._run_context import ModelToolHooks, model_run_context +from .run_steps import NextStepInterruption, NextStepRunAgain, SingleStepResult +from .turn_preparation import get_all_tools, get_model + + +def native_receipts_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any] | None: + """Select the immutable native proposals while leaving decisions mutable.""" + processed = snapshot.get("last_processed_response") or {} + approvals = processed.get("native_approvals", []) + if not approvals: + return None + call_ids = {item["raw_item"]["call_id"] for item in approvals} + step = snapshot.get("current_step") or {} + pending = (step.get("data") or {}).get("interruptions", []) + return { + "plans": approvals, + "pending": [ + item for item in pending if item.get("raw_item", {}).get("call_id") in call_ids + ], + "generated": [ + item + for item in snapshot.get("generated_items", []) + if item.get("raw_item", {}).get("call_id") in call_ids + ], + } + + +def _validate_native_plan(processed: Any) -> None: + if len(processed.native_approvals) != 1 or any( + ( + processed.functions, + processed.custom_tool_calls, + processed.handoffs, + processed.computer_actions, + processed.local_shell_calls, + processed.shell_calls, + processed.apply_patch_calls, + processed.function_tools_not_found, + processed.mcp_approval_requests, + ) + ): + raise UserError( + "Native approvals require a single pending native call without SDK actions." + ) + item = processed.native_approvals[0] + if native_approval_checkpoint(item.raw_item) is None or item.tool_name != item.raw_item.name: + raise UserError("Invalid native approval plan.") + + +def native_pending_step( + *, + bindings: Any, + original_input: Any, + pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + run_state: Any, +) -> SingleStepResult: + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError( + "Start the run with session=ResumableSession to recover native tool approvals." + ) + pending = list(processed_response.native_approvals) + processed_response.interruptions = pending + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=pre_step_items, + new_step_items=[*processed_response.new_items, *pending], + next_step=NextStepInterruption(interruptions=pending), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + + +async def resolve_native_approval_step( + *, + bindings: Any, + original_input: Any, + original_pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + context_wrapper: Any, + hooks: Any, + run_config: Any, + run_state: Any, +) -> SingleStepResult: + from ..models.codex import _CodexModel + + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError("Native approval recovery requires a claimed ResumableSession.") + pending = processed_response.native_approvals[0] + checkpoint = native_approval_checkpoint(pending.raw_item) + assert checkpoint is not None + call = pending.raw_item + key = context_wrapper._resolve_approval_key(pending) + key = context_wrapper._function_approval_key(key, pending) or key + decision = context_wrapper._approvals.get(key) + if decision is not None and (decision.approved is True or decision.rejected is True): + raise UserError("Native approvals require a decision for the original call only.") + approved = context_wrapper.get_approval_status( + pending.tool_name, + call.call_id, + existing_pending=pending, + current_invocation=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + if approved is None: + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[], + next_step=NextStepInterruption(interruptions=[pending]), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + # A new checkpoint can carry the same model call ID. Claiming that checkpoint + # doesn't authorize repeating an invocation that already used its decision. + status = context_wrapper._tool_invocation_status( + call, tool_lookup_key=pending.tool_lookup_key, tool_name=pending.tool_name + ) + if status is None: + raise UserError("Cannot match the native approval to a pending invocation.") + _, completed, executed = status + if executed or completed: + raise UserError( + "Cannot reuse a native call ID after the call has executed or completed. " + "Start a new invocation with a new call ID." + ) + if approved: + from ..tracing.model_tracing import get_model_tracing_impl + + model = bindings.resolved_model or get_model(bindings.execution_agent, run_config) + if not isinstance(model, _CodexModel): + raise UserError("A pending native approval requires its current Codex model.") + tools = await get_all_tools(bindings.execution_agent, context_wrapper) + with model_run_context( + run_state, + ModelToolHooks( + bindings.public_agent, context_wrapper, hooks, run_config.tool_execution + ), + ): + output = await model._run_approved_native_call( + checkpoint, + tools=tools, + tracing=get_model_tracing_impl( + run_config.tracing_disabled, run_config.trace_include_sensitive_data + ), + ) + else: + from .tool_execution import resolve_approval_rejection_message + + output = await resolve_approval_rejection_message( + context_wrapper=context_wrapper, + run_config=run_config, + tool_type="shell" if call.name == "exec_command" else "apply_patch", + tool_name=call.name, + call_id=call.call_id, + tool_call=call, + existing_pending=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + result_item = ToolCallOutputItem( + agent=bindings.public_agent, + raw_item=cast( + ToolCallOutputTypes, + { + "type": "function_call_output" + if call.type == "function_call" + else "custom_tool_call_output", + "call_id": call.call_id, + "output": output, + }, + ), + output=output, + ) + context_wrapper._mark_tool_call_completed(result_item.raw_item) + processed_response.interruptions = [] + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[result_item], + next_step=NextStepRunAgain(), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) diff --git a/src/agents/run_internal/resumable_session.py b/src/agents/run_internal/resumable_session.py index a55fba5c31..9754d31c09 100644 --- a/src/agents/run_internal/resumable_session.py +++ b/src/agents/run_internal/resumable_session.py @@ -37,6 +37,13 @@ async def begin(self, input: Any) -> None: await self._source._begin_run(input) self._items = self._source._get_history() + def bind_state(self, state: Any) -> None: + """Attach the current store claim to the internal execution state.""" + assert self._source is not None + state._resumable_binding = _ResumableSessionBinding( + self._source, self._source._checkpoint.revision + ) + async def finish(self, result: RunResult | RunResultStreaming) -> None: assert self._source is not None state = result.to_state() if result.interruptions else None diff --git a/src/agents/run_internal/run_steps.py b/src/agents/run_internal/run_steps.py index e98177a1d4..4e786e46e4 100644 --- a/src/agents/run_internal/run_steps.py +++ b/src/agents/run_internal/run_steps.py @@ -137,6 +137,7 @@ class ProcessedResponse: # Original invocation recipients, independent of the currently available callables. tool_output_guardrail_result_start: int | None = None # First checkpointed tool output guardrail result owned by this response. + native_approvals: list[ToolApprovalItem] = dataclasses.field(default_factory=list) def has_tools_or_approvals_to_run(self) -> bool: # Handoffs, functions and computer actions need local processing @@ -152,6 +153,7 @@ def has_tools_or_approvals_to_run(self) -> bool: self.apply_patch_calls, self.mcp_approval_requests, self.function_tools_not_found, + self.native_approvals, ] ) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 04fad384e2..b85fb88d64 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -101,7 +101,13 @@ ) from ..lifecycle import RunHooks from ..logger import log_tool_action_error, logger -from ..models._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ..models._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -151,6 +157,7 @@ run_item_to_input_item, shell_rejection_item, ) +from .native_approvals import native_pending_step, resolve_native_approval_step from .run_steps import ( NOT_FINAL_OUTPUT, NextStepFinalOutput, @@ -987,6 +994,17 @@ async def execute_tools_and_side_effects( """Run one turn of the loop, coordinating tools, approvals, guardrails, and handoffs.""" public_agent = bindings.public_agent + if processed_response.native_approvals: + _register_tool_call_items(context_wrapper, processed_response.new_items) + return native_pending_step( + bindings=bindings, + original_input=original_input, + pre_step_items=pre_step_items, + new_response=new_response, + processed_response=processed_response, + run_state=run_state, + ) + execute_final_output_call = execute_final_output execute_handoffs_call = execute_handoffs @@ -1406,6 +1424,18 @@ async def resolve_interrupted_turn( """Continue a turn that was previously interrupted waiting for tool approval.""" public_agent = bindings.public_agent execution_agent = bindings.execution_agent + if processed_response.native_approvals: + return await resolve_native_approval_step( + bindings=bindings, + original_input=original_input, + original_pre_step_items=original_pre_step_items, + new_response=new_response, + processed_response=processed_response, + context_wrapper=context_wrapper, + hooks=hooks, + run_config=run_config, + run_state=run_state, + ) output_index = _build_tool_output_index(original_pre_step_items) response_boundary = _current_response_boundary((), processed_response, run_state) generated_output_start = response_boundary.generated_start @@ -3172,6 +3202,7 @@ def process_model_response( allow_apply_patch_function_fallback: bool = True, ) -> ProcessedResponse: items: list[RunItem] = [] + native_approvals: list[ToolApprovalItem] = [] # Only the Codex adapter creates these types, after validating native call/output pairing. provider_completed = { @@ -3276,6 +3307,12 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: output.__class__.__name__ if hasattr(output, "__class__") else type(output), ) caller = get_mapping_or_attr(output, "caller") + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + items.append(ToolCallItem(raw_item=cast(Any, output), agent=agent)) + native_approvals.append(ToolApprovalItem(agent=agent, raw_item=output)) + tools_used.append(output.name) + continue is_program_owned = get_mapping_or_attr(caller, "type") == "program" if ( not is_program_owned @@ -3964,6 +4001,7 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tools_used=tools_used, mcp_approval_requests=mcp_approval_requests, interruptions=[], + native_approvals=native_approvals, function_tools_not_found=function_tools_not_found, mcp_tool_bindings={ run.tool_call.call_id: binding @@ -3999,6 +4037,9 @@ def _preflight_response_invocations( if isinstance(item, (CodexFunctionToolCallOutput, CodexCustomToolCallOutput)) } for index, output in enumerate(response.output): + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + continue if isinstance(output, (ResponseFunctionToolCall, ResponseCustomToolCall)) and ( output.call_id in provider_completed ): diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 20c8ce2755..d5c8f51f9b 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -110,6 +110,10 @@ log_model_and_tool_data_warning, logger, ) +from .models._codex_items import ( + native_approval_checkpoint, + restore_native_approval_call, +) from .run_context import RunContextWrapper, _ApprovalRecord, _FunctionToolApprovalKey from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -312,7 +316,7 @@ def _default_run_state_validation_error( ), "1.23": ( "Binds prepared tool inputs to their original proposals across retries and resume, " - "and records required ResumableSession ownership for paused runs." + "records required ResumableSession ownership, and preserves native approval receipts." ), } SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES) @@ -785,6 +789,11 @@ def _restore_pydantic_fields_set(value: Any, source: Any) -> None: def _copy_tool_approval_raw_item(raw_item: Any) -> Any: """Copy a supported approval raw item through public Pydantic APIs.""" + native_checkpoint = native_approval_checkpoint(raw_item) + if native_checkpoint is not None: + return restore_native_approval_call( + raw_item.model_dump(exclude_unset=True), native_checkpoint + ) active_container_ids: set[int] = set() raw_item_mro = _static_type_mro(raw_item) for model_type, model_adapter in _TOOL_APPROVAL_MODEL_ADAPTERS: @@ -1410,6 +1419,12 @@ def approve(self, approval_item: ToolApprovalItem, always_approve: bool = False) nested_state.approve(nested_item, always_approve=always_approve) return current_approval_item = self._find_current_approval_item(approval_item) + native_checkpoint = native_approval_checkpoint(approval_item.raw_item) + if native_checkpoint is not None and ( + current_approval_item is None + or native_approval_checkpoint(current_approval_item.raw_item) != native_checkpoint + ): + raise UserError("Native approval does not match the current pending call.") self._context.approve_tool( current_approval_item or approval_item, always_approve=always_approve, @@ -1440,8 +1455,15 @@ def reject( rejection_message=rejection_message, ) return + current_approval_item = self._find_current_approval_item(approval_item) + native_checkpoint = native_approval_checkpoint(approval_item.raw_item) + if native_checkpoint is not None and ( + current_approval_item is None + or native_approval_checkpoint(current_approval_item.raw_item) != native_checkpoint + ): + raise UserError("Native approval does not match the current pending call.") self._context.reject_tool( - self._find_current_approval_item(approval_item) or approval_item, + current_approval_item or approval_item, always_reject=always_reject, rejection_message=rejection_message, ) @@ -2167,6 +2189,15 @@ def _serialize_processed_response( for call_id, binding in processed_response.mcp_tool_bindings.items() }, } + if processed_response.native_approvals: + result["native_approvals"] = [ + _serialize_tool_approval_interruption( + item, + include_tool_name=True, + agent_identity_keys_by_id=agent_identity_keys_by_id, + ) + for item in processed_response.native_approvals + ] if processed_response.tool_output_guardrail_result_start is not None: result["tool_output_guardrail_result_start"] = ( processed_response.tool_output_guardrail_result_start @@ -2226,6 +2257,9 @@ def _serialize_item( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(item.raw_item) + if native_checkpoint is not None: + result["codex_native_checkpoint"] = native_checkpoint if isinstance(item, InputItem): result["input_id"] = item.input_id @@ -2896,6 +2930,9 @@ def _serialize_tool_approval_interruption( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(interruption.raw_item) + if native_checkpoint is not None: + interruption_dict["codex_native_checkpoint"] = native_checkpoint if include_tool_name and interruption.tool_name is not None: interruption_dict["tool_name"] = interruption.tool_name if interruption.tool_namespace is not None: @@ -3643,6 +3680,19 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: if approval_item is not None: interruptions.append(approval_item) + native_approvals: list[ToolApprovalItem] = [] + for approval_data in processed_response_data.get("native_approvals", []): + approval = _deserialize_tool_approval_item( + approval_data, + agent_map=agent_map, + agent_identity_map=agent_identity_map, + fallback_agent=current_agent, + validation_error_factory=validation_error_factory, + ) + if approval is None or native_approval_checkpoint(approval.raw_item) is None: + raise validation_error_factory("Invalid native approval plan", UserError) + native_approvals.append(approval) + saved_bindings = processed_response_data.get("mcp_tool_bindings", {}) mcp_tool_bindings = { call_id: cast(tuple[str, str, int | None], tuple(binding)) @@ -3656,6 +3706,7 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: return ProcessedResponse( new_items=new_items, + native_approvals=native_approvals, handoffs=handoffs, functions=functions, computer_actions=computer_actions, @@ -3835,6 +3886,8 @@ def _deserialize_tool_approval_item( tool_lookup_key = deserialize_function_tool_lookup_key(item_data.get("tool_lookup_key")) allow_bare_name_alias = item_data.get("allow_bare_name_alias") is True raw_item = _deserialize_tool_approval_raw_item(raw_item_data) + if "codex_native_checkpoint" in item_data: + raw_item = restore_native_approval_call(raw_item_data, item_data["codex_native_checkpoint"]) return ToolApprovalItem( agent=agent, raw_item=raw_item, @@ -4236,6 +4289,10 @@ async def _build_run_state_from_json( validation_error_factory=validation_error_factory, ) schema_major, schema_minor = (int(part) for part in schema_version.split(".", maxsplit=1)) + if (schema_major, schema_minor) < (1, 23) and ( + (state_json.get("last_processed_response") or {}).get("native_approvals") + ): + raise validation_error_factory("Native approvals require RunState schema 1.23.", UserError) programmatic_major, programmatic_minor = ( int(part) for part in _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION.split(".", maxsplit=1) ) @@ -5343,6 +5400,10 @@ def _resolve_agent_info( # Tool call items can be function calls, shell calls, apply_patch calls, # MCP calls, etc. Check the type field to determine which type to deserialize as raw_item_tool = _deserialize_tool_call_raw_item(normalized_raw_item) + if "codex_native_checkpoint" in item_data: + raw_item_tool = restore_native_approval_call( + normalized_raw_item, item_data["codex_native_checkpoint"] + ) # Preserve display metadata if it was stored with the item. description = item_data.get("description") title = item_data.get("title") diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py new file mode 100644 index 0000000000..456c8c1a3f --- /dev/null +++ b/tests/memory/test_native_approval_recovery.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import asyncio +import copy +import json + +import pytest + +from agents import Agent, ResumableSession, RunConfig, Runner, RunState +from agents.exceptions import AgentsException, UserError +from agents.items import ModelResponse +from agents.memory.resumable_session import ResumableSessionError +from agents.models._codex_items import CodexPendingCustomCall, CodexPendingFunctionCall +from agents.models.codex import CodexProvider +from agents.testing import ScriptedModel +from agents.usage import Usage +from tests.memory.test_resumable_session import MemoryStore +from tests.models.test_codex_provider import FakeThread, threads as threads +from tests.test_responses import get_text_message +from tests.testing_processor import fetch_ordered_spans + + +def pending_call(tmp_path, kind): + call = { + "type": "function_call" if kind == "exec_command" else "custom_tool_call", + "id": "native-item", + "call_id": "native-call", + "name": kind, + "status": "in_progress", + } + if kind == "exec_command": + call["arguments"] = '{"cmd":"echo harmless"}' + item_type = CodexPendingFunctionCall + else: + call["input"] = "*** Begin Patch\n*** Add File: harmless.txt\n+safe\n*** End Patch\n" + item_type = CodexPendingCustomCall + checkpoint = { + "version": 1, + "cwd": str(tmp_path.resolve()), + "call": copy.deepcopy(call), + "request": {"type": "native_approval", "call_id": "native-call"}, + "model": "fixture-model", + } + item = item_type(**call, checkpoint=checkpoint) + return item.model_copy(update={"codex_reasoning_summary": "Review this effect."}) + + +async def setup_native(tmp_path, monkeypatch, kind="exec_command"): + provider = CodexProvider(cwd=tmp_path) + model = provider.get_model("fixture-model") + pending = pending_call(tmp_path, kind) + calls = [] + requests = [] + + async def respond(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = [pending] if len(requests) == 1 else [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint) + return "native accepted" + + monkeypatch.setattr(model, "get_response", respond) + monkeypatch.setattr(model, "_run_approved_native_call", execute, raising=False) + agent = Agent(name="worker", model=model) + store = MemoryStore() + session = await ResumableSession.create(store) + config = RunConfig(tracing_disabled=True) + return agent, store, session, config, pending, calls, requests + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("approved", [True, False]) +async def test_native_approval_round_trip_uses_original_runtime_only( + tmp_path, monkeypatch, kind, approved +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + first = await Runner.run(agent, "do the native operation", session=session, run_config=config) + assert len(first.interruptions) == 1 + assert calls == [] + assert store.checkpoint(session.session_id)["status"] == "paused" + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + interruption = state.get_interruptions()[0] + assert interruption.arguments == ( + pending.arguments if kind == "exec_command" else pending.input + ) + if approved: + state.approve(interruption) + else: + state.reject(interruption, rejection_message="Leave the release branch alone.") + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert result.final_output == "finished" + assert calls == ([pending.checkpoint] if approved else []) + assert "checkpoint" not in repr(requests[-1]) + expected_message = "native accepted" if approved else "Leave the release branch alone." + assert any( + item.get("call_id") == interruption.call_id and item.get("output") == expected_message + for item in requests[-1] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_plain_runner_rejects_unrecoverable_native_pause(tmp_path, monkeypatch, streamed): + agent, _, _, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + with pytest.raises(UserError, match="session=ResumableSession"): + if streamed: + result = Runner.run_streamed(agent, "start", run_config=config) + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "start", run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("field", ["name", "call_id", "id", "payload"]) +async def test_display_mutation_cannot_be_approved(tmp_path, monkeypatch, kind, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch, kind) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + item = state.get_interruptions()[0] + field = ("arguments" if kind == "exec_command" else "input") if field == "payload" else field + setattr(item.raw_item, field, "changed") + with pytest.raises(UserError): + state.approve(item) + assert not calls + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["payload", "cwd", "request", "model"]) +async def test_coordinated_receipt_mutation_fails_before_claim(tmp_path, monkeypatch, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + # Rewrite every serialized copy consistently; the store's original still binds authority. + snapshot = state.to_json() + + def rewrite(value): + if isinstance(value, dict): + if field == "payload" and value.get("type") == "function_call": + value["arguments"] = '{"cmd":"changed"}' + if value.get("version") == 1 and "cwd" in value and "request" in value: + if field == "cwd": + value["cwd"] = str(tmp_path / "other") + elif field == "request": + value["request"] = {"type": "changed"} + elif field == "model": + value["model"] = "changed" + for child in value.values(): + rewrite(child) + elif isinstance(value, list): + for child in value: + rewrite(child) + + rewrite(snapshot) + changed = await RunState.from_string(agent, json.dumps(snapshot)) + changed._resumable_binding = state._resumable_binding + original = store.records[session.session_id] + with pytest.raises(ResumableSessionError): + await Runner.run(agent, changed, session=recovered, run_config=config) + assert store.records[session.session_id] == original + assert not calls + + +@pytest.mark.asyncio +async def test_no_decision_repauses_and_unbound_copy_never_dispatches(tmp_path, monkeypatch): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + first = await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(result.interruptions) == 1 + assert not calls + copied = await RunState.from_string(agent, first.to_state().to_string()) + copied.approve(copied.get_interruptions()[0]) + with pytest.raises(UserError): + await Runner.run(agent, copied, run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +async def test_completed_native_call_id_cannot_reuse_a_previous_decision( + tmp_path, monkeypatch, kind +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + + async def repeat_native_call(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + if len(requests) <= 2: + # The adapter can attest a new native request with a repeated model call ID. + repeated = pending.model_copy(deep=True) + repeated.checkpoint["request"]["turn_id"] = f"native-turn-{len(requests)}" + output = [repeated] + else: + output = [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + monkeypatch.setattr(agent.model, "get_response", repeat_native_call) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + + # Native execution stays at the adapter seam while public resume owns approval state. + with pytest.raises(UserError, match="completed"): + repeated_result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(repeated_result.interruptions) == 1 + assert len(calls) == 1 + replacement = await ResumableSession.resume(store, session.session_id) + repeated_state = await replacement.load_state(agent) + await Runner.run(agent, repeated_state, session=replacement, run_config=config) + assert len(calls) == 1, "The previous decision authorized another native dispatch." + + assert len(calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("approved", [True, False]) +async def test_sticky_native_decisions_are_rejected(tmp_path, monkeypatch, approved): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + state = await session.load_state(agent) + item = state.get_interruptions()[0] + with pytest.raises(UserError, match="original call only"): + if approved: + state.approve(item, always_approve=True) + else: + state.reject(item, always_reject=True) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_waits_for_checkpoint_ack(tmp_path, monkeypatch, streamed): + agent, store, session, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + store.block_checkpoint = True + events = [] + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + result = Runner.run_streamed(agent, "start", session=session, run_config=config) + + async def consume(): + async for event in result.stream_events(): + events.append(event) + return result + + task = asyncio.create_task(consume()) + else: + task = asyncio.create_task(Runner.run(agent, "start", session=session, run_config=config)) + try: + await asyncio.wait_for(store.saving.wait(), 5) + assert not task.done() + assert store.checkpoint(session.session_id)["status"] == "active" + assert not calls + if streamed: + assert not result.interruptions + assert not any( + getattr(event, "name", None) == "tool_approval_requested" for event in events + ) + with pytest.raises(AgentsException): + result.to_state() + finally: + store.release.set() + finished = await asyncio.wait_for(task, 5) + assert len(finished.interruptions) == 1 + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_data", [False, True]) +async def test_resume_exports_the_original_native_operation_under_current_trace_policy( + tmp_path, monkeypatch, threads, include_data +): + class RestoredThread(FakeThread): + async def resolve_approval(self, call_json, request, preconditions, capture): + assert capture == ("content" if include_data else "metadata") + operation = { + "threadId": self.thread_id, + "turnId": "replacement-turn", + "operationId": "from-native-dispatch", + "kind": "tool", + "toolName": "exec_command", + "callId": json.loads(call_json)["call_id"], + "startedAt": "2026-11-20T06:00:00.000Z", + "input": {"value": "private commit message"}, + } + completed = dict(operation, completedAt="2026-11-20T06:00:02.000Z", outcome="success") + events = [ + json.dumps({"method": f"turn/operation/{kind}", "params": params}) + for kind, params in [("started", operation), ("completed", completed)] + ] + return "native accepted", "replacement-turn", events, None + + agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + pending = state.get_interruptions()[0] + state.approve(pending) + monkeypatch.delattr(agent.model, "_run_approved_native_call") + threads[0].append(RestoredThread()) + result = await Runner.run( + agent, + state, + run_config=RunConfig( + tracing={"operation_detail": "content"}, trace_include_sensitive_data=include_data + ), + ) + assert result.final_output == "finished" + (operation_span,) = [ + span.span_data + for span in fetch_ordered_spans() + if span.span_data.type == "function" + and span.span_data.operation.operation_id == "from-native-dispatch" + ] + assert operation_span.operation.outcome == "completed" + assert operation_span.operation.tool_call_id == pending.call_id + assert ("private commit message" in repr(operation_span.input)) == include_data diff --git a/tests/models/test_codex_pending_approval.py b/tests/models/test_codex_pending_approval.py new file mode 100644 index 0000000000..396404ce47 --- /dev/null +++ b/tests/models/test_codex_pending_approval.py @@ -0,0 +1,166 @@ +import json + +import pytest + +from agents import ModelSettings, UserError +from agents.models._codex_items import ( + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) +from agents.models.codex import CodexProvider +from agents.models.interface import ModelTracing + +from .test_codex_provider import FakeThread, native_pair, threads as threads + + +class ApprovalThread(FakeThread): + async def next_event(self): + encoded = await super().next_event() + if encoded is None: + return None + event = json.loads(encoded) + if event["msg"]["type"] in ( + "exec_approval_request", + "apply_patch_approval_request", + ): + event["preconditions"] = {"version": 1, "first_approval": True} + return json.dumps(event) + + +def approval(call_id="pending"): + return { + "type": "exec_approval_request", + "call_id": call_id, + "turn_id": "turn_1", + "kind": "command", + "command": ["sh", "-c", "printf native-result"], + "cwd": "/workspace", + } + + +def raw(item): + return {"type": "raw_response_item", "item": item} + + +async def response(tmp_path, settings=None): + return ( + await CodexProvider(cwd=tmp_path) + .get_model(None) + .get_response( + None, + "Continue", + settings or ModelSettings(), + [], + None, + [], + ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ) + + +@pytest.mark.asyncio +async def test_approval_waits_for_sampling_completion_and_rejects_late_sibling(tmp_path, threads): + call, _ = native_pair(call_id="pending") + sibling, _ = native_pair(call_id="later") + thread = ApprovalThread( + events=[ + raw(call), + approval(), + raw(sibling), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + with pytest.raises(UserError, match="one outstanding native call"): + await response(tmp_path) + assert thread.closed + + +@pytest.mark.asyncio +async def test_completed_read_stays_completed_while_next_native_call_pauses(tmp_path, threads): + read, read_output = native_pair(call_id="read") + read["arguments"] = json.dumps({"cmd": "cat README.md"}) + pending, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + raw(read), + raw(read_output), + raw(pending), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert [item.call_id for item in result.output] == ["read", "read", "pending"] + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert result.output[1].output == read_output["output"] + assert isinstance(result.output[2], CodexPendingFunctionCall) + assert result.output[2].checkpoint["call"]["call_id"] == "pending" + assert thread.closed + + +@pytest.mark.asyncio +async def test_raw_model_metadata_cannot_forge_a_pending_native_approval(tmp_path, threads): + call, output = native_pair() + call["checkpoint"] = {"request": approval("native_1"), "version": 1} + call["native_approval"] = True + thread = FakeThread(raw_items=[call, output]) + threads[0].append(thread) + result = await response(tmp_path) + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert not hasattr(result.output[0], "checkpoint") + assert "native_approval" not in result.output[0].model_dump() + assert thread.closed + + +@pytest.mark.asyncio +async def test_patch_begin_does_not_mean_an_awaiting_patch_has_executed(tmp_path, threads): + call, _ = native_pair("custom_tool_call", "pending", "apply_patch") + + thread = ApprovalThread( + events=[ + raw(call), + {"type": "patch_apply_begin", "call_id": "pending"}, + { + "type": "apply_patch_approval_request", + "call_id": "pending", + "turn_id": "turn_1", + "changes": {}, + }, + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert isinstance(result.output[0], CodexPendingCustomCall) + assert result.output[0].checkpoint["preconditions"]["first_approval"] is True + assert thread.closed + + +@pytest.mark.asyncio +async def test_reasoning_summary_does_not_change_the_approved_action(tmp_path, threads): + call, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + { + "type": "item_completed", + "item": {"type": "Reasoning", "summary_text": ["Reviewing the command."]}, + }, + raw(call), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path, ModelSettings(reasoning={"summary": "auto"})) + pending = result.output[-1] + assert pending.codex_reasoning_summary == ["Reviewing the command."] + checkpoint = native_approval_checkpoint(pending) + assert checkpoint is not None + assert checkpoint["call"]["call_id"] == "pending" + assert thread.closed From 5d26364e989dc2f7d5e912c2776aaf3fb5913004 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 20:54:39 -0800 Subject: [PATCH 19/33] fix(codex): bound file reads when saving and checking patch approvals --- .../codex/0002-native-approval-recovery.patch | 496 +++++++++++++++++- 1 file changed, 488 insertions(+), 8 deletions(-) diff --git a/patches/codex/0002-native-approval-recovery.patch b/patches/codex/0002-native-approval-recovery.patch index 683d169073..74de4cc1bc 100644 --- a/patches/codex/0002-native-approval-recovery.patch +++ b/patches/codex/0002-native-approval-recovery.patch @@ -37,7 +37,7 @@ index 968d331..9d0bd2e 100644 @@ -474,10 +475,39 @@ impl CodexThread { .await } - + + /// Replays one native call without model sampling, requiring the original human approval. + pub async fn restore_tool_call_if_idle( + &self, @@ -91,10 +91,9 @@ index dcd01e4..dc14807 100644 mod compact_model_fallback; diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs new file mode 100644 -index 0000000..2e9781e --- /dev/null +++ b/codex-rs/core/src/patch_preconditions.rs -@@ -0,0 +1,306 @@ +@@ -0,0 +1,331 @@ +//! Filesystem preconditions for restoring an interrupted native patch approval. + +use crate::session::step_context::StepContext; @@ -118,6 +117,9 @@ index 0000000..2e9781e +use std::path::Path; +use std::path::PathBuf; + ++// Approval recovery fails closed for larger preimages instead of retaining large reads. ++const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; ++ +#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct PatchPreconditions { @@ -254,11 +256,12 @@ index 0000000..2e9781e + } + Ok(_) => { + let bytes = fs -+ .read_file( ++ .read_file_bounded( + &uri, + ReadFileOptions { + follow_symlinks: false, + }, ++ MAX_PATCH_PREIMAGE_BYTES, + ) + .await + .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; @@ -400,6 +403,27 @@ index 0000000..2e9781e + ); + assert!(capture(&fs, &request).await.is_err()); + } ++ #[tokio::test] ++ async fn rejects_preimage_larger_than_recovery_limit() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().canonicalize().unwrap().join("large.txt"); ++ std::fs::File::create(&path) ++ .unwrap() ++ .set_len(MAX_PATCH_PREIMAGE_BYTES + 1) ++ .unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let request = approval( ++ path, ++ FileChange::Update { ++ unified_diff: "-old\n+new\n".into(), ++ move_path: None, ++ }, ++ ); ++ let error = capture(&fs, &request) ++ .await ++ .expect_err("large preimages must fail closed"); ++ assert!(error.to_string().contains("limit is 8388608 bytes")); ++ } +} diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs new file mode 100644 @@ -1056,7 +1080,7 @@ index 66476ae..c092ec5 100644 @@ -2930,7 +2930,7 @@ impl Session { .map(|task| Arc::clone(&task.turn_context)) } - + - async fn active_turn_context_and_cancellation_token( + pub(crate) async fn active_turn_context_and_cancellation_token( &self, @@ -1077,7 +1101,7 @@ index 66476ae..c092ec5 100644 + rx_approve.await.unwrap_or(ReviewDecision::Abort), + ) } - + #[expect( @@ -3219,9 +3225,15 @@ impl Session { reason, @@ -1094,7 +1118,7 @@ index 66476ae..c092ec5 100644 + rx_approve.await.unwrap_or(ReviewDecision::Abort), + ) } - + #[expect( diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 6c6f3d1..1309ac1 100644 @@ -1210,7 +1234,7 @@ index d9afeb6..32a5820 100644 ExecApprovalRequirement::Skip { .. } => { if strict_auto_review { @@ -228,12 +243,24 @@ impl ToolOrchestrator { - + // 2) First attempt under the selected sandbox. let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); - let sandbox_override = sandbox_override_for_first_attempt( @@ -1240,3 +1264,459 @@ index d9afeb6..32a5820 100644 let (escalated_profile, writable_root_metadata) = if sandbox_override == SandboxOverride::EscalatedSandboxWithRestrictions { +diff --git a/codex-rs/exec-server/src/fs_helper.rs b/codex-rs/exec-server/src/fs_helper.rs +--- a/codex-rs/exec-server/src/fs_helper.rs ++++ b/codex-rs/exec-server/src/fs_helper.rs +@@ -71,6 +71,12 @@ + Open(FsHelperOpenParams), + #[serde(rename = "fs/readFile")] + ReadFile(FsReadFileParams), ++ #[serde(rename = "fs/readFileBounded")] ++ ReadFileBounded { ++ path: PathUri, ++ follow_symlinks: bool, ++ max_bytes: u64, ++ }, + #[serde(rename = "fs/writeFile")] + WriteFile(FsWriteFileParams), + #[serde(rename = "fs/createDirectory")] +@@ -311,6 +317,19 @@ + data_base64: STANDARD.encode(data), + })) + } ++ FsHelperRequest::ReadFileBounded { ++ path, ++ follow_symlinks, ++ max_bytes, ++ } => { ++ let data = file_system ++ .read_file_bounded(&path, ReadFileOptions { follow_symlinks }, max_bytes, None) ++ .await ++ .map_err(map_fs_error)?; ++ Ok(FsHelperPayload::ReadFile(FsReadFileResponse { ++ data_base64: STANDARD.encode(data), ++ })) ++ } + FsHelperRequest::WriteFile(params) => { + let bytes = STANDARD.decode(params.data_base64).map_err(|err| { + invalid_request(format!( +diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs +--- a/codex-rs/exec-server/src/local_file_system.rs ++++ b/codex-rs/exec-server/src/local_file_system.rs +@@ -304,6 +304,20 @@ + Box::pin(LocalFileSystem::read_file(self, path, options, sandbox)) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ let (fs, sandbox) = self.file_system_for_reads(sandbox)?; ++ fs.read_file_bounded(path, options, max_bytes, sandbox) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +@@ -517,6 +531,21 @@ + Box::pin(UnsandboxedFileSystem::read_file( + self, path, options, sandbox, + )) ++ } ++ ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ reject_platform_sandbox_context(sandbox)?; ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, None) ++ .await ++ }) + } + + fn read_file_stream<'a>( +@@ -998,6 +1027,24 @@ + Box::pin(DirectFileSystem::read_file(self, path, options, sandbox)) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ reject_sandbox_context(sandbox)?; ++ crate::local_file_system_read::read_file_bounded( ++ path.to_abs_path()?, ++ options, ++ max_bytes, ++ ) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +diff --git a/codex-rs/exec-server/src/local_file_system_read.rs b/codex-rs/exec-server/src/local_file_system_read.rs +--- a/codex-rs/exec-server/src/local_file_system_read.rs ++++ b/codex-rs/exec-server/src/local_file_system_read.rs +@@ -16,6 +16,15 @@ + path: AbsolutePathBuf, + options: ReadFileOptions, + ) -> io::Result> { ++ read_file_bounded(path, options, MAX_READ_FILE_BYTES).await ++} ++ ++pub(super) async fn read_file_bounded( ++ path: AbsolutePathBuf, ++ options: ReadFileOptions, ++ max_bytes: u64, ++) -> io::Result> { ++ let max_bytes = max_bytes.min(MAX_READ_FILE_BYTES); + let cancelled = CancellationToken::new(); + let _cancel_on_drop = cancelled.clone().drop_guard(); + tokio::task::spawn_blocking(move || { +@@ -28,22 +37,31 @@ + no_follow::open_file(path.as_path())? + }; + let metadata = file.metadata()?; +- if metadata.len() > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- let mut bytes = Vec::with_capacity(metadata.len() as usize); +- CancellableReader { +- reader: file.take(MAX_READ_FILE_BYTES + 1), +- cancelled, +- } +- .read_to_end(&mut bytes)?; +- if bytes.len() as u64 > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- Ok(bytes) ++ read_with_limit(file, metadata.len(), max_bytes, cancelled) + }) + .await + .map_err(|error| io::Error::other(format!("filesystem task failed: {error}")))? ++} ++ ++fn read_with_limit( ++ reader: impl Read, ++ initial_len: u64, ++ max_bytes: u64, ++ cancelled: CancellationToken, ++) -> io::Result> { ++ if initial_len > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ let mut bytes = Vec::with_capacity(initial_len as usize); ++ CancellableReader { ++ reader: reader.take(max_bytes + 1), ++ cancelled, ++ } ++ .read_to_end(&mut bytes)?; ++ if bytes.len() as u64 > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ Ok(bytes) + } + + struct CancellableReader { +@@ -62,10 +80,10 @@ + } + } + +-fn file_too_large_error() -> io::Error { ++fn file_too_large_error(max_bytes: u64) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, +- format!("file is too large to read: limit is {MAX_READ_FILE_BYTES} bytes"), ++ format!("file is too large to read: limit is {max_bytes} bytes"), + ) + } + +diff --git a/codex-rs/exec-server/src/local_file_system_read_tests.rs b/codex-rs/exec-server/src/local_file_system_read_tests.rs +--- a/codex-rs/exec-server/src/local_file_system_read_tests.rs ++++ b/codex-rs/exec-server/src/local_file_system_read_tests.rs +@@ -89,3 +89,64 @@ + assert_eq!((first, error.kind()), (*b"x", std::io::ErrorKind::Other)); + Ok(()) + } ++ ++#[tokio::test] ++async fn caller_limit_preserves_regular_read_and_rejects_larger_preimages() -> anyhow::Result<()> { ++ let directory = TempDir::new()?; ++ let path = directory.path().canonicalize()?.join("preimage"); ++ let uri = PathUri::from_host_native_path(&path)?; ++ let contents = vec![0x5a; 16]; ++ std::fs::write(&path, &contents)?; ++ assert_eq!( ++ LOCAL_FS ++ .read_file_bounded( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false ++ }, ++ 16, ++ None ++ ) ++ .await?, ++ contents ++ ); ++ assert!( ++ LOCAL_FS ++ .read_file_bounded( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false ++ }, ++ 15, ++ None ++ ) ++ .await ++ .is_err() ++ ); ++ assert_eq!( ++ LOCAL_FS ++ .read_file( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false ++ }, ++ None ++ ) ++ .await?, ++ contents ++ ); ++ Ok(()) ++} ++ ++#[test] ++fn growth_or_misleading_metadata_cannot_read_past_limit_plus_one() { ++ use std::io::Cursor; ++ use tokio_util::sync::CancellationToken; ++ for initial_len in [0, 8, 16] { ++ let mut reader = Cursor::new(vec![0x5a; 1024]); ++ let error = super::read_with_limit(&mut reader, initial_len, 16, CancellationToken::new()) ++ .unwrap_err(); ++ assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); ++ assert_eq!(reader.position(), 17); ++ } ++} +diff --git a/codex-rs/exec-server/src/sandboxed_file_system.rs b/codex-rs/exec-server/src/sandboxed_file_system.rs +--- a/codex-rs/exec-server/src/sandboxed_file_system.rs ++++ b/codex-rs/exec-server/src/sandboxed_file_system.rs +@@ -145,16 +145,34 @@ + options: ReadFileOptions, + sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { ++ self.read_file_with_limit(path, options, None, sandbox) ++ .await ++ } ++ ++ async fn read_file_with_limit( ++ &self, ++ path: &PathUri, ++ options: ReadFileOptions, ++ max_bytes: Option, ++ sandbox: Option<&FileSystemSandboxContext>, ++ ) -> FileSystemResult> { + let sandbox = require_platform_sandbox(sandbox)?; + validate_native_path(path)?; + let response = self + .run_sandboxed( + sandbox, +- FsHelperRequest::ReadFile(FsReadFileParams { +- path: path.clone(), +- follow_symlinks: (!options.follow_symlinks).then_some(false), +- sandbox: None, +- }), ++ match max_bytes { ++ Some(max_bytes) => FsHelperRequest::ReadFileBounded { ++ path: path.clone(), ++ follow_symlinks: options.follow_symlinks, ++ max_bytes, ++ }, ++ None => FsHelperRequest::ReadFile(FsReadFileParams { ++ path: path.clone(), ++ follow_symlinks: (!options.follow_symlinks).then_some(false), ++ sandbox: None, ++ }), ++ }, + ) + .await? + .expect_read_file() +@@ -364,6 +382,16 @@ + Box::pin(SandboxedFileSystem::read_file(self, path, options, sandbox)) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(self.read_file_with_limit(path, options, Some(max_bytes), sandbox)) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +diff --git a/codex-rs/exec-server/tests/file_system_unix.rs b/codex-rs/exec-server/tests/file_system_unix.rs +--- a/codex-rs/exec-server/tests/file_system_unix.rs ++++ b/codex-rs/exec-server/tests/file_system_unix.rs +@@ -1571,3 +1571,71 @@ + + Ok(()) + } ++ ++#[tokio::test(flavor = "multi_thread", worker_threads = 2)] ++async fn bounded_file_reads_preserve_sandbox_and_no_follow() -> Result<()> { ++ let context = create_file_system_context(FileSystemImplementation::Local).await?; ++ let temp = TempDir::new()?; ++ let root = temp.path().canonicalize()?; ++ let allowed = root.join("allowed"); ++ std::fs::create_dir(&allowed)?; ++ let file = allowed.join("file"); ++ std::fs::write(&file, b"hello")?; ++ let secret = root.join("secret"); ++ std::fs::write(&secret, b"secret")?; ++ let link = allowed.join("link"); ++ symlink(&file, &link)?; ++ let directory_link = allowed.join("directory-link"); ++ symlink(&allowed, &directory_link)?; ++ let sandbox = read_only_sandbox(allowed); ++ let options = ReadFileOptions { ++ follow_symlinks: false, ++ }; ++ for permissions in [None, Some(&sandbox)] { ++ let fs = &context.file_system; ++ assert_eq!( ++ fs.read_file_bounded( ++ &PathUri::from_host_native_path(&file)?, ++ options, ++ 5, ++ permissions ++ ) ++ .await?, ++ b"hello" ++ ); ++ let oversized = fs ++ .read_file_bounded( ++ &PathUri::from_host_native_path(&file)?, ++ options, ++ 4, ++ permissions, ++ ) ++ .await ++ .unwrap_err(); ++ assert!(oversized.to_string().contains("limit is 4 bytes")); ++ for path in [&link, &directory_link.join("file")] { ++ assert!( ++ fs.read_file_bounded( ++ &PathUri::from_host_native_path(path)?, ++ options, ++ 16, ++ permissions ++ ) ++ .await ++ .is_err() ++ ); ++ } ++ } ++ let error = context ++ .file_system ++ .read_file_bounded( ++ &PathUri::from_host_native_path(&secret)?, ++ options, ++ 16, ++ Some(&sandbox), ++ ) ++ .await ++ .unwrap_err(); ++ assert_sandbox_denied(&error); ++ Ok(()) ++} +diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs +--- a/codex-rs/file-system/src/environment_accessor.rs ++++ b/codex-rs/file-system/src/environment_accessor.rs +@@ -40,6 +40,21 @@ + options: ReadFileOptions, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads a file with a hard byte limit, preserving this accessor's permissions. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(std::io::Error::new( ++ std::io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Opens an owned stream; subsequent reads do not borrow or reauthorize through the accessor. + fn read_file_stream<'a>( + &'a self, +@@ -212,6 +227,16 @@ + .read_file(path, options, self.key.sandbox.as_deref()) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, self.key.sandbox.as_deref()) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs +--- a/codex-rs/file-system/src/lib.rs ++++ b/codex-rs/file-system/src/lib.rs +@@ -667,6 +667,23 @@ + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads at most `max_bytes + 1` bytes, rejecting files larger than `max_bytes`. ++ /// Backends must enforce the bound during I/O rather than truncate a full read. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ _sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(io::Error::new( ++ io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Reads a file as a stream of chunks no larger than [`FILE_READ_CHUNK_SIZE`]. + fn read_file_stream<'a>( + &'a self, From 188ef30707bacdcc8c168a52a9a068af6d601738 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 20:54:45 -0800 Subject: [PATCH 20/33] fix(codex): report native model usage with the SDK token schema --- src/agents/models/_codex_tracing.py | 4 ++- src/agents/models/_codex_usage.py | 25 +++++++++++++++++++ src/agents/models/codex.py | 21 +--------------- tests/models/test_codex_provider.py | 23 ++++++++++++++--- tests/models/test_operation_tracing_parity.py | 8 ++++++ 5 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 src/agents/models/_codex_usage.py diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py index dc8cc5189e..b18ed5e0b9 100644 --- a/src/agents/models/_codex_tracing.py +++ b/src/agents/models/_codex_tracing.py @@ -19,6 +19,8 @@ from ..tracing.provider import _is_noop_span, _is_noop_trace from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData from ..tracing.util import _parse_iso_timestamp, time_iso +from ..usage import _response_usage_to_usage, model_usage_to_span_usage +from ._codex_usage import _usage from .interface import ModelTracing @@ -174,7 +176,7 @@ def _metadata(self, operation: _Operation, event: dict[str, Any]) -> None: if isinstance(data, GenerationSpanData): data.model = metadata.model if isinstance(usage, dict): - data.usage = {key: value for key, value in usage.items() if isinstance(value, int)} + data.usage = model_usage_to_span_usage(_response_usage_to_usage(_usage(usage))) def _content(self, operation: _Operation, event: dict[str, Any]) -> None: if self._detail != "content": diff --git a/src/agents/models/_codex_usage.py b/src/agents/models/_codex_usage.py new file mode 100644 index 0000000000..5c30dc5443 --- /dev/null +++ b/src/agents/models/_codex_usage.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import Any + +from openai.types.responses import ResponseUsage +from openai.types.responses.response_usage import OutputTokensDetails + +from ..usage import _make_input_tokens_details + + +def _usage(total: dict[str, Any]) -> ResponseUsage: + return ResponseUsage( + input_tokens=total.get("input_tokens", 0), + output_tokens=total.get("output_tokens", 0), + total_tokens=total.get( + "total_tokens", total.get("input_tokens", 0) + total.get("output_tokens", 0) + ), + input_tokens_details=_make_input_tokens_details( + cached_tokens=total.get("cached_input_tokens", 0), + cache_write_tokens=total.get("cache_write_input_tokens"), + ), + output_tokens_details=OutputTokensDetails( + reasoning_tokens=total.get("reasoning_output_tokens", 0) + ), + ) diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 4589219f5d..43e8b036b4 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -31,11 +31,9 @@ ResponseTextDeltaEvent, ResponseToolSearchCall, ResponseToolSearchOutputItem, - ResponseUsage, ) from openai.types.responses.response_prompt_param import ResponsePromptParam from openai.types.responses.response_reasoning_summary_part_added_event import Part as SummaryPart -from openai.types.responses.response_usage import OutputTokensDetails from ..agent_output import AgentOutputSchemaBase from ..exceptions import UserError @@ -48,7 +46,6 @@ Usage, _attach_raw_usage_snapshot, _extract_raw_usage_snapshot, - _make_input_tokens_details, _response_usage_to_usage, ) from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call @@ -60,6 +57,7 @@ ) from ._codex_tool_hooks import CodexToolHooks from ._codex_tracing import CodexOperationTracing +from ._codex_usage import _usage from ._run_context import ModelToolHooks, get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing @@ -888,20 +886,3 @@ async def interrupt_when_ready() -> None: yield ResponseCompletedEvent( response=response, sequence_number=sequence, type="response.completed" ) - - -def _usage(total: dict[str, Any]) -> ResponseUsage: - return ResponseUsage( - input_tokens=total.get("input_tokens", 0), - output_tokens=total.get("output_tokens", 0), - total_tokens=total.get( - "total_tokens", total.get("input_tokens", 0) + total.get("output_tokens", 0) - ), - input_tokens_details=_make_input_tokens_details( - cached_tokens=total.get("cached_input_tokens", 0), - cache_write_tokens=total.get("cache_write_input_tokens"), - ), - output_tokens_details=OutputTokensDetails( - reasoning_tokens=total.get("reasoning_output_tokens", 0) - ), - ) diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 8699f6ab90..4930397e1f 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1677,8 +1677,19 @@ async def test_codex_operations_preserve_source_spans_and_capture_policy( assert models[0].started_at == "2026-11-20T00:00:01.000000000Z" assert models[0].ended_at == "2026-11-20T00:00:01.250Z" assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" - assert models[0].span_data.usage == {"input_tokens": 10, "output_tokens": 5} - assert models[-1].span_data.usage == {"input_tokens": 30, "output_tokens": 15} + for index, span in [(1, models[0]), (3, models[-1])]: + usage = span.span_data.usage + assert { + key: usage[key] for key in ("requests", "input_tokens", "output_tokens", "total_tokens") + } == { + "requests": 1, + "input_tokens": 10 * index, + "output_tokens": 5 * index, + "total_tokens": 15 * index, + } + assert usage["input_tokens_details"]["cached_tokens"] == 0 + assert usage["input_tokens_details"]["cache_write_tokens"] == 0 + assert usage["output_tokens_details"]["reasoning_tokens"] == 0 assert models[-1].span_data.operation.response_model == "observed-model" assert models[-1].span_data.operation.outcome == "completed" if detail is None: @@ -1894,7 +1905,13 @@ def observe_end(span): assert response.id == "turn_1" assert response.output[0].content[0].text == "A Codex response" assert ended_at == "2026-11-20T00:00:03.250Z" - assert usage == {"input_tokens": 30, "output_tokens": 15} + assert usage["requests"] == 1 + assert usage["input_tokens"] == 30 + assert usage["output_tokens"] == 15 + assert usage["total_tokens"] == 45 + assert usage["input_tokens_details"]["cached_tokens"] == 0 + assert usage["input_tokens_details"]["cache_write_tokens"] == 0 + assert usage["output_tokens_details"]["reasoning_tokens"] == 0 @pytest.mark.asyncio diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py index 05291cf533..030e06a71f 100644 --- a/tests/models/test_operation_tracing_parity.py +++ b/tests/models/test_operation_tracing_parity.py @@ -429,6 +429,14 @@ async def lookup(key: str) -> str: assert span.span_data.operation.response_id == f"resp_parity_{index}" assert span.span_data.usage["input_tokens"] == 10 * index assert span.span_data.usage["output_tokens"] == 5 * index + assert span.span_data.usage["requests"] == 1 + assert span.span_data.usage["input_tokens_details"]["cached_tokens"] == index + assert ( + span.span_data.usage["input_tokens_details"].get("cache_write_tokens", 0) == 0 + ) + assert span.span_data.usage["output_tokens_details"] == { + "reasoning_tokens": index, + } started, ended = server.intervals[index - 1] assert _ns(span.started_at) <= started + 1_000_000 assert _ns(span.ended_at) >= ended - 1_000_000 From f1bdc657cdf5907bb413c4bf482786c2621dca53 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 20:54:51 -0800 Subject: [PATCH 21/33] fix(tracing): observe native approval decisions and resumed tools as they run --- native/codex/src/lib.rs | 35 ++++-- native/codex/src/operation_events.rs | 81 ++++++++++++- src/agents/models/codex.py | 45 ++++++-- src/agents/run_internal/native_approvals.py | 29 +++-- src/agents/run_internal/turn_resolution.py | 1 + tests/memory/test_native_approval_recovery.py | 107 ++++++++++++++++++ tests/memory/test_native_approval_tracing.py | 90 +++++++++++++++ 7 files changed, 357 insertions(+), 31 deletions(-) create mode 100644 tests/memory/test_native_approval_tracing.py diff --git a/native/codex/src/lib.rs b/native/codex/src/lib.rs index acafa6ea45..04342ee245 100644 --- a/native/codex/src/lib.rs +++ b/native/codex/src/lib.rs @@ -499,6 +499,18 @@ impl Thread { }) } + // The approval resolver owns Core events; this reader consumes only observations. + fn next_operation_event<'py>(&self, py: Python<'py>) -> PyResult> { + let lease = self.lease.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + lease.ensure_open().map_err(py_error)?; + Ok(lease + .operations + .next_notification(&lease.id.to_string()) + .await) + }) + } + #[pyo3(signature = (call_json, request_json, preconditions_json, operation_capture=None))] fn resolve_approval<'py>( &self, @@ -516,17 +528,18 @@ impl Thread { .map_err(|_| py_error("Invalid saved Codex file preconditions."))?; let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - lease.ensure_open().map_err(py_error)?; - // Restoration owns a separate capture queue, leaving normal next_event delivery intact. - let operations = operation_events::OperationEvents::new(); - let capture = operations - .initializer(operation_capture.as_deref()) - .map_err(py_error)?; - let (output, turn_id, failure) = - native_approval::resolve(&lease.thread, call, request, preconditions, capture) - .await - .map_err(py_error)?; - let notifications = operations.drain_notifications(&lease.id.to_string()).await; + let outcome = async { + lease.ensure_open()?; + let capture = lease.operations.initializer(operation_capture.as_deref())?; + native_approval::resolve(&lease.thread, call, request, preconditions, capture).await + } + .await; + lease.operations.finish_resolution(); + let (output, turn_id, failure) = outcome.map_err(py_error)?; + let notifications = lease + .operations + .drain_notifications(&lease.id.to_string()) + .await; Ok((output, turn_id, notifications, failure)) }) } diff --git a/native/codex/src/operation_events.rs b/native/codex/src/operation_events.rs index a3589860ee..c6533c327a 100644 --- a/native/codex/src/operation_events.rs +++ b/native/codex/src/operation_events.rs @@ -12,7 +12,7 @@ use codex_core_api::CodexThread; use codex_extension_api::ExtensionDataInit; use codex_protocol::protocol::EventMsg; use serde_json::json; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::{Mutex, mpsc, watch}; const QUEUE_CAPACITY: usize = 64; @@ -66,11 +66,13 @@ pub(crate) struct OperationEvents { sender: mpsc::Sender, dropped: Arc, state: Mutex, + resolution_finished: watch::Sender, } impl OperationEvents { pub(crate) fn new() -> Self { let (sender, receiver) = mpsc::channel(QUEUE_CAPACITY); + let (resolution_finished, _) = watch::channel(false); Self { sender, dropped: Arc::new(AtomicU64::new(0)), @@ -78,6 +80,7 @@ impl OperationEvents { receiver, pending: None, }), + resolution_finished, } } @@ -101,6 +104,37 @@ impl OperationEvents { Ok(init) } + // Release an idle observation reader before the resolver drains the final queue. + pub(crate) fn finish_resolution(&self) { + self.resolution_finished.send_replace(true); + } + + pub(crate) async fn next_notification(&self, thread_id: &str) -> Option { + let mut finished = self.resolution_finished.subscribe(); + let mut state = self.state.lock().await; + if let Ok(event) = state.receiver.try_recv() { + return Some(event); + } + let dropped = self.dropped.swap(0, Ordering::Relaxed); + if dropped != 0 { + return Some( + json!({ + "method": "turn/operation/dropped", + "params": {"threadId": thread_id, "droppedCount": dropped}, + }) + .to_string(), + ); + } + if *finished.borrow() { + return None; + } + tokio::select! { + biased; + event = state.receiver.recv() => event, + _ = finished.changed() => None, + } + } + pub(crate) async fn drain_notifications(&self, thread_id: &str) -> Vec { let mut state = self.state.lock().await; let mut events = Vec::new(); @@ -181,6 +215,51 @@ mod tests { use super::*; use codex_core::operation_observation::{OperationKind, OperationMetadata, OperationOutcome}; + #[tokio::test] + async fn restoration_delivers_start_before_completion_and_drains_once() { + let events = OperationEvents::new(); + let init = events.initializer(Some("metadata")).unwrap(); + let observation = init.get::().unwrap(); + let guard = observation.start( + "thread_1".into(), + "turn_1".into(), + OperationKind::Tool, + OperationMetadata::default(), + ); + let start = events.next_notification("thread_1").await.unwrap(); + assert_eq!( + serde_json::from_str::(&start).unwrap()["method"], + "turn/operation/started" + ); + guard.finish(OperationOutcome::Success); + events.finish_resolution(); + let terminal = events.drain_notifications("thread_1").await; + assert_eq!(terminal.len(), 1); + assert_eq!( + serde_json::from_str::(&terminal[0]).unwrap()["method"], + "turn/operation/completed" + ); + assert!(events.next_notification("thread_1").await.is_none()); + } + + #[tokio::test] + async fn restoration_completion_releases_waiting_reader_before_drain() { + let events = OperationEvents::new(); + let reader = async { + assert!(events.next_notification("thread_1").await.is_none()); + }; + let finish = async { + tokio::task::yield_now().await; + events.finish_resolution(); + assert!(events.drain_notifications("thread_1").await.is_empty()); + }; + tokio::time::timeout(std::time::Duration::from_secs(1), async { + tokio::join!(reader, finish); + }) + .await + .unwrap(); + } + #[tokio::test] async fn delivery_preserves_source_identity_and_per_turn_capture() { for mode in ["metadata", "content"] { diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 43e8b036b4..a89b487e51 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -126,26 +126,55 @@ async def _run_approved_native_call( observations = CodexOperationTracing(tracing, None, checkpoint["model"]) thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) interrupted = True + execution: asyncio.Future[Any] | None = None + observed_turn: str | None = None + observations.bind(thread.thread_id, "") + + def observe(encoded: str) -> None: + nonlocal observed_turn + event = json.loads(encoded) + params = event.get("params") + if observed_turn is None and isinstance(params, dict): + turn = params.get("turnId") + if params.get("threadId") == thread.thread_id and isinstance(turn, str): + observed_turn = turn + observations.bind(thread.thread_id, turn) + observations.observe(event["method"], params) + try: - output, turn_id, notifications, failure = await thread.resolve_approval( - json.dumps(checkpoint["call"]), - json.dumps(checkpoint["request"]), - json.dumps(checkpoint.get("preconditions")), - observations.request_mode(), + execution = asyncio.ensure_future( + thread.resolve_approval( + json.dumps(checkpoint["call"]), + json.dumps(checkpoint["request"]), + json.dumps(checkpoint.get("preconditions")), + observations.request_mode(), + ) ) + while (encoded := await thread.next_operation_event()) is not None: + observe(encoded) + output, turn_id, notifications, failure = await execution observations.bind(thread.thread_id, turn_id) for encoded in notifications: - event = json.loads(encoded) - observations.observe(event["method"], event.get("params")) + observe(encoded) if failure is not None: raise RuntimeError(failure) interrupted = False return str(output) finally: + if execution is not None: + execution.cancel() + + async def close() -> None: + try: + await thread.close() + finally: + if execution is not None: + await asyncio.gather(execution, return_exceptions=True) + try: observations.close(interrupted=interrupted) finally: - await _join_close(thread.close()) + await _join_close(close()) def _pending_native_call( diff --git a/src/agents/run_internal/native_approvals.py b/src/agents/run_internal/native_approvals.py index 2709737ac5..2e9b379d19 100644 --- a/src/agents/run_internal/native_approvals.py +++ b/src/agents/run_internal/native_approvals.py @@ -10,6 +10,7 @@ from ..models._codex_items import native_approval_checkpoint from ..models._run_context import ModelToolHooks, model_run_context from .run_steps import NextStepInterruption, NextStepRunAgain, SingleStepResult +from .tool_execution import pending_tool_span from .turn_preparation import get_all_tools, get_model @@ -65,6 +66,7 @@ def native_pending_step( new_response: Any, processed_response: Any, run_state: Any, + run_config: Any, ) -> SingleStepResult: _validate_native_plan(processed_response) binding = run_state._resumable_binding if run_state is not None else None @@ -77,6 +79,8 @@ def native_pending_step( ) pending = list(processed_response.native_approvals) processed_response.interruptions = pending + with pending_tool_span(run_config, pending[0].tool_name, pending[0].raw_item.call_id) as span: + span.span_data.operation.outcome = "paused" return SingleStepResult( original_input=original_input, model_response=new_response, @@ -128,6 +132,8 @@ async def resolve_native_approval_step( tool_namespace=pending.tool_namespace, ) if approved is None: + with pending_tool_span(run_config, pending.tool_name, call.call_id) as span: + span.span_data.operation.outcome = "paused" return SingleStepResult( original_input=original_input, model_response=new_response, @@ -174,17 +180,18 @@ async def resolve_native_approval_step( else: from .tool_execution import resolve_approval_rejection_message - output = await resolve_approval_rejection_message( - context_wrapper=context_wrapper, - run_config=run_config, - tool_type="shell" if call.name == "exec_command" else "apply_patch", - tool_name=call.name, - call_id=call.call_id, - tool_call=call, - existing_pending=pending, - tool_lookup_key=pending.tool_lookup_key, - tool_namespace=pending.tool_namespace, - ) + with pending_tool_span(run_config, pending.tool_name, call.call_id): + output = await resolve_approval_rejection_message( + context_wrapper=context_wrapper, + run_config=run_config, + tool_type="shell" if call.name == "exec_command" else "apply_patch", + tool_name=call.name, + call_id=call.call_id, + tool_call=call, + existing_pending=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) result_item = ToolCallOutputItem( agent=bindings.public_agent, raw_item=cast( diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index b85fb88d64..e847c9bddd 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -1003,6 +1003,7 @@ async def execute_tools_and_side_effects( new_response=new_response, processed_response=processed_response, run_state=run_state, + run_config=run_config, ) execute_final_output_call = execute_final_output diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py index 456c8c1a3f..f2a822ef26 100644 --- a/tests/memory/test_native_approval_recovery.py +++ b/tests/memory/test_native_approval_recovery.py @@ -290,6 +290,9 @@ async def test_resume_exports_the_original_native_operation_under_current_trace_ tmp_path, monkeypatch, threads, include_data ): class RestoredThread(FakeThread): + async def next_operation_event(self): + return None + async def resolve_approval(self, call_json, request, preconditions, capture): assert capture == ("content" if include_data else "metadata") operation = { @@ -334,3 +337,107 @@ async def resolve_approval(self, call_json, request, preconditions, capture): assert operation_span.operation.outcome == "completed" assert operation_span.operation.tool_call_id == pending.call_id assert ("private commit message" in repr(operation_span.input)) == include_data + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_data,cancel", [(False, False), (True, False), (False, True)]) +async def test_restored_native_execution_observes_live_start_and_interruption( + tmp_path, monkeypatch, threads, include_data, cancel +): + from agents.tracing import get_current_span + from tests.testing_processor import SPAN_PROCESSOR_TESTING + + started = asyncio.Event() + release = asyncio.Event() + closed = asyncio.Event() + notifications = asyncio.Queue() + calls = [] + original_start = SPAN_PROCESSOR_TESTING.on_span_start + caller_spans = [] + + def on_start(span): + original_start(span) + if span.span_data.type == "function": + caller_spans.append(get_current_span()) + started.set() + + monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_start", on_start) + + # The FFI double preserves suspension and cancellation at the native execution boundary. + class RestoredThread(FakeThread): + async def next_operation_event(self): + return await notifications.get() + + async def resolve_approval(self, call_json, request, preconditions, capture): + calls.append(capture) + operation = { + "threadId": self.thread_id, + "turnId": "restored-turn", + "operationId": "live-native-operation", + "kind": "tool", + "toolName": "exec_command", + "callId": json.loads(call_json)["call_id"], + "startedAt": "2026-11-20T06:00:00.000Z", + } + if capture == "content": + operation["input"] = {"value": "private command"} + await notifications.put( + json.dumps({"method": "turn/operation/started", "params": operation}) + ) + await release.wait() + completed = dict(operation, completedAt="2026-11-20T06:00:02.000Z", outcome="success") + await notifications.put( + json.dumps({"method": "turn/operation/completed", "params": completed}) + ) + await notifications.put(None) + return "native accepted", "restored-turn", [], None + + async def close(self): + await super().close() + closed.set() + + agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + monkeypatch.delattr(agent.model, "_run_approved_native_call") + threads[0].append(RestoredThread()) + task = asyncio.create_task( + Runner.run( + agent, + state, + run_config=RunConfig( + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=include_data, + ), + ) + ) + try: + await asyncio.wait_for(started.wait(), 2) + assert not task.done() + assert calls == ["content" if include_data else "metadata"] + assert caller_spans[0] is not None + assert caller_spans[0].span_data.type == "agent" + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + result = await asyncio.wait_for(task, 2) + assert result.final_output == "finished" + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert closed.is_set() + (span,) = [ + span + for span in fetch_ordered_spans() + if span.span_data.type == "function" + and span.span_data.operation.operation_id == "live-native-operation" + ] + assert span.span_data.operation.outcome == ("interrupted" if cancel else "completed") + assert span.started_at == "2026-11-20T06:00:00.000Z" + assert ("private command" in repr(span.span_data.input)) == include_data + assert span.parent_id == caller_spans[0].span_id diff --git a/tests/memory/test_native_approval_tracing.py b/tests/memory/test_native_approval_tracing.py new file mode 100644 index 0000000000..2a563b0ba9 --- /dev/null +++ b/tests/memory/test_native_approval_tracing.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agents import ResumableSession, RunConfig, Runner +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.testing import ScriptedModel +from agents.tracing import set_trace_processors +from tests.memory.test_native_approval_recovery import setup_native +from tests.test_responses import get_text_message +from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["default", "metadata", "content", "veto", "disabled"]) +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_and_denial_emit_only_approval_metadata( + tmp_path, monkeypatch, mode, streamed +): + agent, store, session, _, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + script = ScriptedModel([[pending], [get_text_message("finished")]]) + monkeypatch.setattr(agent.model, "stream_response", script.stream_response) + config = RunConfig( + tracing={} + if mode == "default" + else {"operation_detail": "metadata" if mode == "metadata" else "content"}, + tracing_disabled=mode == "disabled", + trace_include_sensitive_data=mode != "veto", + ) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([SPAN_PROCESSOR_TESTING, processor]) + + async def run(input, current): + if streamed: + result = Runner.run_streamed(agent, input, session=current, run_config=config) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, input, session=current, run_config=config) + + try: + first = await run("start", session) + assert len(first.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + second = await run(await recovered.load_state(agent), recovered) + assert len(second.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.reject(state.get_interruptions()[0], rejection_message="DENIAL_CANARY") + result = await run(state, recovered) + assert result.final_output == "finished" + assert not calls + spans = fetch_ordered_spans() + if mode == "default": + decisions = [s.span_data for s in spans if s.span_data.type == "function"] + assert [s.operation.outcome for s in decisions] == ["paused", "paused", "rejected"] + assert all(s.operation.tool_call_id == pending.call_id for s in decisions) + assert all(s.input is None and s.output is None for s in decisions) + else: + assert not [s for s in spans if s.span_data.type == "function"] + decisions = [ + s.span_data.data + for s in spans + if getattr(s.span_data, "name", None) == "tool_approval" + ] + assert decisions == ( + [] + if mode == "disabled" + else [ + {"tool_name": pending.name, "tool_call_id": pending.call_id, "outcome": outcome} + for outcome in ["paused", "paused", "rejected"] + ] + ) + otel = exporter.get_finished_spans() + approvals = [s for s in otel if s.name == "openai.agents.tool_approval exec_command"] + assert len(approvals) == (0 if mode == "disabled" else 3) + assert not [s for s in otel if s.name.startswith("execute_tool")] + assert "DENIAL_CANARY" not in json.dumps([dict(s.attributes or {}) for s in approvals]) + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() From f6b6848b66fda8182b77337d5ddcd71653e3067a Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:15:02 -0800 Subject: [PATCH 22/33] fix(codex): bound aggregate file hashing for approvals --- .../codex/0002-native-approval-recovery.patch | 555 +++++++++++++++--- 1 file changed, 459 insertions(+), 96 deletions(-) diff --git a/patches/codex/0002-native-approval-recovery.patch b/patches/codex/0002-native-approval-recovery.patch index 74de4cc1bc..e6e37aeac1 100644 --- a/patches/codex/0002-native-approval-recovery.patch +++ b/patches/codex/0002-native-approval-recovery.patch @@ -91,9 +91,10 @@ index dcd01e4..dc14807 100644 mod compact_model_fallback; diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs new file mode 100644 +index 0000000..3b4ae7b --- /dev/null +++ b/codex-rs/core/src/patch_preconditions.rs -@@ -0,0 +1,331 @@ +@@ -0,0 +1,643 @@ +//! Filesystem preconditions for restoring an interrupted native patch approval. + +use crate::session::step_context::StepContext; @@ -106,22 +107,25 @@ new file mode 100644 +use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; +use codex_protocol::protocol::FileChange; +use codex_utils_path_uri::PathUri; -+use serde::Deserialize; +use serde::Serialize; +use serde_json::Value; +use sha2::Digest; +use sha2::Sha256; +use std::collections::BTreeMap; ++use std::collections::HashMap; +use std::io::ErrorKind; +use std::path::Component; +use std::path::Path; +use std::path::PathBuf; ++use std::time::Duration; + +// Approval recovery fails closed for larger preimages instead of retaining large reads. +const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; ++const MAX_PATCH_PATHS: usize = 32; ++const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; ++const MAX_PATCH_SCAN_DURATION: Duration = Duration::from_secs(10); + -+#[derive(Debug, Deserialize, Serialize, PartialEq)] -+#[serde(deny_unknown_fields)] ++#[derive(Debug, Serialize, PartialEq)] +struct PatchPreconditions { + version: u32, + files: BTreeMap>, @@ -133,7 +137,7 @@ new file mode 100644 + +pub(crate) async fn capture_patch_approval_preconditions( + context: &ApprovalContext, -+ approval: &ApplyPatchApprovalRequestEvent, ++ changes: &HashMap, +) -> CodexResult { + let review = &context.review_context; + let environment = review @@ -145,13 +149,13 @@ new file mode 100644 + let grants = review + .turn() + .granted_permissions(&environment.selection.environment_id); -+ serde_json::to_value(capture(&environment.fs_accessor(grants), approval).await?) ++ serde_json::to_value(capture(&environment.fs_accessor(grants), changes).await?) + .map_err(|error| invalid(error.to_string())) +} + +async fn capture_step( + step: &StepContext, -+ approval: &ApplyPatchApprovalRequestEvent, ++ changes: &HashMap, +) -> CodexResult { + let environment = step + .environments @@ -159,7 +163,7 @@ new file mode 100644 + .ok_or_else(|| { + invalid("Restoring patch approvals requires a single selected local environment") + })?; -+ capture(&step.fs_accessor(environment), approval).await ++ capture(&step.fs_accessor(environment), changes).await +} + +pub(crate) async fn validate_patch_preconditions( @@ -167,10 +171,15 @@ new file mode 100644 + approval: &ApplyPatchApprovalRequestEvent, + checkpoint: &Value, +) -> CodexResult<()> { -+ let expected: PatchPreconditions = serde_json::from_value(checkpoint.clone()) -+ .map_err(|error| invalid(format!("Invalid patch approval preconditions: {error}")))?; -+ let observed = capture_step(step, approval).await?; -+ if expected != observed { ++ let expected = validate_checkpoint(&approval.changes, checkpoint)?; ++ let observed = capture_step(step, &approval.changes).await?; ++ if observed.files.iter().any(|(path, digest)| { ++ match (digest, path.to_str().and_then(|path| expected.get(path))) { ++ (Some(digest), Some(Value::String(expected))) => digest != expected, ++ (None, Some(Value::Null)) => false, ++ _ => true, ++ } ++ }) { + return Err(invalid( + "The pending patch or its files have changed since the original approval request", + )); @@ -178,42 +187,126 @@ new file mode 100644 + Ok(()) +} + -+async fn capture( -+ fs: &impl EnvironmentAccess, -+ approval: &ApplyPatchApprovalRequestEvent, -+) -> CodexResult { -+ if approval.changes.is_empty() { ++fn patch_paths(changes: &HashMap) -> CodexResult> { ++ if changes.is_empty() { + return Err(invalid("A pending patch must include file changes")); + } -+ let mut files = BTreeMap::new(); -+ for (path, change) in &approval.changes { -+ let is_add = matches!(change, FileChange::Add { .. }); -+ let original = observe_file(fs, path, is_add).await?; -+ if let FileChange::Delete { content } = change -+ && original.as_deref() != Some(&digest(content.as_bytes())) -+ { -+ return Err(invalid( -+ "The pending deletion no longer matches its original file", -+ )); ++ if changes.len() > MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } ++ let mut count = changes.len(); ++ for change in changes.values() { ++ if matches!( ++ change, ++ FileChange::Update { ++ move_path: Some(_), ++ .. ++ } ++ ) { ++ count += 1; ++ if count > MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } + } -+ files.insert(path.clone(), original); ++ } ++ let mut paths: BTreeMap<_, _> = changes ++ .iter() ++ .map(|(path, change)| (path, matches!(change, FileChange::Add { .. }))) ++ .collect(); ++ for change in changes.values() { + if let FileChange::Update { + move_path: Some(destination), + .. + } = change ++ && paths.insert(destination, true).is_some() + { -+ if approval.changes.contains_key(destination) || files.contains_key(destination) { ++ return Err(invalid( ++ "Pending patch move destinations must be distinct from changed files", ++ )); ++ } ++ } ++ Ok(paths) ++} ++ ++fn validate_checkpoint<'a>( ++ changes: &HashMap, ++ checkpoint: &'a Value, ++) -> CodexResult<&'a serde_json::Map> { ++ let paths = patch_paths(changes)?; ++ let root = checkpoint ++ .as_object() ++ .ok_or_else(|| invalid("Invalid patch approval preconditions"))?; ++ let files = root ++ .get("files") ++ .and_then(Value::as_object) ++ .ok_or_else(|| invalid("Invalid patch approval file map"))?; ++ if root.len() != 2 ++ || root.get("version").and_then(Value::as_u64) != Some(1) ++ || files.len() > MAX_PATCH_PATHS ++ || files.len() != paths.len() ++ { ++ return Err(invalid("Invalid patch approval version or file count")); ++ } ++ for (path, absent) in paths { ++ let value = path ++ .to_str() ++ .and_then(|path| files.get(path)) ++ .ok_or_else(|| invalid("Patch approval file paths do not match the pending patch"))?; ++ if absent { ++ if !value.is_null() { + return Err(invalid( -+ "Pending patch move destinations must be distinct from changed files", ++ "Pending patch destinations must have absent preimages", + )); + } -+ files.insert( -+ destination.clone(), -+ observe_file(fs, destination, true).await?, -+ ); ++ } else if !value.as_str().is_some_and(|value| { ++ value.len() == 64 ++ && value ++ .bytes() ++ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) ++ }) { ++ return Err(invalid("Invalid patch approval preimage digest")); + } + } -+ Ok(PatchPreconditions { version: 1, files }) ++ Ok(files) ++} ++ ++async fn capture( ++ fs: &impl EnvironmentAccess, ++ changes: &HashMap, ++) -> CodexResult { ++ patch_paths(changes)?; ++ tokio::time::timeout(MAX_PATCH_SCAN_DURATION, async { ++ let mut files = BTreeMap::new(); ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for (path, change) in changes { ++ let deleted = match change { ++ FileChange::Delete { content } => Some(content.as_bytes()), ++ _ => None, ++ }; ++ let original = observe_file( ++ fs, ++ path, ++ matches!(change, FileChange::Add { .. }), ++ &mut remaining, ++ deleted, ++ ) ++ .await?; ++ files.insert(path.clone(), original); ++ if let FileChange::Update { ++ move_path: Some(destination), ++ .. ++ } = change ++ { ++ files.insert( ++ destination.clone(), ++ observe_file(fs, destination, true, &mut remaining, None).await?, ++ ); ++ } ++ } ++ Ok(PatchPreconditions { version: 1, files }) ++ }) ++ .await ++ .map_err(|_| invalid("Pending patch scan exceeded the 10 second limit"))? +} + +fn digest(bytes: &[u8]) -> String { @@ -224,6 +317,8 @@ new file mode 100644 + fs: &impl EnvironmentAccess, + path: &Path, + must_be_absent: bool, ++ remaining: &mut u64, ++ deleted: Option<&[u8]>, +) -> CodexResult> { + if !path.is_absolute() + || path @@ -261,10 +356,16 @@ new file mode 100644 + ReadFileOptions { + follow_symlinks: false, + }, -+ MAX_PATCH_PREIMAGE_BYTES, ++ MAX_PATCH_PREIMAGE_BYTES.min(*remaining), + ) + .await + .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; ++ *remaining -= bytes.len() as u64; ++ if deleted.is_some_and(|expected| expected != bytes.as_slice()) { ++ return Err(invalid( ++ "The pending deletion no longer matches its original file", ++ )); ++ } + Ok(Some(digest(&bytes))) + } + } @@ -300,10 +401,10 @@ new file mode 100644 + move_path: None, + }, + ); -+ let original = capture(&fs, &request).await.unwrap(); -+ assert_eq!(original, capture(&fs, &request).await.unwrap()); ++ let original = capture(&fs, &request.changes).await.unwrap(); ++ assert_eq!(original, capture(&fs, &request.changes).await.unwrap()); + std::fs::write(&path, "old\nchanged outside patch context\n").unwrap(); -+ assert_ne!(original, capture(&fs, &request).await.unwrap()); ++ assert_ne!(original, capture(&fs, &request.changes).await.unwrap()); + } + + #[tokio::test] @@ -317,9 +418,9 @@ new file mode 100644 + content: "new\n".into(), + }, + ); -+ assert!(capture(&fs, &request).await.is_ok()); ++ assert!(capture(&fs, &request.changes).await.is_ok()); + std::fs::write(path, "new\n").unwrap(); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + } + + #[tokio::test] @@ -336,9 +437,9 @@ new file mode 100644 + move_path: Some(destination.clone()), + }, + ); -+ assert!(capture(&fs, &request).await.is_ok()); ++ assert!(capture(&fs, &request.changes).await.is_ok()); + std::fs::write(destination, "pre-existing work\n").unwrap(); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + } + + #[tokio::test] @@ -353,9 +454,9 @@ new file mode 100644 + content: "original\n".into(), + }, + ); -+ assert!(capture(&fs, &request).await.is_ok()); ++ assert!(capture(&fs, &request.changes).await.is_ok()); + std::fs::write(path, "changed\n").unwrap(); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + } + + #[cfg(unix)] @@ -375,14 +476,14 @@ new file mode 100644 + move_path: None, + }, + ); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + let request = approval( + link.join("new"), + FileChange::Add { + content: "new".into(), + }, + ); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + let file_link = dir.path().join("file-link"); + std::os::unix::fs::symlink(original.join("file"), &file_link).unwrap(); + let request = approval( @@ -392,7 +493,7 @@ new file mode 100644 + move_path: None, + }, + ); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + let dangling_link = dir.path().join("dangling-link"); + std::os::unix::fs::symlink(dir.path().join("absent"), &dangling_link).unwrap(); + let request = approval( @@ -401,7 +502,7 @@ new file mode 100644 + content: "new".into(), + }, + ); -+ assert!(capture(&fs, &request).await.is_err()); ++ assert!(capture(&fs, &request.changes).await.is_err()); + } + #[tokio::test] + async fn rejects_preimage_larger_than_recovery_limit() { @@ -419,21 +520,234 @@ new file mode 100644 + move_path: None, + }, + ); -+ let error = capture(&fs, &request) ++ let error = capture(&fs, &request.changes) + .await + .expect_err("large preimages must fail closed"); + assert!(error.to_string().contains("limit is 8388608 bytes")); + } ++ fn update(move_path: Option) -> FileChange { ++ FileChange::Update { ++ unified_diff: "-old\n+new\n".into(), ++ move_path, ++ } ++ } ++ ++ #[tokio::test] ++ async fn bounds_all_paths_before_inspecting_any_file() { ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let mut changes: HashMap<_, _> = (0..MAX_PATCH_PATHS + 1) ++ .map(|index| { ++ ( ++ PathBuf::from(format!("relative-{index}")), ++ FileChange::Add { ++ content: String::new(), ++ }, ++ ) ++ }) ++ .collect(); ++ assert!( ++ capture(&fs, &changes) ++ .await ++ .unwrap_err() ++ .to_string() ++ .contains("32 paths") ++ ); ++ changes.remove(&PathBuf::from(format!("relative-{MAX_PATCH_PATHS}"))); ++ changes.insert( ++ PathBuf::from("relative-0"), ++ update(Some(PathBuf::from("move-destination"))), ++ ); ++ assert!( ++ capture(&fs, &changes) ++ .await ++ .unwrap_err() ++ .to_string() ++ .contains("32 paths") ++ ); ++ changes.remove(&PathBuf::from("relative-1")); ++ assert_eq!(patch_paths(&changes).unwrap().len(), MAX_PATCH_PATHS); ++ } ++ ++ #[tokio::test] ++ async fn accepts_exact_path_limit_of_missing_additions() { ++ let dir = tempfile::tempdir().unwrap(); ++ let root = dir.path().canonicalize().unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let changes: HashMap<_, _> = (0..MAX_PATCH_PATHS) ++ .map(|index| { ++ ( ++ root.join(format!("missing-{index}")), ++ FileChange::Add { ++ content: String::new(), ++ }, ++ ) ++ }) ++ .collect(); ++ let checkpoint = capture(&fs, &changes).await.unwrap(); ++ assert_eq!(checkpoint.files.len(), MAX_PATCH_PATHS); ++ assert!(checkpoint.files.values().all(Option::is_none)); ++ } ++ ++ #[tokio::test] ++ async fn move_sources_share_total_byte_budget_with_other_files() { ++ let dir = tempfile::tempdir().unwrap(); ++ let root = dir.path().canonicalize().unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let mut changes = HashMap::new(); ++ for index in 0..2 { ++ let path = root.join(format!("source-{index}")); ++ std::fs::File::create(&path) ++ .unwrap() ++ .set_len(MAX_PATCH_PREIMAGE_BYTES) ++ .unwrap(); ++ changes.insert( ++ path, ++ update(Some(root.join(format!("destination-{index}")))), ++ ); ++ } ++ let empty = root.join("empty"); ++ std::fs::write(&empty, []).unwrap(); ++ changes.insert(empty, update(None)); ++ changes.insert( ++ root.join("missing-addition"), ++ FileChange::Add { ++ content: "new".into(), ++ }, ++ ); ++ assert_eq!(capture(&fs, &changes).await.unwrap().files.len(), 6); ++ let extra = root.join("one-more-byte"); ++ std::fs::write(&extra, b"x").unwrap(); ++ changes.insert(extra, update(None)); ++ assert!( ++ capture(&fs, &changes) ++ .await ++ .unwrap_err() ++ .to_string() ++ .contains("limit is") ++ ); ++ } ++ ++ #[tokio::test] ++ async fn exhausted_budget_still_allows_empty_and_missing_destinations() { ++ let dir = tempfile::tempdir().unwrap(); ++ let root = dir.path().canonicalize().unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let empty = root.join("empty"); ++ std::fs::write(&empty, []).unwrap(); ++ let mut remaining = 0; ++ assert_eq!( ++ observe_file(&fs, &empty, false, &mut remaining, None) ++ .await ++ .unwrap(), ++ Some(digest(b"")) ++ ); ++ assert_eq!( ++ observe_file(&fs, &root.join("missing"), true, &mut remaining, None) ++ .await ++ .unwrap(), ++ None ++ ); ++ assert!( ++ observe_file(&fs, &root.join("missing"), false, &mut remaining, None) ++ .await ++ .is_err() ++ ); ++ std::fs::write(&empty, b"x").unwrap(); ++ assert!( ++ observe_file(&fs, &empty, false, &mut remaining, None) ++ .await ++ .is_err() ++ ); ++ assert_eq!(remaining, 0); ++ } ++ ++ #[tokio::test] ++ async fn deletion_rejects_oversized_event_content_against_small_preimage() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir ++ .path() ++ .canonicalize() ++ .unwrap() ++ .join("changed-before-capture"); ++ std::fs::write(&path, b"small replacement").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let changes = [( ++ path, ++ FileChange::Delete { ++ content: "x".repeat(MAX_PATCH_SCAN_BYTES as usize + 1), ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ assert!( ++ capture(&fs, &changes) ++ .await ++ .unwrap_err() ++ .to_string() ++ .contains("pending deletion") ++ ); ++ } ++ ++ #[test] ++ fn checkpoint_requires_bounded_exact_path_set_and_version_without_cloning() { ++ let source = PathBuf::from("/source"); ++ let destination = PathBuf::from("/destination"); ++ let changes = [(source.clone(), update(Some(destination.clone())))] ++ .into_iter() ++ .collect(); ++ let mut files = serde_json::Map::new(); ++ files.insert( ++ source.to_str().unwrap().into(), ++ Value::String(digest(b"old")), ++ ); ++ files.insert(destination.to_str().unwrap().into(), Value::Null); ++ let valid = serde_json::json!({"version": 1, "files": files}); ++ assert!(validate_checkpoint(&changes, &valid).is_ok()); ++ let mut invalid = valid.clone(); ++ invalid["version"] = serde_json::json!(2); ++ assert!(validate_checkpoint(&changes, &invalid).is_err()); ++ let mut invalid = valid.clone(); ++ invalid["files"] ++ .as_object_mut() ++ .unwrap() ++ .remove(destination.to_str().unwrap()); ++ assert!(validate_checkpoint(&changes, &invalid).is_err()); ++ let mut invalid = valid.clone(); ++ invalid["files"] ++ .as_object_mut() ++ .unwrap() ++ .remove(destination.to_str().unwrap()); ++ invalid["files"]["/unexpected"] = Value::Null; ++ assert!(validate_checkpoint(&changes, &invalid).is_err()); ++ let mut invalid = valid.clone(); ++ invalid["files"][destination.to_str().unwrap()] = Value::String(digest(b"occupied")); ++ assert!(validate_checkpoint(&changes, &invalid).is_err()); ++ let mut invalid = valid; ++ invalid["files"][source.to_str().unwrap()] = ++ Value::String("x".repeat(MAX_PATCH_SCAN_BYTES as usize + 1)); ++ assert!(validate_checkpoint(&changes, &invalid).is_err()); ++ let oversized: serde_json::Map<_, _> = (0..MAX_PATCH_PATHS + 1) ++ .map(|index| (format!("/{index}"), Value::Null)) ++ .collect(); ++ assert!( ++ validate_checkpoint( ++ &changes, ++ &serde_json::json!({"version":1,"files":oversized}) ++ ) ++ .is_err() ++ ); ++ } +} diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs new file mode 100644 -index 0000000..984cbce +index 0000000..b04d400 --- /dev/null +++ b/codex-rs/core/src/restore_tool_call.rs -@@ -0,0 +1,642 @@ +@@ -0,0 +1,692 @@ +//! Model-free replay of one native call, requiring the same fresh human approval. + +use std::collections::HashMap; ++use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + @@ -442,7 +756,7 @@ index 0000000..984cbce +use codex_protocol::error::{CodexErr, Result as CodexResult}; +use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::ToolMode; -+use codex_protocol::protocol::{EventMsg, ReviewDecision}; ++use codex_protocol::protocol::{EventMsg, FileChange, ReviewDecision}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; @@ -460,11 +774,43 @@ index 0000000..984cbce + CodexErr::InvalidRequest(format!("Native tool restoration refused: {message}")) +} + -+#[derive(Debug, Clone)] ++#[derive(Debug)] ++struct CapturedPatch { ++ changes: Arc>, ++ preconditions: Value, ++} ++ ++#[derive(Debug)] +struct CapturableApproval { + event: Option, + permissions: Value, -+ patch: Option, ++ patch: Option, ++} ++ ++impl CapturableApproval { ++ fn checkpoint(&self, expected: &EventMsg) -> CodexResult { ++ // Reuse the pre-approval scan only for the exact changes on this live lease. ++ let patch = match (&self.patch, expected) { ++ (Some(patch), EventMsg::ApplyPatchApprovalRequest(event)) ++ if patch.changes.as_ref() == &event.changes => ++ { ++ Some(&patch.preconditions) ++ } ++ (None, EventMsg::ExecApprovalRequest(_)) => None, ++ _ => return Err(refusal("approval does not match the captured action")), ++ }; ++ let expected = ++ serde_json::to_value(expected).map_err(|_| refusal("invalid approval event"))?; ++ if self ++ .event ++ .as_ref() ++ .and_then(|event| serde_json::to_value(event).ok()) ++ != Some(expected) ++ { ++ return Err(refusal("approval does not match the live request")); ++ } ++ Ok(json!({"version":1,"first_approval":true,"permissions":self.permissions,"patch":patch})) ++ } +} + +#[derive(Debug, Default)] @@ -539,17 +885,14 @@ index 0000000..984cbce + .extension_data + .get::()?; + let patch = if let ApprovalAction::ApplyPatch { changes, .. } = action { -+ let event = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { -+ call_id: ctx.call_id.clone(), -+ turn_id: ctx.review_context.turn().sub_id.clone(), -+ started_at_ms: 0, -+ changes: changes.as_ref().clone(), -+ reason: ctx.approval_reason.clone(), -+ grant_root: None, -+ }; -+ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, &event) -+ .await -+ .ok() ++ let preconditions = ++ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, changes) ++ .await ++ .ok()?; ++ Some(CapturedPatch { ++ changes: Arc::clone(changes), ++ preconditions, ++ }) + } else { + None + }; @@ -770,38 +1113,13 @@ index 0000000..984cbce + .extension_data + .get::() + .ok_or_else(|| refusal("not a first pre-execution approval"))?; -+ let recorded = state -+ .0 -+ .lock() -+ .unwrap() -+ .get(id) -+ .cloned() -+ .ok_or_else(|| refusal("not a first pre-execution approval"))?; -+ if recorded -+ .event -+ .as_ref() -+ .and_then(|event| serde_json::to_value(event).ok()) -+ != serde_json::to_value(&expected).ok() -+ { -+ return Err(refusal("approval does not match the live request")); -+ } + let step = session.capture_step_context(turn, &cancellation).await?; + validate_step(session, &step).await?; -+ let patch = match &expected { -+ EventMsg::ApplyPatchApprovalRequest(event) => { -+ let patch = recorded -+ .patch -+ .clone() -+ .ok_or_else(|| refusal("patch preimages could not be captured before approval"))?; -+ crate::patch_preconditions::validate_patch_preconditions(&step, event, &patch).await?; -+ Some(patch) -+ } -+ _ => None, -+ }; -+ if !state.0.lock().unwrap().contains_key(id) { -+ return Err(refusal("approval is no longer pending")); -+ } -+ Ok(json!({"version":1,"first_approval":true,"permissions":recorded.permissions,"patch":patch})) ++ let approvals = state.0.lock().unwrap(); ++ approvals ++ .get(id) ++ .ok_or_else(|| refusal("not a pending first pre-execution approval"))? ++ .checkpoint(&expected) +} + +pub(crate) fn compare_request(turn: &TurnContext, event: &EventMsg) -> bool { @@ -912,6 +1230,51 @@ index 0000000..984cbce + } + + #[test] ++ fn reuses_patch_snapshot_only_for_the_live_event_and_original_changes() { ++ let path = PathBuf::from("/workspace/new"); ++ let changes = HashMap::from([( ++ path.clone(), ++ FileChange::Add { ++ content: "first".into(), ++ }, ++ )]); ++ let mut request = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { ++ call_id: "original".into(), ++ turn_id: "first-turn".into(), ++ started_at_ms: 1, ++ changes: changes.clone(), ++ reason: None, ++ grant_root: None, ++ }; ++ let original = EventMsg::ApplyPatchApprovalRequest(request.clone()); ++ let snapshot = json!({"version":1,"files":{"/workspace/new":null}}); ++ let mut lease = CapturableApproval { ++ event: Some(original.clone()), ++ permissions: json!("test-fingerprint"), ++ patch: Some(CapturedPatch { ++ changes: Arc::new(changes), ++ preconditions: snapshot.clone(), ++ }), ++ }; ++ assert_eq!(lease.checkpoint(&original).unwrap()["patch"], snapshot); ++ request.started_at_ms += 1; ++ assert!( ++ lease ++ .checkpoint(&EventMsg::ApplyPatchApprovalRequest(request.clone())) ++ .is_err() ++ ); ++ request.changes.insert( ++ path, ++ FileChange::Add { ++ content: "second".into(), ++ }, ++ ); ++ let changed = EventMsg::ApplyPatchApprovalRequest(request); ++ lease.event = Some(changed.clone()); ++ assert!(lease.checkpoint(&changed).is_err()); ++ } ++ ++ #[test] + fn rejects_unrelated_calls_and_noninitial_approvals() { + let mut call = command(); + if let ResponseItem::FunctionCall { call_id, .. } = &mut call { From 85f402e1f7dd58422d17c9b08a288505e2593211 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:15:08 -0800 Subject: [PATCH 23/33] fix(tracing): export cache and reasoning token counts through OpenTelemetry --- .../extensions/tracing/opentelemetry.py | 21 +++++++++++ tests/models/test_operation_tracing_parity.py | 3 ++ tests/tracing/test_opentelemetry.py | 36 ++++++++++++++++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 2883788459..08057cdfc3 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -328,6 +328,27 @@ def _attributes(self, span: Span[Any]) -> dict[str, Any]: ): if isinstance(usage.get(source), int): attributes[destination] = usage[source] + for group, source, destination in ( + ( + "input_tokens_details", + "cached_tokens", + "gen_ai.usage.cache_read.input_tokens", + ), + ( + "input_tokens_details", + "cache_write_tokens", + "gen_ai.usage.cache_write.input_tokens", + ), + ( + "output_tokens_details", + "reasoning_tokens", + "gen_ai.usage.reasoning.output_tokens", + ), + ): + details = usage.get(group) + value = details.get(source) if isinstance(details, Mapping) else None + if isinstance(value, int) and value != 0: + attributes[destination] = value if data.operation.response_model: attributes["gen_ai.response.model"] = data.operation.response_model if data.operation.response_id: diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py index 030e06a71f..7cae606914 100644 --- a/tests/models/test_operation_tracing_parity.py +++ b/tests/models/test_operation_tracing_parity.py @@ -475,6 +475,9 @@ async def lookup(key: str) -> str: assert span.attributes["gen_ai.response.model"] == observed_model assert span.attributes["gen_ai.usage.input_tokens"] == 10 * index assert span.attributes["gen_ai.usage.output_tokens"] == 5 * index + assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == index + assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == index + assert "gen_ai.usage.cache_write.input_tokens" not in span.attributes assert span.attributes["openai.api.type"] == "responses" assert span.kind.name == "CLIENT" started, ended = server.intervals[index - 1] diff --git a/tests/tracing/test_opentelemetry.py b/tests/tracing/test_opentelemetry.py index 1c65745494..523d769872 100644 --- a/tests/tracing/test_opentelemetry.py +++ b/tests/tracing/test_opentelemetry.py @@ -46,7 +46,12 @@ def test_mapping_and_legacy_content_privacy(otel) -> None: with trace("group"), task_span("assistant"), agent_span("Worker"), turn_span(1, "Worker"): with generation_span(model="test-model", input=[{"content": "private-input"}]) as model: model.span_data.output = [{"content": "private-output"}] - model.span_data.usage = {"input_tokens": 13, "output_tokens": 5} + model.span_data.usage = { + "input_tokens": 13, + "output_tokens": 5, + "input_tokens_details": {"cached_tokens": 3, "cache_write_tokens": 4}, + "output_tokens_details": {"reasoning_tokens": 2}, + } with function_span("lookup", input="private-arguments") as tool: tool.span_data.output = "private-result" spans = exporter.get_finished_spans() @@ -63,6 +68,9 @@ def test_mapping_and_legacy_content_privacy(otel) -> None: assert inference.name == "chat test-model" assert inference.attributes["gen_ai.usage.input_tokens"] == 13 assert inference.attributes["gen_ai.usage.output_tokens"] == 5 + assert inference.attributes["gen_ai.usage.cache_read.input_tokens"] == 3 + assert inference.attributes["gen_ai.usage.cache_write.input_tokens"] == 4 + assert inference.attributes["gen_ai.usage.reasoning.output_tokens"] == 2 assert inference.parent.span_id == by_operation["openai.agents.turn"].context.span_id assert by_operation["execute_tool"].parent.span_id == inference.parent.span_id assert "private-" not in str([span.attributes for span in spans]) @@ -500,3 +508,29 @@ async def chunks(): {"type": "text", "content": "answer"}, {"type": "tool_call", "id": "call-lookup", "name": "lookup", "arguments": {"key": "value"}}, ] + + +@pytest.mark.parametrize("details", [None, 0]) +def test_missing_or_zero_usage_details_do_not_add_optional_counts(otel, details) -> None: + _, exporter = otel + usage: dict[str, object] = {"input_tokens": 13, "output_tokens": 5} + if details is not None: + usage.update( + { + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + ) + with trace("usage"), generation_span(model="test-model") as model: + model.span_data.usage = usage + (span,) = exporter.get_finished_spans() + assert span.attributes["gen_ai.usage.input_tokens"] == 13 + assert span.attributes["gen_ai.usage.output_tokens"] == 5 + assert not any( + key in span.attributes + for key in ( + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cache_write.input_tokens", + "gen_ai.usage.reasoning.output_tokens", + ) + ) From 26a7cef98a771ad05b04a599985394a074f5acaf Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:33:06 -0800 Subject: [PATCH 24/33] fix(codex): keep workspace preimages out of approval checkpoints --- native/codex/src/operation_events.rs | 6 +- .../codex/0002-native-approval-recovery.patch | 542 ++++++++++++++++-- 2 files changed, 509 insertions(+), 39 deletions(-) diff --git a/native/codex/src/operation_events.rs b/native/codex/src/operation_events.rs index c6533c327a..4f1ade2ef7 100644 --- a/native/codex/src/operation_events.rs +++ b/native/codex/src/operation_events.rs @@ -182,13 +182,13 @@ impl OperationEvents { let event = event.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; match &event.msg { EventMsg::ExecApprovalRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) => { - let preconditions = thread - .capture_tool_approval_preconditions(event.msg.clone()) + let (request, preconditions) = thread + .capture_tool_approval(event.msg) .await .map_err(|_| "This Codex approval cannot be safely resumed.".to_owned())?; state.pending = Some(json!({ "id": event.id, - "msg": event.msg, + "msg": request, "preconditions": preconditions, }).to_string()); continue; diff --git a/patches/codex/0002-native-approval-recovery.patch b/patches/codex/0002-native-approval-recovery.patch index e6e37aeac1..0ef0c97d57 100644 --- a/patches/codex/0002-native-approval-recovery.patch +++ b/patches/codex/0002-native-approval-recovery.patch @@ -23,7 +23,7 @@ index 542abf9..5ffc5cf 100644 similar = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs -index 968d331..9d0bd2e 100644 +index 968d331..a10c351 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -45,6 +45,7 @@ use codex_protocol::protocol::AskForApproval; @@ -60,11 +60,11 @@ index 968d331..9d0bd2e 100644 + .await + } + -+ /// Captures live preconditions before retiring a worker waiting for its first native approval. -+ pub async fn capture_tool_approval_preconditions( ++ /// Returns the live approval without file preimages, paired with its verified preconditions. ++ pub async fn capture_tool_approval( + &self, + approval: EventMsg, -+ ) -> CodexResult { ++ ) -> CodexResult<(EventMsg, serde_json::Value)> { + crate::restore_tool_call::capture(&self.session, approval).await + } + @@ -91,10 +91,10 @@ index dcd01e4..dc14807 100644 mod compact_model_fallback; diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs new file mode 100644 -index 0000000..3b4ae7b +index 0000000..d32a51c --- /dev/null +++ b/codex-rs/core/src/patch_preconditions.rs -@@ -0,0 +1,643 @@ +@@ -0,0 +1,978 @@ +//! Filesystem preconditions for restoring an interrupted native patch approval. + +use crate::session::step_context::StepContext; @@ -120,9 +120,9 @@ index 0000000..3b4ae7b +use std::time::Duration; + +// Approval recovery fails closed for larger preimages instead of retaining large reads. -+const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; ++pub(crate) const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; +const MAX_PATCH_PATHS: usize = 32; -+const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; ++pub(crate) const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; +const MAX_PATCH_SCAN_DURATION: Duration = Duration::from_secs(10); + +#[derive(Debug, Serialize, PartialEq)] @@ -156,6 +156,7 @@ index 0000000..3b4ae7b +async fn capture_step( + step: &StepContext, + changes: &HashMap, ++ check_deleted_content: bool, +) -> CodexResult { + let environment = step + .environments @@ -163,7 +164,12 @@ index 0000000..3b4ae7b + .ok_or_else(|| { + invalid("Restoring patch approvals requires a single selected local environment") + })?; -+ capture(&step.fs_accessor(environment), changes).await ++ capture_with_deletion_check( ++ &step.fs_accessor(environment), ++ changes, ++ check_deleted_content, ++ ) ++ .await +} + +pub(crate) async fn validate_patch_preconditions( @@ -172,7 +178,8 @@ index 0000000..3b4ae7b + checkpoint: &Value, +) -> CodexResult<()> { + let expected = validate_checkpoint(&approval.changes, checkpoint)?; -+ let observed = capture_step(step, &approval.changes).await?; ++ let check_deleted_content = checkpoint.get("version").and_then(Value::as_u64) == Some(1); ++ let observed = capture_step(step, &approval.changes, check_deleted_content).await?; + if observed.files.iter().any(|(path, digest)| { + match (digest, path.to_str().and_then(|path| expected.get(path))) { + (Some(digest), Some(Value::String(expected))) => digest != expected, @@ -228,10 +235,70 @@ index 0000000..3b4ae7b + Ok(paths) +} + ++fn valid_digest(value: &str) -> bool { ++ value.len() == 64 ++ && value ++ .bytes() ++ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) ++} ++ +fn validate_checkpoint<'a>( + changes: &HashMap, + checkpoint: &'a Value, +) -> CodexResult<&'a serde_json::Map> { ++ let files = validate_checkpoint_shape(changes, checkpoint)?; ++ if checkpoint.get("version").and_then(Value::as_u64) == Some(2) { ++ for change in changes.values() { ++ match change { ++ FileChange::Delete { content } if !content.is_empty() => { ++ return Err(invalid("Version 2 patch deletions must omit raw content")); ++ } ++ FileChange::Update { unified_diff, .. } if !valid_digest(unified_diff) => { ++ return Err(invalid( ++ "Version 2 patch updates require a SHA-256 diff digest", ++ )); ++ } ++ _ => {} ++ } ++ } ++ } ++ Ok(files) ++} ++ ++/// Binds a live native deletion to its saved digest before redacting its content. ++/// No file I/O or content cloning occurs here; callers bound any other live fields separately. ++pub(crate) fn matches_delete_preimages( ++ changes: &HashMap, ++ checkpoint: &Value, ++) -> bool { ++ let Ok(files) = validate_checkpoint_shape(changes, checkpoint) else { ++ return false; ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for change in changes.values() { ++ if let FileChange::Delete { content } = change { ++ let size = content.len() as u64; ++ if size > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return false; ++ } ++ remaining -= size; ++ } ++ } ++ changes.iter().all(|(path, change)| { ++ let FileChange::Delete { content } = change else { ++ return true; ++ }; ++ path.to_str() ++ .and_then(|path| files.get(path)) ++ .and_then(Value::as_str) ++ .is_some_and(|expected| digest(content.as_bytes()) == expected) ++ }) ++} ++ ++fn validate_checkpoint_shape<'a>( ++ changes: &HashMap, ++ checkpoint: &'a Value, ++) -> CodexResult<&'a serde_json::Map> { + let paths = patch_paths(changes)?; + let root = checkpoint + .as_object() @@ -241,7 +308,7 @@ index 0000000..3b4ae7b + .and_then(Value::as_object) + .ok_or_else(|| invalid("Invalid patch approval file map"))?; + if root.len() != 2 -+ || root.get("version").and_then(Value::as_u64) != Some(1) ++ || !matches!(root.get("version").and_then(Value::as_u64), Some(1 | 2)) + || files.len() > MAX_PATCH_PATHS + || files.len() != paths.len() + { @@ -258,12 +325,7 @@ index 0000000..3b4ae7b + "Pending patch destinations must have absent preimages", + )); + } -+ } else if !value.as_str().is_some_and(|value| { -+ value.len() == 64 -+ && value -+ .bytes() -+ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) -+ }) { ++ } else if !value.as_str().is_some_and(valid_digest) { + return Err(invalid("Invalid patch approval preimage digest")); + } + } @@ -274,13 +336,21 @@ index 0000000..3b4ae7b + fs: &impl EnvironmentAccess, + changes: &HashMap, +) -> CodexResult { ++ capture_with_deletion_check(fs, changes, true).await ++} ++ ++async fn capture_with_deletion_check( ++ fs: &impl EnvironmentAccess, ++ changes: &HashMap, ++ check_deleted_content: bool, ++) -> CodexResult { + patch_paths(changes)?; + tokio::time::timeout(MAX_PATCH_SCAN_DURATION, async { + let mut files = BTreeMap::new(); + let mut remaining = MAX_PATCH_SCAN_BYTES; + for (path, change) in changes { + let deleted = match change { -+ FileChange::Delete { content } => Some(content.as_bytes()), ++ FileChange::Delete { content } if check_deleted_content => Some(content.as_bytes()), + _ => None, + }; + let original = observe_file( @@ -704,7 +774,7 @@ index 0000000..3b4ae7b + let valid = serde_json::json!({"version": 1, "files": files}); + assert!(validate_checkpoint(&changes, &valid).is_ok()); + let mut invalid = valid.clone(); -+ invalid["version"] = serde_json::json!(2); ++ invalid["version"] = serde_json::json!(3); + assert!(validate_checkpoint(&changes, &invalid).is_err()); + let mut invalid = valid.clone(); + invalid["files"] @@ -737,13 +807,278 @@ index 0000000..3b4ae7b + .is_err() + ); + } ++ fn deletion_checkpoint(path: &Path, content: &[u8], version: u32) -> Value { ++ let files = [(path.to_path_buf(), Some(digest(content)))] ++ .into_iter() ++ .collect(); ++ let mut checkpoint = ++ serde_json::to_value(PatchPreconditions { version: 1, files }).unwrap(); ++ checkpoint["version"] = serde_json::json!(version); ++ checkpoint ++ } ++ ++ #[test] ++ fn v2_deletion_requires_redaction_and_mandatory_digest() { ++ let path = PathBuf::from("/deleted"); ++ let mut changes = [( ++ path.clone(), ++ FileChange::Delete { ++ content: "private contents".into(), ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ let checkpoint = deletion_checkpoint(&path, b"private contents", 2); ++ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); ++ changes.insert( ++ path.clone(), ++ FileChange::Delete { ++ content: String::new(), ++ }, ++ ); ++ assert!(validate_checkpoint(&changes, &checkpoint).is_ok()); ++ for value in [ ++ Value::Null, ++ Value::String(String::new()), ++ Value::String("A".repeat(64)), ++ ] { ++ let mut malformed = checkpoint.clone(); ++ malformed["files"][path.to_str().unwrap()] = value; ++ assert!(validate_checkpoint(&changes, &malformed).is_err()); ++ } ++ let mut malformed = checkpoint; ++ malformed["files"].as_object_mut().unwrap().clear(); ++ assert!(validate_checkpoint(&changes, &malformed).is_err()); ++ } ++ ++ #[test] ++ fn v2_update_requires_diff_digest_but_live_delete_matcher_accepts_raw_updates() { ++ let path = PathBuf::from("/updated"); ++ let raw_diff = " private neighboring line\n-old\n+new"; ++ let mut changes = [( ++ path.clone(), ++ FileChange::Update { ++ unified_diff: raw_diff.into(), ++ move_path: None, ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ let checkpoint = deletion_checkpoint(&path, b"original file", 2); ++ assert!(matches_delete_preimages(&changes, &checkpoint)); ++ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); ++ for diff in [String::new(), "A".repeat(64), "z".repeat(64)] { ++ changes.insert( ++ path.clone(), ++ FileChange::Update { ++ unified_diff: diff, ++ move_path: None, ++ }, ++ ); ++ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); ++ } ++ changes.insert( ++ path.clone(), ++ FileChange::Update { ++ unified_diff: digest(raw_diff.as_bytes()), ++ move_path: None, ++ }, ++ ); ++ assert!(validate_checkpoint(&changes, &checkpoint).is_ok()); ++ let mut legacy = checkpoint; ++ legacy["version"] = serde_json::json!(1); ++ changes.insert( ++ path, ++ FileChange::Update { ++ unified_diff: raw_diff.into(), ++ move_path: None, ++ }, ++ ); ++ assert!(validate_checkpoint(&changes, &legacy).is_ok()); ++ } ++ ++ #[test] ++ fn live_delete_matcher_binds_raw_content_without_reading_the_path() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().canonicalize().unwrap().join("not-on-disk"); ++ let mut changes = [( ++ path.clone(), ++ FileChange::Delete { ++ content: "original".into(), ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ let checkpoint = deletion_checkpoint(&path, b"original", 2); ++ assert!(matches_delete_preimages(&changes, &checkpoint)); ++ assert!(!path.exists()); ++ changes.insert( ++ path.clone(), ++ FileChange::Delete { ++ content: "forged".into(), ++ }, ++ ); ++ assert!(!matches_delete_preimages(&changes, &checkpoint)); ++ changes.insert( ++ path.clone(), ++ FileChange::Delete { ++ content: "original".into(), ++ }, ++ ); ++ let mut wrong_digest = checkpoint; ++ wrong_digest["files"][path.to_str().unwrap()] = Value::String(digest(b"different")); ++ assert!(!matches_delete_preimages(&changes, &wrong_digest)); ++ } ++ ++ #[test] ++ fn live_delete_matcher_enforces_content_budgets_before_hashing() { ++ let path = PathBuf::from("/large-delete"); ++ let checkpoint = deletion_checkpoint(&path, b"irrelevant", 2); ++ let changes = [( ++ path, ++ FileChange::Delete { ++ content: "x".repeat(MAX_PATCH_PREIMAGE_BYTES as usize + 1), ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ assert!(!matches_delete_preimages(&changes, &checkpoint)); ++ let changes: HashMap<_, _> = (0..3) ++ .map(|index| { ++ ( ++ PathBuf::from(format!("/delete-{index}")), ++ FileChange::Delete { ++ content: "x".repeat(MAX_PATCH_PREIMAGE_BYTES as usize), ++ }, ++ ) ++ }) ++ .collect(); ++ let files: BTreeMap<_, _> = changes ++ .keys() ++ .map(|path| (path.clone(), Some(digest(b"irrelevant")))) ++ .collect(); ++ let checkpoint = serde_json::to_value(PatchPreconditions { version: 2, files }).unwrap(); ++ assert!(!matches_delete_preimages(&changes, &checkpoint)); ++ let path = PathBuf::from("/empty-delete"); ++ let checkpoint = deletion_checkpoint(&path, b"", 2); ++ let changes = [( ++ path, ++ FileChange::Delete { ++ content: String::new(), ++ }, ++ )] ++ .into_iter() ++ .collect(); ++ assert!(matches_delete_preimages(&changes, &checkpoint)); ++ } ++ ++ #[tokio::test] ++ async fn v2_redacted_delete_validates_full_preimage_and_rejects_forged_digest_or_drift() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().canonicalize().unwrap().join("deleted"); ++ std::fs::write(&path, b"private original file").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let mut request = approval( ++ path.clone(), ++ FileChange::Delete { ++ content: "private original file".into(), ++ }, ++ ); ++ let initial = capture(&fs, &request.changes).await.unwrap(); ++ assert_eq!(initial.version, 1); ++ let mut checkpoint = serde_json::to_value(initial).unwrap(); ++ checkpoint["version"] = serde_json::json!(2); ++ request.changes.insert( ++ path.clone(), ++ FileChange::Delete { ++ content: String::new(), ++ }, ++ ); ++ let (session, turn) = crate::session::tests::make_session_and_context().await; ++ let session = std::sync::Arc::new(session); ++ let step = session ++ .capture_step_context( ++ std::sync::Arc::new(turn), ++ &tokio_util::sync::CancellationToken::new(), ++ ) ++ .await ++ .unwrap(); ++ validate_patch_preconditions(&step, &request, &checkpoint) ++ .await ++ .unwrap(); ++ let mut forged = checkpoint.clone(); ++ forged["files"][path.to_str().unwrap()] = Value::String(digest(b"forged")); ++ assert!( ++ validate_patch_preconditions(&step, &request, &forged) ++ .await ++ .is_err() ++ ); ++ std::fs::write(&path, b"stale replacement").unwrap(); ++ assert!( ++ validate_patch_preconditions(&step, &request, &checkpoint) ++ .await ++ .is_err() ++ ); ++ } ++ ++ #[tokio::test] ++ async fn legacy_delete_keeps_raw_equality_and_initial_capture() { ++ let dir = tempfile::tempdir().unwrap(); ++ let path = dir.path().canonicalize().unwrap().join("legacy-deleted"); ++ std::fs::write(&path, b"legacy contents").unwrap(); ++ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); ++ let mut request = approval( ++ path.clone(), ++ FileChange::Delete { ++ content: "legacy contents".into(), ++ }, ++ ); ++ let checkpoint = ++ serde_json::to_value(capture(&fs, &request.changes).await.unwrap()).unwrap(); ++ assert_eq!(checkpoint["version"], 1); ++ let (session, turn) = crate::session::tests::make_session_and_context().await; ++ let session = std::sync::Arc::new(session); ++ let step = session ++ .capture_step_context( ++ std::sync::Arc::new(turn), ++ &tokio_util::sync::CancellationToken::new(), ++ ) ++ .await ++ .unwrap(); ++ validate_patch_preconditions(&step, &request, &checkpoint) ++ .await ++ .unwrap(); ++ request.changes.insert( ++ path.clone(), ++ FileChange::Delete { ++ content: "forged".into(), ++ }, ++ ); ++ assert!( ++ validate_patch_preconditions(&step, &request, &checkpoint) ++ .await ++ .is_err() ++ ); ++ assert!(capture(&fs, &request.changes).await.is_err()); ++ request.changes.insert( ++ path, ++ FileChange::Delete { ++ content: String::new(), ++ }, ++ ); ++ assert!( ++ validate_patch_preconditions(&step, &request, &checkpoint) ++ .await ++ .is_err() ++ ); ++ } +} diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs new file mode 100644 -index 0000000..b04d400 +index 0000000..2818bcc --- /dev/null +++ b/codex-rs/core/src/restore_tool_call.rs -@@ -0,0 +1,692 @@ +@@ -0,0 +1,809 @@ +//! Model-free replay of one native call, requiring the same fresh human approval. + +use std::collections::HashMap; @@ -761,6 +1096,7 @@ index 0000000..b04d400 +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; + ++use crate::patch_preconditions::{MAX_PATCH_PREIMAGE_BYTES, MAX_PATCH_SCAN_BYTES}; +use crate::session::session::Session; +use crate::session::step_context::StepContext; +use crate::session::turn_context::TurnContext; @@ -1103,7 +1439,39 @@ index 0000000..b04d400 + Ok(()) +} + -+pub(crate) async fn capture(session: &Arc, expected: EventMsg) -> CodexResult { ++// Prepared deletes include the complete file. Prepared diffs can include neighboring ++// lines absent from the submitted patch. Keep those bytes out of the SDK checkpoint. ++fn compact_patch(event: &mut EventMsg) -> CodexResult { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return Ok(false); ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ let mut compacted = false; ++ for change in request.changes.values_mut() { ++ match change { ++ FileChange::Add { .. } => continue, ++ FileChange::Delete { content } => { ++ // The trusted file snapshot already binds the entire deletion. ++ *content = String::new(); ++ } ++ FileChange::Update { unified_diff, .. } => { ++ let bytes = unified_diff.len() as u64; ++ if bytes > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return Err(refusal("prepared patch diff exceeds the recovery limit")); ++ } ++ remaining -= bytes; ++ *unified_diff = format!("{:x}", Sha256::digest(unified_diff.as_bytes())); ++ } ++ } ++ compacted = true; ++ } ++ Ok(compacted) ++} ++ ++pub(crate) async fn capture( ++ session: &Arc, ++ mut expected: EventMsg, ++) -> CodexResult<(EventMsg, Value)> { + let (turn, cancellation) = session + .active_turn_context_and_cancellation_token() + .await @@ -1116,20 +1484,40 @@ index 0000000..b04d400 + let step = session.capture_step_context(turn, &cancellation).await?; + validate_step(session, &step).await?; + let approvals = state.0.lock().unwrap(); -+ approvals ++ let mut checkpoint = approvals + .get(id) + .ok_or_else(|| refusal("not a pending first pre-execution approval"))? -+ .checkpoint(&expected) ++ .checkpoint(&expected)?; ++ if compact_patch(&mut expected)? { ++ checkpoint["patch"]["version"] = json!(2); ++ } ++ Ok((expected, checkpoint)) +} + -+pub(crate) fn compare_request(turn: &TurnContext, event: &EventMsg) -> bool { ++pub(crate) fn compare_request(turn: &TurnContext, event: &mut EventMsg) -> bool { + record_capture_event(turn, event); + let Some(restore) = active(turn) else { + return true; + }; -+ !restore.requested.swap(true, Ordering::SeqCst) -+ && approval_id(event) == approval_id(&restore.expected) -+ && matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected) ++ if restore.requested.swap(true, Ordering::SeqCst) ++ || approval_id(event) != approval_id(&restore.expected) ++ { ++ return false; ++ } ++ // Version 1 keeps comparing the original request verbatim, including any ++ // digest-looking text. Only v2 uses a compact request and file fingerprints. ++ let patch = &restore.checkpoint["patch"]; ++ if patch["version"] == 2 { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return false; ++ }; ++ if !crate::patch_preconditions::matches_delete_preimages(&request.changes, patch) ++ || !matches!(compact_patch(event), Ok(true)) ++ { ++ return false; ++ } ++ } ++ matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected) +} + +pub(crate) fn record_decision(turn: &TurnContext, decision: ReviewDecision) -> ReviewDecision { @@ -1229,6 +1617,70 @@ index 0000000..b04d400 + json!({"version":1,"first_approval":true,"permissions":"test-fingerprint","patch":null}) + } + ++ #[tokio::test] ++ async fn compact_approval_hides_prepared_file_text_and_matches_only_the_original_action() { ++ let deleted = PathBuf::from("/workspace/delete"); ++ let updated = PathBuf::from("/workspace/update"); ++ let raw = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { ++ call_id: "original".into(), ++ turn_id: "first-turn".into(), ++ started_at_ms: 1, ++ changes: HashMap::from([ ++ ( ++ deleted.clone(), ++ FileChange::Delete { ++ content: "private original".into(), ++ }, ++ ), ++ ( ++ updated.clone(), ++ FileChange::Update { ++ unified_diff: "-old\n+new\n private neighbor".into(), ++ move_path: None, ++ }, ++ ), ++ ]), ++ reason: None, ++ grant_root: None, ++ }; ++ let mut expected = EventMsg::ApplyPatchApprovalRequest(raw.clone()); ++ assert!(compact_patch(&mut expected).unwrap()); ++ assert!( ++ !serde_json::to_string(&expected) ++ .unwrap() ++ .contains("private") ++ ); ++ let mut state = checkpoint(); ++ state["patch"] = json!({"version":2,"files":{ ++ "/workspace/delete": format!("{:x}", Sha256::digest(b"private original")), ++ "/workspace/update": format!("{:x}", Sha256::digest(b"old\nprivate neighbor")), ++ }}); ++ let call: ResponseItem = serde_json::from_value(json!({ ++ "type":"custom_tool_call", "name":"apply_patch", "call_id":"original", ++ "input":"original patch", "status":"completed" ++ })) ++ .unwrap(); ++ let (_, turn) = crate::session::tests::make_session_and_context().await; ++ for changed_path in [None, Some(&deleted), Some(&updated)] { ++ let restore = ++ RestoreToolCall::new(call.clone(), expected.clone(), Some(state.clone())).unwrap(); ++ assert!(restore.claim(&turn)); ++ turn.extension_data.insert(restore); ++ let mut fresh = raw.clone(); ++ if let Some(path) = changed_path { ++ match fresh.changes.get_mut(path).unwrap() { ++ FileChange::Delete { content } => content.push('!'), ++ FileChange::Update { unified_diff, .. } => unified_diff.push('!'), ++ _ => unreachable!(), ++ } ++ } ++ assert_eq!( ++ compare_request(&turn, &mut EventMsg::ApplyPatchApprovalRequest(fresh)), ++ changed_path.is_none() ++ ); ++ } ++ } ++ + #[test] + fn reuses_patch_snapshot_only_for_the_live_event_and_original_changes() { + let path = PathBuf::from("/workspace/new"); @@ -1311,14 +1763,14 @@ index 0000000..b04d400 + assert!(restore.claim(&turn)); + assert!(!restore.claim(&turn)); + assert!(validate_before_execution(&session, &step).await.is_err()); -+ assert!(compare_request(&turn, &approval())); ++ assert!(compare_request(&turn, &mut approval())); + assert_eq!( + record_decision(&turn, ReviewDecision::Approved), + ReviewDecision::Approved + ); + assert!(validate_before_execution(&session, &step).await.is_ok()); + assert!(validate_before_execution(&session, &step).await.is_err()); -+ assert!(!compare_request(&turn, &approval())); ++ assert!(!compare_request(&turn, &mut approval())); + } + + #[tokio::test] @@ -1336,8 +1788,8 @@ index 0000000..b04d400 + if let EventMsg::ExecApprovalRequest(event) = &mut changed { + event.command.push("unexpected".into()); + } -+ assert!(!compare_request(&turn, &changed)); -+ assert!(!compare_request(&turn, &approval())); ++ assert!(!compare_request(&turn, &mut changed)); ++ assert!(!compare_request(&turn, &mut approval())); + assert!(!active(&turn).unwrap().approved.load(Ordering::SeqCst)); + } + @@ -1372,7 +1824,7 @@ index 0000000..b04d400 + .unwrap() + .claim(&turn) + ); -+ assert!(compare_request(&turn, &approval())); ++ assert!(compare_request(&turn, &mut approval())); + record_decision(&turn, ReviewDecision::Approved); + assert!(validate_before_execution(&session, &step).await.is_err()); + } @@ -1437,7 +1889,7 @@ index 0000000..b04d400 + } +} diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs -index 66476ae..c092ec5 100644 +index 66476ae..6bfe356 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2930,7 +2930,7 @@ impl Session { @@ -1449,11 +1901,20 @@ index 66476ae..c092ec5 100644 &self, ) -> Option<(Arc, CancellationToken)> { let active = self.active_turn.lock().await; +@@ -3155,7 +3155,7 @@ impl Session { + .as_ref() + .map(PluginCommandAttribution::serialized_fields) + .unzip(); +- let event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { ++ let mut event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + model_context: Some(model_context), + kind, + call_id, @@ -3175,9 +3175,15 @@ impl Session { available_decisions: Some(available_decisions), parsed_cmd, }); -+ if !crate::restore_tool_call::compare_request(turn_context, &event) { ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { + return ReviewDecision::Abort; + } let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); @@ -1466,11 +1927,20 @@ index 66476ae..c092ec5 100644 } #[expect( +@@ -3211,7 +3217,7 @@ impl Session { + warn!("Overwriting existing pending approval for call_id: {approval_id}"); + } + +- let event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { ++ let mut event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { + call_id, + turn_id: turn_context.sub_id.clone(), + started_at_ms: now_unix_timestamp_ms(), @@ -3219,9 +3225,15 @@ impl Session { reason, grant_root, }); -+ if !crate::restore_tool_call::compare_request(turn_context, &event) { ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { + return ReviewDecision::Abort; + } let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); From 6fdda2c30a1dc04524fafb3f5c30e79d93059f5e Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:33:11 -0800 Subject: [PATCH 25/33] fix(codex): preserve cancellation and native failures during cleanup --- src/agents/models/codex.py | 31 ++++++------ tests/models/test_codex_provider.py | 77 +++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index a89b487e51..3ce0c4dc2e 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -4,7 +4,7 @@ import json import os import time -from collections.abc import AsyncIterator, Awaitable, Sequence +from collections.abc import AsyncIterator, Sequence from contextlib import AsyncExitStack from dataclasses import fields from importlib import import_module @@ -39,6 +39,7 @@ from ..exceptions import UserError from ..handoffs import Handoff from ..items import ModelResponse, TResponseInputItem, TResponseStreamEvent +from ..memory.session import await_session_mutation as _join_close from ..model_settings import ModelSettings from ..retry import ModelRetryAdvice, ModelRetryAdviceRequest from ..tool import ApplyPatchTool, ShellTool, Tool, WebSearchTool @@ -126,6 +127,7 @@ async def _run_approved_native_call( observations = CodexOperationTracing(tracing, None, checkpoint["model"]) thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) interrupted = True + cancellation: asyncio.CancelledError | None = None execution: asyncio.Future[Any] | None = None observed_turn: str | None = None observations.bind(thread.thread_id, "") @@ -160,6 +162,9 @@ def observe(encoded: str) -> None: raise RuntimeError(failure) interrupted = False return str(output) + except asyncio.CancelledError as error: + cancellation = error + raise finally: if execution is not None: execution.cancel() @@ -174,7 +179,15 @@ async def close() -> None: try: observations.close(interrupted=interrupted) finally: - await _join_close(close()) + try: + await _join_close(close()) + except asyncio.CancelledError: + if cancellation is not None: + raise cancellation from None + raise + except Exception: + if not interrupted: + raise def _pending_native_call( @@ -259,20 +272,6 @@ def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: return (reasoning.effort if reasoning else None, reasoning.summary if reasoning else None) -async def _join_close(operation: Awaitable[None]) -> None: - """Keep cleanup owned through repeated cancellation of the awaiting task.""" - task = asyncio.ensure_future(operation) - cancelled = None - while not task.done(): - try: - await asyncio.shield(task) - except asyncio.CancelledError as error: - cancelled = error - task.result() - if cancelled is not None: - raise cancelled - - class _CodexModel(Model): _supports_stored_response_id = False _sandbox_model_capabilities: ClassVar[dict[str, Any]] = { diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 4930397e1f..841313b162 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1207,6 +1207,83 @@ async def close(self): assert thread.closed +@pytest.mark.asyncio +@pytest.mark.parametrize("restored", [False, True]) +@pytest.mark.parametrize("primary", ["success", "failure", "cancel"]) +@pytest.mark.parametrize("late_cancel", [False, True]) +async def test_native_cleanup_preserves_cancellation_and_primary_failure( + tmp_path, threads, restored, primary, late_cancel +): + from agents.models.interface import ModelTracing + + close_started, release_close = asyncio.Event(), asyncio.Event() + closed = [] + + class HeldCloseThread(FakeThread): + async def next_operation_event(self): + await self.started.wait() + return None + + async def resolve_approval(self, *args): + self.started.set() + if primary == "cancel": + await asyncio.Event().wait() + if primary == "failure": + raise ValueError("primary failure") + return "accepted", "turn_1", [], None + + async def close(self): + close_started.set() + try: + await release_close.wait() + raise RuntimeError("cleanup failure") + finally: + closed.append(True) + await super().close() + + thread = HeldCloseThread( + events=[{"type": "error"}] if primary == "failure" else [], + gate=asyncio.Event() if primary == "cancel" else None, + ) + threads[0].append(thread) + provider = CodexProvider(cwd=tmp_path) + checkpoint = {"version": 1, "cwd": str(tmp_path), "model": "fixture", "call": {}, "request": {}} + operation = ( + provider._run_approved_native_call(checkpoint, ModelTracing.DISABLED) + if restored + else Runner.run(Agent(name="test"), "Answer", run_config=RunConfig(model_provider=provider)) + ) + task = asyncio.create_task(operation) + try: + await asyncio.wait_for(thread.started.wait(), 2) + if primary == "cancel": + task.cancel("primary cancellation") + await asyncio.wait_for(close_started.wait(), 2) + if late_cancel: + task.cancel("cleanup cancellation") + await asyncio.sleep(0) + task.cancel("repeated cancellation") + await asyncio.sleep(0) + assert not task.done() + release_close.set() + error, reason = RuntimeError, "cleanup failure" + if primary == "failure": + error = ValueError if restored else RuntimeError + reason = "primary failure" if restored else "Codex model turn failed" + if primary == "cancel" or late_cancel: + error = asyncio.CancelledError + first_cancel = primary == "cancel" and (restored or not late_cancel) + reason = "primary cancellation" if first_cancel else "cleanup cancellation" + with pytest.raises(error, match=reason): + await task + assert closed == [True] + finally: + release_close.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await provider.aclose() + + @pytest.mark.asyncio async def test_closing_stream_during_tool_publication_retires_pending_native_call( tmp_path, threads From 1406c8bc837c2dfa19b0dab51011409fc7d22b1f Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:49:18 -0800 Subject: [PATCH 26/33] fix(codex): count every completed inference before an approval pause --- src/agents/models/_codex_usage.py | 17 +++- src/agents/models/codex.py | 8 +- tests/memory/test_native_approval_recovery.py | 88 ++++++++++++++++++- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/src/agents/models/_codex_usage.py b/src/agents/models/_codex_usage.py index 5c30dc5443..11f2608efb 100644 --- a/src/agents/models/_codex_usage.py +++ b/src/agents/models/_codex_usage.py @@ -5,7 +5,7 @@ from openai.types.responses import ResponseUsage from openai.types.responses.response_usage import OutputTokensDetails -from ..usage import _make_input_tokens_details +from ..usage import _make_input_tokens_details, _response_usage_to_usage def _usage(total: dict[str, Any]) -> ResponseUsage: @@ -23,3 +23,18 @@ def _usage(total: dict[str, Any]) -> ResponseUsage: reasoning_tokens=total.get("reasoning_output_tokens", 0) ), ) + + +def _add_sample_usage(previous: ResponseUsage | None, sample: dict[str, Any]) -> ResponseUsage: + current = _usage(sample) + if previous is None: + return current + total = _response_usage_to_usage(previous) + total.add(_response_usage_to_usage(current)) + return ResponseUsage( + input_tokens=total.input_tokens, + output_tokens=total.output_tokens, + total_tokens=total.total_tokens, + input_tokens_details=total.input_tokens_details, + output_tokens_details=total.output_tokens_details, + ) diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 3ce0c4dc2e..933eaa0402 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -58,7 +58,7 @@ ) from ._codex_tool_hooks import CodexToolHooks from ._codex_tracing import CodexOperationTracing -from ._codex_usage import _usage +from ._codex_usage import _add_sample_usage, _usage from ._run_context import ModelToolHooks, get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing @@ -642,8 +642,10 @@ async def interrupt_when_ready() -> None: raw_usage["token_count"] = info continue if kind == "raw_response_completed": - if usage is None and message.get("token_usage") is not None: - usage = _usage(message["token_usage"]) + # An approval can pause before Core sends the cumulative token_count. + # Include each completed sample now. The next count replaces this total. + if message.get("token_usage") is not None: + usage = _add_sample_usage(usage, message["token_usage"]) if ( model_settings.preserve_raw_usage and message.get("usage_metadata") is not None diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py index f2a822ef26..4abe3dbcaa 100644 --- a/tests/memory/test_native_approval_recovery.py +++ b/tests/memory/test_native_approval_recovery.py @@ -14,7 +14,7 @@ from agents.models.codex import CodexProvider from agents.testing import ScriptedModel from agents.usage import Usage -from tests.memory.test_resumable_session import MemoryStore +from tests.memory.test_resumable_session import MemoryStore, run_agent from tests.models.test_codex_provider import FakeThread, threads as threads from tests.test_responses import get_text_message from tests.testing_processor import fetch_ordered_spans @@ -70,6 +70,92 @@ async def execute(checkpoint, *, tools, tracing): return agent, store, session, config, pending, calls, requests +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"]) +@pytest.mark.parametrize("approval_after_count", [False, True], ids=["before-count", "after-count"]) +async def test_native_pause_preserves_all_sampling_usage_across_session_resume( + tmp_path, monkeypatch, threads, streamed, approval_after_count +): + from tests.models.test_codex_pending_approval import ApprovalThread, approval, raw + from tests.models.test_codex_provider import native_pair + + earlier = { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + "cached_input_tokens": 1, + "cache_write_input_tokens": 1, + "reasoning_output_tokens": 1, + } + current = { + "input_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + "cached_input_tokens": 2, + "cache_write_input_tokens": 3, + "reasoning_output_tokens": 2, + } + cumulative = {key: earlier[key] + current[key] for key in earlier} + + # The wire double preserves the completion/count ordering around a real Runner pause. + class SamplingApprovalThread(ApprovalThread): + async def events(self): + yield { + "type": "raw_response_completed", + "response_id": "earlier", + "token_usage": earlier, + } + yield {"type": "token_count", "info": {"total_token_usage": earlier}} + call, _ = native_pair(call_id="pending") + yield raw(call) + if not approval_after_count: + yield approval() + yield { + "type": "raw_response_completed", + "response_id": "current", + "token_usage": current, + } + yield {"type": "token_count", "info": {"total_token_usage": cumulative}} + if approval_after_count: + yield approval() + + def tokens(usage): + return ( + usage.input_tokens, + usage.output_tokens, + usage.total_tokens, + usage.input_tokens_details.cached_tokens, + usage.input_tokens_details.cache_write_tokens, + usage.output_tokens_details.reasoning_tokens, + ) + + model = CodexProvider(cwd=tmp_path).get_model("fixture-model") + agent = Agent(name="worker", model=model) + first = SamplingApprovalThread() + threads[0].extend([first, FakeThread(text="finished")]) + calls = [] + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint["call"]["call_id"]) + return "native accepted" + + monkeypatch.setattr(model, "_run_approved_native_call", execute) + store = MemoryStore() + session = await ResumableSession.create(store) + paused = await run_agent(agent, "start", session, streamed) + assert first.closed and len(paused.interruptions) == 1 and calls == [] + assert tokens(paused.context_wrapper.usage) == (15, 5, 20, 3, 4, 3) + assert store.checkpoint(session.session_id)["status"] == "paused" + + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + completed = await run_agent(agent, state, None, streamed) + assert completed.final_output == "finished" and calls == ["pending"] + # The fresh native thread adds its own 10 input and 7 output tokens exactly once. + assert tokens(completed.context_wrapper.usage) == (25, 12, 37, 6, 4, 5) + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) @pytest.mark.parametrize("approved", [True, False]) From f31f5a81f7dc3792213b8afe7dda6e14b615e2b1 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:49:23 -0800 Subject: [PATCH 27/33] test(codex): check cancellation reasons on supported Python versions --- tests/models/test_codex_provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 841313b162..7e38eefcd0 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -1274,7 +1274,9 @@ async def close(self): error = asyncio.CancelledError first_cancel = primary == "cancel" and (restored or not late_cancel) reason = "primary cancellation" if first_cancel else "cleanup cancellation" - with pytest.raises(error, match=reason): + # Python 3.10 has not been forwarding cancellation reasons across task boundaries. + match = None if error is asyncio.CancelledError and sys.version_info < (3, 11) else reason + with pytest.raises(error, match=match): await task assert closed == [True] finally: From dd901e1b294cc2dfe7864c6b2715c4976f2148ab Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Fri, 20 Nov 2026 21:55:05 -0800 Subject: [PATCH 28/33] fix(codex): bound native tool results before preparing trace content --- patches/codex/0001-operation-tracing.patch | 601 ++++++++++++++++++++- patches/codex/README.md | 14 +- 2 files changed, 599 insertions(+), 16 deletions(-) diff --git a/patches/codex/0001-operation-tracing.patch b/patches/codex/0001-operation-tracing.patch index ecd02a17a1..494e19f304 100644 --- a/patches/codex/0001-operation-tracing.patch +++ b/patches/codex/0001-operation-tracing.patch @@ -218,10 +218,10 @@ index 9f4ef92..34a88e0 100644 }; diff --git a/codex-rs/core/src/operation_observation.rs b/codex-rs/core/src/operation_observation.rs new file mode 100644 -index 0000000..cea2a98 +index 0000000..ccea56b --- /dev/null +++ b/codex-rs/core/src/operation_observation.rs -@@ -0,0 +1,1970 @@ +@@ -0,0 +1,1987 @@ +//! Opt-in, in-memory operation facts for the host that owns a turn. +//! +//! These observations never enter the rollout or the telemetry logger. The host @@ -459,6 +459,23 @@ index 0000000..cea2a98 + self.finish_at(outcome, completed_at, duration_ms); + } + ++ pub(crate) fn finish_with_bounded_output_fn( ++ &self, ++ outcome: OperationOutcome, ++ output: impl FnOnce() -> Result, ++ ) { ++ let completed_at = timestamp(); ++ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; ++ if self.observation.captures_content() { ++ let content = match output() { ++ Ok(value) => project_content(&value), ++ Err(reason) => omitted_content(reason), ++ }; ++ self.update(|event| event.output = Some(content)); ++ } ++ self.finish_at(outcome, completed_at, duration_ms); ++ } ++ + pub fn finish_with_output(&self, outcome: OperationOutcome, output: &Value) { + let completed_at = timestamp(); + let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; @@ -2315,8 +2332,27 @@ index 317e029..4354977 100644 let ToolInvocation { session, turn, +diff --git a/codex-rs/core/src/tools/code_mode/output.rs b/codex-rs/core/src/tools/code_mode/output.rs +index a85b77a..f264419 100644 +--- a/codex-rs/core/src/tools/code_mode/output.rs ++++ b/codex-rs/core/src/tools/code_mode/output.rs +@@ -40,6 +40,14 @@ impl CodeModeToolOutput { + } + + impl ToolOutput for CodeModeToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ self.output.bounded_code_mode_result(payload, max_bytes) ++ } ++ + fn log_output(&self) -> String { + self.output.log_output() + } diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs -index fe7de2c..60aa2b8 100644 +index fe7de2c..718ea5c 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -125,6 +125,8 @@ pub struct ToolInvocation { @@ -2336,6 +2372,188 @@ index fe7de2c..60aa2b8 100644 item_id, window_id: self.session.current_window_id().await, }) +@@ -176,6 +179,14 @@ pub struct McpToolOutput { + } + + impl ToolOutput for McpToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ self.result.bounded_code_mode_result(payload, max_bytes) ++ } ++ + fn log_output(&self) -> String { + // Logging has its own budget; do not first apply the model-context budget. + let output = self.result.log_output(); +@@ -262,6 +273,15 @@ pub struct ToolSearchOutput { + } + + impl ToolOutput for ToolSearchOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ let _ = payload; ++ codex_tools::bounded_tool_output(&self.tools, max_bytes) ++ } ++ + fn log_output(&self) -> String { + let tools = self + .tools +@@ -329,6 +349,17 @@ impl FunctionToolOutput { + } + + impl ToolOutput for FunctionToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ // The normal conversion clones and joins body items. Admit only a bounded ++ // serialized source first, including item count and embedded media bytes. ++ codex_tools::bounded_tool_output(&self.body, max_bytes)?; ++ Ok(self.code_mode_result(payload)) ++ } ++ + fn log_output(&self) -> String { + function_call_output_content_items_to_text(&self.body).unwrap_or_default() + } +@@ -357,6 +388,15 @@ impl ApplyPatchToolOutput { + } + + impl ToolOutput for ApplyPatchToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ let _ = max_bytes; ++ Ok(self.code_mode_result(payload)) ++ } ++ + fn log_output(&self) -> String { + self.text.clone() + } +@@ -390,6 +430,15 @@ pub struct AbortedToolOutput { + } + + impl ToolOutput for AbortedToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ codex_tools::bounded_tool_output(&self.message, max_bytes)?; ++ Ok(self.code_mode_result(payload)) ++ } ++ + fn log_output(&self) -> String { + self.message.clone() + } +@@ -437,6 +486,17 @@ pub struct ExecCommandToolOutput { + } + + impl ToolOutput for ExecCommandToolOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ // Rendering may clone raw text even when the model token budget is small. ++ // Establish a source bound before calling the existing formatter. ++ codex_tools::bounded_tool_output(&(&self.chunk_id, &self.raw_output.text), max_bytes)?; ++ Ok(self.code_mode_result(payload)) ++ } ++ + fn log_output(&self) -> String { + // The telemetry budget must not inherit the model's output-token limit. + let output = self +diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs +index aa9a5f3..c62943d 100644 +--- a/codex-rs/core/src/tools/context_tests.rs ++++ b/codex-rs/core/src/tools/context_tests.rs +@@ -554,3 +554,78 @@ fn exec_command_tool_output_preserves_omission_metadata_when_truncated() { + assert!(text.contains("Warning: truncated output (original token count: 42000)")); + assert_eq!(text.matches(&marker).count(), 1); + } ++ ++#[test] ++fn bounded_capture_checks_function_source_before_cloning_and_joining() { ++ let payload = ToolPayload::Function { ++ arguments: "{}".into(), ++ }; ++ let mut output = FunctionToolOutput::from_content( ++ vec![ ++ FunctionCallOutputContentItem::InputText { ++ text: "first".into(), ++ }, ++ FunctionCallOutputContentItem::InputText { ++ text: "second".into(), ++ }, ++ ], ++ Some(true), ++ ); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Ok(json!("first\nsecond")) ++ ); ++ output.body.push(FunctionCallOutputContentItem::InputImage { ++ image: ImageReference::Inline { ++ image_url: "x".repeat(1024 * 1024), ++ }, ++ detail: None, ++ }); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++ output.body = (0..10000) ++ .map(|_| FunctionCallOutputContentItem::InputText { ++ text: String::new(), ++ }) ++ .collect(); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++} ++ ++#[test] ++fn bounded_capture_checks_exec_source_before_rendering() { ++ let payload = ToolPayload::Function { ++ arguments: "{}".into(), ++ }; ++ let mut output = ExecCommandToolOutput { ++ event_call_id: "call-42".into(), ++ chunk_id: "chunk".into(), ++ wall_time: Duration::from_millis(25), ++ raw_output: "visible".into(), ++ truncation_policy: TruncationPolicy::Bytes(10000), ++ max_output_tokens: None, ++ process_id: None, ++ exit_code: Some(0), ++ original_token_count: Some(1), ++ hook_command: None, ++ tool_cwd: None, ++ }; ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Ok(output.code_mode_result(&payload)) ++ ); ++ output.raw_output.text = "x".repeat(1024 * 1024); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++ output.max_output_tokens = Some(1); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++} diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs index 64d6cc1..0c1c08a 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch.rs @@ -2410,10 +2628,10 @@ index 3344ed8..ea173f5 100644 pub(crate) mod registry; diff --git a/codex-rs/core/src/tools/operation_observation.rs b/codex-rs/core/src/tools/operation_observation.rs new file mode 100644 -index 0000000..2c84e35 +index 0000000..5ecf36d --- /dev/null +++ b/codex-rs/core/src/tools/operation_observation.rs -@@ -0,0 +1,184 @@ +@@ -0,0 +1,207 @@ +//! Execution observations, separate from admission, approval, and result hooks. + +use std::sync::Arc; @@ -2504,9 +2722,32 @@ index 0000000..2c84e35 + } else { + OperationOutcome::Error + }; -+ guard.finish_with_output_fn(outcome, || match result { -+ Ok(output) => output.code_mode_result(&invocation.payload), -+ Err(error) => serde_json::json!({"error": error.to_string()}), ++ guard.finish_with_bounded_output_fn(outcome, || match result { ++ Ok(output) => { ++ output.bounded_code_mode_result(&invocation.payload, CONTENT_VALUE_LIMIT_BYTES) ++ } ++ Err(error) => { ++ let message = match error { ++ FunctionCallError::RespondToModel(message) ++ | FunctionCallError::Fatal(message) => message, ++ }; ++ // Bound source text before Display allocates the optional fatal prefix. ++ if message.len() > CONTENT_VALUE_LIMIT_BYTES { ++ return Err("capture_budget"); ++ } ++ #[derive(serde::Serialize)] ++ struct ErrorOutput<'a> { ++ error: &'a str, ++ } ++ let rendered; ++ let error = if matches!(error, FunctionCallError::Fatal(_)) { ++ rendered = error.to_string(); ++ rendered.as_str() ++ } else { ++ message.as_str() ++ }; ++ codex_tools::bounded_tool_output(&ErrorOutput { error }, CONTENT_VALUE_LIMIT_BYTES) ++ } + }); + } +} @@ -2600,10 +2841,10 @@ index 0000000..2c84e35 +mod tests; diff --git a/codex-rs/core/src/tools/operation_observation_tests.rs b/codex-rs/core/src/tools/operation_observation_tests.rs new file mode 100644 -index 0000000..b94ecd9 +index 0000000..ed180f3 --- /dev/null +++ b/codex-rs/core/src/tools/operation_observation_tests.rs -@@ -0,0 +1,348 @@ +@@ -0,0 +1,455 @@ +use super::*; +use crate::operation_observation::OperationCaptureMode; +use crate::operation_observation::OperationEvent; @@ -2952,6 +3193,113 @@ index 0000000..b94ecd9 + <= crate::operation_observation::CONTENT_LIMIT_BYTES + ); +} ++ ++#[tokio::test] ++async fn bounded_capture_marks_unknown_outputs_and_large_errors_without_materialization() { ++ struct Unknown(Arc); ++ impl ToolOutput for Unknown { ++ fn bounded_code_mode_result( ++ &self, ++ _: &ToolPayload, ++ _: usize, ++ ) -> Result { ++ self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); ++ Err("unsupported_output") ++ } ++ fn log_output(&self) -> String { ++ panic!("unexpected conversion") ++ } ++ fn success_for_logging(&self) -> bool { ++ true ++ } ++ fn to_response_item( ++ &self, ++ _: &str, ++ _: &ToolPayload, ++ ) -> codex_protocol::models::ResponseInputItem { ++ panic!("unexpected conversion") ++ } ++ fn code_mode_result(&self, _: &ToolPayload) -> Value { ++ panic!("unexpected conversion") ++ } ++ } ++ for mode in [ ++ OperationCaptureMode::Metadata, ++ OperationCaptureMode::Content, ++ ] { ++ let (invocation, events) = fixture(mode).await; ++ let observation = invocation ++ .turn ++ .extension_data ++ .get::() ++ .unwrap(); ++ let execution = ExecutionObservation { ++ observation, ++ thread_id: "thread".into(), ++ turn_id: "turn".into(), ++ metadata: OperationMetadata::default(), ++ input: None, ++ guard: Mutex::new(None), ++ }; ++ execution.start(None); ++ let captures = Arc::new(std::sync::atomic::AtomicUsize::new(0)); ++ execution.finish(&invocation, &Ok(Box::new(Unknown(captures.clone())))); ++ assert_eq!( ++ captures.load(std::sync::atomic::Ordering::Relaxed), ++ usize::from(mode == OperationCaptureMode::Content) ++ ); ++ let output = events.0.lock().unwrap()[1].output.clone(); ++ if mode == OperationCaptureMode::Content { ++ assert_eq!( ++ output.unwrap().reason.as_deref(), ++ Some("unsupported_output") ++ ); ++ } else { ++ assert!(output.is_none()); ++ } ++ } ++ for (error, expected) in [ ++ ( ++ FunctionCallError::RespondToModel("x".repeat(1024 * 1024)), ++ None, ++ ), ++ (FunctionCallError::Fatal("x".repeat(1024 * 1024)), None), ++ ( ++ FunctionCallError::RespondToModel("visible".into()), ++ Some("visible"), ++ ), ++ ( ++ FunctionCallError::Fatal("visible".into()), ++ Some("Fatal error: visible"), ++ ), ++ ] { ++ let (invocation, events) = fixture(OperationCaptureMode::Content).await; ++ let execution = ExecutionObservation { ++ observation: invocation ++ .turn ++ .extension_data ++ .get::() ++ .unwrap(), ++ thread_id: "thread".into(), ++ turn_id: "turn".into(), ++ metadata: OperationMetadata::default(), ++ input: None, ++ guard: Mutex::new(None), ++ }; ++ execution.start(None); ++ execution.finish(&invocation, &Err(error)); ++ let events = events.0.lock().unwrap(); ++ let output = events[1].output.as_ref().unwrap(); ++ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); ++ if let Some(expected) = expected { ++ assert_eq!(output.value, serde_json::json!({"error": expected})); ++ assert!(!output.omitted && !output.truncated); ++ } else { ++ assert!(output.omitted && output.truncated); ++ assert_eq!(output.reason.as_deref(), Some("capture_budget")); ++ } ++ } ++} diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index a1a88e4..d9afeb6 100644 --- a/codex-rs/core/src/tools/orchestrator.rs @@ -2965,7 +3313,7 @@ index a1a88e4..d9afeb6 100644 .run(req, &attempt_with_network_approval, &attempt_tool_ctx) .await; diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs -index 3dccc2e..bbbf568 100644 +index 3dccc2e..e1db6de 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -62,6 +62,16 @@ pub use codex_tools::ToolExposure; @@ -2985,7 +3333,22 @@ index 3dccc2e..bbbf568 100644 /// Opts this invocation into synchronous native-call review before the handler runs. fn requires_guardian_review(&self, _invocation: &ToolInvocation) -> bool { false -@@ -1050,7 +1060,7 @@ async fn handle_any_tool( +@@ -335,6 +345,14 @@ struct PostToolUseFeedbackOutput { + } + + impl ToolOutput for PostToolUseFeedbackOutput { ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ self.original.bounded_code_mode_result(payload, max_bytes) ++ } ++ + fn log_output(&self) -> String { + self.original.log_output() + } +@@ -1050,7 +1068,7 @@ async fn handle_any_tool( ) -> Result { let call_id = invocation.call_id.clone(); let payload = invocation.payload.clone(); @@ -3021,3 +3384,217 @@ index ced543a..193452e 100644 ResponseEvent::ModelVerifications(_) => "model_verifications".into(), ResponseEvent::TurnModerationMetadata(_) => "turn_moderation_metadata".into(), ResponseEvent::GuardianAssessment(_) => "guardian_assessment".into(), +diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs +index 8661659..1d31dfa 100644 +--- a/codex-rs/tools/src/lib.rs ++++ b/codex-rs/tools/src/lib.rs +@@ -104,6 +104,7 @@ pub use tool_executor::ToolExposure; + pub use tool_executor::ToolExposures; + pub use tool_output::JsonToolOutput; + pub use tool_output::ToolOutput; ++pub use tool_output::bounded_tool_output; + pub use tool_payload::ToolPayload; + pub use tool_search::ToolSearchEntry; + pub use tool_search::ToolSearchInfo; +diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs +index 3a4c1e8..6335093 100644 +--- a/codex-rs/tools/src/tool_output.rs ++++ b/codex-rs/tools/src/tool_output.rs +@@ -68,6 +68,17 @@ pub trait ToolOutput: Send { + response_input_to_code_mode_result(self.to_response_item("", payload)) + } + ++ /// Produces capture data without first materializing an unbounded owned result. ++ /// Implementations must bound borrowed source data before cloning or rendering it. ++ /// Unknown output types omit capture; execution and normal model output are unchanged. ++ fn bounded_code_mode_result( ++ &self, ++ _payload: &ToolPayload, ++ _max_bytes: usize, ++ ) -> Result { ++ Err("unsupported_output") ++ } ++ + /// Borrows original host-only metadata for recording, not for model output or logging. + fn tool_result_metadata(&self) -> Option<&JsonValue> { + None +@@ -122,6 +133,14 @@ where + (**self).code_mode_result(payload) + } + ++ fn bounded_code_mode_result( ++ &self, ++ payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ (**self).bounded_code_mode_result(payload, max_bytes) ++ } ++ + fn tool_result_metadata(&self) -> Option<&JsonValue> { + (**self).tool_result_metadata() + } +@@ -197,9 +216,42 @@ impl ToolOutput for JsonToolOutput { + fn code_mode_result(&self, _payload: &ToolPayload) -> JsonValue { + self.value.clone() + } ++ ++ fn bounded_code_mode_result( ++ &self, ++ _payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ bounded_tool_output(&self.value, max_bytes) ++ } + } + + impl ToolOutput for codex_protocol::mcp::CallToolResult { ++ fn bounded_code_mode_result( ++ &self, ++ _payload: &ToolPayload, ++ max_bytes: usize, ++ ) -> Result { ++ // Borrow only public result fields: private _meta must not be cloned or serialized. ++ #[derive(serde::Serialize)] ++ #[serde(rename_all = "camelCase")] ++ struct PublicResult<'a> { ++ content: &'a [JsonValue], ++ #[serde(skip_serializing_if = "Option::is_none")] ++ structured_content: &'a Option, ++ #[serde(skip_serializing_if = "Option::is_none")] ++ is_error: Option, ++ } ++ bounded_tool_output( ++ &PublicResult { ++ content: &self.content, ++ structured_content: &self.structured_content, ++ is_error: self.is_error, ++ }, ++ max_bytes, ++ ) ++ } ++ + fn log_output(&self) -> String { + let output = self.as_function_call_output_payload(); + // Do not fall back to serializing media or encrypted content into logs. +@@ -286,3 +338,32 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> + #[cfg(test)] + #[path = "tool_output_tests.rs"] + mod tests; ++ ++/// Serializes borrowed capture data through a capped buffer before creating an owned tree. ++pub fn bounded_tool_output( ++ value: &(impl serde::Serialize + ?Sized), ++ max_bytes: usize, ++) -> Result { ++ struct CappedWriter { ++ bytes: Vec, ++ limit: usize, ++ } ++ impl std::io::Write for CappedWriter { ++ fn write(&mut self, bytes: &[u8]) -> std::io::Result { ++ if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { ++ return Err(std::io::Error::other("capture budget exceeded")); ++ } ++ self.bytes.extend_from_slice(bytes); ++ Ok(bytes.len()) ++ } ++ fn flush(&mut self) -> std::io::Result<()> { ++ Ok(()) ++ } ++ } ++ let mut writer = CappedWriter { ++ bytes: Vec::new(), ++ limit: max_bytes, ++ }; ++ serde_json::to_writer(&mut writer, value).map_err(|_| "capture_budget")?; ++ serde_json::from_slice(&writer.bytes).map_err(|_| "unsupported_output") ++} +diff --git a/codex-rs/tools/src/tool_output_tests.rs b/codex-rs/tools/src/tool_output_tests.rs +index bca0e81..111131b 100644 +--- a/codex-rs/tools/src/tool_output_tests.rs ++++ b/codex-rs/tools/src/tool_output_tests.rs +@@ -44,3 +44,84 @@ fn code_mode_text_preserves_message_content() { + ) + ); + } ++ ++#[test] ++fn bounded_capture_preserves_json_and_rejects_large_nested_values() { ++ use super::{JsonToolOutput, ToolOutput}; ++ let payload = crate::ToolPayload::Function { ++ arguments: "{}".into(), ++ }; ++ let expected = serde_json::json!({"structured": ["α", {"value": true}]}); ++ let output: Box = Box::new(JsonToolOutput::new(expected.clone())); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Ok(expected) ++ ); ++ let output = JsonToolOutput::new(serde_json::json!({"nested": ["x".repeat(1024 * 1024)]})); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++} ++ ++#[test] ++fn bounded_capture_omits_private_mcp_metadata_before_serializing() { ++ use super::ToolOutput; ++ let payload = crate::ToolPayload::Function { ++ arguments: "{}".into(), ++ }; ++ let mut output = codex_protocol::mcp::CallToolResult { ++ content: vec![serde_json::json!({"type":"text", "text":"visible"})], ++ structured_content: Some(serde_json::json!({"answer": 42})), ++ is_error: Some(false), ++ meta: Some(serde_json::json!({"private": "x".repeat(1024 * 1024)})), ++ }; ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Ok(serde_json::json!({ ++ "content": [{"type":"text", "text":"visible"}], ++ "structuredContent": {"answer":42}, "isError":false ++ })) ++ ); ++ output.structured_content = Some(serde_json::json!({"large": "x".repeat(1024 * 1024)})); ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++ output.structured_content = None; ++ output.content = vec![serde_json::json!({"type":"image", "data":"x".repeat(1024 * 1024)})]; ++ assert_eq!( ++ output.bounded_code_mode_result(&payload, 1024), ++ Err("capture_budget") ++ ); ++} ++ ++#[test] ++fn bounded_capture_never_calls_unknown_legacy_output_conversion() { ++ use super::ToolOutput; ++ struct Unknown; ++ impl ToolOutput for Unknown { ++ fn log_output(&self) -> String { ++ panic!("unbounded log") ++ } ++ fn success_for_logging(&self) -> bool { ++ true ++ } ++ fn to_response_item(&self, _: &str, _: &crate::ToolPayload) -> ResponseInputItem { ++ panic!("unbounded model conversion") ++ } ++ fn code_mode_result(&self, _: &crate::ToolPayload) -> serde_json::Value { ++ panic!("unbounded result conversion") ++ } ++ } ++ let output: Box = Box::new(Unknown); ++ assert_eq!( ++ output.bounded_code_mode_result( ++ &crate::ToolPayload::Function { ++ arguments: "{}".into() ++ }, ++ 1024 ++ ), ++ Err("unsupported_output") ++ ); ++} diff --git a/patches/codex/README.md b/patches/codex/README.md index e7e5da4e92..07876f9771 100644 --- a/patches/codex/README.md +++ b/patches/codex/README.md @@ -48,7 +48,7 @@ export RUST_MIN_STACK=8388608 python scripts/build_codex_wheel.py --profile dev --test-native --out "$SDK_ROOT/dist" ``` -The script has been copying the exact pinned upstream source into a temporary build directory and applying the checked-in patch there. Cargo source overrides have been selecting that patched workspace for the extension and execution helper. The wheel has been including the helper assets, licenses, source revision, patch, and this guide. For a release build, omit `--profile dev`. +The script has been copying the exact pinned upstream source into a temporary build directory and applying the checked-in patches in filename order there. Cargo source overrides have been selecting that patched workspace for the extension and execution helper. The wheel has been including the helper assets, licenses, source revision, patches, and this guide. For a release build, omit `--profile dev`. Set `WHEEL` to the absolute wheel path printed by the build. Create a clean environment and run the installed-package verifier outside the checkout. @@ -91,9 +91,12 @@ test "$(git rev-parse HEAD)" = 6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 test -z "$(git status --porcelain)" git apply --check "$PATCH_FILE" git apply "$PATCH_FILE" +git apply --check "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" +git apply "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" git diff --check cd codex-rs cargo +1.95.0 test -p codex-core --lib operation_observation +cargo +1.95.0 test -p codex-core -p codex-tools --lib bounded_capture cargo +1.95.0 test -p codex-core --lib mcp_tool_call_ids_are_added_to_request_meta cargo +1.95.0 test -p codex-api --lib process_sse_reports_body_model_without_routing_authority cargo +1.95.0 fmt --all -- --check @@ -111,7 +114,7 @@ The [binding observer](../../native/codex/src/operation_events.rs) has been prov | Content | Permitted input and output with a separate 32 KiB UTF-8 JSON budget for each direction | | Disabled | No observer | -Raw strings that resemble JSON have been receiving a byte check before parsing. Typed values have been passing through capped serialization before capture has been allocating JSON. Oversized text has been retaining a bounded UTF-8 prefix with truncation markers. Structured values and media have been receiving omission markers. Request identity hashing has been borrowing prepared fields. The upstream `ToolOutput::code_mode_result` conversion has been remaining outside auxiliary capture-memory accounting. +Raw strings that resemble JSON have been receiving a byte check before parsing. Typed values have been passing through capped serialization before capture has been allocating JSON. Oversized text has been retaining a bounded UTF-8 prefix with truncation markers. Structured values and media have been receiving omission markers. Request identity hashing has been borrowing prepared fields. Tool-result capture has been using the opt-in `ToolOutput::bounded_code_mode_result` path before any owned result conversion. JSON and public MCP fields have been passing through capped serialization while borrowed; MCP `_meta` has been remaining outside that serializer. Common function, search, shell, aborted, patch, and wrapper outputs have been bounding their source before cloning or rendering. Oversized source has been producing `capture_budget` omission markers. Other output implementations have been producing `unsupported_output` omission markers without invoking their general output converter. Metadata mode has been skipping output conversion entirely. Tool errors have been receiving a source-byte check before error formatting. Content has been staying in memory, outside rollout storage and diagnostic logs. HTTP and WebSocket observations have been including the final submitted input and continuation fields. Identical-input retries have been sharing one model operation. Changed semantic input has been starting another. Response-body model names have been supplying observation metadata without changing routing authority. Stream completion has been ending the model observation before session code has been draining tool futures. @@ -119,21 +122,24 @@ Native tool observations have been measuring handler execution after authorizati ## Prepare an upstream change -Keep maintained Rust source and tests in the patch. Leave local Cargo lock changes out. Refresh the patch from the pinned checkout after reviewing source changes. +Keep maintained Rust source and tests in the patch. Leave local Cargo lock changes out. Refresh the patch from the pinned checkout after reviewing source changes. The refresh below has been temporarily removing `0002-native-approval-recovery.patch` so the tracing patch has been containing only its own changes. ```sh cd "$CODEX_CHECKOUT" base=6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 +git apply -R "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" git add -N codex-rs/core/src/operation_observation.rs \ codex-rs/core/src/tools/operation_observation.rs \ codex-rs/core/src/tools/operation_observation_tests.rs git diff --check "$base" candidate=$(mktemp "${PATCH_FILE}.XXXXXX") git -c diff.suppressBlankEmpty=true diff --binary "$base" -- \ - codex-rs/core codex-rs/codex-api codex-rs/otel > "$candidate" + codex-rs/core codex-rs/codex-api codex-rs/otel codex-rs/tools > "$candidate" test -s "$candidate" git apply --stat "$candidate" mv "$candidate" "$PATCH_FILE" +git apply --check "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" +git apply "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" ``` Before opening an upstream PR, apply the refreshed patch in another clean checkout and rerun the Rust checks above. Include the pinned revision, observer API example, capture contract, source hashes, and actual source and installed-wheel test results in the PR. Follow the upstream repository's contribution instructions. From fe89afa2b1560bae3cc3f65a00e9f2861a19576b Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 05:18:48 -0800 Subject: [PATCH 29/33] fix(codex): use stock runtime callbacks and reject unsupported approval recovery --- .github/workflows/codex-preview.yml | 43 +- native/codex/Cargo.toml | 5 +- native/codex/src/bin/codex-sdk-runtime.rs | 3 +- native/codex/src/config.rs | 22 +- native/codex/src/lib.rs | 153 +- native/codex/src/native_approval.rs | 157 - native/codex/src/operation_events.rs | 343 -- native/codex/src/operation_tracing.rs | 456 +++ native/codex/verify_installed.py | 19 +- patches/codex/0001-operation-tracing.patch | 3600 ----------------- .../codex/0002-native-approval-recovery.patch | 2555 ------------ patches/codex/README.md | 151 - scripts/build_codex_wheel.py | 80 +- src/agents/memory/resumable_session.py | 17 - src/agents/models/_codex_items.py | 76 +- src/agents/models/_codex_tracing.py | 326 +- src/agents/models/codex.py | 312 +- src/agents/run.py | 4 - src/agents/run_context.py | 4 - src/agents/run_internal/native_approvals.py | 220 - src/agents/run_internal/resumable_session.py | 7 - src/agents/run_internal/run_steps.py | 2 - src/agents/run_internal/turn_resolution.py | 44 +- src/agents/run_state.py | 73 +- tests/memory/test_native_approval_recovery.py | 529 --- tests/memory/test_native_approval_tracing.py | 90 - tests/memory/test_resumable_session.py | 41 +- .../test_codex_operation_retry_boundary.py | 103 - tests/models/test_codex_pending_approval.py | 166 - tests/models/test_codex_provider.py | 973 +---- tests/models/test_operation_tracing_parity.py | 761 ---- tests/test_build_codex_wheel.py | 47 - 32 files changed, 881 insertions(+), 10501 deletions(-) delete mode 100644 native/codex/src/native_approval.rs delete mode 100644 native/codex/src/operation_events.rs create mode 100644 native/codex/src/operation_tracing.rs delete mode 100644 patches/codex/0001-operation-tracing.patch delete mode 100644 patches/codex/0002-native-approval-recovery.patch delete mode 100644 patches/codex/README.md delete mode 100644 src/agents/run_internal/native_approvals.py delete mode 100644 tests/memory/test_native_approval_recovery.py delete mode 100644 tests/memory/test_native_approval_tracing.py delete mode 100644 tests/models/test_codex_operation_retry_boundary.py delete mode 100644 tests/models/test_codex_pending_approval.py delete mode 100644 tests/models/test_operation_tracing_parity.py delete mode 100644 tests/test_build_codex_wheel.py diff --git a/.github/workflows/codex-preview.yml b/.github/workflows/codex-preview.yml index d48a62e74e..385a1475f4 100644 --- a/.github/workflows/codex-preview.yml +++ b/.github/workflows/codex-preview.yml @@ -6,13 +6,6 @@ on: paths: &wheel_paths - 'src/**' - 'native/**' - - 'patches/codex/**' - - 'tests/models/test_operation_tracing_parity.py' - - 'tests/models/test_codex_mutable_tool_hooks.py' - - 'tests/mcp/helpers.py' - - 'tests/mcp/model_compat.py' - - 'tests/testing_processor.py' - - 'tests/conftest.py' - 'scripts/**' - 'pyproject.toml' - 'uv.lock' @@ -169,7 +162,10 @@ jobs: options=() if [[ "$RUNNER_OS" == Linux ]]; then options+=(--compatibility manylinux_2_28); fi uv run --no-sync python scripts/build_codex_wheel.py \ - --test-native --out "$GITHUB_WORKSPACE/dist" --target "$WHEEL_TARGET" "${options[@]}" + --out "$GITHUB_WORKSPACE/dist" --target "$WHEEL_TARGET" "${options[@]}" + - name: Test native policy and configuration + if: matrix.platform == 'manylinux_2_28_x86_64' + run: cargo test --locked --release --lib --manifest-path native/codex/Cargo.toml - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: codex-wheel-${{ matrix.platform }} @@ -196,18 +192,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - sparse-checkout: | - native/codex/verify_installed.py - pyproject.toml - tests/__init__.py - tests/models/__init__.py - tests/testing_processor.py - tests/conftest.py - tests/models/test_operation_tracing_parity.py - tests/models/test_codex_mutable_tool_hooks.py - tests/mcp/__init__.py - tests/mcp/helpers.py - tests/mcp/model_compat.py + sparse-checkout: native/codex/verify_installed.py sparse-checkout-cone-mode: false - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 with: @@ -229,24 +214,6 @@ jobs: subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', str(wheel)], check=True) shutil.copy2(os.environ['VERIFIER'], 'verify_installed.py') subprocess.run([str(python), '-I', 'verify_installed.py', '--output', 'installed.json'], check=True) - - name: Verify tracing and native tool hooks through the installed native wheel - working-directory: ${{ runner.temp }} - env: - CODEX_TRACING_NATIVE_TEST: '1' - PARITY_TESTS: ${{ github.workspace }}/tests/models/test_operation_tracing_parity.py - run: | - import os, subprocess, sys - from pathlib import Path - python = Path('codex-venv') / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') - subprocess.run([str(python), '-I', '-m', 'pip', 'install', '--only-binary=:all:', - 'pytest>=9.0.3', 'pytest-asyncio', 'opentelemetry-sdk>=1.45.0,<2'], check=True) - tests = Path(os.environ['PARITY_TESTS']) - selected = [str(tests)] - if sys.platform == 'darwin': - # Hosted macOS has been providing the native sandbox required by these tests. - selected.append(str(tests.with_name('test_codex_mutable_tool_hooks.py'))) - subprocess.run([str(python), '-I', '-m', 'pytest', *selected, - '--import-mode=importlib', '-q'], check=True) - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: codex-installed-${{ matrix.platform }}-${{ matrix.python }} diff --git a/native/codex/Cargo.toml b/native/codex/Cargo.toml index 4dd9f8702e..2721505279 100644 --- a/native/codex/Cargo.toml +++ b/native/codex/Cargo.toml @@ -5,8 +5,6 @@ edition = "2024" license = "MIT" publish = false -[workspace] - [lib] name = "_codex_native" crate-type = ["cdylib"] @@ -29,6 +27,7 @@ codex-config = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8 codex-core = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-core-api = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-exec-server = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } +codex-extension-api = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-features = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-http-client = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-hooks = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } @@ -44,8 +43,6 @@ codex-code-mode = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30 codex-model-provider = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-guardian-v2 = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } -codex-extension-api = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } - [dev-dependencies] tempfile = "3" codex-sandboxing = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } diff --git a/native/codex/src/bin/codex-sdk-runtime.rs b/native/codex/src/bin/codex-sdk-runtime.rs index 6c59c1acf0..aec395687e 100644 --- a/native/codex/src/bin/codex-sdk-runtime.rs +++ b/native/codex/src/bin/codex-sdk-runtime.rs @@ -16,8 +16,7 @@ fn relay_stdio(port: u16) -> Result<(), Box> { let _ = requests.shutdown(Shutdown::Write); }); // The process exits when the relay closes, even if Codex still holds stdin open. - // Forward each reply before waiting for another request on the same connection. - let mut stdout = io::stdout(); + let mut stdout = io::stdout().lock(); io::copy(&mut connection, &mut stdout)?; stdout.flush()?; Ok(()) diff --git a/native/codex/src/config.rs b/native/codex/src/config.rs index 0b9c53ebbe..fbf7b44353 100644 --- a/native/codex/src/config.rs +++ b/native/codex/src/config.rs @@ -34,6 +34,7 @@ type Result = std::result::Result; pub struct CoreServices { pub config: Config, pub manager: Arc, + pub operations: Arc, home: PathBuf, cwd: PathBuf, overrides: Vec<(String, toml::Value)>, @@ -447,8 +448,11 @@ pub async fn build_services( } else { "codex-code-mode-host" }); + let operations = Arc::new(crate::operation_tracing::Observers::default()); let manager = Arc::new_cyclic(|weak| { let mut extensions = ExtensionRegistryBuilder::::new(); + extensions.model_request_contributor(operations.clone()); + extensions.tool_lifecycle_contributor(operations.clone()); codex_guardian_v2::install(&mut extensions, auth.clone(), weak.clone()); codex_core::install_agent_message_board(&mut extensions, weak.clone()); codex_mcp_extension::install(&mut extensions); @@ -515,6 +519,7 @@ pub async fn build_services( Ok(CoreServices { config, manager, + operations, home, cwd, overrides, @@ -957,11 +962,6 @@ mod tests { } fn fixture_builder(root: &Path, overrides: &[&str]) -> ConfigBuilder { - let mut overrides: Vec = overrides.iter().map(|s| (*s).to_owned()).collect(); - if cfg!(target_os = "windows") { - // Keep managed workspace-write fixtures from falling back to read-only on Windows. - overrides.insert(0, "windows.sandbox=\"elevated\"".to_owned()); - } ConfigBuilder::default() .codex_home(root.to_path_buf()) .harness_overrides(ConfigOverrides { @@ -971,17 +971,15 @@ mod tests { .loader_overrides(LoaderOverrides::with_managed_config_path_for_tests( root.join("managed.toml"), )) - .cli_overrides(parse_overrides(overrides).unwrap()) + .cli_overrides( + parse_overrides(overrides.iter().map(|s| (*s).to_owned()).collect()).unwrap(), + ) } async fn fixture_config(requirements: &str, overrides: &[&str]) -> (tempfile::TempDir, Config) { let root = tempfile::tempdir().unwrap(); - let canonical_root = codex_core_api::AbsolutePathBuf::from_absolute_path(root.path()) - .unwrap() - .canonicalize() - .unwrap(); - std::fs::write(canonical_root.join("requirements.toml"), requirements).unwrap(); - let config = fixture_builder(canonical_root.as_path(), overrides) + std::fs::write(root.path().join("requirements.toml"), requirements).unwrap(); + let config = fixture_builder(root.path(), overrides) .build() .await .unwrap(); diff --git a/native/codex/src/lib.rs b/native/codex/src/lib.rs index 04342ee245..553efea382 100644 --- a/native/codex/src/lib.rs +++ b/native/codex/src/lib.rs @@ -1,7 +1,6 @@ //! Private, asynchronous Core bindings for the opt-in Codex model provider. mod config; -mod native_approval; -mod operation_events; +mod operation_tracing; mod tool_hooks; use std::path::PathBuf; @@ -10,12 +9,11 @@ use std::sync::{Arc, Once, OnceLock}; use codex_core_api::{ CodexThread, StartIfIdleSubmission, StartThreadOptions, TurnInputRequest, TurnStartOptions, - WithTurnExtensionData, }; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::normalize_dynamic_tool_specs; use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::Op; +use codex_protocol::protocol::{EventMsg, Op}; use codex_protocol::user_input::UserInput; use futures::FutureExt; use futures::future::{BoxFuture, Shared, join_all}; @@ -48,8 +46,8 @@ struct ThreadRecord { thread: Arc, id: ThreadId, model: String, + operations: Arc, cleanup: OnceLock, - operations: operation_events::OperationEvents, } impl ThreadRecord { @@ -59,33 +57,42 @@ impl ThreadRecord { let thread = self.thread.clone(); let services = self.services.clone(); let id = self.id; + let operations = self.operations.clone(); let task = runtime().spawn(async move { - // Each fresh root owns an independent local agent runtime. Fence that - // exact tree before waiting, so native subagents cannot outlive an attempt. - let shutdown = services.manager.request_agent_tree_shutdown(id).await; - let result = match shutdown { - Ok(shutdown) => shutdown.wait().await, - Err(error) => { - let _ = thread.shutdown_and_wait().await; - Err(error) - } - }; - if result.is_ok() { - let ids = services - .manager - .list_agent_subtree_thread_ids(id) - .await - .map_err(|_| "Unable to release the Codex agent tree.".to_owned())?; - for id in ids { - if let Ok(current) = services.manager.get_thread(id).await { - services - .manager - .remove_thread_if_matches(&id, ¤t) - .await; + let result = async { + // Each fresh root owns an independent local agent runtime. Fence that + // exact tree before waiting, so native subagents cannot outlive an attempt. + let shutdown = services.manager.request_agent_tree_shutdown(id).await; + let result = match shutdown { + Ok(shutdown) => shutdown.wait().await, + Err(error) => { + let _ = thread.shutdown_and_wait().await; + Err(error) + } + }; + if result.is_ok() { + let ids = services + .manager + .list_agent_subtree_thread_ids(id) + .await + .map_err(|_| { + "Unable to release the Codex agent tree.".to_owned() + })?; + for id in ids { + if let Ok(current) = services.manager.get_thread(id).await { + services + .manager + .remove_thread_if_matches(&id, ¤t) + .await; + } } } + result.map_err(|_| "Unable to shut down the Codex agent tree.".to_owned()) } - result.map_err(|_| "Unable to shut down the Codex agent tree.".to_owned()) + .await; + // Detach even when a joined shutdown returns an error; the buffer remains drainable. + operations.close(); + result }); async move { task.await @@ -192,14 +199,15 @@ async fn start_thread( let created = Box::pin(services.manager.start_thread(options)) .await .map_err(|_| "Unable to start the Codex thread.".to_owned())?; + let operations = services.operations.root(created.thread_id.to_string()); Ok(Arc::new(ThreadLease { record: Arc::new(ThreadRecord { model: created.session_configured.model, id: created.thread_id, thread: created.thread, services, + operations, cleanup: OnceLock::new(), - operations: operation_events::OperationEvents::new(), }), })) } @@ -391,14 +399,22 @@ struct Thread { #[pymethods] impl Thread { - #[getter] - fn model(&self) -> String { - self.lease.model.clone() + fn enable_tracing(&self) -> PyResult<()> { + self.lease.ensure_open().map_err(py_error)?; + self.lease + .operations + .enable(self.lease.thread.thread_extension_data()); + Ok(()) + } + + fn drain_operations(&self) -> Option { + // Draining a closed lease is intentional: shutdown can finish or cancel observations. + self.lease.operations.buffer.drain() } #[getter] - fn thread_id(&self) -> String { - self.lease.id.to_string() + fn model(&self) -> String { + self.lease.model.clone() } fn tool_hook_orders(&self, server: &str) -> PyResult> { @@ -428,13 +444,11 @@ impl Thread { }) } - #[pyo3(signature = (input_json, output_schema_json, operation_capture=None))] fn start_turn<'py>( &self, py: Python<'py>, input_json: String, output_schema_json: String, - operation_capture: Option, ) -> PyResult> { let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { @@ -445,13 +459,10 @@ impl Thread { let schema: Option = serde_json::from_str(&output_schema_json) .map_err(|_| "Invalid Codex output schema.".to_owned())?; - let capture = lease.operations.initializer(operation_capture.as_deref())?; let request = TurnInputRequest::user_input(input).on_start(TurnStartOptions { final_output_json_schema: schema, ..Default::default() }); - let request = - WithTurnExtensionData::new(request, capture).capture_native_approvals(); match Box::pin(lease.thread.start_turn_if_idle(request)) .await .map_err(|_| "Unable to start the Codex turn.".to_owned())? @@ -472,12 +483,19 @@ impl Thread { pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = async { lease.ensure_open()?; - lease - .operations - .next_event(&lease.thread, &lease.id.to_string()) - .await - } - .await; + let event = lease.thread.next_event().await.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; + match &event.msg { + EventMsg::ExecApprovalRequest(_) + | EventMsg::ApplyPatchApprovalRequest(_) + | EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) => { + return Err("Codex requested interactive approval. This provider supports non-interactive execution only.".to_owned()); + } + _ => {} + } + serde_json::to_string(&event).map_err(|_| "Unable to decode the Codex event.".to_owned()) + }.await; result.map_err(py_error) }) } @@ -499,51 +517,6 @@ impl Thread { }) } - // The approval resolver owns Core events; this reader consumes only observations. - fn next_operation_event<'py>(&self, py: Python<'py>) -> PyResult> { - let lease = self.lease.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - lease.ensure_open().map_err(py_error)?; - Ok(lease - .operations - .next_notification(&lease.id.to_string()) - .await) - }) - } - - #[pyo3(signature = (call_json, request_json, preconditions_json, operation_capture=None))] - fn resolve_approval<'py>( - &self, - py: Python<'py>, - call_json: String, - request_json: String, - preconditions_json: String, - operation_capture: Option, - ) -> PyResult> { - let call = serde_json::from_str(&call_json) - .map_err(|_| py_error("Invalid saved Codex tool call."))?; - let request = serde_json::from_str(&request_json) - .map_err(|_| py_error("Invalid saved Codex approval request."))?; - let preconditions = serde_json::from_str(&preconditions_json) - .map_err(|_| py_error("Invalid saved Codex file preconditions."))?; - let lease = self.lease.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let outcome = async { - lease.ensure_open()?; - let capture = lease.operations.initializer(operation_capture.as_deref())?; - native_approval::resolve(&lease.thread, call, request, preconditions, capture).await - } - .await; - lease.operations.finish_resolution(); - let (output, turn_id, failure) = outcome.map_err(py_error)?; - let notifications = lease - .operations - .drain_notifications(&lease.id.to_string()) - .await; - Ok((output, turn_id, notifications, failure)) - }) - } - fn close<'py>(&self, py: Python<'py>) -> PyResult> { let cleanup = self.lease.begin_close(); pyo3_async_runtimes::tokio::future_into_py( diff --git a/native/codex/src/native_approval.rs b/native/codex/src/native_approval.rs deleted file mode 100644 index d192567b11..0000000000 --- a/native/codex/src/native_approval.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! Resume one native call after the SDK has atomically claimed its saved approval. - -use codex_core_api::{CodexThread, StartIfIdleSubmission}; -use codex_extension_api::ExtensionDataInit; -use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::{EventMsg, Op, ReviewDecision}; - -use crate::NativeResult; - -pub(super) async fn resolve( - thread: &CodexThread, - call: ResponseItem, - expected: EventMsg, - preconditions: Option, - capture: ExtensionDataInit, -) -> NativeResult<(Option, String, Option)> { - let call_id = match (&call, &expected) { - (ResponseItem::FunctionCall { call_id, .. }, EventMsg::ExecApprovalRequest(request)) - if call_id == &request.call_id => - { - call_id.clone() - } - ( - ResponseItem::CustomToolCall { call_id, .. }, - EventMsg::ApplyPatchApprovalRequest(request), - ) if call_id == &request.call_id => call_id.clone(), - _ => return Err("The saved Codex approval does not match its tool call.".into()), - }; - let patch = matches!(expected, EventMsg::ApplyPatchApprovalRequest(_)); - // Restoration dispatches this call through Core. Core checks the saved request - // against its freshly prepared action before it can ask us for the decision. - let StartIfIdleSubmission::Started { turn_id, .. } = - Box::pin(thread.restore_tool_call_if_idle(call, expected, preconditions, Some(capture))) - .await - .map_err(|_| "Could not restore the pending Codex approval.")? - else { - return Err("The Codex thread is busy.".into()); - }; - - let outcome: NativeResult = async { - let mut answered = false; - let mut finished = false; - let mut result = None; - loop { - let event = thread - .next_event() - .await - .map_err(|_| "The restored Codex turn ended unexpectedly.")?; - if event.id != turn_id { - continue; - } - let decision = match event.msg { - EventMsg::ExecApprovalRequest(request) - if !patch - && !answered - && request.call_id == call_id - && request.approval_id.is_none() - && request.turn_id == turn_id - && request - .effective_available_decisions() - .contains(&ReviewDecision::Approved) => - { - Op::ExecApproval { - id: request.call_id, - turn_id: Some(turn_id.clone()), - decision: ReviewDecision::Approved, - } - } - EventMsg::ApplyPatchApprovalRequest(request) - if patch - && !answered - && request.call_id == call_id - && request.turn_id == turn_id => - { - Op::PatchApproval { - id: request.call_id, - decision: ReviewDecision::Approved, - } - } - EventMsg::ExecApprovalRequest(_) - | EventMsg::ApplyPatchApprovalRequest(_) - | EventMsg::RequestPermissions(_) - | EventMsg::RequestUserInput(_) - | EventMsg::ElicitationRequest(_) - | EventMsg::DynamicToolCallRequest(_) => { - return Err("Codex requested another decision while restoring the call.".into()); - } - EventMsg::ExecCommandEnd(end) - if answered && !patch && end.call_id == call_id && end.turn_id == turn_id => - { - finished = true; - continue; - } - EventMsg::PatchApplyEnd(end) - if answered && patch && end.call_id == call_id && end.turn_id == turn_id => - { - finished = true; - continue; - } - EventMsg::RawResponseItem(item) => { - let output = match item.item { - ResponseItem::FunctionCallOutput { - call_id: Some(id), - output, - .. - } if !patch && id == call_id => output, - ResponseItem::CustomToolCallOutput { - call_id: id, - output, - .. - } if patch && id == call_id => output, - _ => continue, - }; - if !answered || result.is_some() { - return Err( - "Codex could not confirm the saved approval before dispatch.".into(), - ); - } - result = output.body.to_text(); - continue; - } - EventMsg::TurnComplete(_) => { - if !finished { - // An exec can yield a handle to a still-running process. That handle - // dies with this thread. Don't checkpoint its uncertain effect as done. - return Err( - "Codex could not confirm that the approved call finished.".into() - ); - } - return result - .ok_or_else(|| "Codex did not complete the approved call.".into()); - } - EventMsg::TurnAborted(_) | EventMsg::Error(_) | EventMsg::ShutdownComplete => { - return Err( - "Codex could not complete the restored approval under current policy." - .into(), - ); - } - _ => continue, - }; - thread - .submit(decision) - .await - .map_err(|_| "Could not submit the saved approval to Codex.")?; - answered = true; - } - } - .await; - match outcome { - Ok(output) => Ok((Some(output), turn_id, None)), - Err(failure) => { - // Close real Core work before the caller drains its cancelled/failed observations. - let _ = thread.shutdown_and_wait().await; - Ok((None, turn_id, Some(failure))) - } - } -} diff --git a/native/codex/src/operation_events.rs b/native/codex/src/operation_events.rs deleted file mode 100644 index 4f1ade2ef7..0000000000 --- a/native/codex/src/operation_events.rs +++ /dev/null @@ -1,343 +0,0 @@ -//! Bounded, best-effort delivery on the thread's existing event stream. - -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex as StdMutex}; - -use codex_core::operation_observation::{ - OperationCaptureMode, OperationEvent, OperationObservation, OperationObserver, OperationPhase, -}; -use codex_core_api::CodexThread; -use codex_extension_api::ExtensionDataInit; -use codex_protocol::protocol::EventMsg; -use serde_json::json; -use tokio::sync::{Mutex, mpsc, watch}; - -const QUEUE_CAPACITY: usize = 64; - -struct Sink { - sender: mpsc::Sender, - dropped: Arc, - accepting: StdMutex, -} - -impl OperationObserver for Sink { - fn emit(&self, event: OperationEvent) { - let method = match event.phase { - OperationPhase::Started => "turn/operation/started", - OperationPhase::Completed => "turn/operation/completed", - }; - let serialized = serde_json::to_string(&json!({"method": method, "params": event})); - let Ok(accepting) = self.accepting.lock() else { - return; - }; - if !*accepting { - return; - } - if serialized.is_err() - || self - .sender - .try_send(serialized.unwrap_or_default()) - .is_err() - { - self.dropped.fetch_add(1, Ordering::Relaxed); - } - } - - fn turn_finished<'a>( - &'a self, - _thread_id: &'a str, - _turn_id: &'a str, - ) -> Pin + Send + 'a>> { - if let Ok(mut accepting) = self.accepting.lock() { - *accepting = false; - } - Box::pin(async {}) - } -} - -struct State { - receiver: mpsc::Receiver, - pending: Option, -} - -pub(crate) struct OperationEvents { - sender: mpsc::Sender, - dropped: Arc, - state: Mutex, - resolution_finished: watch::Sender, -} - -impl OperationEvents { - pub(crate) fn new() -> Self { - let (sender, receiver) = mpsc::channel(QUEUE_CAPACITY); - let (resolution_finished, _) = watch::channel(false); - Self { - sender, - dropped: Arc::new(AtomicU64::new(0)), - state: Mutex::new(State { - receiver, - pending: None, - }), - resolution_finished, - } - } - - pub(crate) fn initializer(&self, mode: Option<&str>) -> Result { - let mode = match mode { - None => None, - Some("metadata") => Some(OperationCaptureMode::Metadata), - Some("content") => Some(OperationCaptureMode::Content), - Some(_) => return Err("Invalid Codex operation capture mode.".to_owned()), - }; - // An empty initializer explicitly clears the previous turn's capture policy. - let mut init = ExtensionDataInit::default(); - if let Some(mode) = mode { - let sink = Arc::new(Sink { - sender: self.sender.clone(), - dropped: self.dropped.clone(), - accepting: StdMutex::new(true), - }); - init.insert(OperationObservation::new(mode, sink)); - } - Ok(init) - } - - // Release an idle observation reader before the resolver drains the final queue. - pub(crate) fn finish_resolution(&self) { - self.resolution_finished.send_replace(true); - } - - pub(crate) async fn next_notification(&self, thread_id: &str) -> Option { - let mut finished = self.resolution_finished.subscribe(); - let mut state = self.state.lock().await; - if let Ok(event) = state.receiver.try_recv() { - return Some(event); - } - let dropped = self.dropped.swap(0, Ordering::Relaxed); - if dropped != 0 { - return Some( - json!({ - "method": "turn/operation/dropped", - "params": {"threadId": thread_id, "droppedCount": dropped}, - }) - .to_string(), - ); - } - if *finished.borrow() { - return None; - } - tokio::select! { - biased; - event = state.receiver.recv() => event, - _ = finished.changed() => None, - } - } - - pub(crate) async fn drain_notifications(&self, thread_id: &str) -> Vec { - let mut state = self.state.lock().await; - let mut events = Vec::new(); - while let Ok(event) = state.receiver.try_recv() { - events.push(event); - } - let dropped = self.dropped.swap(0, Ordering::Relaxed); - if dropped != 0 { - events.push( - json!({ - "method": "turn/operation/dropped", - "params": {"threadId": thread_id, "droppedCount": dropped}, - }) - .to_string(), - ); - } - events - } - - pub(crate) async fn next_event( - &self, - thread: &CodexThread, - thread_id: &str, - ) -> Result { - let mut state = self.state.lock().await; - loop { - if let Ok(event) = state.receiver.try_recv() { - return Ok(event); - } - let dropped = self.dropped.swap(0, Ordering::Relaxed); - if dropped != 0 { - return Ok(json!({ - "method": "turn/operation/dropped", - "params": {"threadId": thread_id, "droppedCount": dropped}, - }) - .to_string()); - } - if let Some(event) = state.pending.take() { - return Ok(event); - } - tokio::select! { - biased; - Some(event) = state.receiver.recv() => return Ok(event), - event = thread.next_event() => { - let event = event.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; - match &event.msg { - EventMsg::ExecApprovalRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) => { - let (request, preconditions) = thread - .capture_tool_approval(event.msg) - .await - .map_err(|_| "This Codex approval cannot be safely resumed.".to_owned())?; - state.pending = Some(json!({ - "id": event.id, - "msg": request, - "preconditions": preconditions, - }).to_string()); - continue; - } - EventMsg::RequestPermissions(_) - | EventMsg::RequestUserInput(_) - | EventMsg::ElicitationRequest(_) => { - return Err("Codex requested an unsupported kind of interactive approval.".to_owned()); - } - _ => {} - } - // Keep the native event across cancellation while draining observations. - // Core emits and closes operation admission before its turn terminal. - state.pending = Some(serde_json::to_string(&event) - .map_err(|_| "Unable to decode the Codex event.".to_owned())?); - } - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use codex_core::operation_observation::{OperationKind, OperationMetadata, OperationOutcome}; - - #[tokio::test] - async fn restoration_delivers_start_before_completion_and_drains_once() { - let events = OperationEvents::new(); - let init = events.initializer(Some("metadata")).unwrap(); - let observation = init.get::().unwrap(); - let guard = observation.start( - "thread_1".into(), - "turn_1".into(), - OperationKind::Tool, - OperationMetadata::default(), - ); - let start = events.next_notification("thread_1").await.unwrap(); - assert_eq!( - serde_json::from_str::(&start).unwrap()["method"], - "turn/operation/started" - ); - guard.finish(OperationOutcome::Success); - events.finish_resolution(); - let terminal = events.drain_notifications("thread_1").await; - assert_eq!(terminal.len(), 1); - assert_eq!( - serde_json::from_str::(&terminal[0]).unwrap()["method"], - "turn/operation/completed" - ); - assert!(events.next_notification("thread_1").await.is_none()); - } - - #[tokio::test] - async fn restoration_completion_releases_waiting_reader_before_drain() { - let events = OperationEvents::new(); - let reader = async { - assert!(events.next_notification("thread_1").await.is_none()); - }; - let finish = async { - tokio::task::yield_now().await; - events.finish_resolution(); - assert!(events.drain_notifications("thread_1").await.is_empty()); - }; - tokio::time::timeout(std::time::Duration::from_secs(1), async { - tokio::join!(reader, finish); - }) - .await - .unwrap(); - } - - #[tokio::test] - async fn delivery_preserves_source_identity_and_per_turn_capture() { - for mode in ["metadata", "content"] { - let events = OperationEvents::new(); - let init = events.initializer(Some(mode)).unwrap(); - let observation = init.get::().unwrap(); - let guard = observation.start( - "thread_1".into(), - "turn_1".into(), - OperationKind::Tool, - OperationMetadata { - tool_name: Some("lookup".into()), - ..Default::default() - }, - ); - guard.finish_with_output(OperationOutcome::Success, &json!({"text": "visible"})); - let notifications = events.drain_notifications("thread_1").await; - assert_eq!(notifications.len(), 2); - let start: serde_json::Value = serde_json::from_str(¬ifications[0]).unwrap(); - let end: serde_json::Value = serde_json::from_str(¬ifications[1]).unwrap(); - assert_eq!(start["method"], "turn/operation/started"); - assert_eq!(end["method"], "turn/operation/completed"); - assert_eq!(start["params"]["operationId"], end["params"]["operationId"]); - assert_eq!(end["params"]["threadId"], "thread_1"); - assert_eq!(end["params"]["turnId"], "turn_1"); - assert_eq!(end["params"]["outcome"], "success"); - assert!(end["params"]["completedAt"].is_string()); - assert_eq!(end["params"].get("output").is_some(), mode == "content"); - assert!( - events - .initializer(None) - .unwrap() - .get::() - .is_none() - ); - } - } - - #[tokio::test] - async fn queue_overflow_stays_bounded_and_reports_loss() { - let events = OperationEvents::new(); - let init = events.initializer(Some("metadata")).unwrap(); - let observation = init.get::().unwrap(); - for _ in 0..QUEUE_CAPACITY { - let guard = observation.start( - "thread_1".into(), - "turn_1".into(), - OperationKind::Model, - OperationMetadata::default(), - ); - guard.finish(OperationOutcome::Success); - } - let state = events.state.lock().await; - assert_eq!(state.receiver.len(), QUEUE_CAPACITY); - assert_eq!( - events.dropped.load(Ordering::Relaxed), - QUEUE_CAPACITY as u64 - ); - } - - #[tokio::test] - async fn closing_turn_admission_suppresses_late_guard_drop() { - let (sender, mut receiver) = mpsc::channel(QUEUE_CAPACITY); - let sink = Arc::new(Sink { - sender, - dropped: Arc::new(AtomicU64::new(0)), - accepting: StdMutex::new(true), - }); - let observation = OperationObservation::new(OperationCaptureMode::Metadata, sink.clone()); - let guard = observation.start( - "thread_1".into(), - "turn_1".into(), - OperationKind::Model, - OperationMetadata::default(), - ); - assert!(receiver.try_recv().is_ok()); - sink.turn_finished("thread_1", "turn_1").await; - drop(guard); - assert!(receiver.try_recv().is_err()); - } -} diff --git a/native/codex/src/operation_tracing.rs b/native/codex/src/operation_tracing.rs new file mode 100644 index 0000000000..638e5a7f92 --- /dev/null +++ b/native/codex/src/operation_tracing.rs @@ -0,0 +1,456 @@ +//! Optional, passive metadata observations for SDK-owned ephemeral root threads. +//! Stock request streams and tool execution remain unchanged. Warmups, child threads, +//! content capture, transport-level timing, and generic tool outputs are out of scope. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use codex_extension_api::{ + ExtensionData, ModelRequestContributor, ModelRequestInput, ModelRequestKind, + ModelResponseError, ModelResponseInterceptor, ModelResponseStream, ResponseEvent, + ToolCallOutcome, ToolFinishInput, ToolLifecycleContributor, ToolLifecycleFuture, + ToolStartInput, +}; +use codex_protocol::ToolName; +use codex_protocol::protocol::TokenUsage; +use futures::StreamExt; +use serde_json::{Value, json}; + +const MAX_EVENTS: usize = 512; +const MAX_STRING_BYTES: usize = 256; + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or_default() +} + +#[derive(Debug, Default)] +pub(crate) struct Buffer { + events: Mutex, + sequence: AtomicU64, +} + +#[derive(Debug, Default)] +struct Events { + items: Vec, + dropped: bool, +} + +impl Buffer { + fn push(&self, event: Value) { + let mut events = self + .events + .lock() + .unwrap_or_else(|error| error.into_inner()); + if events.items.len() < MAX_EVENTS { + events.items.push(event); + } else { + events.dropped = true; + } + } + + fn bounded(&self, value: &str) -> bool { + let valid = !value.is_empty() && value.len() <= MAX_STRING_BYTES; + if !valid { + self.events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .dropped = true; + } + valid + } + + pub(crate) fn drain(&self) -> Option { + let events = { + let mut buffered = self + .events + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut events = std::mem::take(&mut buffered.items); + if std::mem::take(&mut buffered.dropped) { + events.push(json!({"kind": "dropped"})); + } + events + }; + if events.is_empty() { + return None; + } + Some(serde_json::to_string(&events).unwrap_or_else(|_| "[{\"kind\":\"dropped\"}]".into())) + } + + fn tool(&self, phase: &str, call_id: &str, name: &ToolName, outcome: Option<&str>) { + if !self.bounded(call_id) + || !self.bounded(&name.name) + || name + .namespace + .as_deref() + .is_some_and(|value| !self.bounded(value)) + { + return; + } + let mut event = json!({ + "kind": "tool", "phase": phase, "id": call_id, "at": now(), + "name": name.name, "namespace": name.namespace, + }); + if let Some(outcome) = outcome { + event["outcome"] = json!(outcome); + } + self.push(event); + } +} + +#[derive(Debug, Default)] +pub(crate) struct Observers { + roots: Mutex>>, +} + +impl Observers { + pub(crate) fn root(self: &Arc, id: String) -> Arc { + Arc::new(Root { + id, + observers: self.clone(), + buffer: Arc::new(Buffer::default()), + enabled: AtomicBool::new(false), + }) + } + + fn buffer(&self, id: &str) -> Option> { + self.roots + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(id)? + .upgrade() + } +} + +pub(crate) struct Root { + id: String, + observers: Arc, + pub(crate) buffer: Arc, + enabled: AtomicBool, +} + +impl Root { + pub(crate) fn enable(&self, store: &ExtensionData) { + if !self.enabled.swap(true, Ordering::AcqRel) { + store.insert_arc(self.buffer.clone()); + self.observers + .roots + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(self.id.clone(), Arc::downgrade(&self.buffer)); + } + } + + pub(crate) fn close(&self) { + let mut roots = self + .observers + .roots + .lock() + .unwrap_or_else(|error| error.into_inner()); + if roots + .get(&self.id) + .is_some_and(|buffer| buffer.ptr_eq(&Arc::downgrade(&self.buffer))) + { + roots.remove(&self.id); + } + } +} + +impl Drop for Root { + fn drop(&mut self) { + self.close(); + } +} + +struct ModelObservation { + buffer: Arc, + id: String, + finished: bool, +} + +impl ModelObservation { + fn finish(&mut self, outcome: &str, usage: Option<&TokenUsage>) { + if self.finished { + return; + } + self.finished = true; + let mut event = json!({ + "kind": "model", "phase": "end", "id": self.id, + "at": now(), "outcome": outcome, + }); + if let Some(usage) = usage { + event["usage"] = json!(usage); + } + self.buffer.push(event); + } +} + +impl Drop for ModelObservation { + fn drop(&mut self) { + self.finish("interrupted", None); + } +} + +impl ModelResponseInterceptor for ModelObservation { + fn intercept(self: Box, stream: ModelResponseStream) -> ModelResponseStream { + let mut observation = *self; + // Inspect borrows each event and yields the original event without buffering or rewriting. + Box::pin(stream.inspect(move |event| match event { + Ok(ResponseEvent::Completed { token_usage, .. }) => { + observation.finish("completed", token_usage.as_ref()); + } + Err(ModelResponseError::ResponseIncomplete(error)) => { + observation.finish("failed", error.token_usage.as_ref()); + } + Err(_) => observation.finish("failed", None), + _ => {} + })) + } +} + +impl ModelRequestContributor for Observers { + fn request(&self, input: ModelRequestInput<'_>) -> Option> { + if input.kind != ModelRequestKind::Generation { + return None; + } + let buffer = self.buffer(input.thread_id)?; + if !buffer.bounded(input.model) { + return None; + } + let id = format!("model-{}", buffer.sequence.fetch_add(1, Ordering::Relaxed)); + buffer.push(json!({ + "kind": "model", "phase": "start", "id": id, + "at": now(), "model": input.model, + })); + Some(Box::new(ModelObservation { + buffer, + id, + finished: false, + })) + } +} + +impl ToolLifecycleContributor for Observers { + fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() { + // A handler start can still encounter an approval denial before any effect occurs. + buffer.tool("start", input.call_id, input.tool_name, None); + } + Box::pin(std::future::ready(())) + } + + fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() { + let outcome = match input.outcome { + ToolCallOutcome::Completed { success: true } => "completed", + ToolCallOutcome::Aborted => "interrupted", + _ => "failed", + }; + // Stock finish callbacks may arrive without start when policy or cancellation wins. + buffer.tool("end", input.call_id, input.tool_name, Some(outcome)); + } + Box::pin(std::future::ready(())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::stream; + + fn request( + observers: &Observers, + root: &str, + kind: ModelRequestKind, + ) -> Option> { + observers.request(ModelRequestInput { + kind, + thread_id: root, + client_metadata: &mut None, + model: "fixture-model", + }) + } + + fn drain(buffer: &Buffer) -> Vec { + buffer + .drain() + .map(|value| serde_json::from_str(&value).unwrap()) + .unwrap_or_default() + } + + #[tokio::test] + async fn model_observation_forwards_original_events_and_keeps_only_metadata() { + let observers = Arc::new(Observers::default()); + let root = observers.root("root".into()); + assert!(request(&observers, "root", ModelRequestKind::Generation).is_none()); + root.enable(&ExtensionData::new("root")); + assert!(request(&observers, "child", ModelRequestKind::Generation).is_none()); + assert!(request(&observers, "root", ModelRequestKind::Warmup).is_none()); + let interceptor = request(&observers, "root", ModelRequestKind::Generation).unwrap(); + let usage = TokenUsage { + input_tokens: 11, + output_tokens: 3, + total_tokens: 14, + ..Default::default() + }; + let mut observed = interceptor.intercept(Box::pin(stream::iter([ + Ok(ResponseEvent::Created { + response_id: Some("private-response-id".into()), + }), + Ok(ResponseEvent::Completed { + response_id: "private-response-id".into(), + token_usage: Some(usage), + usage_metadata: None, + end_turn: Some(true), + }), + ]))); + assert!( + matches!(observed.next().await, Some(Ok(ResponseEvent::Created { response_id: Some(id) })) if id == "private-response-id") + ); + assert!( + matches!(observed.next().await, Some(Ok(ResponseEvent::Completed { response_id, .. })) if response_id == "private-response-id") + ); + assert!(observed.next().await.is_none()); + drop(observed); + let events = drain(&root.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[0]["id"], events[1]["id"]); + assert_eq!(events[0]["model"], "fixture-model"); + assert_eq!(events[1]["usage"]["total_tokens"], 14); + assert_eq!(events[1]["outcome"], "completed"); + assert!(events[0]["at"].as_u64().unwrap() > 1_000_000_000_000); + assert!(!format!("{events:?}").contains("private-response-id")); + assert!(root.buffer.drain().is_none()); + } + + #[tokio::test] + async fn failure_drop_and_close_remain_isolated_and_drainable() { + let observers = Arc::new(Observers::default()); + let first = observers.root("first".into()); + let second = observers.root("second".into()); + first.enable(&ExtensionData::new("first")); + second.enable(&ExtensionData::new("second")); + let pending = request(&observers, "second", ModelRequestKind::Generation).unwrap(); + let interceptor = request(&observers, "first", ModelRequestKind::Generation).unwrap(); + let mut observed = interceptor.intercept(Box::pin(stream::iter([Err( + ModelResponseError::Stream("private failure detail".into()), + )]))); + assert!( + matches!(observed.next().await, Some(Err(ModelResponseError::Stream(message))) if message == "private failure detail") + ); + drop(observed); + first.close(); + assert!(request(&observers, "first", ModelRequestKind::Generation).is_none()); + let events = drain(&first.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[1]["outcome"], "failed"); + assert!(!format!("{events:?}").contains("private failure detail")); + assert_eq!(drain(&second.buffer).len(), 1); + second.close(); + drop(pending); + assert_eq!(drain(&second.buffer)[0]["outcome"], "interrupted"); + } + + #[tokio::test] + async fn tool_outcomes_preserve_namespace_without_payloads_or_false_success() { + let observers = Observers::default(); + let store = ExtensionData::new("root"); + let buffer = Arc::new(Buffer::default()); + store.insert_arc(buffer.clone()); + let name = ToolName::namespaced("functions", "lookup"); + let outcomes = [ + ToolCallOutcome::Completed { success: true }, + ToolCallOutcome::Completed { success: false }, + ToolCallOutcome::Blocked, + ToolCallOutcome::Failed { + handler_executed: false, + }, + ToolCallOutcome::Aborted, + ]; + for (index, outcome) in outcomes.into_iter().enumerate() { + let id = format!("call-{index}"); + buffer.tool("start", &id, &name, None); + observers + .on_tool_finish(ToolFinishInput { + session_store: &store, + thread_store: &store, + turn_store: &store, + turn_id: "turn", + call_id: &id, + tool_name: &name, + source: codex_extension_api::ToolCallSource::Direct, + outcome, + }) + .await; + } + let events = drain(&buffer); + let outcomes = ["completed", "failed", "failed", "failed", "interrupted"]; + for (pair, outcome) in events.chunks_exact(2).zip(outcomes) { + assert_eq!(pair[0]["id"], pair[1]["id"]); + assert_eq!(pair[0]["namespace"], "functions"); + assert_eq!(pair[1]["namespace"], "functions"); + assert_eq!(pair[1]["name"], "lookup"); + assert_eq!(pair[1]["outcome"], outcome); + assert!( + pair.iter() + .all(|event| event.get("input").is_none() && event.get("output").is_none()) + ); + } + } + + #[test] + fn bounds_events_and_omits_oversized_identifiers_without_truncating() { + let buffer = Buffer::default(); + buffer.tool( + "start", + &"x".repeat(MAX_STRING_BYTES + 1), + &ToolName::plain("tool"), + None, + ); + assert_eq!(drain(&buffer), vec![json!({"kind": "dropped"})]); + for index in 0..MAX_EVENTS + 2 { + buffer.tool("start", &index.to_string(), &ToolName::plain("tool"), None); + } + let events = drain(&buffer); + assert_eq!(events.len(), MAX_EVENTS + 1); + assert_eq!(events.last().unwrap(), &json!({"kind": "dropped"})); + assert!(buffer.drain().is_none()); + } + + #[test] + fn concurrent_handlers_and_drains_do_not_drop_observations() { + let buffer = Arc::new(Buffer::default()); + let completed = Arc::new(AtomicU64::new(0)); + let received = std::thread::scope(|scope| { + for worker in 0..4 { + let buffer = buffer.clone(); + let completed = completed.clone(); + scope.spawn(move || { + for item in 0..32 { + buffer.tool( + "start", + &format!("{worker}-{item}"), + &ToolName::plain("tool"), + None, + ); + } + completed.fetch_add(1, Ordering::Release); + }); + } + let mut events = Vec::new(); + while completed.load(Ordering::Acquire) != 4 { + events.extend(drain(&buffer)); + std::thread::yield_now(); + } + events.extend(drain(&buffer)); + events + }); + assert_eq!(received.len(), 128); + assert!(received.iter().all(|event| event["kind"] == "tool")); + } +} diff --git a/native/codex/verify_installed.py b/native/codex/verify_installed.py index bb46961639..637e2543a9 100644 --- a/native/codex/verify_installed.py +++ b/native/codex/verify_installed.py @@ -135,11 +135,14 @@ def respond(request): async def native_route(folder): + from unittest.mock import Mock + from pydantic import BaseModel from agents import Agent, RunConfig, Runner, RunState from agents.decorators import tool from agents.models.codex import CodexProvider + from agents.tracing import TracingProcessor, set_trace_processors, set_tracing_disabled class Answer(BaseModel): label: str @@ -287,7 +290,15 @@ def lookup(value: str) -> str: encoding="utf-8", ) provider = CodexProvider(cwd=folder / "workspace", api_key="synthetic-installed-key") - config = RunConfig(model_provider=provider, tracing_disabled=True) + # The isolated worker collects spans locally, without a network exporter. + processor = Mock(spec=TracingProcessor) + set_trace_processors([processor]) + set_tracing_disabled(False) + config = RunConfig( + model_provider=provider, + trace_include_sensitive_data=False, + tracing={"operation_detail": "metadata"}, + ) try: paused = Runner.run_streamed(agent, "private-input", run_config=config) async for event in paused.stream_events(): @@ -317,7 +328,13 @@ def lookup(value: str) -> str: ) assert usage.input_tokens_details.cached_tokens == 6 assert usage.output_tokens_details.reasoning_tokens == 3 + spans = [call.args[0].span_data for call in processor.on_span_end.call_args_list] + samples = [span for span in spans if span.type == "generation"] + assert len(samples) == 3 + assert all(span.input is None and span.output is None for span in samples) + assert all(span.usage["input_tokens"] == 12 for span in samples) + config.tracing_disabled = True streamed = Runner.run_streamed(plain, "fresh-input", run_config=config) deltas = [] async for event in streamed.stream_events(): diff --git a/patches/codex/0001-operation-tracing.patch b/patches/codex/0001-operation-tracing.patch deleted file mode 100644 index 494e19f304..0000000000 --- a/patches/codex/0001-operation-tracing.patch +++ /dev/null @@ -1,3600 +0,0 @@ -diff --git a/codex-rs/codex-api/src/common.rs b/codex-rs/codex-api/src/common.rs -index 2510f35..0a4af6c 100644 ---- a/codex-rs/codex-api/src/common.rs -+++ b/codex-rs/codex-api/src/common.rs -@@ -89,6 +89,8 @@ pub enum ResponseEvent { - /// Emitted when the server includes `OpenAI-Model` on the stream response. - /// This can differ from the requested model when backend safety routing applies. - ServerModel(String), -+ /// The response body's model name, for observation only, without routing authority. -+ ModelObservation(String), - /// Emitted when the server recommends additional account verification. - ModelVerifications(Vec), - /// Emitted when the server includes moderation metadata for first-party turn presentation. -diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs -index f43e285..d330da5 100644 ---- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs -+++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs -@@ -907,6 +907,11 @@ async fn run_websocket_response_stream( - } - continue; - } -+ if let Some(model) = event.observation_model() { -+ let _ = tx_event -+ .send(Ok(ResponseEvent::ModelObservation(model))) -+ .await; -+ } - let model = event.response_model(); - // Preserve terminal usage before any accompanying notification can block. - let event = match process_responses_event(event) { -diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs -index 67f627e..1e6c9cd 100644 ---- a/codex-rs/codex-api/src/sse/responses.rs -+++ b/codex-rs/codex-api/src/sse/responses.rs -@@ -190,6 +190,16 @@ impl ResponsesStreamEvent { - &self.kind - } - -+ /// Returns the body's model name for observation, without routing authority. -+ pub fn observation_model(&self) -> Option { -+ self.response -+ .as_ref()? -+ .get("model")? -+ .as_str() -+ .filter(|model| !model.is_empty()) -+ .map(str::to_owned) -+ } -+ - /// Returns the effective model reported by the server, if present. - /// - /// Precedence: -@@ -637,6 +647,15 @@ async fn process_sse_with_treatment( - let safety_buffering = event.safety_buffering(&safety_buffering_treatment); - let metadata_events = event.metadata_events(safety_buffering); - -+ if let Some(model) = event.observation_model() { -+ if tx_event -+ .send(Ok(ResponseEvent::ModelObservation(model))) -+ .await -+ .is_err() -+ { -+ return; -+ } -+ } - if let Some(model) = event.response_model() - && last_server_model.as_deref() != Some(model.as_str()) - { -@@ -1613,7 +1632,7 @@ mod tests { - } - - #[tokio::test] -- async fn process_sse_ignores_response_model_field_in_payload() { -+ async fn process_sse_reports_body_model_without_routing_authority() { - let events = run_sse(vec![ - json!({ - "type": "response.created", -@@ -1632,10 +1651,17 @@ mod tests { - ]) - .await; - -- assert_eq!(events.len(), 2); -- assert_matches!(&events[0], ResponseEvent::Created { .. }); -+ assert_eq!(events.len(), 4); -+ assert_matches!(&events[0], ResponseEvent::ModelObservation(model) if model == CYBER_RESTRICTED_MODEL_FOR_TESTS); -+ assert_matches!(&events[1], ResponseEvent::Created { .. }); -+ assert_matches!(&events[2], ResponseEvent::ModelObservation(model) if model == CYBER_RESTRICTED_MODEL_FOR_TESTS); -+ assert!( -+ !events -+ .iter() -+ .any(|event| matches!(event, ResponseEvent::ServerModel(_))) -+ ); - assert_matches!( -- &events[1], -+ &events[3], - ResponseEvent::Completed { - response_id, - token_usage: None, -diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs -index 2ac9b93..8a01fc3 100644 ---- a/codex-rs/core/src/client.rs -+++ b/codex-rs/core/src/client.rs -@@ -2033,6 +2033,14 @@ impl ResponsesHttpExecutor<'_> { - request.input = input; - } - inference_trace_attempt.record_started(&request); -+ crate::operation_observation::observe_model_request( -+ &crate::operation_observation::ModelRequestContent { -+ input: &request.input, -+ previous_response_id: None, -+ }, -+ &request, -+ &client_setup.api_provider.base_url, -+ ); - let client = ApiResponsesClient::new( - transport, - client_setup.api_provider, -@@ -2185,6 +2193,7 @@ impl ModelClientSession { - } else { - session_telemetry_for_request(session_telemetry, &request) - }; -+ let operation_endpoint = client_setup.api_provider.base_url.clone(); - match self - .websocket_connection(WebsocketConnectParams { - session_telemetry, -@@ -2341,6 +2350,17 @@ impl ModelClientSession { - inference_trace_attempt.record_started(&ws_request); - } - -+ if !warmup { -+ let ResponsesWsRequest::ResponseCreate(payload) = &ws_request; -+ crate::operation_observation::observe_model_request( -+ &crate::operation_observation::ModelRequestContent { -+ input: payload.input, -+ previous_response_id: payload.previous_response_id.as_deref(), -+ }, -+ &request, -+ &operation_endpoint, -+ ); -+ } - let websocket_connection = - self.websocket_session.connection.as_ref().ok_or_else(|| { - self.client.state.provider.map_api_error(ApiError::Stream( -@@ -2686,6 +2706,8 @@ where - mpsc::channel::>(RESPONSE_STREAM_CHANNEL_CAPACITY); - let consumer_dropped = CancellationToken::new(); - let consumer_dropped_for_stream = consumer_dropped.clone(); -+ let operation_observation = crate::operation_observation::current_model_operation() -+ .map(crate::operation_observation::ModelOperation::for_stream); - - tokio::spawn(async move { - let mut logged_error = false; -@@ -2702,6 +2724,7 @@ where - loop { - let event = tokio::select! { - _ = consumer_dropped.cancelled() => { -+ if let Some(operation) = &operation_observation { operation.source_ended(); } - inference_trace_attempt.record_cancelled( - STREAM_DROPPED_REASON, - upstream_request_id, -@@ -2712,8 +2735,21 @@ where - event = api_stream.next() => event, - }; - let Some(event) = event else { -+ if let Some(operation) = &operation_observation { -+ operation.source_ended(); -+ } - break; - }; -+ if let (Some(operation), Err(error)) = (&operation_observation, &event) { -+ operation.source_error(error); -+ } -+ if let (Some(operation), Ok(event)) = (&operation_observation, &event) { -+ operation.event(event); -+ } -+ // Body model names are observation facts, never session-routing updates. -+ if matches!(&event, Ok(ResponseEvent::ModelObservation(_))) { -+ continue; -+ } - match event { - Ok(ResponseEvent::OutputItemDone(mut item)) => { - // Assign IDs after interceptors, which may replace provider items. -diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs -index 61400a1..dcd01e4 100644 ---- a/codex-rs/core/src/lib.rs -+++ b/codex-rs/core/src/lib.rs -@@ -10,6 +10,7 @@ mod apps; - mod client; - mod client_common; - mod model_request; -+pub mod operation_observation; - mod realtime_context; - mod realtime_conversation; - mod realtime_history; -diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs -index 8c4ce1f..c6efa68 100644 ---- a/codex-rs/core/src/mcp_tool_call.rs -+++ b/codex-rs/core/src/mcp_tool_call.rs -@@ -559,6 +559,9 @@ async fn handle_approved_mcp_tool_call( - request_meta, - ) - .await?; -+ crate::tools::operation_observation::start_current_execution_with_input( -+ &tool_input, -+ ); - let mcp_call_trace = sess - .services - .rollout_thread_trace -diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs -index 9f4ef92..34a88e0 100644 ---- a/codex-rs/core/src/mcp_tool_call_tests.rs -+++ b/codex-rs/core/src/mcp_tool_call_tests.rs -@@ -1847,6 +1847,7 @@ async fn codex_apps_auth_elicitation_enabled_by_default_requests_elicitation( - #[test] - fn mcp_tool_call_ids_are_added_to_request_meta() { - let origin = crate::tools::context::ToolCallOrigin { -+ operation_id: None, - item_id: Some(ResponseItemId::from_server("fc-live".to_string())), - window_id: "thread-live:2".to_string(), - }; -diff --git a/codex-rs/core/src/operation_observation.rs b/codex-rs/core/src/operation_observation.rs -new file mode 100644 -index 0000000..ccea56b ---- /dev/null -+++ b/codex-rs/core/src/operation_observation.rs -@@ -0,0 +1,1987 @@ -+//! Opt-in, in-memory operation facts for the host that owns a turn. -+//! -+//! These observations never enter the rollout or the telemetry logger. The host -+//! owns delivery and tracing policy; core owns operation boundaries and content. -+ -+use chrono::SecondsFormat; -+use chrono::Utc; -+use futures::FutureExt; -+use serde::Serialize; -+use serde_json::Value; -+use std::future::Future; -+use std::pin::Pin; -+use std::sync::Arc; -+use std::sync::Mutex; -+use std::time::Instant; -+ -+pub const CONTENT_LIMIT_BYTES: usize = 32 * 1024; -+pub(crate) const CONTENT_VALUE_LIMIT_BYTES: usize = CONTENT_LIMIT_BYTES - 256; -+ -+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub enum OperationCaptureMode { -+ Metadata, -+ Content, -+} -+ -+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub enum OperationKind { -+ Model, -+ Tool, -+} -+ -+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub enum OperationOutcome { -+ Success, -+ Error, -+ Interrupted, -+} -+ -+#[derive(Clone, Copy, Debug, PartialEq, Eq)] -+pub enum OperationPhase { -+ Started, -+ Completed, -+} -+ -+#[derive(Clone, Debug, Default, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub struct OperationMetadata { -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub parent_operation_id: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub requested_model: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub observed_model: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub response_id: Option, -+ pub response_id_trusted: bool, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub usage: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub tool_name: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub call_id: Option, -+} -+ -+#[derive(Clone, Debug, PartialEq, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub struct OperationContent { -+ pub value: Value, -+ pub truncated: bool, -+ pub omitted: bool, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub reason: Option, -+} -+ -+#[derive(Clone, Debug, Serialize)] -+#[serde(rename_all = "camelCase")] -+pub struct OperationEvent { -+ #[serde(skip)] -+ pub phase: OperationPhase, -+ pub operation_id: String, -+ pub thread_id: String, -+ pub turn_id: String, -+ pub kind: OperationKind, -+ pub started_at: String, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub completed_at: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub duration_ms: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub outcome: Option, -+ #[serde(flatten)] -+ pub metadata: OperationMetadata, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub input: Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub output: Option, -+} -+ -+pub trait OperationObserver: Send + Sync { -+ /// Must return promptly. Delivery failures must not affect execution. -+ fn emit(&self, event: OperationEvent); -+ -+ /// Closes admission for this source turn before its terminal event. -+ fn turn_finished<'a>( -+ &'a self, -+ _thread_id: &'a str, -+ _turn_id: &'a str, -+ ) -> Pin + Send + 'a>> { -+ Box::pin(async {}) -+ } -+} -+ -+#[derive(Clone)] -+pub struct OperationObservation { -+ mode: OperationCaptureMode, -+ sink: Arc, -+} -+ -+impl std::fmt::Debug for OperationObservation { -+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -+ f.debug_struct("OperationObservation") -+ .field("mode", &self.mode) -+ .finish_non_exhaustive() -+ } -+} -+ -+impl OperationObservation { -+ pub fn new(mode: OperationCaptureMode, sink: Arc) -> Self { -+ Self { mode, sink } -+ } -+ -+ pub fn captures_content(&self) -> bool { -+ self.mode == OperationCaptureMode::Content -+ } -+ -+ pub fn start( -+ &self, -+ thread_id: String, -+ turn_id: String, -+ kind: OperationKind, -+ metadata: OperationMetadata, -+ ) -> OperationGuard { -+ let started = Instant::now(); -+ let event = OperationEvent { -+ phase: OperationPhase::Started, -+ operation_id: uuid::Uuid::new_v4().to_string(), -+ thread_id, -+ turn_id, -+ kind, -+ started_at: timestamp(), -+ completed_at: None, -+ duration_ms: None, -+ outcome: None, -+ metadata, -+ input: None, -+ output: None, -+ }; -+ self.emit(event.clone()); -+ OperationGuard { -+ operation_id: event.operation_id.clone(), -+ observation: self.clone(), -+ started, -+ state: Mutex::new(event), -+ } -+ } -+ -+ pub async fn turn_finished(&self, thread_id: &str, turn_id: &str) { -+ let future = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { -+ self.sink.turn_finished(thread_id, turn_id) -+ })); -+ if let Ok(future) = future { -+ let _ = std::panic::AssertUnwindSafe(future).catch_unwind().await; -+ } -+ } -+ -+ fn emit(&self, event: OperationEvent) { -+ // An optional host observer must never unwind an agent operation. -+ let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.sink.emit(event))); -+ } -+} -+ -+tokio::task_local! { -+ static CURRENT_MODEL_OPERATION: ModelOperation; -+} -+ -+pub struct OperationGuard { -+ operation_id: String, -+ observation: OperationObservation, -+ started: Instant, -+ state: Mutex, -+} -+ -+impl OperationGuard { -+ pub fn operation_id(&self) -> &str { -+ &self.operation_id -+ } -+ -+ pub fn capture_input(&self, value: &Value) { -+ if self.observation.mode == OperationCaptureMode::Content { -+ self.update(|event| event.input = Some(project_content(value))); -+ } -+ } -+ -+ pub(crate) fn capture_input_content(&self, content: OperationContent) { -+ if self.observation.captures_content() { -+ self.update(|event| event.input = Some(content)); -+ } -+ } -+ -+ pub fn capture_output(&self, value: &Value) { -+ if self.observation.mode == OperationCaptureMode::Content { -+ self.update(|event| event.output = Some(project_content(value))); -+ } -+ } -+ -+ fn update(&self, update: impl FnOnce(&mut OperationEvent)) { -+ let mut event = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if event.phase == OperationPhase::Started { -+ update(&mut event); -+ } -+ } -+ -+ pub fn finish_with_output_fn(&self, outcome: OperationOutcome, output: impl FnOnce() -> Value) { -+ let completed_at = timestamp(); -+ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; -+ if self.observation.captures_content() { -+ self.capture_output(&output()); -+ } -+ self.finish_at(outcome, completed_at, duration_ms); -+ } -+ -+ pub(crate) fn finish_with_bounded_output_fn( -+ &self, -+ outcome: OperationOutcome, -+ output: impl FnOnce() -> Result, -+ ) { -+ let completed_at = timestamp(); -+ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; -+ if self.observation.captures_content() { -+ let content = match output() { -+ Ok(value) => project_content(&value), -+ Err(reason) => omitted_content(reason), -+ }; -+ self.update(|event| event.output = Some(content)); -+ } -+ self.finish_at(outcome, completed_at, duration_ms); -+ } -+ -+ pub fn finish_with_output(&self, outcome: OperationOutcome, output: &Value) { -+ let completed_at = timestamp(); -+ let duration_ms = self.started.elapsed().as_secs_f64() * 1000.0; -+ self.capture_output(output); -+ self.finish_at(outcome, completed_at, duration_ms); -+ } -+ -+ pub fn finish(&self, outcome: OperationOutcome) { -+ self.finish_at( -+ outcome, -+ timestamp(), -+ self.started.elapsed().as_secs_f64() * 1000.0, -+ ); -+ } -+ -+ fn finish_at(&self, outcome: OperationOutcome, completed_at: String, duration_ms: f64) { -+ let terminal = { -+ let mut event = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if event.phase == OperationPhase::Completed { -+ return; -+ } -+ event.phase = OperationPhase::Completed; -+ event.completed_at = Some(completed_at); -+ event.duration_ms = Some(duration_ms); -+ event.outcome = Some(outcome); -+ event.clone() -+ }; -+ self.observation.emit(terminal); -+ } -+} -+ -+impl Drop for OperationGuard { -+ fn drop(&mut self) { -+ self.finish(OperationOutcome::Interrupted); -+ } -+} -+ -+fn timestamp() -> String { -+ Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true) -+} -+ -+/// Projects JSON in memory, omitting binary/hidden reasoning and transport secrets. -+/// Inline markers survive consumers that extract values from the wire envelope. -+pub fn project_content(value: &Value) -> OperationContent { -+ project_with(|projection| projection.project(value, CONTENT_VALUE_LIMIT_BYTES, 0)) -+} -+ -+pub(crate) fn project_text(text: &str) -> OperationContent { -+ project_with(|projection| projection.text(text, CONTENT_VALUE_LIMIT_BYTES, 0)) -+} -+ -+/// Typed adapters stop serialization before constructing an oversized owned JSON value. -+fn bounded_value(value: &(impl Serialize + ?Sized)) -> Result { -+ struct BoundedWriter(Vec); -+ impl std::io::Write for BoundedWriter { -+ fn write(&mut self, bytes: &[u8]) -> std::io::Result { -+ if bytes.len() > CONTENT_VALUE_LIMIT_BYTES.saturating_sub(self.0.len()) { -+ return Err(std::io::Error::other("capture_budget")); -+ } -+ self.0.extend_from_slice(bytes); -+ Ok(bytes.len()) -+ } -+ fn flush(&mut self) -> std::io::Result<()> { -+ Ok(()) -+ } -+ } -+ let mut writer = BoundedWriter(Vec::new()); -+ serde_json::to_writer(&mut writer, value).map_err(|error| { -+ if error.is_io() { -+ "capture_budget" -+ } else { -+ "serialization_failed" -+ } -+ })?; -+ serde_json::from_slice(&writer.0).map_err(|_| "serialization_failed") -+} -+ -+pub(crate) fn project_serializable_content(value: &(impl Serialize + ?Sized)) -> OperationContent { -+ match bounded_value(value) { -+ Ok(value) => project_content(&value), -+ Err(reason) => omitted_content(reason), -+ } -+} -+ -+fn project_with(project: impl FnOnce(&mut ContentProjection) -> Value) -> OperationContent { -+ let mut projection = ContentProjection { -+ truncated: false, -+ omitted: false, -+ }; -+ let value = project(&mut projection); -+ let content = OperationContent { -+ value, -+ truncated: projection.truncated, -+ omitted: projection.omitted, -+ reason: if projection.truncated { -+ Some("capture_budget".into()) -+ } else if projection.omitted { -+ Some("unsupported_content".into()) -+ } else { -+ None -+ }, -+ }; -+ if json_size(&content) <= CONTENT_LIMIT_BYTES { -+ content -+ } else { -+ omitted_content("capture_budget") -+ } -+} -+ -+fn omitted_content(reason: &str) -> OperationContent { -+ OperationContent { -+ value: if reason == "capture_budget" { -+ serde_json::json!({"omitted":true,"truncated":true,"reason":reason}) -+ } else { -+ serde_json::json!({"omitted":true,"reason":reason}) -+ }, -+ truncated: reason == "capture_budget", -+ omitted: true, -+ reason: Some(reason.into()), -+ } -+} -+ -+fn json_size(value: &(impl Serialize + ?Sized)) -> usize { -+ struct Counter(usize); -+ impl std::io::Write for Counter { -+ fn write(&mut self, bytes: &[u8]) -> std::io::Result { -+ self.0 = self.0.saturating_add(bytes.len()); -+ Ok(bytes.len()) -+ } -+ fn flush(&mut self) -> std::io::Result<()> { -+ Ok(()) -+ } -+ } -+ let mut counter = Counter(0); -+ serde_json::to_writer(&mut counter, value).map_or(usize::MAX, |_| counter.0) -+} -+ -+struct ContentProjection { -+ truncated: bool, -+ omitted: bool, -+} -+ -+impl ContentProjection { -+ fn project(&mut self, value: &Value, budget: usize, depth: usize) -> Value { -+ if depth > 32 { -+ self.truncated = true; -+ return Value::String("[truncated]".into()); -+ } -+ match value { -+ Value::String(text) => self.text(text, budget, depth), -+ Value::Array(items) => { -+ let mut result = Vec::new(); -+ let mut used: usize = 2; -+ for item in items { -+ let available = budget.saturating_sub(used + 34); -+ if available < 84 { -+ self.truncated = true; -+ result.push(serde_json::json!({"truncated":true})); -+ break; -+ } -+ let item = self.project(item, available, depth + 1); -+ used += json_size(&item) + 2; -+ result.push(item); -+ } -+ Value::Array(result) -+ } -+ Value::Object(items) => { -+ self.truncated |= items.get("truncated") == Some(&Value::Bool(true)); -+ self.omitted |= items.get("omitted") == Some(&Value::Bool(true)); -+ let kind = items.get("type").and_then(Value::as_str); -+ let binary_resource = kind == Some("resource") -+ && items -+ .get("resource") -+ .and_then(|value| value.get("blob")) -+ .is_some(); -+ if binary_resource -+ || kind.is_some_and(|kind| { -+ matches!( -+ kind, -+ "reasoning" -+ | "reasoning_text" -+ | "thinking" -+ | "redacted_thinking" -+ | "compaction" -+ | "encrypted_content" -+ | "image" -+ | "input_image" -+ | "output_image" -+ | "image_url" -+ | "computer_screenshot" -+ | "image_generation_call" -+ | "audio" -+ | "input_audio" -+ | "output_audio" -+ | "file" -+ | "input_file" -+ | "video" -+ | "input_video" -+ ) -+ }) -+ { -+ self.omitted = true; -+ return serde_json::json!({"type":kind,"omitted":true,"reason":"unsupported_content"}); -+ } -+ let mut result = serde_json::Map::new(); -+ let mut used: usize = 2; -+ // Preserve structural keys without collecting an unbounded list of references. -+ let structural = -+ |key: &str| matches!(key, "type" | "role" | "id" | "call_id" | "name"); -+ let ordered = items -+ .iter() -+ .filter(|(key, _)| structural(key)) -+ .chain(items.iter().filter(|(key, _)| !structural(key))); -+ for (key, value) in ordered { -+ let available = budget.saturating_sub(used + json_size(&key) + 36); -+ if available < 84 { -+ self.truncated = true; -+ result.insert("truncated".into(), Value::Bool(true)); -+ break; -+ } -+ let normalized = key.to_ascii_lowercase().replace(['-', '_'], ""); -+ let value = if matches!( -+ normalized.as_str(), -+ "apikey" -+ | "xapikey" -+ | "authorization" -+ | "accesstoken" -+ | "refreshtoken" -+ | "password" -+ | "headers" -+ | "clientmetadata" -+ | "encryptedcontent" -+ | "reasoning" -+ | "reasoningcontent" -+ | "reasoningdetails" -+ | "thinking" -+ | "imageurl" -+ | "filedata" -+ | "imagedata" -+ | "audiodata" -+ | "b64json" -+ ) { -+ self.omitted = true; -+ Value::String("[omitted]".into()) -+ } else { -+ self.project(value, available, depth + 1) -+ }; -+ used += json_size(&key) + json_size(&value) + 4; -+ result.insert(key.clone(), value); -+ } -+ Value::Object(result) -+ } -+ value => value.clone(), -+ } -+ } -+ -+ fn response_item( -+ &mut self, -+ item: &codex_protocol::models::ResponseItem, -+ budget: usize, -+ ) -> Value { -+ use codex_protocol::models::ContentItem; -+ use codex_protocol::models::ResponseItem; -+ match bounded_value(item) { -+ Ok(value) => return self.project(&value, budget, 0), -+ Err("capture_budget") => {} -+ Err(reason) => { -+ self.omitted = true; -+ return omitted_content(reason).value; -+ } -+ } -+ if let Some(value) = self.oversized_tool_result(item, budget) { -+ return value; -+ } -+ let ResponseItem::Message { -+ role, -+ content, -+ id, -+ status, -+ phase, -+ encrypted_content, -+ internal_chat_message_metadata_passthrough, -+ .. -+ } = item -+ else { -+ self.truncated = true; -+ self.omitted = true; -+ return omitted_content("capture_budget").value; -+ }; -+ // Large messages retain borrowed visible text without cloning media or the full item. -+ let mut result = serde_json::Map::new(); -+ result.insert("type".into(), Value::String("message".into())); -+ result.insert("role".into(), self.text(role, budget.min(256), 0)); -+ for (key, value) in [ -+ ("id", bounded_value(id)), -+ ("status", bounded_value(status)), -+ ("phase", bounded_value(phase)), -+ ] { -+ if let Ok(value) = value { -+ if !value.is_null() { -+ result.insert(key.into(), self.project(&value, 256, 0)); -+ } -+ } else { -+ self.omitted = true; -+ result.insert("omitted".into(), Value::Bool(true)); -+ } -+ } -+ if encrypted_content.is_some() || internal_chat_message_metadata_passthrough.is_some() { -+ self.omitted = true; -+ result.insert("omitted".into(), Value::Bool(true)); -+ } -+ let mut used = json_size(&result) + 24; -+ let mut parts = Vec::new(); -+ for part in content { -+ let available = budget.saturating_sub(used + 96); -+ if available < 84 { -+ self.truncated = true; -+ parts.push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); -+ break; -+ } -+ let value = match part { -+ ContentItem::InputText { text } => { -+ serde_json::json!({"type":"input_text","text":self.text(text, available, 0)}) -+ } -+ ContentItem::OutputText { -+ text, -+ annotations, -+ logprobs, -+ } => { -+ let mut value = serde_json::json!({"type":"output_text","text":self.text(text, available, 0)}); -+ if annotations.is_some() || logprobs.is_some() { -+ self.omitted = true; -+ value["omitted"] = Value::Bool(true); -+ } -+ value -+ } -+ ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => { -+ self.omitted = true; -+ serde_json::json!({"type":if matches!(part, ContentItem::InputImage { .. }) {"input_image"} else {"input_audio"},"omitted":true,"reason":"unsupported_content"}) -+ } -+ }; -+ used += json_size(&value) + 2; -+ parts.push(value); -+ } -+ result.insert("content".into(), Value::Array(parts)); -+ Value::Object(result) -+ } -+ -+ fn oversized_tool_result( -+ &mut self, -+ item: &codex_protocol::models::ResponseItem, -+ budget: usize, -+ ) -> Option { -+ use codex_protocol::models::{ -+ FunctionCallOutputBody, FunctionCallOutputContentItem, ResponseItem, -+ }; -+ let (kind, call_id, name, output) = match item { -+ ResponseItem::FunctionCallOutput { -+ call_id, -+ name, -+ output, -+ .. -+ } => ( -+ "function_call_output", -+ call_id.as_deref(), -+ name.as_deref(), -+ output, -+ ), -+ ResponseItem::CustomToolCallOutput { -+ call_id, -+ name, -+ output, -+ .. -+ } => ( -+ "custom_tool_call_output", -+ Some(call_id.as_str()), -+ name.as_deref(), -+ output, -+ ), -+ _ => return None, -+ }; -+ self.omitted = true; -+ let mut result = serde_json::json!({"type":kind,"omitted":true,"reason":"capture_budget"}); -+ for (key, value) in [("call_id", call_id), ("name", name)] { -+ if let Some(value) = value { -+ result[key] = self.text(value, budget.min(256), 0); -+ } -+ } -+ let mut used = json_size(&result) + 24; -+ result["output"] = match &output.body { -+ FunctionCallOutputBody::Text(text) => { -+ self.text(text, budget.saturating_sub(used + 64), 0) -+ } -+ FunctionCallOutputBody::ContentItems(items) => { -+ let mut parts = Vec::new(); -+ for part in items { -+ let available = budget.saturating_sub(used + 96); -+ if available < 84 { -+ self.truncated = true; -+ parts.push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); -+ break; -+ } -+ let value = match part { -+ FunctionCallOutputContentItem::InputText { text } => { -+ serde_json::json!({"type":"input_text","text":self.text(text, available, 0)}) -+ } -+ _ => { -+ let kind = match part { -+ FunctionCallOutputContentItem::InputImage { .. } => "input_image", -+ FunctionCallOutputContentItem::InputAudio { .. } => "input_audio", -+ FunctionCallOutputContentItem::EncryptedContent { .. } => { -+ "encrypted_content" -+ } -+ FunctionCallOutputContentItem::InputText { .. } => unreachable!(), -+ }; -+ serde_json::json!({"type":kind,"omitted":true,"reason":"unsupported_content"}) -+ } -+ }; -+ used += json_size(&value) + 2; -+ parts.push(value); -+ } -+ Value::Array(parts) -+ } -+ }; -+ Some(self.project(&result, budget, 0)) -+ } -+ -+ fn text(&mut self, text: &str, budget: usize, depth: usize) -> Value { -+ static DATA_URI: std::sync::LazyLock = std::sync::LazyLock::new(|| { -+ regex_lite::Regex::new(r#"(?i)data:[^\s;,]+(?:;[^\s,]*)?,[^\s\"'<>]+"#) -+ .expect("valid data URI pattern") -+ }); -+ self.truncated |= text.ends_with("[truncated]"); -+ self.omitted |= text.contains("[omitted]") || text.contains("[omitted: media]"); -+ // Classification borrows the text; bound raw bytes before allocating a JSON tree. -+ let parse_limit = budget.min(CONTENT_LIMIT_BYTES); -+ if text.trim_start().starts_with(['{', '[']) { -+ if text.len() > parse_limit { -+ self.truncated = true; -+ self.omitted = true; -+ return Value::String( -+ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) -+ .to_string(), -+ ); -+ } -+ if let Ok(embedded) = serde_json::from_str::(&text) { -+ let safe = self.project(&embedded, budget, depth + 1); -+ if safe == embedded && json_size(&text) <= budget { -+ return Value::String(text.to_owned()); -+ } -+ let mut encoded = safe.to_string(); -+ if json_size(&encoded) > budget { -+ // Rebudget the value so escaped JSON remains valid. -+ encoded = self -+ .project(&embedded, budget.saturating_sub(2) / 2, depth + 1) -+ .to_string(); -+ } -+ return Value::String(encoded); -+ } -+ } -+ // Regex replacement and JSON sizing only see a bounded UTF-8 prefix. -+ let mut end = text.len().min(budget); -+ while !text.is_char_boundary(end) { -+ end -= 1; -+ } -+ let source_truncated = end < text.len(); -+ self.truncated |= source_truncated; -+ let text = &text[..end]; -+ let projected = DATA_URI.replace_all(text, "[omitted: media]"); -+ self.omitted |= projected != text; -+ let text = projected.into_owned(); -+ if !source_truncated && json_size(&text) <= budget { -+ return Value::String(text); -+ } -+ self.truncated = true; -+ let mut low = 0; -+ let mut high = text.len().min(budget); -+ while low < high { -+ let middle = low + (high - low + 1) / 2; -+ let mut end = middle; -+ while !text.is_char_boundary(end) { -+ end -= 1; -+ } -+ if json_size(&&text[..end]).saturating_add("[truncated]".len()) <= budget { -+ low = middle; -+ } else { -+ high = middle - 1; -+ } -+ } -+ while !text.is_char_boundary(low) { -+ low -= 1; -+ } -+ Value::String(format!("{}[truncated]", &text[..low])) -+ } -+} -+ -+#[derive(Clone)] -+pub(crate) struct ModelOperation { -+ observation: Arc, -+ thread_id: String, -+ turn_id: String, -+ requested_model: String, -+ stream_generation: Option, -+ state: Arc>, -+} -+ -+#[derive(Default)] -+struct ModelOperationState { -+ guard: Option, -+ output: Vec, -+ output_bytes: usize, -+ partial_text: String, -+ partial_truncated: bool, -+ fingerprint: Option, -+ generation: u64, -+ source_end: Option<(String, f64)>, -+ retry_waiting: bool, -+ authoritative_model: bool, -+} -+ -+impl ModelOperation { -+ pub(crate) fn new( -+ observation: Arc, -+ thread_id: String, -+ turn_id: String, -+ requested_model: String, -+ ) -> Self { -+ Self { -+ observation, -+ thread_id, -+ turn_id, -+ requested_model, -+ stream_generation: None, -+ state: Arc::new(Mutex::new(ModelOperationState::default())), -+ } -+ } -+ -+ pub(crate) async fn scope(&self, future: F) -> F::Output { -+ CURRENT_MODEL_OPERATION.scope(self.clone(), future).await -+ } -+ -+ pub(crate) fn finish(&self, outcome: OperationOutcome) { -+ // Only the source response.completed event establishes inference success. -+ // An outer successful turn handoff can still have preempted this stream. -+ let outcome = if outcome == OperationOutcome::Success { -+ OperationOutcome::Interrupted -+ } else { -+ outcome -+ }; -+ let state = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if let Some(guard) = &state.guard { -+ if let Some((completed_at, duration_ms)) = &state.source_end -+ && (outcome != OperationOutcome::Interrupted || !state.retry_waiting) -+ { -+ if self.observation.captures_content() { -+ guard.capture_output(&model_output(&state)); -+ } -+ guard.finish_at(outcome, completed_at.clone(), *duration_ms); -+ } else { -+ guard.finish_with_output_fn(outcome, || model_output(&state)); -+ } -+ } -+ } -+ -+ pub(crate) fn interruption_guard(&self) -> ModelInterruptionGuard { -+ ModelInterruptionGuard(self.clone()) -+ } -+ -+ pub(crate) fn retry_waiting(&self, waiting: bool) { -+ self.state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner) -+ .retry_waiting = waiting; -+ } -+ -+ pub(crate) fn source_ended(&self) { -+ let mut state = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if self -+ .stream_generation -+ .is_some_and(|generation| generation != state.generation) -+ { -+ return; -+ } -+ if state.source_end.is_none() { -+ let source_end = state -+ .guard -+ .as_ref() -+ .map(|guard| (timestamp(), guard.started.elapsed().as_secs_f64() * 1000.0)); -+ if source_end.is_some() { -+ state.source_end = source_end; -+ } -+ } -+ } -+ -+ pub(crate) fn source_error(&self, error: &codex_api::ApiError) { -+ self.source_ended(); -+ let state = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if self -+ .stream_generation -+ .is_some_and(|generation| generation != state.generation) -+ { -+ return; -+ } -+ if let codex_api::ApiError::ResponseIncomplete(incomplete) = error -+ && let Some(guard) = &state.guard -+ { -+ guard.update(|event| { -+ event.metadata.response_id = incomplete.response_id.clone(); -+ event.metadata.usage = incomplete -+ .token_usage -+ .as_ref() -+ .and_then(|usage| serde_json::to_value(usage).ok()); -+ }); -+ } -+ } -+ -+ fn request( -+ &self, -+ request: &T, -+ identity: &codex_api::ResponsesApiRequest, -+ endpoint: &str, -+ ) { -+ let mut state = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ state.generation = state.generation.wrapping_add(1); -+ let previous_source_end = state.source_end.take(); -+ state.retry_waiting = false; -+ state.authoritative_model = false; -+ // Exclude routing metadata and changing transport fields from retry identity. -+ let fingerprint = request_fingerprint(identity); -+ if state.guard.is_some() && state.fingerprint == Some(fingerprint) { -+ if let Some(guard) = &state.guard { -+ guard.update(|event| { -+ event.metadata.response_id = None; -+ event.metadata.observed_model = None; -+ event.metadata.usage = None; -+ event.metadata.response_id_trusted = trusted_response_endpoint(endpoint); -+ }); -+ if self.observation.captures_content() { -+ guard.capture_input(&model_request_content(request)); -+ } -+ } -+ // A retried stream replaces any partial output from the failed attempt. -+ state.output.clear(); -+ state.output_bytes = 0; -+ state.partial_text.clear(); -+ state.partial_truncated = false; -+ return; -+ } -+ if let Some(previous) = state.guard.take() { -+ if self.observation.captures_content() { -+ previous.capture_output(&model_output(&state)); -+ } -+ if let Some((completed_at, duration_ms)) = previous_source_end { -+ previous.finish_at(OperationOutcome::Error, completed_at, duration_ms); -+ } else { -+ previous.finish(OperationOutcome::Error); -+ } -+ } -+ state.output.clear(); -+ state.output_bytes = 0; -+ state.partial_text.clear(); -+ state.partial_truncated = false; -+ state.fingerprint = Some(fingerprint); -+ let guard = self.observation.start( -+ self.thread_id.clone(), -+ self.turn_id.clone(), -+ OperationKind::Model, -+ OperationMetadata { -+ requested_model: Some(self.requested_model.clone()), -+ response_id_trusted: trusted_response_endpoint(endpoint), -+ ..Default::default() -+ }, -+ ); -+ // Capture the assembled wire body, rather than the caller's input. -+ if self.observation.captures_content() { -+ guard.capture_input(&model_request_content(request)); -+ } -+ state.guard = Some(guard); -+ } -+ -+ pub(crate) fn for_stream(mut self) -> Self { -+ self.stream_generation = Some( -+ self.state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner) -+ .generation, -+ ); -+ self -+ } -+ -+ pub(crate) fn event(&self, event: &codex_api::ResponseEvent) { -+ let mut state = self -+ .state -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ if self -+ .stream_generation -+ .is_some_and(|generation| generation != state.generation) -+ || state.source_end.is_some() -+ { -+ return; -+ } -+ match event { -+ codex_api::ResponseEvent::OutputTextDelta(text) -+ if self.observation.captures_content() => -+ { -+ let mut end = text -+ .len() -+ .min(CONTENT_LIMIT_BYTES.saturating_sub(state.partial_text.len())); -+ while !text.is_char_boundary(end) { -+ end -= 1; -+ } -+ state.partial_text.push_str(&text[..end]); -+ state.partial_truncated |= end < text.len(); -+ } -+ codex_api::ResponseEvent::OutputItemDone(item) -+ if self.observation.mode == OperationCaptureMode::Content => -+ { -+ if matches!(item, codex_protocol::models::ResponseItem::Message { .. }) { -+ state.partial_text.clear(); -+ state.partial_truncated = false; -+ } -+ if state.output_bytes < CONTENT_LIMIT_BYTES { -+ let content = project_response_item(item); -+ state.output_bytes += serde_json::to_vec(&content.value) -+ .map_or(CONTENT_LIMIT_BYTES, |value| value.len()); -+ state.output.push(content.value); -+ if content.truncated || content.omitted { -+ state.output.push(serde_json::json!({"omitted":content.omitted,"truncated":content.truncated,"reason":content.reason})); -+ } -+ } else if state -+ .output -+ .last() -+ .and_then(|item| item.get("reason")) -+ .and_then(Value::as_str) -+ != Some("capture_budget") -+ { -+ state -+ .output -+ .push(serde_json::json!({"truncated":true,"reason":"capture_budget"})); -+ } -+ } -+ codex_api::ResponseEvent::ModelObservation(model) => { -+ if !state.authoritative_model -+ && let Some(guard) = &state.guard -+ { -+ guard.update(|event| event.metadata.observed_model = Some(model.clone())); -+ } -+ } -+ codex_api::ResponseEvent::ServerModel(model) => { -+ state.authoritative_model = true; -+ if let Some(guard) = &state.guard { -+ guard.update(|event| event.metadata.observed_model = Some(model.clone())); -+ } -+ } -+ codex_api::ResponseEvent::Created { response_id } => { -+ if let Some(guard) = &state.guard { -+ guard.update(|event| event.metadata.response_id = response_id.clone()); -+ } -+ } -+ codex_api::ResponseEvent::Completed { -+ response_id, -+ token_usage, -+ .. -+ } => { -+ if let Some(guard) = &state.guard { -+ guard.update(|event| { -+ event.metadata.response_id = Some(response_id.clone()); -+ event.metadata.usage = token_usage -+ .as_ref() -+ .and_then(|usage| serde_json::to_value(usage).ok()); -+ }); -+ guard.finish_with_output_fn(OperationOutcome::Success, || model_output(&state)); -+ } -+ } -+ _ => {} -+ } -+ } -+} -+ -+fn model_output(state: &ModelOperationState) -> Value { -+ let mut output = state.output.clone(); -+ if !state.partial_text.is_empty() { -+ output.push(serde_json::json!({"type":"output_text","text":state.partial_text,"partial":true,"truncated":state.partial_truncated})); -+ } -+ Value::Array(output) -+} -+ -+pub(crate) fn current_model_operation() -> Option { -+ CURRENT_MODEL_OPERATION.try_with(Clone::clone).ok() -+} -+ -+/// Only wire messages and continuation identity consume the content budget. -+#[derive(Serialize)] -+pub(crate) struct ModelRequestContent<'a> { -+ #[serde(serialize_with = "serialize_model_inputs")] -+ pub input: &'a [codex_protocol::models::ResponseItem], -+ #[serde(skip_serializing_if = "Option::is_none")] -+ pub previous_response_id: Option<&'a str>, -+} -+ -+fn serialize_model_inputs( -+ input: &[codex_protocol::models::ResponseItem], -+ serializer: S, -+) -> Result { -+ use serde::ser::SerializeSeq; -+ let mut sequence = serializer.serialize_seq(None)?; -+ let mut projection = ContentProjection { -+ truncated: false, -+ omitted: false, -+ }; -+ let mut used = 0; -+ for item in input { -+ let available = CONTENT_VALUE_LIMIT_BYTES.saturating_sub(used + 512); -+ if available < 84 { -+ sequence.serialize_element( -+ &serde_json::json!({"truncated":true,"reason":"capture_budget"}), -+ )?; -+ break; -+ } -+ let value = projection.response_item(item, available); -+ used += json_size(&value) + 2; -+ sequence.serialize_element(&value)?; -+ } -+ sequence.end() -+} -+ -+fn project_response_item(item: &codex_protocol::models::ResponseItem) -> OperationContent { -+ project_with(|projection| projection.response_item(item, CONTENT_VALUE_LIMIT_BYTES)) -+} -+ -+pub(crate) fn observe_model_request( -+ request: &T, -+ identity: &codex_api::ResponsesApiRequest, -+ endpoint: &str, -+) { -+ if let Some(operation) = current_model_operation() { -+ operation.request(request, identity, endpoint); -+ } -+} -+ -+pub(crate) struct ModelInterruptionGuard(ModelOperation); -+ -+impl Drop for ModelInterruptionGuard { -+ fn drop(&mut self) { -+ self.0.finish(OperationOutcome::Interrupted); -+ } -+} -+ -+fn model_request_content(request: &(impl Serialize + ?Sized)) -> Value { -+ let mut value = match bounded_value(request) { -+ Ok(value) => value, -+ Err(reason) => return omitted_content(reason).value, -+ }; -+ if let Value::Object(body) = &mut value { -+ // The per-direction content budget belongs to messages and instructions. -+ // Tool definitions affect request identity but must not consume that budget. -+ body.retain(|key, _| { -+ matches!( -+ key.as_str(), -+ "instructions" | "input" | "previous_response_id" -+ ) -+ }); -+ } -+ value -+} -+ -+fn model_request_identity(request: &codex_api::ResponsesApiRequest) -> impl Serialize + '_ { -+ #[derive(Serialize)] -+ struct Identity<'a> { -+ input: &'a [codex_protocol::models::ResponseItem], -+ model: &'a str, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ text: Option<&'a codex_api::TextControls>, -+ tool_choice: &'a str, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ tools: Option<&'a codex_api::ResponsesApiTools>, -+ } -+ Identity { -+ input: &request.input, -+ model: &request.model, -+ text: request.text.as_ref(), -+ tool_choice: &request.tool_choice, -+ tools: request.tools.as_ref(), -+ } -+} -+ -+fn request_fingerprint(request: &codex_api::ResponsesApiRequest) -> u64 { -+ use std::hash::Hasher; -+ struct Digest(std::collections::hash_map::DefaultHasher); -+ impl std::io::Write for Digest { -+ fn write(&mut self, bytes: &[u8]) -> std::io::Result { -+ self.0.write(bytes); -+ Ok(bytes.len()) -+ } -+ fn flush(&mut self) -> std::io::Result<()> { -+ Ok(()) -+ } -+ } -+ let mut digest = Digest(std::collections::hash_map::DefaultHasher::new()); -+ // Typed semantic fields borrow the prepared body. Transport metadata is excluded; -+ // request retries clone history and retain its map ordering and raw tool JSON. -+ let _ = serde_json::to_writer(&mut digest, &model_request_identity(request)); -+ digest.0.finish() -+} -+ -+fn trusted_response_endpoint(endpoint: &str) -> bool { -+ url::Url::parse(endpoint) -+ .is_ok_and(|url| url.scheme() == "https" && url.host_str() == Some("api.openai.com")) -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use pretty_assertions::assert_eq; -+ -+ #[derive(Default)] -+ struct RecordingObserver(Mutex>); -+ impl OperationObserver for RecordingObserver { -+ fn emit(&self, event: OperationEvent) { -+ self.0.lock().unwrap().push(event); -+ } -+ } -+ -+ fn observation(mode: OperationCaptureMode) -> (OperationObservation, Arc) { -+ let sink = Arc::new(RecordingObserver::default()); -+ (OperationObservation::new(mode, sink.clone()), sink) -+ } -+ -+ // Capture fixtures remain independent from the prepared transport identity. -+ fn test_request(value: &Value) -> codex_api::ResponsesApiRequest { -+ codex_api::ResponsesApiRequest { -+ model: value["model"].as_str().unwrap_or("model").to_owned(), -+ input: vec![codex_protocol::models::ResponseItem::Message { -+ role: "user".into(), -+ content: vec![codex_protocol::models::ContentItem::InputText { -+ text: value["input"].to_string(), -+ }], -+ encrypted_content: None, -+ status: None, -+ id: None, -+ phase: None, -+ internal_chat_message_metadata_passthrough: None, -+ }], -+ tools: value.get("tools").map(|tools| { -+ Arc::::from( -+ serde_json::value::to_raw_value(tools).unwrap(), -+ ) -+ .into() -+ }), -+ tool_choice: value["tool_choice"].as_str().unwrap_or("auto").to_owned(), -+ text: None, -+ stream: true, -+ service_tier: None, -+ parallel_tool_calls: true, -+ reasoning: None, -+ store: false, -+ stream_options: None, -+ include: vec![], -+ prompt_cache_key: None, -+ client_metadata: None, -+ access_programs: None, -+ } -+ } -+ -+ #[test] -+ fn fingerprint_ignores_transport_metadata_and_tracks_semantic_fields() { -+ let old_identity = |request: &codex_api::ResponsesApiRequest| { -+ let mut value = serde_json::to_value(request).unwrap(); -+ value.as_object_mut().unwrap().retain(|key, _| { -+ matches!( -+ key.as_str(), -+ "model" | "input" | "tools" | "tool_choice" | "text" -+ ) -+ }); -+ value -+ }; -+ let request = test_request(&serde_json::json!({"input":["same"]})); -+ let original = request_fingerprint(&request); -+ let mut retry = request.clone(); -+ retry.client_metadata = Some(std::collections::HashMap::from([ -+ ("routing".into(), "new route".into()), -+ ("trace".into(), "new trace".into()), -+ ])); -+ retry.service_tier = Some("priority".into()); -+ retry.prompt_cache_key = Some("new cache key".into()); -+ assert_eq!(old_identity(&request), old_identity(&retry)); -+ assert_eq!(original, request_fingerprint(&retry)); -+ let mut changed_model = request.clone(); -+ changed_model.model = "different".into(); -+ let mut changed_input = request.clone(); -+ changed_input.input.clear(); -+ let mut changed_choice = request.clone(); -+ changed_choice.tool_choice = "none".into(); -+ let mut changed_text = request.clone(); -+ changed_text.text = Some(codex_api::TextControls::default()); -+ let changed_tools = test_request(&serde_json::json!({"input":["same"],"tools":[]})); -+ for changed in [ -+ changed_model, -+ changed_input, -+ changed_choice, -+ changed_text, -+ changed_tools, -+ ] { -+ assert_ne!(old_identity(&request), old_identity(&changed)); -+ assert_ne!(original, request_fingerprint(&changed)); -+ } -+ } -+ -+ #[test] -+ fn metadata_never_captures_payload_and_finalizes_once() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ { -+ let operation = observation.start( -+ "thread".into(), -+ "turn".into(), -+ OperationKind::Tool, -+ OperationMetadata::default(), -+ ); -+ operation.capture_input(&serde_json::json!({"text":"INPUT_CANARY"})); -+ operation.finish_with_output( -+ OperationOutcome::Success, -+ &Value::String("OUTPUT_CANARY".into()), -+ ); -+ operation.finish(OperationOutcome::Error); -+ } -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[0].phase, OperationPhase::Started); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); -+ assert_eq!(events[0].operation_id, events[1].operation_id); -+ assert_eq!(events[1].input, None); -+ assert_eq!(events[1].output, None); -+ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); -+ } -+ -+ #[test] -+ fn dropped_operation_reports_interruption() { -+ let (observation, sink) = observation(OperationCaptureMode::Content); -+ drop(observation.start( -+ "thread".into(), -+ "turn".into(), -+ OperationKind::Tool, -+ OperationMetadata::default(), -+ )); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); -+ assert!(events[1].duration_ms.unwrap() >= 0.0); -+ assert!(chrono::DateTime::parse_from_rfc3339(&events[1].started_at).is_ok()); -+ assert!( -+ chrono::DateTime::parse_from_rfc3339(events[1].completed_at.as_ref().unwrap()).is_ok() -+ ); -+ } -+ -+ #[test] -+ fn projection_bounds_utf8_and_excludes_hidden_or_binary_parts() { -+ let content = project_content(&serde_json::json!({ -+ "input":[{"role":"user","content":[{"type":"input_text","text":"hello"},{"type":"input_image","image_url":"BINARY_CANARY"}]},{"type":"reasoning","summary":[{"text":"REASONING_CANARY"}]}], -+ "authorization":"SECRET_CANARY", -+ "output":"é🙂".repeat(CONTENT_LIMIT_BYTES) -+ })); -+ let serialized = serde_json::to_string(&content).unwrap(); -+ assert!(serialized.len() <= CONTENT_LIMIT_BYTES); -+ assert!(content.truncated); -+ assert!(content.omitted); -+ assert!(!serialized.contains("CANARY")); -+ assert_eq!(content.value["input"][0]["role"], "user"); -+ assert_eq!(content.value["input"][0]["content"][0]["text"], "hello"); -+ } -+ -+ #[test] -+ fn mcp_binary_content_is_omitted_but_text_resources_remain_visible() { -+ let content = project_content(&serde_json::json!([ -+ {"type":"image","data":"IMAGE_CANARY","mimeType":"image/png"}, -+ {"type":"resource","resource":{"uri":"file:///image","blob":"BLOB_CANARY"}}, -+ {"type":"audio","data":"AUDIO_CANARY","mimeType":"audio/wav"}, -+ {"type":"resource","resource":{"uri":"file:///text","text":"visible text"}}, -+ {"type":"text","text":"visible output"} -+ ])); -+ let serialized = serde_json::to_string(&content).unwrap(); -+ assert!(content.omitted); -+ assert!(!serialized.contains("CANARY")); -+ assert_eq!(content.value[3]["resource"]["text"], "visible text"); -+ assert_eq!(content.value[4]["text"], "visible output"); -+ } -+ -+ #[test] -+ fn oversized_text_keeps_a_useful_utf8_prefix_and_inline_marker() { -+ let content = project_text(&format!( -+ "file prefix: {}", -+ "é🙂".repeat(CONTENT_LIMIT_BYTES) -+ )); -+ assert!(content.value.as_str().unwrap().starts_with("file prefix: ")); -+ assert!(content.value.as_str().unwrap().ends_with("[truncated]")); -+ assert!(content.truncated); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[test] -+ fn native_tool_catalog_does_not_truncate_intact_model_input() { -+ let input = serde_json::json!([ -+ {"role":"user","content":[{"type":"input_text","text":"x".repeat(27_000)}]}, -+ ]); -+ let request = serde_json::json!({ -+ "model":"fixture-model", -+ "instructions":"worker instructions", -+ "input": input, -+ "tools":[{"type":"function","name":"native_tool","description":"catalog".repeat(20_000)}], -+ "text":{"format":{"type":"json_schema","schema":{"description":"schema".repeat(10_000)}}}, -+ "tool_choice":"auto", -+ }); -+ let selected = -+ serde_json::json!({"input": request["input"], "instructions":request["instructions"]}); -+ let content = project_content(&model_request_content(&selected)); -+ assert!(!content.truncated); -+ assert!(!content.omitted); -+ assert_eq!(content.value["input"], input); -+ assert_eq!(content.value["instructions"], "worker instructions"); -+ assert!(content.value.get("tools").is_none()); -+ assert!(content.value.get("text").is_none()); -+ let mut changed = request.clone(); -+ changed["tools"][0]["name"] = Value::String("different_tool".into()); -+ assert_ne!( -+ request_fingerprint(&test_request(&request)), -+ request_fingerprint(&test_request(&changed)) -+ ); -+ } -+ -+ #[test] -+ fn embedded_json_that_fits_the_budget_remains_intact() { -+ let raw = serde_json::json!({"text":"x".repeat(27_000)}).to_string(); -+ let content = project_text(&raw); -+ assert_eq!(content.value, Value::String(raw)); -+ assert!(!content.truncated); -+ } -+ -+ #[test] -+ fn oversized_messages_preserve_roles_and_part_identity() { -+ let content = project_content(&serde_json::json!([ -+ {"content":[{"text":format!("prefix {}", "x".repeat(80_000)),"type":"input_text"}],"role":"user"}, -+ ])); -+ assert_eq!(content.value[0]["role"], "user"); -+ assert_eq!(content.value[0]["content"][0]["type"], "input_text"); -+ assert!( -+ content.value[0]["content"][0]["text"] -+ .as_str() -+ .unwrap() -+ .starts_with("prefix ") -+ ); -+ assert!( -+ content.value[0]["content"][0]["text"] -+ .as_str() -+ .unwrap() -+ .ends_with("[truncated]") -+ ); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[test] -+ fn embedded_json_redacts_in_budget_sensitive_fields() { -+ let input = -+ serde_json::json!({"accessToken":"TOKEN_CANARY","result":"visible"}).to_string(); -+ let content = project_text(&input); -+ let embedded: Value = serde_json::from_str(content.value.as_str().unwrap()).unwrap(); -+ assert_eq!(embedded["accessToken"], "[omitted]"); -+ assert_eq!(embedded["result"], "visible"); -+ assert!(!content.truncated); -+ assert!(content.omitted); -+ assert!(!content.value.to_string().contains("CANARY")); -+ } -+ -+ #[test] -+ fn oversized_json_text_is_omitted_before_structural_projection() { -+ let inputs = [ -+ serde_json::json!({"authorization":"LARGE_JSON_CANARY","items":vec![0; CONTENT_LIMIT_BYTES * 8]}).to_string(), -+ format!("{}{}", " ".repeat(CONTENT_LIMIT_BYTES * 2), r#"{"authorization":"WHITESPACE_CANARY","result":"visible"}"#), -+ ]; -+ for raw in inputs { -+ let content = project_text(&raw); -+ let marker: Value = serde_json::from_str(content.value.as_str().unwrap()).unwrap(); -+ assert_eq!( -+ marker, -+ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) -+ ); -+ assert!(content.omitted); -+ assert!(content.truncated); -+ assert_eq!(content.reason.as_deref(), Some("capture_budget")); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ assert!(!content.value.to_string().contains("CANARY")); -+ } -+ } -+ -+ #[test] -+ fn embedded_json_parse_limit_follows_remaining_direction_budget() { -+ let raw = -+ serde_json::json!({"authorization":"NESTED_JSON_CANARY","text":"y".repeat(8_000)}) -+ .to_string(); -+ assert!(raw.len() < CONTENT_LIMIT_BYTES); -+ let content = project_content( -+ &serde_json::json!({"a":"x".repeat(CONTENT_VALUE_LIMIT_BYTES - 1_200),"z":raw}), -+ ); -+ let marker: Value = serde_json::from_str(content.value["z"].as_str().unwrap()).unwrap(); -+ assert_eq!( -+ marker, -+ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) -+ ); -+ assert!(content.omitted); -+ assert!(content.truncated); -+ assert!(!content.value.to_string().contains("CANARY")); -+ } -+ -+ #[test] -+ fn embedded_media_and_container_budget_have_visible_markers() { -+ let content = project_content(&serde_json::json!({ -+ "message":"before data:image/png;base64,BINARY_CANARY after", -+ "nested": [{"Authorization":"AUTH_CANARY","visible":"hello"}], -+ "nested_encoded": serde_json::json!({"request":serde_json::json!({"apiKey":"NESTED_CANARY","uri":"data:image/png;base64,MEDIA_CANARY","text":"visible nested"}).to_string()}).to_string(), -+ "results": vec![serde_json::json!({"text":"x".repeat(180)}); 3000], -+ })); -+ let serialized = serde_json::to_string(&content).unwrap(); -+ assert!(!serialized.contains("CANARY")); -+ assert!( -+ content.value["message"] -+ .as_str() -+ .unwrap() -+ .contains("[omitted: media]") -+ ); -+ assert_eq!(content.value["nested"][0]["visible"], "hello"); -+ assert!( -+ content.value["nested_encoded"] -+ .as_str() -+ .unwrap() -+ .contains("visible nested") -+ ); -+ assert!(content.value["results"].to_string().contains("truncated")); -+ assert!(serialized.len() <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[test] -+ fn borrowed_large_plain_text_retains_prefix_and_redacts_media() { -+ let value = serde_json::json!({ -+ "role":"tool", -+ "text":format!("visible data:image/png;base64,MEDIA_CANARY {}", "é🙂".repeat(CONTENT_LIMIT_BYTES * 8)), -+ "authorization":"TOKEN_CANARY", -+ }); -+ let content = project_content(&value); -+ assert_eq!(content.value["role"], "tool"); -+ assert!( -+ content.value["text"] -+ .as_str() -+ .unwrap() -+ .starts_with("visible [omitted: media] ") -+ ); -+ assert!( -+ content.value["text"] -+ .as_str() -+ .unwrap() -+ .ends_with("[truncated]") -+ ); -+ assert!(content.truncated); -+ assert!(content.omitted); -+ assert!(!content.value.to_string().contains("CANARY")); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ fn tool_result_item( -+ output: codex_protocol::models::FunctionCallOutputPayload, -+ custom: bool, -+ ) -> codex_protocol::models::ResponseItem { -+ use codex_protocol::models::ResponseItem; -+ if custom { -+ ResponseItem::CustomToolCallOutput { -+ id: None, -+ call_id: "call-file".into(), -+ name: Some("read_file".into()), -+ output, -+ internal_chat_message_metadata_passthrough: None, -+ } -+ } else { -+ ResponseItem::FunctionCallOutput { -+ id: None, -+ call_id: Some("call-file".into()), -+ name: Some("read_file".into()), -+ namespace: None, -+ output, -+ internal_chat_message_metadata_passthrough: None, -+ } -+ } -+ } -+ -+ #[test] -+ fn oversized_tool_results_preserve_call_identity_and_readable_utf8_prefix() { -+ for custom in [false, true] { -+ let item = tool_result_item( -+ codex_protocol::models::FunctionCallOutputPayload::from_text(format!( -+ "file prefix: {}", -+ "é🙂".repeat(CONTENT_LIMIT_BYTES) -+ )), -+ custom, -+ ); -+ let content = project_with(|projection| { -+ projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES) -+ }); -+ assert_eq!(content.value["call_id"], "call-file"); -+ assert_eq!( -+ content.value["type"], -+ if custom { -+ "custom_tool_call_output" -+ } else { -+ "function_call_output" -+ } -+ ); -+ let output = content.value["output"].as_str().unwrap(); -+ assert!(output.starts_with("file prefix: ")); -+ assert!(output.ends_with("[truncated]")); -+ assert!(content.truncated); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ } -+ -+ #[test] -+ fn oversized_multimodal_tool_result_keeps_text_and_omits_binary_parts() { -+ use codex_protocol::models::{ -+ FunctionCallOutputContentItem, FunctionCallOutputPayload, ImageReference, -+ }; -+ let item = tool_result_item( -+ FunctionCallOutputPayload::from_content_items(vec![ -+ FunctionCallOutputContentItem::InputImage { -+ image: ImageReference::Inline { -+ image_url: "IMAGE_CANARY".repeat(CONTENT_LIMIT_BYTES), -+ }, -+ detail: None, -+ }, -+ FunctionCallOutputContentItem::InputAudio { -+ audio_url: "AUDIO_CANARY".repeat(CONTENT_LIMIT_BYTES), -+ }, -+ FunctionCallOutputContentItem::InputText { -+ text: format!("file prefix: {}", "x".repeat(CONTENT_LIMIT_BYTES * 4)), -+ }, -+ ]), -+ false, -+ ); -+ let content = -+ project_with(|projection| projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES)); -+ assert_eq!(content.value["call_id"], "call-file"); -+ assert_eq!(content.value["output"][0]["omitted"], true); -+ assert_eq!(content.value["output"][1]["omitted"], true); -+ assert!( -+ content.value["output"][2]["text"] -+ .as_str() -+ .unwrap() -+ .starts_with("file prefix: ") -+ ); -+ assert!(!content.value.to_string().contains("CANARY")); -+ assert!(content.omitted); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[test] -+ fn oversized_json_tool_result_uses_the_existing_raw_json_guard() { -+ let raw = serde_json::json!({"authorization":"TOOL_JSON_CANARY","text":"x".repeat(CONTENT_LIMIT_BYTES * 4)}).to_string(); -+ let item = tool_result_item( -+ codex_protocol::models::FunctionCallOutputPayload::from_text(raw), -+ true, -+ ); -+ let content = -+ project_with(|projection| projection.response_item(&item, CONTENT_VALUE_LIMIT_BYTES)); -+ let marker: Value = -+ serde_json::from_str(content.value["output"].as_str().unwrap()).unwrap(); -+ assert_eq!(marker["reason"], "capture_budget"); -+ assert_eq!(marker["omitted"], true); -+ assert_eq!(content.value["call_id"], "call-file"); -+ assert!(!content.value.to_string().contains("CANARY")); -+ assert!(content.omitted); -+ assert!(content.truncated); -+ } -+ -+ #[test] -+ fn typed_oversized_items_stop_serialization_with_explicit_omission() { -+ #[derive(Serialize)] -+ struct TypedItem<'a> { -+ text: &'a str, -+ } -+ let text = "x".repeat(CONTENT_LIMIT_BYTES * 128); -+ let content = project_serializable_content(&TypedItem { text: &text }); -+ assert!(content.omitted); -+ assert!(content.truncated); -+ assert_eq!(content.value["truncated"], true); -+ assert_eq!(content.reason.as_deref(), Some("capture_budget")); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[test] -+ fn oversized_typed_messages_keep_plain_prefix_and_omit_structured_text() { -+ for kind in ["input_text", "output_text"] { -+ for structured in [false, true] { -+ let text = if structured { -+ serde_json::json!({"authorization":"MESSAGE_CANARY","text":"x".repeat(96 * 1024)}).to_string() -+ } else { -+ format!("TEXT_PREFIX {}", "é".repeat(96 * 1024)) -+ }; -+ let item: codex_protocol::models::ResponseItem = serde_json::from_value(serde_json::json!({ -+ "type":"message","id":"msg_test","role":"user","content":[{"type":kind,"text":text}] -+ })).unwrap(); -+ let output = project_response_item(&item); -+ let request = ModelRequestContent { -+ input: std::slice::from_ref(&item), -+ previous_response_id: Some("previous"), -+ }; -+ let input = project_content(&model_request_content(&request)); -+ for (content, value) in -+ [(&output, &output.value), (&input, &input.value["input"][0])] -+ { -+ assert_eq!(value["role"], "user"); -+ assert_eq!(value["id"], "msg_test"); -+ let text = value["content"][0]["text"].as_str().unwrap(); -+ if structured { -+ let marker: Value = serde_json::from_str(text).unwrap(); -+ assert_eq!(marker["reason"], "capture_budget"); -+ assert!(content.omitted); -+ assert!(!text.contains("CANARY")); -+ } else { -+ assert!(text.starts_with("TEXT_PREFIX ")); -+ assert!(text.ends_with("[truncated]")); -+ } -+ assert!(content.truncated); -+ assert!(json_size(content) <= CONTENT_LIMIT_BYTES); -+ } -+ assert_eq!(input.value["previous_response_id"], "previous"); -+ } -+ } -+ } -+ -+ #[test] -+ fn oversized_media_does_not_consume_visible_message_text_budget() { -+ let item: codex_protocol::models::ResponseItem = serde_json::from_value(serde_json::json!({ -+ "type":"message","role":"user","content":[ -+ {"type":"input_image","image_url":format!("data:image/png;base64,{}", "x".repeat(2_000_000))}, -+ {"type":"input_text","text":"VISIBLE_AFTER_MEDIA"} -+ ] -+ })).unwrap(); -+ let content = project_response_item(&item); -+ assert_eq!(content.value["content"][0]["omitted"], true); -+ assert_eq!(content.value["content"][1]["text"], "VISIBLE_AFTER_MEDIA"); -+ assert!(content.omitted); -+ assert!(json_size(&content) <= CONTENT_LIMIT_BYTES); -+ } -+ -+ #[tokio::test] -+ async fn same_input_retry_has_one_live_start_and_changed_input_splits() { -+ let (observation, sink) = observation(OperationCaptureMode::Content); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ model.scope(async { -+ let request = serde_json::json!({"model":"model","input":[{"role":"user","content":"first"}],"client_metadata":{"secret":"META_CANARY"}}); -+ observe_model_request(&request, &test_request(&request), "http://localhost:8080"); -+ assert_eq!(sink.0.lock().unwrap().len(), 1); -+ observe_model_request(&request, &test_request(&request), "http://localhost:8080"); -+ assert_eq!(sink.0.lock().unwrap().len(), 1); -+ let changed = serde_json::json!({"model":"model","input":[{"role":"user","content":"second"}]}); -+ observe_model_request(&changed, &test_request(&changed), "http://localhost:8080"); -+ model.event(&codex_api::ResponseEvent::Completed { response_id:"response".into(),token_usage:None,usage_metadata:None,end_turn:Some(true) }); -+ }).await; -+ model.finish(OperationOutcome::Error); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 4); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); -+ assert_eq!(events[3].outcome, Some(OperationOutcome::Success)); -+ assert_eq!(events[3].metadata.response_id.as_deref(), Some("response")); -+ assert!(!events[3].metadata.response_id_trusted); -+ assert!( -+ !serde_json::to_string(&*events) -+ .unwrap() -+ .contains("META_CANARY") -+ ); -+ } -+ -+ #[test] -+ fn body_model_fallback_never_overrides_authoritative_header_model() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "requested".into(), -+ ); -+ let request = serde_json::json!({"model":"requested","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ model.event(&codex_api::ResponseEvent::ModelObservation( -+ "body-first".into(), -+ )); -+ assert_eq!( -+ model -+ .state -+ .lock() -+ .unwrap() -+ .guard -+ .as_ref() -+ .unwrap() -+ .state -+ .lock() -+ .unwrap() -+ .metadata -+ .observed_model -+ .as_deref(), -+ Some("body-first") -+ ); -+ model.event(&codex_api::ResponseEvent::ServerModel("header".into())); -+ model.event(&codex_api::ResponseEvent::ModelObservation( -+ "body-last".into(), -+ )); -+ model.event(&codex_api::ResponseEvent::Completed { -+ response_id: "response".into(), -+ token_usage: None, -+ usage_metadata: None, -+ end_turn: Some(true), -+ }); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events[1].metadata.observed_model.as_deref(), Some("header")); -+ } -+ -+ #[test] -+ fn stale_stream_cannot_finish_a_new_attempt() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ let old = model.clone().for_stream(); -+ old.source_ended(); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ old.event(&codex_api::ResponseEvent::Completed { -+ response_id: "old".into(), -+ token_usage: None, -+ usage_metadata: None, -+ end_turn: Some(true), -+ }); -+ old.source_ended(); -+ assert_eq!(sink.0.lock().unwrap().len(), 1); -+ model.event(&codex_api::ResponseEvent::Completed { -+ response_id: "new".into(), -+ token_usage: None, -+ usage_metadata: None, -+ end_turn: Some(true), -+ }); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].metadata.response_id.as_deref(), Some("new")); -+ } -+ -+ #[test] -+ fn final_error_uses_stream_end_before_later_cleanup() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ model.source_ended(); -+ let expected = model.state.lock().unwrap().source_end.clone().unwrap(); -+ // A later EOF/drop must not move the source error boundary to tool cleanup. -+ model.source_ended(); -+ model.finish(OperationOutcome::Error); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events[1].completed_at.as_ref(), Some(&expected.0)); -+ assert_eq!(events[1].duration_ms, Some(expected.1)); -+ } -+ -+ #[test] -+ fn successful_handoff_without_response_completion_marks_inference_interrupted() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ model.finish(OperationOutcome::Success); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); -+ } -+ -+ #[test] -+ fn interruption_during_retry_wait_includes_backoff() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ model.source_ended(); -+ model.state.lock().unwrap().source_end = Some(("2000-01-01T00:00:00.000Z".into(), 0.0)); -+ model.retry_waiting(true); -+ model.finish(OperationOutcome::Interrupted); -+ let events = sink.0.lock().unwrap(); -+ assert_ne!( -+ events[1].completed_at.as_deref(), -+ Some("2000-01-01T00:00:00.000Z") -+ ); -+ assert!(events[1].duration_ms.unwrap() > 0.0); -+ } -+ -+ #[test] -+ fn changed_input_closes_previous_inference_at_saved_source_end() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ model.source_ended(); -+ let expected = model.state.lock().unwrap().source_end.clone().unwrap(); -+ let changed = serde_json::json!({"model":"model","input":["changed"]}); -+ model.request(&changed, &test_request(&changed), "http://localhost"); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 3); -+ assert_eq!(events[1].completed_at.as_ref(), Some(&expected.0)); -+ assert_eq!(events[1].duration_ms, Some(expected.1)); -+ } -+ -+ #[test] -+ fn incomplete_response_preserves_usage_without_error_payload() { -+ let (observation, sink) = observation(OperationCaptureMode::Metadata); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost"); -+ let error = codex_api::ApiError::ResponseIncomplete(Box::new( -+ codex_protocol::error::ResponseIncompleteError { -+ reason: "ERROR_PAYLOAD_CANARY".into(), -+ response_id: Some("partial-response".into()), -+ token_usage: Some(codex_protocol::protocol::TokenUsage { -+ input_tokens: 7, -+ output_tokens: 3, -+ total_tokens: 10, -+ ..Default::default() -+ }), -+ }, -+ )); -+ model.source_error(&error); -+ model.finish(OperationOutcome::Error); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!( -+ events[1].metadata.response_id.as_deref(), -+ Some("partial-response") -+ ); -+ assert_eq!( -+ events[1].metadata.usage.as_ref().unwrap()["input_tokens"], -+ 7 -+ ); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); -+ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); -+ } -+ -+ #[test] -+ fn model_interruption_preserves_bounded_visible_partial_output() { -+ let (observation, sink) = observation(OperationCaptureMode::Content); -+ let model = ModelOperation::new( -+ Arc::new(observation), -+ "thread".into(), -+ "turn".into(), -+ "model".into(), -+ ); -+ let interrupted = model.interruption_guard(); -+ let request = serde_json::json!({"model":"model","input":[]}); -+ model.request(&request, &test_request(&request), "http://localhost:8080"); -+ model.event(&codex_api::ResponseEvent::OutputTextDelta( -+ "partial answer".into(), -+ )); -+ drop(interrupted); -+ let events = sink.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); -+ assert_eq!( -+ events[1].output.as_ref().unwrap().value[0]["text"], -+ "partial answer" -+ ); -+ assert_eq!(events[1].output.as_ref().unwrap().value[0]["partial"], true); -+ } -+ -+ #[test] -+ fn response_id_trust_requires_exact_official_https_host() { -+ assert!(trusted_response_endpoint("https://api.openai.com/v1")); -+ assert!(!trusted_response_endpoint( -+ "https://chatgpt.com/backend-api/codex" -+ )); -+ assert!(!trusted_response_endpoint( -+ "https://api.openai.com.example.test/v1" -+ )); -+ assert!(!trusted_response_endpoint("http://api.openai.com/v1")); -+ assert!(!trusted_response_endpoint("http://127.0.0.1:9000/v1")); -+ } -+} -diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs -index cc418ca..66476ae 100644 ---- a/codex-rs/core/src/session/mod.rs -+++ b/codex-rs/core/src/session/mod.rs -@@ -2526,6 +2526,15 @@ impl Session { - - /// Persist the event to rollout and send it to clients. - pub(crate) async fn send_event(&self, turn_context: &TurnContext, msg: EventMsg) { -+ if matches!(&msg, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_)) -+ && let Some(observation) = turn_context -+ .extension_data -+ .get::() -+ { -+ observation -+ .turn_finished(&self.thread_id.to_string(), &turn_context.sub_id) -+ .await; -+ } - if let Some((item_id, started_at_ms)) = msg.command_approval_start() { - // The client can see the command before execution. Keep its first start - // for the canonical lifecycle events that will be saved in history. -diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs -index e4a092e..871cdc9 100644 ---- a/codex-rs/core/src/session/turn.rs -+++ b/codex-rs/core/src/session/turn.rs -@@ -1745,6 +1745,53 @@ async fn run_sampling_request( - client_session: &mut ModelClientSession, - input: Vec, - cancellation_token: CancellationToken, -+) -> CodexResult<(SamplingRequestResult, Vec)> { -+ let observation = step_context -+ .turn -+ .extension_data -+ .get::(); -+ let operation = observation.map(|observation| { -+ crate::operation_observation::ModelOperation::new( -+ observation, -+ sess.thread_id.to_string(), -+ step_context.turn.sub_id.clone(), -+ step_context.settings.model_info.slug.clone(), -+ ) -+ }); -+ let future = run_sampling_request_observed( -+ sess, -+ step_context, -+ turn_store, -+ turn_diff_tracker, -+ client_session, -+ input, -+ cancellation_token, -+ ); -+ if let Some(operation) = operation { -+ let _interrupted = operation.interruption_guard(); -+ let result = operation.scope(future).await; -+ operation.finish(match &result { -+ Ok(_) => crate::operation_observation::OperationOutcome::Success, -+ Err(error) if matches!(error.details(), CodexErrorDetails::TurnAborted) => { -+ crate::operation_observation::OperationOutcome::Interrupted -+ } -+ Err(_) => crate::operation_observation::OperationOutcome::Error, -+ }); -+ result -+ } else { -+ future.await -+ } -+} -+ -+#[allow(clippy::too_many_arguments)] -+async fn run_sampling_request_observed( -+ sess: Arc, -+ step_context: Arc, -+ turn_store: Arc, -+ turn_diff_tracker: SharedTurnDiffTracker, -+ client_session: &mut ModelClientSession, -+ input: Vec, -+ cancellation_token: CancellationToken, - ) -> CodexResult<(SamplingRequestResult, Vec)> { - let turn_context = Arc::clone(&step_context.turn); - let preempt = step_context.preempt.clone().unwrap_or_default(); -@@ -1854,6 +1901,9 @@ async fn run_sampling_request( - }, - }; - -+ if let Some(operation) = crate::operation_observation::current_model_operation() { -+ operation.retry_waiting(true); -+ } - let retry = handle_response_stream_error( - &mut retry_state, - max_retries, -@@ -1866,6 +1916,9 @@ async fn run_sampling_request( - .or_cancel(&preempt) - .or_cancel(&cancellation_token) - .await?; -+ if let Some(operation) = crate::operation_observation::current_model_operation() { -+ operation.retry_waiting(false); -+ } - if cancellation_token.is_cancelled() { - return Err(CodexErr::TurnAborted); - } -@@ -3170,6 +3223,8 @@ async fn try_run_sampling_request( - active_item_is_streaming_to_client = stream_item_to_client; - } - } -+ // Consumed by the client observer before session events are forwarded. -+ ResponseEvent::ModelObservation(_) => {} - ResponseEvent::ServerModel(server_model) => { - if !turn_context - .server_model_warning_emitted -diff --git a/codex-rs/core/src/tools/code_mode/execute_handler.rs b/codex-rs/core/src/tools/code_mode/execute_handler.rs -index 317e029..4354977 100644 ---- a/codex-rs/core/src/tools/code_mode/execute_handler.rs -+++ b/codex-rs/core/src/tools/code_mode/execute_handler.rs -@@ -208,7 +208,10 @@ impl CodeModeExecuteHandler { - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - let handler_span = tracing::Span::current(); -- let originating_call = invocation.originating_call().await; -+ let mut originating_call = invocation.originating_call().await; -+ if let Some(origin) = originating_call.as_mut() { -+ origin.operation_id = crate::tools::operation_observation::current_tool_operation_id(); -+ } - let ToolInvocation { - session, - turn, -diff --git a/codex-rs/core/src/tools/code_mode/output.rs b/codex-rs/core/src/tools/code_mode/output.rs -index a85b77a..f264419 100644 ---- a/codex-rs/core/src/tools/code_mode/output.rs -+++ b/codex-rs/core/src/tools/code_mode/output.rs -@@ -40,6 +40,14 @@ impl CodeModeToolOutput { - } - - impl ToolOutput for CodeModeToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ self.output.bounded_code_mode_result(payload, max_bytes) -+ } -+ - fn log_output(&self) -> String { - self.output.log_output() - } -diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs -index fe7de2c..718ea5c 100644 ---- a/codex-rs/core/src/tools/context.rs -+++ b/codex-rs/core/src/tools/context.rs -@@ -125,6 +125,8 @@ pub struct ToolInvocation { - /// Identity of the model call that started a tool invocation, retained across code-mode waits. - #[derive(Clone)] - pub(crate) struct ToolCallOrigin { -+ /// Execution observation retained across code-mode yields and detached callbacks. -+ pub(crate) operation_id: Option, - /// Best-effort lookup: invocations without a matching history item still - /// retain their known window ID. - pub(crate) item_id: Option, -@@ -158,6 +160,7 @@ impl ToolInvocation { - _ => None, - }); - Some(ToolCallOrigin { -+ operation_id: super::operation_observation::current_tool_operation_id(), - item_id, - window_id: self.session.current_window_id().await, - }) -@@ -176,6 +179,14 @@ pub struct McpToolOutput { - } - - impl ToolOutput for McpToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ self.result.bounded_code_mode_result(payload, max_bytes) -+ } -+ - fn log_output(&self) -> String { - // Logging has its own budget; do not first apply the model-context budget. - let output = self.result.log_output(); -@@ -262,6 +273,15 @@ pub struct ToolSearchOutput { - } - - impl ToolOutput for ToolSearchOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ let _ = payload; -+ codex_tools::bounded_tool_output(&self.tools, max_bytes) -+ } -+ - fn log_output(&self) -> String { - let tools = self - .tools -@@ -329,6 +349,17 @@ impl FunctionToolOutput { - } - - impl ToolOutput for FunctionToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ // The normal conversion clones and joins body items. Admit only a bounded -+ // serialized source first, including item count and embedded media bytes. -+ codex_tools::bounded_tool_output(&self.body, max_bytes)?; -+ Ok(self.code_mode_result(payload)) -+ } -+ - fn log_output(&self) -> String { - function_call_output_content_items_to_text(&self.body).unwrap_or_default() - } -@@ -357,6 +388,15 @@ impl ApplyPatchToolOutput { - } - - impl ToolOutput for ApplyPatchToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ let _ = max_bytes; -+ Ok(self.code_mode_result(payload)) -+ } -+ - fn log_output(&self) -> String { - self.text.clone() - } -@@ -390,6 +430,15 @@ pub struct AbortedToolOutput { - } - - impl ToolOutput for AbortedToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ codex_tools::bounded_tool_output(&self.message, max_bytes)?; -+ Ok(self.code_mode_result(payload)) -+ } -+ - fn log_output(&self) -> String { - self.message.clone() - } -@@ -437,6 +486,17 @@ pub struct ExecCommandToolOutput { - } - - impl ToolOutput for ExecCommandToolOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ // Rendering may clone raw text even when the model token budget is small. -+ // Establish a source bound before calling the existing formatter. -+ codex_tools::bounded_tool_output(&(&self.chunk_id, &self.raw_output.text), max_bytes)?; -+ Ok(self.code_mode_result(payload)) -+ } -+ - fn log_output(&self) -> String { - // The telemetry budget must not inherit the model's output-token limit. - let output = self -diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs -index aa9a5f3..c62943d 100644 ---- a/codex-rs/core/src/tools/context_tests.rs -+++ b/codex-rs/core/src/tools/context_tests.rs -@@ -554,3 +554,78 @@ fn exec_command_tool_output_preserves_omission_metadata_when_truncated() { - assert!(text.contains("Warning: truncated output (original token count: 42000)")); - assert_eq!(text.matches(&marker).count(), 1); - } -+ -+#[test] -+fn bounded_capture_checks_function_source_before_cloning_and_joining() { -+ let payload = ToolPayload::Function { -+ arguments: "{}".into(), -+ }; -+ let mut output = FunctionToolOutput::from_content( -+ vec![ -+ FunctionCallOutputContentItem::InputText { -+ text: "first".into(), -+ }, -+ FunctionCallOutputContentItem::InputText { -+ text: "second".into(), -+ }, -+ ], -+ Some(true), -+ ); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Ok(json!("first\nsecond")) -+ ); -+ output.body.push(FunctionCallOutputContentItem::InputImage { -+ image: ImageReference::Inline { -+ image_url: "x".repeat(1024 * 1024), -+ }, -+ detail: None, -+ }); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+ output.body = (0..10000) -+ .map(|_| FunctionCallOutputContentItem::InputText { -+ text: String::new(), -+ }) -+ .collect(); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+} -+ -+#[test] -+fn bounded_capture_checks_exec_source_before_rendering() { -+ let payload = ToolPayload::Function { -+ arguments: "{}".into(), -+ }; -+ let mut output = ExecCommandToolOutput { -+ event_call_id: "call-42".into(), -+ chunk_id: "chunk".into(), -+ wall_time: Duration::from_millis(25), -+ raw_output: "visible".into(), -+ truncation_policy: TruncationPolicy::Bytes(10000), -+ max_output_tokens: None, -+ process_id: None, -+ exit_code: Some(0), -+ original_token_count: Some(1), -+ hook_command: None, -+ tool_cwd: None, -+ }; -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Ok(output.code_mode_result(&payload)) -+ ); -+ output.raw_output.text = "x".repeat(1024 * 1024); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+ output.max_output_tokens = Some(1); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+} -diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs -index 64d6cc1..0c1c08a 100644 ---- a/codex-rs/core/src/tools/handlers/apply_patch.rs -+++ b/codex-rs/core/src/tools/handlers/apply_patch.rs -@@ -338,6 +338,10 @@ impl ApplyPatchHandler { - } - - impl CoreToolRuntime for ApplyPatchHandler { -+ fn approves_inside_handler(&self) -> bool { -+ true -+ } -+ - fn environments_for_input( - &self, - invocation: &ToolInvocation, -diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs -index 0eac846..59c158c 100644 ---- a/codex-rs/core/src/tools/handlers/dynamic.rs -+++ b/codex-rs/core/src/tools/handlers/dynamic.rs -@@ -174,6 +174,10 @@ impl DynamicToolHandler { - } - - impl CoreToolRuntime for DynamicToolHandler { -+ fn is_client_tool_handoff(&self) -> bool { -+ true -+ } -+ - fn finishes_on_cancellation(&self) -> bool { - true - } -diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs -index 8ac6845..a040228 100644 ---- a/codex-rs/core/src/tools/handlers/mcp.rs -+++ b/codex-rs/core/src/tools/handlers/mcp.rs -@@ -307,6 +307,10 @@ impl McpHandler { - } - - impl CoreToolRuntime for McpHandler { -+ fn approves_inside_handler(&self) -> bool { -+ true -+ } -+ - fn environments_for_input( - &self, - invocation: &ToolInvocation, -diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs -index 6a81b62..00af650 100644 ---- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs -+++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs -@@ -309,6 +309,10 @@ fn one_shot_exec_command_spec(spec: ToolSpec) -> ToolSpec { - } - - impl CoreToolRuntime for ExecCommandHandler { -+ fn approves_inside_handler(&self) -> bool { -+ true -+ } -+ - fn telemetry_tags(&self, invocation: &ToolInvocation) -> ToolTelemetryTags { - // Record the configured mode so fallback calls remain in their rollout cohort. - // This does not indicate whether the command actually used a snapshot. -diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs -index 3344ed8..ea173f5 100644 ---- a/codex-rs/core/src/tools/mod.rs -+++ b/codex-rs/core/src/tools/mod.rs -@@ -14,6 +14,7 @@ mod multi_agent_tool; - mod native_tool_review; - pub(crate) use native_tool_review::review as review_native_tool_call; - pub(crate) mod network_approval; -+pub(crate) mod operation_observation; - pub(crate) mod orchestrator; - pub(crate) mod parallel; - pub(crate) mod registry; -diff --git a/codex-rs/core/src/tools/operation_observation.rs b/codex-rs/core/src/tools/operation_observation.rs -new file mode 100644 -index 0000000..5ecf36d ---- /dev/null -+++ b/codex-rs/core/src/tools/operation_observation.rs -@@ -0,0 +1,207 @@ -+//! Execution observations, separate from admission, approval, and result hooks. -+ -+use std::sync::Arc; -+use std::sync::Mutex; -+ -+use serde_json::Value; -+ -+use crate::function_tool::FunctionCallError; -+use crate::operation_observation::CONTENT_VALUE_LIMIT_BYTES; -+use crate::operation_observation::OperationContent; -+use crate::operation_observation::OperationGuard; -+use crate::operation_observation::OperationKind; -+use crate::operation_observation::OperationMetadata; -+use crate::operation_observation::OperationObservation; -+use crate::operation_observation::OperationOutcome; -+use crate::operation_observation::project_content; -+use crate::operation_observation::project_serializable_content; -+use crate::operation_observation::project_text; -+use crate::tools::context::ToolCallSource; -+use crate::tools::context::ToolInvocation; -+use crate::tools::context::ToolOutput; -+use crate::tools::context::ToolPayload; -+use crate::tools::registry::CoreToolRuntime; -+ -+tokio::task_local! { -+ static CURRENT_EXECUTION: Arc; -+} -+ -+struct ExecutionObservation { -+ observation: Arc, -+ thread_id: String, -+ turn_id: String, -+ metadata: OperationMetadata, -+ input: Option, -+ guard: Mutex>, -+} -+ -+impl ExecutionObservation { -+ fn start(&self, input: Option) { -+ let mut guard = self -+ .guard -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ // Sandbox retries still belong to the same native tool execution. -+ if guard.is_some() { -+ return; -+ } -+ let started = self.observation.start( -+ self.thread_id.clone(), -+ self.turn_id.clone(), -+ OperationKind::Tool, -+ self.metadata.clone(), -+ ); -+ if let Some(input) = input.or_else(|| self.input.clone()) { -+ started.capture_input_content(input); -+ } -+ *guard = Some(started); -+ } -+ -+ fn operation_id(&self) -> Option { -+ self.guard -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner) -+ .as_ref() -+ .map(|guard| guard.operation_id().to_string()) -+ } -+ -+ fn finish( -+ &self, -+ invocation: &ToolInvocation, -+ result: &Result, FunctionCallError>, -+ ) { -+ let guard = self -+ .guard -+ .lock() -+ .unwrap_or_else(std::sync::PoisonError::into_inner); -+ let Some(guard) = guard.as_ref() else { -+ // A rejected approval has not executed a tool. -+ return; -+ }; -+ let outcome = if invocation.cancellation_token.is_cancelled() { -+ OperationOutcome::Interrupted -+ } else if result -+ .as_ref() -+ .is_ok_and(|output| output.success_for_logging()) -+ { -+ OperationOutcome::Success -+ } else { -+ OperationOutcome::Error -+ }; -+ guard.finish_with_bounded_output_fn(outcome, || match result { -+ Ok(output) => { -+ output.bounded_code_mode_result(&invocation.payload, CONTENT_VALUE_LIMIT_BYTES) -+ } -+ Err(error) => { -+ let message = match error { -+ FunctionCallError::RespondToModel(message) -+ | FunctionCallError::Fatal(message) => message, -+ }; -+ // Bound source text before Display allocates the optional fatal prefix. -+ if message.len() > CONTENT_VALUE_LIMIT_BYTES { -+ return Err("capture_budget"); -+ } -+ #[derive(serde::Serialize)] -+ struct ErrorOutput<'a> { -+ error: &'a str, -+ } -+ let rendered; -+ let error = if matches!(error, FunctionCallError::Fatal(_)) { -+ rendered = error.to_string(); -+ rendered.as_str() -+ } else { -+ message.as_str() -+ }; -+ codex_tools::bounded_tool_output(&ErrorOutput { error }, CONTENT_VALUE_LIMIT_BYTES) -+ } -+ }); -+ } -+} -+ -+/// Called immediately before an approved sandbox or MCP execution starts. -+pub(crate) fn start_current_execution() { -+ let _ = CURRENT_EXECUTION.try_with(|execution| execution.start(None)); -+} -+ -+/// MCP preparation can rewrite arguments before sending them to the server. -+pub(crate) fn start_current_execution_with_input(input: &Value) { -+ let _ = CURRENT_EXECUTION.try_with(|execution| { -+ let input = execution -+ .observation -+ .captures_content() -+ .then(|| project_content(input)); -+ execution.start(input); -+ }); -+} -+ -+pub(crate) fn current_tool_operation_id() -> Option { -+ CURRENT_EXECUTION -+ .try_with(|execution| execution.operation_id()) -+ .ok() -+ .flatten() -+} -+ -+pub(crate) async fn observe_execution( -+ tool: &dyn CoreToolRuntime, -+ invocation: &ToolInvocation, -+) -> Result, FunctionCallError> { -+ let Some(observation) = invocation.turn.extension_data.get::() else { -+ return tool.handle(invocation.clone()).await; -+ }; -+ if tool.is_client_tool_handoff() { -+ return tool.handle(invocation.clone()).await; -+ } -+ -+ let parent_operation_id = match &invocation.source { -+ ToolCallSource::CodeMode { cell_id, .. } => invocation -+ .session -+ .services -+ .code_mode_service -+ .cell_originating_call(&codex_code_mode::CellId::new(cell_id.clone())) -+ .and_then(|origin| origin.operation_id), -+ _ => current_tool_operation_id(), -+ }; -+ let input = observation -+ .captures_content() -+ .then(|| match &invocation.payload { -+ ToolPayload::Function { arguments } => { -+ // Trace-only parsing must not allocate a JSON tree beyond the capture budget. -+ if arguments.len() > CONTENT_VALUE_LIMIT_BYTES { -+ project_content(&serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"})) -+ } else if let Ok(value) = serde_json::from_str::(arguments) { -+ project_content(&value) -+ } else { -+ project_text(arguments) -+ } -+ } -+ ToolPayload::Custom { input } => project_text(input), -+ ToolPayload::ToolSearch { arguments } => { -+ project_serializable_content(arguments) -+ } -+ }); -+ let execution = Arc::new(ExecutionObservation { -+ observation, -+ thread_id: invocation.session.thread_id.to_string(), -+ turn_id: invocation.turn.sub_id.clone(), -+ metadata: OperationMetadata { -+ tool_name: Some(crate::tools::flat_tool_name(&invocation.tool_name).into_owned()), -+ call_id: Some(invocation.call_id.clone()), -+ parent_operation_id, -+ ..Default::default() -+ }, -+ input, -+ guard: Mutex::new(None), -+ }); -+ if !tool.approves_inside_handler() { -+ execution.start(None); -+ } -+ let result = CURRENT_EXECUTION -+ .scope(Arc::clone(&execution), tool.handle(invocation.clone())) -+ .await; -+ execution.finish(invocation, &result); -+ result -+} -+ -+#[cfg(test)] -+#[path = "operation_observation_tests.rs"] -+mod tests; -diff --git a/codex-rs/core/src/tools/operation_observation_tests.rs b/codex-rs/core/src/tools/operation_observation_tests.rs -new file mode 100644 -index 0000000..ed180f3 ---- /dev/null -+++ b/codex-rs/core/src/tools/operation_observation_tests.rs -@@ -0,0 +1,455 @@ -+use super::*; -+use crate::operation_observation::OperationCaptureMode; -+use crate::operation_observation::OperationEvent; -+use crate::operation_observation::OperationObserver; -+use crate::operation_observation::OperationPhase; -+use crate::session::step_context::StepContext; -+use crate::session::tests::make_session_and_context; -+use crate::tools::context::FunctionToolOutput; -+use crate::tools::context::ToolCallOrigin; -+use crate::tools::registry::ToolExecutor; -+use crate::turn_diff_tracker::TurnDiffTracker; -+use pretty_assertions::assert_eq; -+use tokio::sync::Notify; -+use tokio_util::sync::CancellationToken; -+ -+#[derive(Default)] -+struct Events(Mutex>); -+ -+impl OperationObserver for Events { -+ fn emit(&self, event: OperationEvent) { -+ self.0.lock().unwrap().push(event); -+ } -+} -+ -+struct Handler { -+ approved: Option>, -+ entered: Option>, -+ complete: Option>, -+ denied: bool, -+ handoff: bool, -+ success: bool, -+} -+ -+impl Default for Handler { -+ fn default() -> Self { -+ Self { -+ approved: None, -+ entered: None, -+ complete: None, -+ denied: false, -+ handoff: false, -+ success: true, -+ } -+ } -+} -+ -+impl ToolExecutor for Handler { -+ fn tool_name(&self) -> codex_tools::ToolName { -+ codex_tools::ToolName::plain("test_tool") -+ } -+ -+ fn spec(&self) -> codex_tools::ToolSpec { -+ codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool { -+ name: "test_tool".to_string(), -+ description: "Observation fixture.".to_string(), -+ strict: false, -+ defer_loading: None, -+ parameters: codex_tools::JsonSchema::default(), -+ output_schema: None, -+ }) -+ } -+ -+ fn handle<'a>(&'a self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'a> -+ where -+ ToolInvocation: 'a, -+ { -+ Box::pin(async move { -+ if let Some(approved) = &self.approved { -+ approved.notified().await; -+ } -+ if self.denied { -+ return Err(FunctionCallError::RespondToModel( -+ "Denied fixture.".to_string(), -+ )); -+ } -+ if self.approved.is_some() { -+ start_current_execution(); -+ } -+ if let Some(entered) = &self.entered { -+ entered.notify_one(); -+ } -+ if let Some(complete) = &self.complete { -+ complete.notified().await; -+ } -+ Ok(Box::new(FunctionToolOutput::from_text( -+ "private-tool-result".to_string(), -+ Some(self.success), -+ )) as Box) -+ }) -+ } -+} -+ -+impl CoreToolRuntime for Handler { -+ fn approves_inside_handler(&self) -> bool { -+ self.approved.is_some() || self.denied -+ } -+ -+ fn is_client_tool_handoff(&self) -> bool { -+ self.handoff -+ } -+} -+ -+async fn fixture(mode: OperationCaptureMode) -> (ToolInvocation, Arc) { -+ let (session, turn) = make_session_and_context().await; -+ let events = Arc::new(Events::default()); -+ turn.extension_data -+ .insert(OperationObservation::new(mode, events.clone())); -+ let turn = Arc::new(turn); -+ let invocation = ToolInvocation { -+ session: Arc::new(session), -+ step_context: StepContext::for_test(Arc::clone(&turn)), -+ turn, -+ cancellation_token: CancellationToken::new(), -+ tracker: Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())), -+ call_id: "call-1".to_string(), -+ tool_name: codex_tools::ToolName::plain("test_tool"), -+ source: ToolCallSource::Direct, -+ payload: ToolPayload::Function { -+ arguments: r#"{"query":"private-tool-argument"}"#.to_string(), -+ }, -+ environments: Default::default(), -+ }; -+ (invocation, events) -+} -+ -+#[tokio::test] -+async fn approval_precedes_live_execution_start() { -+ let (invocation, events) = fixture(OperationCaptureMode::Content).await; -+ let approved = Arc::new(Notify::new()); -+ let entered = Arc::new(Notify::new()); -+ let complete = Arc::new(Notify::new()); -+ let handler = Handler { -+ approved: Some(Arc::clone(&approved)), -+ entered: Some(Arc::clone(&entered)), -+ complete: Some(Arc::clone(&complete)), -+ ..Default::default() -+ }; -+ let task = tokio::spawn(async move { observe_execution(&handler, &invocation).await }); -+ tokio::task::yield_now().await; -+ assert!(events.0.lock().unwrap().is_empty()); -+ approved.notify_one(); -+ entered.notified().await; -+ { -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 1); -+ assert_eq!(events[0].phase, OperationPhase::Started); -+ assert_eq!(events[0].metadata.call_id.as_deref(), Some("call-1")); -+ } -+ complete.notify_one(); -+ assert!(task.await.unwrap().is_ok()); -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[0].operation_id, events[1].operation_id); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); -+ let content = serde_json::to_string(&events[1]).unwrap(); -+ assert!(content.contains("private-tool-argument")); -+ assert!(content.contains("private-tool-result")); -+} -+ -+#[tokio::test] -+async fn denied_execution_and_client_handoff_emit_no_native_operation() { -+ let (invocation, events) = fixture(OperationCaptureMode::Content).await; -+ assert!( -+ observe_execution( -+ &Handler { -+ denied: true, -+ ..Default::default() -+ }, -+ &invocation -+ ) -+ .await -+ .is_err() -+ ); -+ assert!( -+ observe_execution( -+ &Handler { -+ handoff: true, -+ ..Default::default() -+ }, -+ &invocation -+ ) -+ .await -+ .is_ok() -+ ); -+ assert!(events.0.lock().unwrap().is_empty()); -+} -+ -+#[tokio::test] -+async fn metadata_omits_payloads_and_retains_failed_execution() { -+ let (invocation, events) = fixture(OperationCaptureMode::Metadata).await; -+ let registry = crate::tools::registry::ToolRegistry::with_handler_for_test(Arc::new(Handler { -+ success: false, -+ ..Default::default() -+ })); -+ assert!( -+ registry -+ .dispatch_any_with_state(invocation, None) -+ .await -+ .is_ok() -+ ); -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); -+ assert!(events[1].input.is_none()); -+ assert!(events[1].output.is_none()); -+ let serialized = serde_json::to_string(&*events).unwrap(); -+ assert!(!serialized.contains("private-tool-argument")); -+ assert!(!serialized.contains("private-tool-result")); -+} -+ -+#[tokio::test] -+async fn abort_emits_one_interrupted_terminal() { -+ let (invocation, events) = fixture(OperationCaptureMode::Metadata).await; -+ let entered = Arc::new(Notify::new()); -+ let handler = Handler { -+ entered: Some(Arc::clone(&entered)), -+ complete: Some(Arc::new(Notify::new())), -+ ..Default::default() -+ }; -+ let task = tokio::spawn(async move { observe_execution(&handler, &invocation).await }); -+ entered.notified().await; -+ task.abort(); -+ assert!(task.await.err().unwrap().is_cancelled()); -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[0].operation_id, events[1].operation_id); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Interrupted)); -+} -+ -+#[tokio::test] -+async fn detached_parallel_code_cell_tools_retain_the_issuing_operation() { -+ let (mut first, events) = fixture(OperationCaptureMode::Metadata).await; -+ let cell_id = codex_code_mode::CellId::new("cell-1".to_string()); -+ first -+ .session -+ .services -+ .code_mode_service -+ .mark_cell_ready_for_dispatch( -+ &cell_id, -+ Some(ToolCallOrigin { -+ operation_id: Some("issuing-exec-operation".to_string()), -+ item_id: None, -+ window_id: "window-1".to_string(), -+ }), -+ ); -+ first.source = ToolCallSource::CodeMode { -+ cell_id: "cell-1".to_string(), -+ runtime_tool_call_id: "nested-1".to_string(), -+ }; -+ let mut second = first.clone(); -+ second.call_id = "call-2".to_string(); -+ second.source = ToolCallSource::CodeMode { -+ cell_id: "cell-1".to_string(), -+ runtime_tool_call_id: "nested-2".to_string(), -+ }; -+ let first_entered = Arc::new(Notify::new()); -+ let second_entered = Arc::new(Notify::new()); -+ let complete = Arc::new(Notify::new()); -+ let first_handler = Handler { -+ entered: Some(Arc::clone(&first_entered)), -+ complete: Some(Arc::clone(&complete)), -+ ..Default::default() -+ }; -+ let second_handler = Handler { -+ entered: Some(Arc::clone(&second_entered)), -+ complete: Some(Arc::clone(&complete)), -+ ..Default::default() -+ }; -+ let first_task = tokio::spawn(async move { observe_execution(&first_handler, &first).await }); -+ let second_task = -+ tokio::spawn(async move { observe_execution(&second_handler, &second).await }); -+ first_entered.notified().await; -+ second_entered.notified().await; -+ assert_eq!(events.0.lock().unwrap().len(), 2); -+ complete.notify_waiters(); -+ assert!(first_task.await.unwrap().is_ok()); -+ assert!(second_task.await.unwrap().is_ok()); -+ let events = events.0.lock().unwrap(); -+ let starts: Vec<_> = events -+ .iter() -+ .filter(|event| event.phase == OperationPhase::Started) -+ .collect(); -+ assert_eq!(starts.len(), 2); -+ assert_ne!(starts[0].operation_id, starts[1].operation_id); -+ for event in starts { -+ assert_eq!( -+ event.metadata.parent_operation_id.as_deref(), -+ Some("issuing-exec-operation") -+ ); -+ } -+} -+ -+#[tokio::test] -+async fn oversized_function_arguments_are_omitted_without_changing_execution() { -+ let (mut invocation, events) = fixture(OperationCaptureMode::Content).await; -+ invocation.payload = ToolPayload::Function { -+ arguments: serde_json::json!({"authorization":"TOOL_ARGUMENT_CANARY","items":vec![0; CONTENT_VALUE_LIMIT_BYTES * 8]}).to_string(), -+ }; -+ assert!( -+ observe_execution(&Handler::default(), &invocation) -+ .await -+ .is_ok() -+ ); -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); -+ let content = events[1].input.as_ref().unwrap(); -+ assert_eq!( -+ content.value, -+ serde_json::json!({"omitted":true,"truncated":true,"reason":"capture_budget"}) -+ ); -+ assert!(content.omitted); -+ assert!(content.truncated); -+ assert!(!serde_json::to_string(&*events).unwrap().contains("CANARY")); -+} -+ -+#[tokio::test] -+async fn oversized_custom_input_keeps_prefix_without_changing_execution() { -+ let (mut invocation, events) = fixture(OperationCaptureMode::Content).await; -+ invocation.payload = ToolPayload::Custom { -+ input: format!( -+ "visible custom input {}", -+ "é🙂".repeat(CONTENT_VALUE_LIMIT_BYTES * 8) -+ ), -+ }; -+ assert!( -+ observe_execution(&Handler::default(), &invocation) -+ .await -+ .is_ok() -+ ); -+ let events = events.0.lock().unwrap(); -+ assert_eq!(events.len(), 2); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Success)); -+ let content = events[1].input.as_ref().unwrap(); -+ assert!( -+ content -+ .value -+ .as_str() -+ .unwrap() -+ .starts_with("visible custom input ") -+ ); -+ assert!(content.value.as_str().unwrap().ends_with("[truncated]")); -+ assert!(content.truncated); -+ assert!( -+ serde_json::to_vec(content).unwrap().len() -+ <= crate::operation_observation::CONTENT_LIMIT_BYTES -+ ); -+} -+ -+#[tokio::test] -+async fn bounded_capture_marks_unknown_outputs_and_large_errors_without_materialization() { -+ struct Unknown(Arc); -+ impl ToolOutput for Unknown { -+ fn bounded_code_mode_result( -+ &self, -+ _: &ToolPayload, -+ _: usize, -+ ) -> Result { -+ self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); -+ Err("unsupported_output") -+ } -+ fn log_output(&self) -> String { -+ panic!("unexpected conversion") -+ } -+ fn success_for_logging(&self) -> bool { -+ true -+ } -+ fn to_response_item( -+ &self, -+ _: &str, -+ _: &ToolPayload, -+ ) -> codex_protocol::models::ResponseInputItem { -+ panic!("unexpected conversion") -+ } -+ fn code_mode_result(&self, _: &ToolPayload) -> Value { -+ panic!("unexpected conversion") -+ } -+ } -+ for mode in [ -+ OperationCaptureMode::Metadata, -+ OperationCaptureMode::Content, -+ ] { -+ let (invocation, events) = fixture(mode).await; -+ let observation = invocation -+ .turn -+ .extension_data -+ .get::() -+ .unwrap(); -+ let execution = ExecutionObservation { -+ observation, -+ thread_id: "thread".into(), -+ turn_id: "turn".into(), -+ metadata: OperationMetadata::default(), -+ input: None, -+ guard: Mutex::new(None), -+ }; -+ execution.start(None); -+ let captures = Arc::new(std::sync::atomic::AtomicUsize::new(0)); -+ execution.finish(&invocation, &Ok(Box::new(Unknown(captures.clone())))); -+ assert_eq!( -+ captures.load(std::sync::atomic::Ordering::Relaxed), -+ usize::from(mode == OperationCaptureMode::Content) -+ ); -+ let output = events.0.lock().unwrap()[1].output.clone(); -+ if mode == OperationCaptureMode::Content { -+ assert_eq!( -+ output.unwrap().reason.as_deref(), -+ Some("unsupported_output") -+ ); -+ } else { -+ assert!(output.is_none()); -+ } -+ } -+ for (error, expected) in [ -+ ( -+ FunctionCallError::RespondToModel("x".repeat(1024 * 1024)), -+ None, -+ ), -+ (FunctionCallError::Fatal("x".repeat(1024 * 1024)), None), -+ ( -+ FunctionCallError::RespondToModel("visible".into()), -+ Some("visible"), -+ ), -+ ( -+ FunctionCallError::Fatal("visible".into()), -+ Some("Fatal error: visible"), -+ ), -+ ] { -+ let (invocation, events) = fixture(OperationCaptureMode::Content).await; -+ let execution = ExecutionObservation { -+ observation: invocation -+ .turn -+ .extension_data -+ .get::() -+ .unwrap(), -+ thread_id: "thread".into(), -+ turn_id: "turn".into(), -+ metadata: OperationMetadata::default(), -+ input: None, -+ guard: Mutex::new(None), -+ }; -+ execution.start(None); -+ execution.finish(&invocation, &Err(error)); -+ let events = events.0.lock().unwrap(); -+ let output = events[1].output.as_ref().unwrap(); -+ assert_eq!(events[1].outcome, Some(OperationOutcome::Error)); -+ if let Some(expected) = expected { -+ assert_eq!(output.value, serde_json::json!({"error": expected})); -+ assert!(!output.omitted && !output.truncated); -+ } else { -+ assert!(output.omitted && output.truncated); -+ assert_eq!(output.reason.as_deref(), Some("capture_budget")); -+ } -+ } -+} -diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs -index a1a88e4..d9afeb6 100644 ---- a/codex-rs/core/src/tools/orchestrator.rs -+++ b/codex-rs/core/src/tools/orchestrator.rs -@@ -102,6 +102,7 @@ impl ToolOrchestrator { - .as_ref() - .map(ActiveNetworkApproval::execution_proxy), - }; -+ super::operation_observation::start_current_execution(); - let run_result = tool - .run(req, &attempt_with_network_approval, &attempt_tool_ctx) - .await; -diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs -index 3dccc2e..e1db6de 100644 ---- a/codex-rs/core/src/tools/registry.rs -+++ b/codex-rs/core/src/tools/registry.rs -@@ -62,6 +62,16 @@ pub use codex_tools::ToolExposure; - /// Implementers provide the shared `ToolExecutor` behavior plus optional - /// core-owned metadata for hooks, telemetry, tool search, and argument diffs. - pub(crate) trait CoreToolRuntime: ToolExecutor { -+ /// Client callbacks execute outside this process and are observed by their owner. -+ fn is_client_tool_handoff(&self) -> bool { -+ false -+ } -+ -+ /// Defers the execution observation until the handler's approval has completed. -+ fn approves_inside_handler(&self) -> bool { -+ false -+ } -+ - /// Opts this invocation into synchronous native-call review before the handler runs. - fn requires_guardian_review(&self, _invocation: &ToolInvocation) -> bool { - false -@@ -335,6 +345,14 @@ struct PostToolUseFeedbackOutput { - } - - impl ToolOutput for PostToolUseFeedbackOutput { -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ self.original.bounded_code_mode_result(payload, max_bytes) -+ } -+ - fn log_output(&self) -> String { - self.original.log_output() - } -@@ -1050,7 +1068,7 @@ async fn handle_any_tool( - ) -> Result { - let call_id = invocation.call_id.clone(); - let payload = invocation.payload.clone(); -- let output = tool.handle(invocation.clone()).await?; -+ let output = super::operation_observation::observe_execution(tool, &invocation).await?; - let post_tool_use_payload = - CoreToolRuntime::post_tool_use_payload(tool, &invocation, output.as_ref()); - let result = AnyToolResult { -diff --git a/codex-rs/core/src/turn_timing.rs b/codex-rs/core/src/turn_timing.rs -index de45b74..9dd25e6 100644 ---- a/codex-rs/core/src/turn_timing.rs -+++ b/codex-rs/core/src/turn_timing.rs -@@ -492,6 +492,7 @@ fn response_event_records_turn_ttft(event: &ResponseEvent) -> bool { - | ResponseEvent::ReasoningContentDelta { .. } => true, - ResponseEvent::Created { .. } - | ResponseEvent::ServerModel(_) -+ | ResponseEvent::ModelObservation(_) - | ResponseEvent::ModelVerifications(_) - | ResponseEvent::TurnModerationMetadata(_) - | ResponseEvent::GuardianAssessment(_) -diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs -index ced543a..193452e 100644 ---- a/codex-rs/otel/src/events/session_telemetry.rs -+++ b/codex-rs/otel/src/events/session_telemetry.rs -@@ -1406,7 +1406,9 @@ impl SessionTelemetry { - ResponseEvent::ReasoningSummaryPartAdded { .. } => { - "reasoning_summary_part_added".into() - } -- ResponseEvent::ServerModel(_) => "server_model".into(), -+ ResponseEvent::ServerModel(_) | ResponseEvent::ModelObservation(_) => { -+ "server_model".into() -+ } - ResponseEvent::ModelVerifications(_) => "model_verifications".into(), - ResponseEvent::TurnModerationMetadata(_) => "turn_moderation_metadata".into(), - ResponseEvent::GuardianAssessment(_) => "guardian_assessment".into(), -diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs -index 8661659..1d31dfa 100644 ---- a/codex-rs/tools/src/lib.rs -+++ b/codex-rs/tools/src/lib.rs -@@ -104,6 +104,7 @@ pub use tool_executor::ToolExposure; - pub use tool_executor::ToolExposures; - pub use tool_output::JsonToolOutput; - pub use tool_output::ToolOutput; -+pub use tool_output::bounded_tool_output; - pub use tool_payload::ToolPayload; - pub use tool_search::ToolSearchEntry; - pub use tool_search::ToolSearchInfo; -diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs -index 3a4c1e8..6335093 100644 ---- a/codex-rs/tools/src/tool_output.rs -+++ b/codex-rs/tools/src/tool_output.rs -@@ -68,6 +68,17 @@ pub trait ToolOutput: Send { - response_input_to_code_mode_result(self.to_response_item("", payload)) - } - -+ /// Produces capture data without first materializing an unbounded owned result. -+ /// Implementations must bound borrowed source data before cloning or rendering it. -+ /// Unknown output types omit capture; execution and normal model output are unchanged. -+ fn bounded_code_mode_result( -+ &self, -+ _payload: &ToolPayload, -+ _max_bytes: usize, -+ ) -> Result { -+ Err("unsupported_output") -+ } -+ - /// Borrows original host-only metadata for recording, not for model output or logging. - fn tool_result_metadata(&self) -> Option<&JsonValue> { - None -@@ -122,6 +133,14 @@ where - (**self).code_mode_result(payload) - } - -+ fn bounded_code_mode_result( -+ &self, -+ payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ (**self).bounded_code_mode_result(payload, max_bytes) -+ } -+ - fn tool_result_metadata(&self) -> Option<&JsonValue> { - (**self).tool_result_metadata() - } -@@ -197,9 +216,42 @@ impl ToolOutput for JsonToolOutput { - fn code_mode_result(&self, _payload: &ToolPayload) -> JsonValue { - self.value.clone() - } -+ -+ fn bounded_code_mode_result( -+ &self, -+ _payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ bounded_tool_output(&self.value, max_bytes) -+ } - } - - impl ToolOutput for codex_protocol::mcp::CallToolResult { -+ fn bounded_code_mode_result( -+ &self, -+ _payload: &ToolPayload, -+ max_bytes: usize, -+ ) -> Result { -+ // Borrow only public result fields: private _meta must not be cloned or serialized. -+ #[derive(serde::Serialize)] -+ #[serde(rename_all = "camelCase")] -+ struct PublicResult<'a> { -+ content: &'a [JsonValue], -+ #[serde(skip_serializing_if = "Option::is_none")] -+ structured_content: &'a Option, -+ #[serde(skip_serializing_if = "Option::is_none")] -+ is_error: Option, -+ } -+ bounded_tool_output( -+ &PublicResult { -+ content: &self.content, -+ structured_content: &self.structured_content, -+ is_error: self.is_error, -+ }, -+ max_bytes, -+ ) -+ } -+ - fn log_output(&self) -> String { - let output = self.as_function_call_output_payload(); - // Do not fall back to serializing media or encrypted content into logs. -@@ -286,3 +338,32 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> - #[cfg(test)] - #[path = "tool_output_tests.rs"] - mod tests; -+ -+/// Serializes borrowed capture data through a capped buffer before creating an owned tree. -+pub fn bounded_tool_output( -+ value: &(impl serde::Serialize + ?Sized), -+ max_bytes: usize, -+) -> Result { -+ struct CappedWriter { -+ bytes: Vec, -+ limit: usize, -+ } -+ impl std::io::Write for CappedWriter { -+ fn write(&mut self, bytes: &[u8]) -> std::io::Result { -+ if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { -+ return Err(std::io::Error::other("capture budget exceeded")); -+ } -+ self.bytes.extend_from_slice(bytes); -+ Ok(bytes.len()) -+ } -+ fn flush(&mut self) -> std::io::Result<()> { -+ Ok(()) -+ } -+ } -+ let mut writer = CappedWriter { -+ bytes: Vec::new(), -+ limit: max_bytes, -+ }; -+ serde_json::to_writer(&mut writer, value).map_err(|_| "capture_budget")?; -+ serde_json::from_slice(&writer.bytes).map_err(|_| "unsupported_output") -+} -diff --git a/codex-rs/tools/src/tool_output_tests.rs b/codex-rs/tools/src/tool_output_tests.rs -index bca0e81..111131b 100644 ---- a/codex-rs/tools/src/tool_output_tests.rs -+++ b/codex-rs/tools/src/tool_output_tests.rs -@@ -44,3 +44,84 @@ fn code_mode_text_preserves_message_content() { - ) - ); - } -+ -+#[test] -+fn bounded_capture_preserves_json_and_rejects_large_nested_values() { -+ use super::{JsonToolOutput, ToolOutput}; -+ let payload = crate::ToolPayload::Function { -+ arguments: "{}".into(), -+ }; -+ let expected = serde_json::json!({"structured": ["α", {"value": true}]}); -+ let output: Box = Box::new(JsonToolOutput::new(expected.clone())); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Ok(expected) -+ ); -+ let output = JsonToolOutput::new(serde_json::json!({"nested": ["x".repeat(1024 * 1024)]})); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+} -+ -+#[test] -+fn bounded_capture_omits_private_mcp_metadata_before_serializing() { -+ use super::ToolOutput; -+ let payload = crate::ToolPayload::Function { -+ arguments: "{}".into(), -+ }; -+ let mut output = codex_protocol::mcp::CallToolResult { -+ content: vec![serde_json::json!({"type":"text", "text":"visible"})], -+ structured_content: Some(serde_json::json!({"answer": 42})), -+ is_error: Some(false), -+ meta: Some(serde_json::json!({"private": "x".repeat(1024 * 1024)})), -+ }; -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Ok(serde_json::json!({ -+ "content": [{"type":"text", "text":"visible"}], -+ "structuredContent": {"answer":42}, "isError":false -+ })) -+ ); -+ output.structured_content = Some(serde_json::json!({"large": "x".repeat(1024 * 1024)})); -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+ output.structured_content = None; -+ output.content = vec![serde_json::json!({"type":"image", "data":"x".repeat(1024 * 1024)})]; -+ assert_eq!( -+ output.bounded_code_mode_result(&payload, 1024), -+ Err("capture_budget") -+ ); -+} -+ -+#[test] -+fn bounded_capture_never_calls_unknown_legacy_output_conversion() { -+ use super::ToolOutput; -+ struct Unknown; -+ impl ToolOutput for Unknown { -+ fn log_output(&self) -> String { -+ panic!("unbounded log") -+ } -+ fn success_for_logging(&self) -> bool { -+ true -+ } -+ fn to_response_item(&self, _: &str, _: &crate::ToolPayload) -> ResponseInputItem { -+ panic!("unbounded model conversion") -+ } -+ fn code_mode_result(&self, _: &crate::ToolPayload) -> serde_json::Value { -+ panic!("unbounded result conversion") -+ } -+ } -+ let output: Box = Box::new(Unknown); -+ assert_eq!( -+ output.bounded_code_mode_result( -+ &crate::ToolPayload::Function { -+ arguments: "{}".into() -+ }, -+ 1024 -+ ), -+ Err("unsupported_output") -+ ); -+} diff --git a/patches/codex/0002-native-approval-recovery.patch b/patches/codex/0002-native-approval-recovery.patch deleted file mode 100644 index 0ef0c97d57..0000000000 --- a/patches/codex/0002-native-approval-recovery.patch +++ /dev/null @@ -1,2555 +0,0 @@ -diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock -index 1d27abb..f8972a1 100644 ---- a/codex-rs/Cargo.lock -+++ b/codex-rs/Cargo.lock -@@ -3051,6 +3051,7 @@ dependencies = [ - "serde_json", - "serial_test", - "sha1 0.10.6", -+ "sha2 0.10.9", - "shlex", - "similar", - "tempfile", -diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml -index 542abf9..5ffc5cf 100644 ---- a/codex-rs/core/Cargo.toml -+++ b/codex-rs/core/Cargo.toml -@@ -114,6 +114,7 @@ rmcp = { workspace = true, default-features = false, features = [ - serde = { workspace = true, features = ["derive"] } - serde_json = { workspace = true } - sha1 = { workspace = true } -+sha2 = { workspace = true } - shlex = { workspace = true } - similar = { workspace = true } - tempfile = { workspace = true } -diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs -index 968d331..a10c351 100644 ---- a/codex-rs/core/src/codex_thread.rs -+++ b/codex-rs/core/src/codex_thread.rs -@@ -45,6 +45,7 @@ use codex_protocol::protocol::AskForApproval; - use codex_protocol::protocol::EnvironmentConfig; - use codex_protocol::protocol::EnvironmentConfigState; - use codex_protocol::protocol::Event; -+use codex_protocol::protocol::EventMsg; - use codex_protocol::protocol::MultiAgentVersion; - use codex_protocol::protocol::Op; - use codex_protocol::protocol::SandboxPolicy; -@@ -474,10 +475,39 @@ impl CodexThread { - .await - } - -+ /// Replays one native call without model sampling, requiring the original human approval. -+ pub async fn restore_tool_call_if_idle( -+ &self, -+ call: ResponseItem, -+ expected_approval: EventMsg, -+ preconditions: Option, -+ turn_extension_init: Option, -+ ) -> CodexResult { -+ let restore = -+ crate::restore_tool_call::RestoreToolCall::new(call, expected_approval, preconditions)?; -+ let mut init = match turn_extension_init { -+ Some(init) => init, -+ None => self.config_snapshot().await.turn_extension_init, -+ }; -+ init.insert(restore); -+ self.start_turn_if_idle(crate::WithTurnExtensionData::new( -+ TurnInputRequest::user_input(Vec::new()), -+ init, -+ )) -+ .await -+ } -+ -+ /// Returns the live approval without file preimages, paired with its verified preconditions. -+ pub async fn capture_tool_approval( -+ &self, -+ approval: EventMsg, -+ ) -> CodexResult<(EventMsg, serde_json::Value)> { -+ crate::restore_tool_call::capture(&self.session, approval).await -+ } -+ - /// Starts a regular turn only when the thread is idle. - /// -- /// Core declines the input without recording or enqueueing it when idle -- /// work cannot start. -+ /// Core declines the input without recording or enqueueing it when idle work cannot start. - pub async fn start_turn_if_idle( - &self, - request: impl Into>, -diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs -index dcd01e4..dc14807 100644 ---- a/codex-rs/core/src/lib.rs -+++ b/codex-rs/core/src/lib.rs -@@ -32,6 +32,8 @@ pub use codex_protocol::turn_input::TurnStartOptions; - pub use responses_metadata::CodexResponsesMetadata; - pub use turn_metadata::detached_memory_responses_metadata; - mod codex_thread; -+mod restore_tool_call; -+mod patch_preconditions; - mod turn_extension_data; - pub use turn_extension_data::WithTurnExtensionData; - mod compact_model_fallback; -diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs -new file mode 100644 -index 0000000..d32a51c ---- /dev/null -+++ b/codex-rs/core/src/patch_preconditions.rs -@@ -0,0 +1,978 @@ -+//! Filesystem preconditions for restoring an interrupted native patch approval. -+ -+use crate::session::step_context::StepContext; -+use crate::tools::ApprovalContext; -+use codex_file_system::EnvironmentAccess; -+use codex_file_system::GetMetadataOptions; -+use codex_file_system::ReadFileOptions; -+use codex_protocol::error::CodexErr; -+use codex_protocol::error::Result as CodexResult; -+use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; -+use codex_protocol::protocol::FileChange; -+use codex_utils_path_uri::PathUri; -+use serde::Serialize; -+use serde_json::Value; -+use sha2::Digest; -+use sha2::Sha256; -+use std::collections::BTreeMap; -+use std::collections::HashMap; -+use std::io::ErrorKind; -+use std::path::Component; -+use std::path::Path; -+use std::path::PathBuf; -+use std::time::Duration; -+ -+// Approval recovery fails closed for larger preimages instead of retaining large reads. -+pub(crate) const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; -+const MAX_PATCH_PATHS: usize = 32; -+pub(crate) const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; -+const MAX_PATCH_SCAN_DURATION: Duration = Duration::from_secs(10); -+ -+#[derive(Debug, Serialize, PartialEq)] -+struct PatchPreconditions { -+ version: u32, -+ files: BTreeMap>, -+} -+ -+fn invalid(message: impl Into) -> CodexErr { -+ CodexErr::InvalidRequest(message.into()) -+} -+ -+pub(crate) async fn capture_patch_approval_preconditions( -+ context: &ApprovalContext, -+ changes: &HashMap, -+) -> CodexResult { -+ let review = &context.review_context; -+ let environment = review -+ .environments() -+ .single_local_environment() -+ .ok_or_else(|| { -+ invalid("Restoring patch approvals requires a single selected local environment") -+ })?; -+ let grants = review -+ .turn() -+ .granted_permissions(&environment.selection.environment_id); -+ serde_json::to_value(capture(&environment.fs_accessor(grants), changes).await?) -+ .map_err(|error| invalid(error.to_string())) -+} -+ -+async fn capture_step( -+ step: &StepContext, -+ changes: &HashMap, -+ check_deleted_content: bool, -+) -> CodexResult { -+ let environment = step -+ .environments -+ .single_local_environment() -+ .ok_or_else(|| { -+ invalid("Restoring patch approvals requires a single selected local environment") -+ })?; -+ capture_with_deletion_check( -+ &step.fs_accessor(environment), -+ changes, -+ check_deleted_content, -+ ) -+ .await -+} -+ -+pub(crate) async fn validate_patch_preconditions( -+ step: &StepContext, -+ approval: &ApplyPatchApprovalRequestEvent, -+ checkpoint: &Value, -+) -> CodexResult<()> { -+ let expected = validate_checkpoint(&approval.changes, checkpoint)?; -+ let check_deleted_content = checkpoint.get("version").and_then(Value::as_u64) == Some(1); -+ let observed = capture_step(step, &approval.changes, check_deleted_content).await?; -+ if observed.files.iter().any(|(path, digest)| { -+ match (digest, path.to_str().and_then(|path| expected.get(path))) { -+ (Some(digest), Some(Value::String(expected))) => digest != expected, -+ (None, Some(Value::Null)) => false, -+ _ => true, -+ } -+ }) { -+ return Err(invalid( -+ "The pending patch or its files have changed since the original approval request", -+ )); -+ } -+ Ok(()) -+} -+ -+fn patch_paths(changes: &HashMap) -> CodexResult> { -+ if changes.is_empty() { -+ return Err(invalid("A pending patch must include file changes")); -+ } -+ if changes.len() > MAX_PATCH_PATHS { -+ return Err(invalid("Pending patch scans are limited to 32 paths")); -+ } -+ let mut count = changes.len(); -+ for change in changes.values() { -+ if matches!( -+ change, -+ FileChange::Update { -+ move_path: Some(_), -+ .. -+ } -+ ) { -+ count += 1; -+ if count > MAX_PATCH_PATHS { -+ return Err(invalid("Pending patch scans are limited to 32 paths")); -+ } -+ } -+ } -+ let mut paths: BTreeMap<_, _> = changes -+ .iter() -+ .map(|(path, change)| (path, matches!(change, FileChange::Add { .. }))) -+ .collect(); -+ for change in changes.values() { -+ if let FileChange::Update { -+ move_path: Some(destination), -+ .. -+ } = change -+ && paths.insert(destination, true).is_some() -+ { -+ return Err(invalid( -+ "Pending patch move destinations must be distinct from changed files", -+ )); -+ } -+ } -+ Ok(paths) -+} -+ -+fn valid_digest(value: &str) -> bool { -+ value.len() == 64 -+ && value -+ .bytes() -+ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) -+} -+ -+fn validate_checkpoint<'a>( -+ changes: &HashMap, -+ checkpoint: &'a Value, -+) -> CodexResult<&'a serde_json::Map> { -+ let files = validate_checkpoint_shape(changes, checkpoint)?; -+ if checkpoint.get("version").and_then(Value::as_u64) == Some(2) { -+ for change in changes.values() { -+ match change { -+ FileChange::Delete { content } if !content.is_empty() => { -+ return Err(invalid("Version 2 patch deletions must omit raw content")); -+ } -+ FileChange::Update { unified_diff, .. } if !valid_digest(unified_diff) => { -+ return Err(invalid( -+ "Version 2 patch updates require a SHA-256 diff digest", -+ )); -+ } -+ _ => {} -+ } -+ } -+ } -+ Ok(files) -+} -+ -+/// Binds a live native deletion to its saved digest before redacting its content. -+/// No file I/O or content cloning occurs here; callers bound any other live fields separately. -+pub(crate) fn matches_delete_preimages( -+ changes: &HashMap, -+ checkpoint: &Value, -+) -> bool { -+ let Ok(files) = validate_checkpoint_shape(changes, checkpoint) else { -+ return false; -+ }; -+ let mut remaining = MAX_PATCH_SCAN_BYTES; -+ for change in changes.values() { -+ if let FileChange::Delete { content } = change { -+ let size = content.len() as u64; -+ if size > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { -+ return false; -+ } -+ remaining -= size; -+ } -+ } -+ changes.iter().all(|(path, change)| { -+ let FileChange::Delete { content } = change else { -+ return true; -+ }; -+ path.to_str() -+ .and_then(|path| files.get(path)) -+ .and_then(Value::as_str) -+ .is_some_and(|expected| digest(content.as_bytes()) == expected) -+ }) -+} -+ -+fn validate_checkpoint_shape<'a>( -+ changes: &HashMap, -+ checkpoint: &'a Value, -+) -> CodexResult<&'a serde_json::Map> { -+ let paths = patch_paths(changes)?; -+ let root = checkpoint -+ .as_object() -+ .ok_or_else(|| invalid("Invalid patch approval preconditions"))?; -+ let files = root -+ .get("files") -+ .and_then(Value::as_object) -+ .ok_or_else(|| invalid("Invalid patch approval file map"))?; -+ if root.len() != 2 -+ || !matches!(root.get("version").and_then(Value::as_u64), Some(1 | 2)) -+ || files.len() > MAX_PATCH_PATHS -+ || files.len() != paths.len() -+ { -+ return Err(invalid("Invalid patch approval version or file count")); -+ } -+ for (path, absent) in paths { -+ let value = path -+ .to_str() -+ .and_then(|path| files.get(path)) -+ .ok_or_else(|| invalid("Patch approval file paths do not match the pending patch"))?; -+ if absent { -+ if !value.is_null() { -+ return Err(invalid( -+ "Pending patch destinations must have absent preimages", -+ )); -+ } -+ } else if !value.as_str().is_some_and(valid_digest) { -+ return Err(invalid("Invalid patch approval preimage digest")); -+ } -+ } -+ Ok(files) -+} -+ -+async fn capture( -+ fs: &impl EnvironmentAccess, -+ changes: &HashMap, -+) -> CodexResult { -+ capture_with_deletion_check(fs, changes, true).await -+} -+ -+async fn capture_with_deletion_check( -+ fs: &impl EnvironmentAccess, -+ changes: &HashMap, -+ check_deleted_content: bool, -+) -> CodexResult { -+ patch_paths(changes)?; -+ tokio::time::timeout(MAX_PATCH_SCAN_DURATION, async { -+ let mut files = BTreeMap::new(); -+ let mut remaining = MAX_PATCH_SCAN_BYTES; -+ for (path, change) in changes { -+ let deleted = match change { -+ FileChange::Delete { content } if check_deleted_content => Some(content.as_bytes()), -+ _ => None, -+ }; -+ let original = observe_file( -+ fs, -+ path, -+ matches!(change, FileChange::Add { .. }), -+ &mut remaining, -+ deleted, -+ ) -+ .await?; -+ files.insert(path.clone(), original); -+ if let FileChange::Update { -+ move_path: Some(destination), -+ .. -+ } = change -+ { -+ files.insert( -+ destination.clone(), -+ observe_file(fs, destination, true, &mut remaining, None).await?, -+ ); -+ } -+ } -+ Ok(PatchPreconditions { version: 1, files }) -+ }) -+ .await -+ .map_err(|_| invalid("Pending patch scan exceeded the 10 second limit"))? -+} -+ -+fn digest(bytes: &[u8]) -> String { -+ format!("{:x}", Sha256::digest(bytes)) -+} -+ -+async fn observe_file( -+ fs: &impl EnvironmentAccess, -+ path: &Path, -+ must_be_absent: bool, -+ remaining: &mut u64, -+ deleted: Option<&[u8]>, -+) -> CodexResult> { -+ if !path.is_absolute() -+ || path -+ .components() -+ .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) -+ { -+ return Err(invalid( -+ "Pending patch paths must be normalized absolute local paths", -+ )); -+ } -+ // The selected local filesystem's no-follow operations reject symlinks in -+ // every component, including directory ancestors, without opening targets. -+ let uri = PathUri::from_host_native_path(path).map_err(|error| invalid(error.to_string()))?; -+ match fs -+ .get_metadata( -+ &uri, -+ GetMetadataOptions { -+ follow_symlinks: false, -+ }, -+ ) -+ .await -+ { -+ Err(error) if must_be_absent && error.kind() == ErrorKind::NotFound => Ok(None), -+ Err(error) => Err(invalid(format!( -+ "Cannot inspect pending patch file: {error}" -+ ))), -+ Ok(_) if must_be_absent => Err(invalid("A pending patch destination already exists")), -+ Ok(metadata) if !metadata.is_file || metadata.is_symlink => { -+ Err(invalid("Pending patch sources must be regular files")) -+ } -+ Ok(_) => { -+ let bytes = fs -+ .read_file_bounded( -+ &uri, -+ ReadFileOptions { -+ follow_symlinks: false, -+ }, -+ MAX_PATCH_PREIMAGE_BYTES.min(*remaining), -+ ) -+ .await -+ .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; -+ *remaining -= bytes.len() as u64; -+ if deleted.is_some_and(|expected| expected != bytes.as_slice()) { -+ return Err(invalid( -+ "The pending deletion no longer matches its original file", -+ )); -+ } -+ Ok(Some(digest(&bytes))) -+ } -+ } -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use codex_exec_server::LOCAL_FS; -+ use codex_file_system::FileSystemEnvironmentAccessor; -+ -+ fn approval(path: PathBuf, change: FileChange) -> ApplyPatchApprovalRequestEvent { -+ ApplyPatchApprovalRequestEvent { -+ call_id: "original-call".into(), -+ turn_id: "original-turn".into(), -+ started_at_ms: 0, -+ changes: [(path, change)].into_iter().collect(), -+ reason: None, -+ grant_root: None, -+ } -+ } -+ -+ #[tokio::test] -+ async fn existing_update_binds_full_original_contents() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().join("existing.txt"); -+ std::fs::write(&path, "old\nuntouched\n").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ path.clone(), -+ FileChange::Update { -+ unified_diff: "-old\n+new\n".into(), -+ move_path: None, -+ }, -+ ); -+ let original = capture(&fs, &request.changes).await.unwrap(); -+ assert_eq!(original, capture(&fs, &request.changes).await.unwrap()); -+ std::fs::write(&path, "old\nchanged outside patch context\n").unwrap(); -+ assert_ne!(original, capture(&fs, &request.changes).await.unwrap()); -+ } -+ -+ #[tokio::test] -+ async fn add_requires_absent_destination_and_blocks_completed_replay() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().join("new.txt"); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ path.clone(), -+ FileChange::Add { -+ content: "new\n".into(), -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_ok()); -+ std::fs::write(path, "new\n").unwrap(); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ } -+ -+ #[tokio::test] -+ async fn move_requires_absent_destination() { -+ let dir = tempfile::tempdir().unwrap(); -+ let source = dir.path().join("old.txt"); -+ let destination = dir.path().join("new.txt"); -+ std::fs::write(&source, "old\n").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ source, -+ FileChange::Update { -+ unified_diff: "-old\n+new\n".into(), -+ move_path: Some(destination.clone()), -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_ok()); -+ std::fs::write(destination, "pre-existing work\n").unwrap(); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ } -+ -+ #[tokio::test] -+ async fn delete_requires_original_contents() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().join("old.txt"); -+ std::fs::write(&path, "original\n").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ path.clone(), -+ FileChange::Delete { -+ content: "original\n".into(), -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_ok()); -+ std::fs::write(path, "changed\n").unwrap(); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ } -+ -+ #[cfg(unix)] -+ #[tokio::test] -+ async fn rejects_symlink_sources_and_ancestors() { -+ let dir = tempfile::tempdir().unwrap(); -+ let original = dir.path().join("original"); -+ std::fs::create_dir(&original).unwrap(); -+ std::fs::write(original.join("file"), "original\n").unwrap(); -+ let link = dir.path().join("link"); -+ std::os::unix::fs::symlink(&original, &link).unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ link.join("file"), -+ FileChange::Update { -+ unified_diff: "-original\n+new\n".into(), -+ move_path: None, -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ let request = approval( -+ link.join("new"), -+ FileChange::Add { -+ content: "new".into(), -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ let file_link = dir.path().join("file-link"); -+ std::os::unix::fs::symlink(original.join("file"), &file_link).unwrap(); -+ let request = approval( -+ file_link, -+ FileChange::Update { -+ unified_diff: "-original\n+new\n".into(), -+ move_path: None, -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ let dangling_link = dir.path().join("dangling-link"); -+ std::os::unix::fs::symlink(dir.path().join("absent"), &dangling_link).unwrap(); -+ let request = approval( -+ dangling_link, -+ FileChange::Add { -+ content: "new".into(), -+ }, -+ ); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ } -+ #[tokio::test] -+ async fn rejects_preimage_larger_than_recovery_limit() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().canonicalize().unwrap().join("large.txt"); -+ std::fs::File::create(&path) -+ .unwrap() -+ .set_len(MAX_PATCH_PREIMAGE_BYTES + 1) -+ .unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let request = approval( -+ path, -+ FileChange::Update { -+ unified_diff: "-old\n+new\n".into(), -+ move_path: None, -+ }, -+ ); -+ let error = capture(&fs, &request.changes) -+ .await -+ .expect_err("large preimages must fail closed"); -+ assert!(error.to_string().contains("limit is 8388608 bytes")); -+ } -+ fn update(move_path: Option) -> FileChange { -+ FileChange::Update { -+ unified_diff: "-old\n+new\n".into(), -+ move_path, -+ } -+ } -+ -+ #[tokio::test] -+ async fn bounds_all_paths_before_inspecting_any_file() { -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let mut changes: HashMap<_, _> = (0..MAX_PATCH_PATHS + 1) -+ .map(|index| { -+ ( -+ PathBuf::from(format!("relative-{index}")), -+ FileChange::Add { -+ content: String::new(), -+ }, -+ ) -+ }) -+ .collect(); -+ assert!( -+ capture(&fs, &changes) -+ .await -+ .unwrap_err() -+ .to_string() -+ .contains("32 paths") -+ ); -+ changes.remove(&PathBuf::from(format!("relative-{MAX_PATCH_PATHS}"))); -+ changes.insert( -+ PathBuf::from("relative-0"), -+ update(Some(PathBuf::from("move-destination"))), -+ ); -+ assert!( -+ capture(&fs, &changes) -+ .await -+ .unwrap_err() -+ .to_string() -+ .contains("32 paths") -+ ); -+ changes.remove(&PathBuf::from("relative-1")); -+ assert_eq!(patch_paths(&changes).unwrap().len(), MAX_PATCH_PATHS); -+ } -+ -+ #[tokio::test] -+ async fn accepts_exact_path_limit_of_missing_additions() { -+ let dir = tempfile::tempdir().unwrap(); -+ let root = dir.path().canonicalize().unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let changes: HashMap<_, _> = (0..MAX_PATCH_PATHS) -+ .map(|index| { -+ ( -+ root.join(format!("missing-{index}")), -+ FileChange::Add { -+ content: String::new(), -+ }, -+ ) -+ }) -+ .collect(); -+ let checkpoint = capture(&fs, &changes).await.unwrap(); -+ assert_eq!(checkpoint.files.len(), MAX_PATCH_PATHS); -+ assert!(checkpoint.files.values().all(Option::is_none)); -+ } -+ -+ #[tokio::test] -+ async fn move_sources_share_total_byte_budget_with_other_files() { -+ let dir = tempfile::tempdir().unwrap(); -+ let root = dir.path().canonicalize().unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let mut changes = HashMap::new(); -+ for index in 0..2 { -+ let path = root.join(format!("source-{index}")); -+ std::fs::File::create(&path) -+ .unwrap() -+ .set_len(MAX_PATCH_PREIMAGE_BYTES) -+ .unwrap(); -+ changes.insert( -+ path, -+ update(Some(root.join(format!("destination-{index}")))), -+ ); -+ } -+ let empty = root.join("empty"); -+ std::fs::write(&empty, []).unwrap(); -+ changes.insert(empty, update(None)); -+ changes.insert( -+ root.join("missing-addition"), -+ FileChange::Add { -+ content: "new".into(), -+ }, -+ ); -+ assert_eq!(capture(&fs, &changes).await.unwrap().files.len(), 6); -+ let extra = root.join("one-more-byte"); -+ std::fs::write(&extra, b"x").unwrap(); -+ changes.insert(extra, update(None)); -+ assert!( -+ capture(&fs, &changes) -+ .await -+ .unwrap_err() -+ .to_string() -+ .contains("limit is") -+ ); -+ } -+ -+ #[tokio::test] -+ async fn exhausted_budget_still_allows_empty_and_missing_destinations() { -+ let dir = tempfile::tempdir().unwrap(); -+ let root = dir.path().canonicalize().unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let empty = root.join("empty"); -+ std::fs::write(&empty, []).unwrap(); -+ let mut remaining = 0; -+ assert_eq!( -+ observe_file(&fs, &empty, false, &mut remaining, None) -+ .await -+ .unwrap(), -+ Some(digest(b"")) -+ ); -+ assert_eq!( -+ observe_file(&fs, &root.join("missing"), true, &mut remaining, None) -+ .await -+ .unwrap(), -+ None -+ ); -+ assert!( -+ observe_file(&fs, &root.join("missing"), false, &mut remaining, None) -+ .await -+ .is_err() -+ ); -+ std::fs::write(&empty, b"x").unwrap(); -+ assert!( -+ observe_file(&fs, &empty, false, &mut remaining, None) -+ .await -+ .is_err() -+ ); -+ assert_eq!(remaining, 0); -+ } -+ -+ #[tokio::test] -+ async fn deletion_rejects_oversized_event_content_against_small_preimage() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir -+ .path() -+ .canonicalize() -+ .unwrap() -+ .join("changed-before-capture"); -+ std::fs::write(&path, b"small replacement").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let changes = [( -+ path, -+ FileChange::Delete { -+ content: "x".repeat(MAX_PATCH_SCAN_BYTES as usize + 1), -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ assert!( -+ capture(&fs, &changes) -+ .await -+ .unwrap_err() -+ .to_string() -+ .contains("pending deletion") -+ ); -+ } -+ -+ #[test] -+ fn checkpoint_requires_bounded_exact_path_set_and_version_without_cloning() { -+ let source = PathBuf::from("/source"); -+ let destination = PathBuf::from("/destination"); -+ let changes = [(source.clone(), update(Some(destination.clone())))] -+ .into_iter() -+ .collect(); -+ let mut files = serde_json::Map::new(); -+ files.insert( -+ source.to_str().unwrap().into(), -+ Value::String(digest(b"old")), -+ ); -+ files.insert(destination.to_str().unwrap().into(), Value::Null); -+ let valid = serde_json::json!({"version": 1, "files": files}); -+ assert!(validate_checkpoint(&changes, &valid).is_ok()); -+ let mut invalid = valid.clone(); -+ invalid["version"] = serde_json::json!(3); -+ assert!(validate_checkpoint(&changes, &invalid).is_err()); -+ let mut invalid = valid.clone(); -+ invalid["files"] -+ .as_object_mut() -+ .unwrap() -+ .remove(destination.to_str().unwrap()); -+ assert!(validate_checkpoint(&changes, &invalid).is_err()); -+ let mut invalid = valid.clone(); -+ invalid["files"] -+ .as_object_mut() -+ .unwrap() -+ .remove(destination.to_str().unwrap()); -+ invalid["files"]["/unexpected"] = Value::Null; -+ assert!(validate_checkpoint(&changes, &invalid).is_err()); -+ let mut invalid = valid.clone(); -+ invalid["files"][destination.to_str().unwrap()] = Value::String(digest(b"occupied")); -+ assert!(validate_checkpoint(&changes, &invalid).is_err()); -+ let mut invalid = valid; -+ invalid["files"][source.to_str().unwrap()] = -+ Value::String("x".repeat(MAX_PATCH_SCAN_BYTES as usize + 1)); -+ assert!(validate_checkpoint(&changes, &invalid).is_err()); -+ let oversized: serde_json::Map<_, _> = (0..MAX_PATCH_PATHS + 1) -+ .map(|index| (format!("/{index}"), Value::Null)) -+ .collect(); -+ assert!( -+ validate_checkpoint( -+ &changes, -+ &serde_json::json!({"version":1,"files":oversized}) -+ ) -+ .is_err() -+ ); -+ } -+ fn deletion_checkpoint(path: &Path, content: &[u8], version: u32) -> Value { -+ let files = [(path.to_path_buf(), Some(digest(content)))] -+ .into_iter() -+ .collect(); -+ let mut checkpoint = -+ serde_json::to_value(PatchPreconditions { version: 1, files }).unwrap(); -+ checkpoint["version"] = serde_json::json!(version); -+ checkpoint -+ } -+ -+ #[test] -+ fn v2_deletion_requires_redaction_and_mandatory_digest() { -+ let path = PathBuf::from("/deleted"); -+ let mut changes = [( -+ path.clone(), -+ FileChange::Delete { -+ content: "private contents".into(), -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ let checkpoint = deletion_checkpoint(&path, b"private contents", 2); -+ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); -+ changes.insert( -+ path.clone(), -+ FileChange::Delete { -+ content: String::new(), -+ }, -+ ); -+ assert!(validate_checkpoint(&changes, &checkpoint).is_ok()); -+ for value in [ -+ Value::Null, -+ Value::String(String::new()), -+ Value::String("A".repeat(64)), -+ ] { -+ let mut malformed = checkpoint.clone(); -+ malformed["files"][path.to_str().unwrap()] = value; -+ assert!(validate_checkpoint(&changes, &malformed).is_err()); -+ } -+ let mut malformed = checkpoint; -+ malformed["files"].as_object_mut().unwrap().clear(); -+ assert!(validate_checkpoint(&changes, &malformed).is_err()); -+ } -+ -+ #[test] -+ fn v2_update_requires_diff_digest_but_live_delete_matcher_accepts_raw_updates() { -+ let path = PathBuf::from("/updated"); -+ let raw_diff = " private neighboring line\n-old\n+new"; -+ let mut changes = [( -+ path.clone(), -+ FileChange::Update { -+ unified_diff: raw_diff.into(), -+ move_path: None, -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ let checkpoint = deletion_checkpoint(&path, b"original file", 2); -+ assert!(matches_delete_preimages(&changes, &checkpoint)); -+ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); -+ for diff in [String::new(), "A".repeat(64), "z".repeat(64)] { -+ changes.insert( -+ path.clone(), -+ FileChange::Update { -+ unified_diff: diff, -+ move_path: None, -+ }, -+ ); -+ assert!(validate_checkpoint(&changes, &checkpoint).is_err()); -+ } -+ changes.insert( -+ path.clone(), -+ FileChange::Update { -+ unified_diff: digest(raw_diff.as_bytes()), -+ move_path: None, -+ }, -+ ); -+ assert!(validate_checkpoint(&changes, &checkpoint).is_ok()); -+ let mut legacy = checkpoint; -+ legacy["version"] = serde_json::json!(1); -+ changes.insert( -+ path, -+ FileChange::Update { -+ unified_diff: raw_diff.into(), -+ move_path: None, -+ }, -+ ); -+ assert!(validate_checkpoint(&changes, &legacy).is_ok()); -+ } -+ -+ #[test] -+ fn live_delete_matcher_binds_raw_content_without_reading_the_path() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().canonicalize().unwrap().join("not-on-disk"); -+ let mut changes = [( -+ path.clone(), -+ FileChange::Delete { -+ content: "original".into(), -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ let checkpoint = deletion_checkpoint(&path, b"original", 2); -+ assert!(matches_delete_preimages(&changes, &checkpoint)); -+ assert!(!path.exists()); -+ changes.insert( -+ path.clone(), -+ FileChange::Delete { -+ content: "forged".into(), -+ }, -+ ); -+ assert!(!matches_delete_preimages(&changes, &checkpoint)); -+ changes.insert( -+ path.clone(), -+ FileChange::Delete { -+ content: "original".into(), -+ }, -+ ); -+ let mut wrong_digest = checkpoint; -+ wrong_digest["files"][path.to_str().unwrap()] = Value::String(digest(b"different")); -+ assert!(!matches_delete_preimages(&changes, &wrong_digest)); -+ } -+ -+ #[test] -+ fn live_delete_matcher_enforces_content_budgets_before_hashing() { -+ let path = PathBuf::from("/large-delete"); -+ let checkpoint = deletion_checkpoint(&path, b"irrelevant", 2); -+ let changes = [( -+ path, -+ FileChange::Delete { -+ content: "x".repeat(MAX_PATCH_PREIMAGE_BYTES as usize + 1), -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ assert!(!matches_delete_preimages(&changes, &checkpoint)); -+ let changes: HashMap<_, _> = (0..3) -+ .map(|index| { -+ ( -+ PathBuf::from(format!("/delete-{index}")), -+ FileChange::Delete { -+ content: "x".repeat(MAX_PATCH_PREIMAGE_BYTES as usize), -+ }, -+ ) -+ }) -+ .collect(); -+ let files: BTreeMap<_, _> = changes -+ .keys() -+ .map(|path| (path.clone(), Some(digest(b"irrelevant")))) -+ .collect(); -+ let checkpoint = serde_json::to_value(PatchPreconditions { version: 2, files }).unwrap(); -+ assert!(!matches_delete_preimages(&changes, &checkpoint)); -+ let path = PathBuf::from("/empty-delete"); -+ let checkpoint = deletion_checkpoint(&path, b"", 2); -+ let changes = [( -+ path, -+ FileChange::Delete { -+ content: String::new(), -+ }, -+ )] -+ .into_iter() -+ .collect(); -+ assert!(matches_delete_preimages(&changes, &checkpoint)); -+ } -+ -+ #[tokio::test] -+ async fn v2_redacted_delete_validates_full_preimage_and_rejects_forged_digest_or_drift() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().canonicalize().unwrap().join("deleted"); -+ std::fs::write(&path, b"private original file").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let mut request = approval( -+ path.clone(), -+ FileChange::Delete { -+ content: "private original file".into(), -+ }, -+ ); -+ let initial = capture(&fs, &request.changes).await.unwrap(); -+ assert_eq!(initial.version, 1); -+ let mut checkpoint = serde_json::to_value(initial).unwrap(); -+ checkpoint["version"] = serde_json::json!(2); -+ request.changes.insert( -+ path.clone(), -+ FileChange::Delete { -+ content: String::new(), -+ }, -+ ); -+ let (session, turn) = crate::session::tests::make_session_and_context().await; -+ let session = std::sync::Arc::new(session); -+ let step = session -+ .capture_step_context( -+ std::sync::Arc::new(turn), -+ &tokio_util::sync::CancellationToken::new(), -+ ) -+ .await -+ .unwrap(); -+ validate_patch_preconditions(&step, &request, &checkpoint) -+ .await -+ .unwrap(); -+ let mut forged = checkpoint.clone(); -+ forged["files"][path.to_str().unwrap()] = Value::String(digest(b"forged")); -+ assert!( -+ validate_patch_preconditions(&step, &request, &forged) -+ .await -+ .is_err() -+ ); -+ std::fs::write(&path, b"stale replacement").unwrap(); -+ assert!( -+ validate_patch_preconditions(&step, &request, &checkpoint) -+ .await -+ .is_err() -+ ); -+ } -+ -+ #[tokio::test] -+ async fn legacy_delete_keeps_raw_equality_and_initial_capture() { -+ let dir = tempfile::tempdir().unwrap(); -+ let path = dir.path().canonicalize().unwrap().join("legacy-deleted"); -+ std::fs::write(&path, b"legacy contents").unwrap(); -+ let fs = FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS); -+ let mut request = approval( -+ path.clone(), -+ FileChange::Delete { -+ content: "legacy contents".into(), -+ }, -+ ); -+ let checkpoint = -+ serde_json::to_value(capture(&fs, &request.changes).await.unwrap()).unwrap(); -+ assert_eq!(checkpoint["version"], 1); -+ let (session, turn) = crate::session::tests::make_session_and_context().await; -+ let session = std::sync::Arc::new(session); -+ let step = session -+ .capture_step_context( -+ std::sync::Arc::new(turn), -+ &tokio_util::sync::CancellationToken::new(), -+ ) -+ .await -+ .unwrap(); -+ validate_patch_preconditions(&step, &request, &checkpoint) -+ .await -+ .unwrap(); -+ request.changes.insert( -+ path.clone(), -+ FileChange::Delete { -+ content: "forged".into(), -+ }, -+ ); -+ assert!( -+ validate_patch_preconditions(&step, &request, &checkpoint) -+ .await -+ .is_err() -+ ); -+ assert!(capture(&fs, &request.changes).await.is_err()); -+ request.changes.insert( -+ path, -+ FileChange::Delete { -+ content: String::new(), -+ }, -+ ); -+ assert!( -+ validate_patch_preconditions(&step, &request, &checkpoint) -+ .await -+ .is_err() -+ ); -+ } -+} -diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs -new file mode 100644 -index 0000000..2818bcc ---- /dev/null -+++ b/codex-rs/core/src/restore_tool_call.rs -@@ -0,0 +1,809 @@ -+//! Model-free replay of one native call, requiring the same fresh human approval. -+ -+use std::collections::HashMap; -+use std::path::PathBuf; -+use std::sync::atomic::{AtomicBool, Ordering}; -+use std::sync::{Arc, Mutex, OnceLock}; -+ -+use codex_protocol::approvals::ExecApprovalKind; -+use codex_protocol::config_types::ApprovalsReviewer; -+use codex_protocol::error::{CodexErr, Result as CodexResult}; -+use codex_protocol::models::ResponseItem; -+use codex_protocol::openai_models::ToolMode; -+use codex_protocol::protocol::{EventMsg, FileChange, ReviewDecision}; -+use serde_json::{Value, json}; -+use sha2::{Digest, Sha256}; -+use tokio_util::sync::CancellationToken; -+ -+use crate::patch_preconditions::{MAX_PATCH_PREIMAGE_BYTES, MAX_PATCH_SCAN_BYTES}; -+use crate::session::session::Session; -+use crate::session::step_context::StepContext; -+use crate::session::turn_context::TurnContext; -+use crate::state::TaskKind; -+use crate::tools::ApprovalContext; -+use crate::tools::parallel::ToolCallRuntime; -+use crate::tools::router::ToolRouter; -+use crate::tools::sandboxing::ApprovalAction; -+ -+fn refusal(message: &str) -> CodexErr { -+ CodexErr::InvalidRequest(format!("Native tool restoration refused: {message}")) -+} -+ -+#[derive(Debug)] -+struct CapturedPatch { -+ changes: Arc>, -+ preconditions: Value, -+} -+ -+#[derive(Debug)] -+struct CapturableApproval { -+ event: Option, -+ permissions: Value, -+ patch: Option, -+} -+ -+impl CapturableApproval { -+ fn checkpoint(&self, expected: &EventMsg) -> CodexResult { -+ // Reuse the pre-approval scan only for the exact changes on this live lease. -+ let patch = match (&self.patch, expected) { -+ (Some(patch), EventMsg::ApplyPatchApprovalRequest(event)) -+ if patch.changes.as_ref() == &event.changes => -+ { -+ Some(&patch.preconditions) -+ } -+ (None, EventMsg::ExecApprovalRequest(_)) => None, -+ _ => return Err(refusal("approval does not match the captured action")), -+ }; -+ let expected = -+ serde_json::to_value(expected).map_err(|_| refusal("invalid approval event"))?; -+ if self -+ .event -+ .as_ref() -+ .and_then(|event| serde_json::to_value(event).ok()) -+ != Some(expected) -+ { -+ return Err(refusal("approval does not match the live request")); -+ } -+ Ok(json!({"version":1,"first_approval":true,"permissions":self.permissions,"patch":patch})) -+ } -+} -+ -+#[derive(Debug, Default)] -+struct CapturableApprovals(Mutex>); -+ -+impl crate::WithTurnExtensionData { -+ /// Enables pre-execution native approval checkpoint capture for SDK-owned turns. -+ pub fn capture_native_approvals(mut self) -> Self { -+ self.turn_extension_init -+ .get_or_insert_with(Default::default) -+ .insert(CapturableApprovals::default()); -+ self -+ } -+} -+ -+fn permission_snapshot( -+ turn: &TurnContext, -+ environments: &crate::environment_selection::TurnEnvironmentSnapshot, -+ policy: codex_protocol::protocol::AskForApproval, -+ reviewer: ApprovalsReviewer, -+) -> Value { -+ let environments: Vec<_> = environments -+ .turn_environments() -+ .map(|environment| { -+ json!({ -+ "id": environment.selection.environment_id, -+ "profile": environment.permission_profile_with_workspace_roots(), -+ "symbolic_profile": environment.permission_profile(), -+ "network": environment.config().network_policy, -+ "grants": turn.granted_permissions(&environment.selection.environment_id), -+ }) -+ }) -+ .collect(); -+ let mut snapshot = json!({ -+ "approval_policy": policy, "reviewer": reviewer, "environments": environments, -+ "managed_network": turn.config.permissions.network.as_ref().map(|network| network.environment_policy()), -+ }); -+ snapshot.sort_all_objects(); -+ json!(format!( -+ "{:x}", -+ Sha256::digest(snapshot.to_string().as_bytes()) -+ )) -+} -+ -+pub(crate) struct CaptureLease { -+ state: Arc, -+ call_id: String, -+} -+ -+impl Drop for CaptureLease { -+ fn drop(&mut self) { -+ self.state.0.lock().unwrap().remove(&self.call_id); -+ } -+} -+ -+pub(crate) async fn capture_lease( -+ action: &ApprovalAction, -+ ctx: &ApprovalContext, -+) -> Option { -+ if ctx.retry_reason.is_some() -+ || ctx.network_approval_context.is_some() -+ || !matches!( -+ action, -+ ApprovalAction::ExecCommand { .. } | ApprovalAction::ApplyPatch { .. } -+ ) -+ { -+ return None; -+ } -+ let state = ctx -+ .review_context -+ .turn() -+ .extension_data -+ .get::()?; -+ let patch = if let ApprovalAction::ApplyPatch { changes, .. } = action { -+ let preconditions = -+ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, changes) -+ .await -+ .ok()?; -+ Some(CapturedPatch { -+ changes: Arc::clone(changes), -+ preconditions, -+ }) -+ } else { -+ None -+ }; -+ state.0.lock().unwrap().insert( -+ ctx.call_id.clone(), -+ CapturableApproval { -+ event: None, -+ patch, -+ permissions: permission_snapshot( -+ ctx.review_context.turn(), -+ ctx.review_context.environments(), -+ ctx.review_context.approval_policy, -+ ctx.review_context.approvals_reviewer, -+ ), -+ }, -+ ); -+ Some(CaptureLease { -+ state, -+ call_id: ctx.call_id.clone(), -+ }) -+} -+ -+fn approval_id(event: &EventMsg) -> Option<&str> { -+ match event { -+ EventMsg::ExecApprovalRequest(event) -+ if event.kind == ExecApprovalKind::Command -+ && event.approval_id.is_none() -+ && event.network_approval_context.is_none() => -+ { -+ Some(&event.call_id) -+ } -+ EventMsg::ApplyPatchApprovalRequest(event) if event.grant_root.is_none() => { -+ Some(&event.call_id) -+ } -+ _ => None, -+ } -+} -+ -+fn comparable(event: &EventMsg) -> CodexResult { -+ let mut value = serde_json::to_value(event).map_err(|_| refusal("invalid approval event"))?; -+ let object = value -+ .as_object_mut() -+ .ok_or_else(|| refusal("invalid approval event"))?; -+ object.remove("turn_id"); -+ object.remove("started_at_ms"); -+ Ok(value) -+} -+ -+pub(crate) fn record_capture_event(turn: &TurnContext, event: &EventMsg) { -+ let Some(id) = approval_id(event) else { return }; -+ if let Some(state) = turn.extension_data.get::() { -+ if let Some(pending) = state.0.lock().unwrap().get_mut(id) { -+ pending.event = Some(event.clone()); -+ } -+ } -+} -+ -+#[derive(Debug)] -+pub(crate) struct RestoreToolCall { -+ call: ResponseItem, -+ expected: EventMsg, -+ checkpoint: Value, -+ owner: OnceLock, -+ requested: AtomicBool, -+ approved: AtomicBool, -+ executed: AtomicBool, -+} -+ -+impl RestoreToolCall { -+ pub(crate) fn new( -+ call: ResponseItem, -+ expected: EventMsg, -+ checkpoint: Option, -+ ) -> CodexResult { -+ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; -+ let supported = match (&call, &expected) { -+ ( -+ ResponseItem::FunctionCall { -+ name, -+ call_id, -+ namespace, -+ .. -+ }, -+ EventMsg::ExecApprovalRequest(_), -+ ) => { -+ name == "exec_command" -+ && call_id == id -+ && namespace.as_deref().is_none_or(|ns| ns == "functions") -+ } -+ ( -+ ResponseItem::CustomToolCall { -+ name, -+ call_id, -+ namespace, -+ .. -+ }, -+ EventMsg::ApplyPatchApprovalRequest(_), -+ ) => { -+ name == "apply_patch" -+ && call_id == id -+ && namespace.as_deref().is_none_or(|ns| ns == "functions") -+ } -+ _ => false, -+ }; -+ if !supported { -+ return Err(refusal( -+ "only one direct exec_command or apply_patch call is supported", -+ )); -+ } -+ let checkpoint = -+ checkpoint.ok_or_else(|| refusal("missing original pre-execution checkpoint"))?; -+ if checkpoint.get("version") != Some(&json!(1)) -+ || checkpoint.get("first_approval") != Some(&json!(true)) -+ { -+ return Err(refusal("checkpoint does not match the original approval")); -+ } -+ Ok(Self { -+ call, -+ expected, -+ checkpoint, -+ owner: OnceLock::new(), -+ requested: AtomicBool::new(false), -+ approved: AtomicBool::new(false), -+ executed: AtomicBool::new(false), -+ }) -+ } -+ -+ pub(crate) fn accepts_approval(&self, action: &ApprovalAction, ctx: &ApprovalContext) -> bool { -+ if ctx.retry_reason.is_some() -+ || ctx.network_approval_context.is_some() -+ || ctx.strict_auto_review -+ || self.requested.load(Ordering::SeqCst) -+ || approval_id(&self.expected) != Some(ctx.call_id.as_str()) -+ { -+ return false; -+ } -+ matches!( -+ (&self.expected, action), -+ ( -+ EventMsg::ExecApprovalRequest(_), -+ ApprovalAction::ExecCommand { .. } -+ ) | ( -+ EventMsg::ApplyPatchApprovalRequest(_), -+ ApprovalAction::ApplyPatch { .. } -+ ) -+ ) -+ } -+ -+ pub(crate) fn is_patch(&self) -> bool { -+ matches!(self.expected, EventMsg::ApplyPatchApprovalRequest(_)) -+ } -+ -+ pub(crate) fn claim(&self, turn: &TurnContext) -> bool { -+ self.owner.set(turn.sub_id.clone()).is_ok() -+ } -+} -+ -+pub(crate) fn active(turn: &TurnContext) -> Option> { -+ turn.extension_data -+ .get::() -+ .filter(|restore| restore.owner.get() == Some(&turn.sub_id)) -+} -+ -+async fn validate_step(session: &Arc, step: &StepContext) -> CodexResult<()> { -+ let config = session.get_config().await; -+ let requirements = config.config_layer_stack.requirements(); -+ if requirements.auto_review_required_for_model(&step.settings.model_info.slug) -+ || requirements -+ .approvals_reviewer -+ .can_set(&ApprovalsReviewer::User) -+ .is_err() -+ { -+ return Err(refusal("managed automatic review is unsupported")); -+ } -+ if step.tool_router.tool_mode() != ToolMode::Direct -+ || step.environments.single_local_environment().is_none() -+ { -+ return Err(refusal( -+ "only direct tools in one local environment are supported", -+ )); -+ } -+ if step.settings.approvals_reviewer() != ApprovalsReviewer::User -+ || step.turn.strict_auto_review_enabled() -+ { -+ return Err(refusal("automatic approval review is unsupported")); -+ } -+ let hooks = session.hooks().list_with_environment_hooks( -+ crate::hook_environment_executor::environment_hook_sources(&step.environments, &[]), -+ ); -+ if !hooks.hooks.is_empty() -+ || !hooks.warnings.is_empty() -+ || !step.executor_hook_sources.is_empty() -+ { -+ return Err(refusal("configured hooks are unsupported")); -+ } -+ if let Some(restore) = active(&step.turn) { -+ if restore.checkpoint.get("permissions") -+ != Some(&permission_snapshot( -+ &step.turn, -+ &step.environments, -+ step.settings.approval_policy(), -+ step.settings.approvals_reviewer(), -+ )) -+ { -+ return Err(refusal("permissions or approval policy changed")); -+ } -+ } -+ Ok(()) -+} -+ -+// Prepared deletes include the complete file. Prepared diffs can include neighboring -+// lines absent from the submitted patch. Keep those bytes out of the SDK checkpoint. -+fn compact_patch(event: &mut EventMsg) -> CodexResult { -+ let EventMsg::ApplyPatchApprovalRequest(request) = event else { -+ return Ok(false); -+ }; -+ let mut remaining = MAX_PATCH_SCAN_BYTES; -+ let mut compacted = false; -+ for change in request.changes.values_mut() { -+ match change { -+ FileChange::Add { .. } => continue, -+ FileChange::Delete { content } => { -+ // The trusted file snapshot already binds the entire deletion. -+ *content = String::new(); -+ } -+ FileChange::Update { unified_diff, .. } => { -+ let bytes = unified_diff.len() as u64; -+ if bytes > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { -+ return Err(refusal("prepared patch diff exceeds the recovery limit")); -+ } -+ remaining -= bytes; -+ *unified_diff = format!("{:x}", Sha256::digest(unified_diff.as_bytes())); -+ } -+ } -+ compacted = true; -+ } -+ Ok(compacted) -+} -+ -+pub(crate) async fn capture( -+ session: &Arc, -+ mut expected: EventMsg, -+) -> CodexResult<(EventMsg, Value)> { -+ let (turn, cancellation) = session -+ .active_turn_context_and_cancellation_token() -+ .await -+ .ok_or_else(|| refusal("no active approval"))?; -+ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; -+ let state = turn -+ .extension_data -+ .get::() -+ .ok_or_else(|| refusal("not a first pre-execution approval"))?; -+ let step = session.capture_step_context(turn, &cancellation).await?; -+ validate_step(session, &step).await?; -+ let approvals = state.0.lock().unwrap(); -+ let mut checkpoint = approvals -+ .get(id) -+ .ok_or_else(|| refusal("not a pending first pre-execution approval"))? -+ .checkpoint(&expected)?; -+ if compact_patch(&mut expected)? { -+ checkpoint["patch"]["version"] = json!(2); -+ } -+ Ok((expected, checkpoint)) -+} -+ -+pub(crate) fn compare_request(turn: &TurnContext, event: &mut EventMsg) -> bool { -+ record_capture_event(turn, event); -+ let Some(restore) = active(turn) else { -+ return true; -+ }; -+ if restore.requested.swap(true, Ordering::SeqCst) -+ || approval_id(event) != approval_id(&restore.expected) -+ { -+ return false; -+ } -+ // Version 1 keeps comparing the original request verbatim, including any -+ // digest-looking text. Only v2 uses a compact request and file fingerprints. -+ let patch = &restore.checkpoint["patch"]; -+ if patch["version"] == 2 { -+ let EventMsg::ApplyPatchApprovalRequest(request) = event else { -+ return false; -+ }; -+ if !crate::patch_preconditions::matches_delete_preimages(&request.changes, patch) -+ || !matches!(compact_patch(event), Ok(true)) -+ { -+ return false; -+ } -+ } -+ matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected) -+} -+ -+pub(crate) fn record_decision(turn: &TurnContext, decision: ReviewDecision) -> ReviewDecision { -+ if let Some(restore) = active(turn) { -+ if decision == ReviewDecision::Approved { -+ restore.approved.store(true, Ordering::SeqCst); -+ } else if !matches!( -+ decision, -+ ReviewDecision::Denied { .. } | ReviewDecision::Abort -+ ) { -+ return ReviewDecision::Abort; -+ } -+ } -+ decision -+} -+ -+pub(crate) async fn validate_before_execution( -+ session: &Arc, -+ step: &StepContext, -+) -> CodexResult<()> { -+ let Some(restore) = active(&step.turn) else { -+ return Ok(()); -+ }; -+ validate_step(session, step).await?; -+ if !restore.approved.load(Ordering::SeqCst) { -+ return Err(refusal("a matching fresh human approval is required")); -+ } -+ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { -+ let checkpoint = restore -+ .checkpoint -+ .get("patch") -+ .ok_or_else(|| refusal("missing patch preimages"))?; -+ crate::patch_preconditions::validate_patch_preconditions(step, event, checkpoint).await?; -+ } -+ if restore.executed.swap(true, Ordering::SeqCst) { -+ return Err(refusal("a restored native call cannot retry execution")); -+ } -+ Ok(()) -+} -+ -+pub(crate) async fn run( -+ session: Arc, -+ turn: Arc, -+ restore: Arc, -+ cancellation: CancellationToken, -+) -> crate::tasks::SessionTaskResult { -+ session.emit_turn_started(&turn, TaskKind::Regular).await; -+ let step = session -+ .capture_step_context(Arc::clone(&turn), &cancellation) -+ .await?; -+ validate_step(&session, &step).await?; -+ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { -+ let checkpoint = restore -+ .checkpoint -+ .get("patch") -+ .ok_or_else(|| refusal("missing patch preimages"))?; -+ crate::patch_preconditions::validate_patch_preconditions(&step, event, checkpoint).await?; -+ } -+ let call = ToolRouter::build_tool_call(restore.call.clone()) -+ .map_err(|_| refusal("invalid native call"))? -+ .ok_or_else(|| refusal("unsupported native call"))?; -+ if !step.tool_router.exposes_tool(&call.tool_name) { -+ return Err(refusal("tool is unavailable in the replacement worker")); -+ } -+ crate::stream_events_utils::record_completed_response_item(&session, &step, &restore.call) -+ .await; -+ let result = ToolCallRuntime::new(Arc::clone(&session), Arc::clone(&step), Default::default()) -+ .handle_tool_call(call, cancellation) -+ .await?; -+ crate::stream_events_utils::record_completed_response_item(&session, &step, &result.item).await; -+ Ok(None) -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use codex_protocol::protocol::AskForApproval; -+ -+ fn command() -> ResponseItem { -+ serde_json::from_value(json!({ -+ "type":"function_call", "call_id":"original", "name":"exec_command", -+ "arguments":"{\"cmd\":\"printf original\"}", -+ })) -+ .unwrap() -+ } -+ -+ fn approval() -> EventMsg { -+ serde_json::from_value(json!({ -+ "type":"exec_approval_request", "kind":"command", "call_id":"original", -+ "turn_id":"first-turn", "started_at_ms":1, "command":["sh","-c","printf original"], -+ "cwd":"/workspace", "environmentId":"local", "parsed_cmd":[], -+ })) -+ .unwrap() -+ } -+ -+ fn checkpoint() -> Value { -+ json!({"version":1,"first_approval":true,"permissions":"test-fingerprint","patch":null}) -+ } -+ -+ #[tokio::test] -+ async fn compact_approval_hides_prepared_file_text_and_matches_only_the_original_action() { -+ let deleted = PathBuf::from("/workspace/delete"); -+ let updated = PathBuf::from("/workspace/update"); -+ let raw = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { -+ call_id: "original".into(), -+ turn_id: "first-turn".into(), -+ started_at_ms: 1, -+ changes: HashMap::from([ -+ ( -+ deleted.clone(), -+ FileChange::Delete { -+ content: "private original".into(), -+ }, -+ ), -+ ( -+ updated.clone(), -+ FileChange::Update { -+ unified_diff: "-old\n+new\n private neighbor".into(), -+ move_path: None, -+ }, -+ ), -+ ]), -+ reason: None, -+ grant_root: None, -+ }; -+ let mut expected = EventMsg::ApplyPatchApprovalRequest(raw.clone()); -+ assert!(compact_patch(&mut expected).unwrap()); -+ assert!( -+ !serde_json::to_string(&expected) -+ .unwrap() -+ .contains("private") -+ ); -+ let mut state = checkpoint(); -+ state["patch"] = json!({"version":2,"files":{ -+ "/workspace/delete": format!("{:x}", Sha256::digest(b"private original")), -+ "/workspace/update": format!("{:x}", Sha256::digest(b"old\nprivate neighbor")), -+ }}); -+ let call: ResponseItem = serde_json::from_value(json!({ -+ "type":"custom_tool_call", "name":"apply_patch", "call_id":"original", -+ "input":"original patch", "status":"completed" -+ })) -+ .unwrap(); -+ let (_, turn) = crate::session::tests::make_session_and_context().await; -+ for changed_path in [None, Some(&deleted), Some(&updated)] { -+ let restore = -+ RestoreToolCall::new(call.clone(), expected.clone(), Some(state.clone())).unwrap(); -+ assert!(restore.claim(&turn)); -+ turn.extension_data.insert(restore); -+ let mut fresh = raw.clone(); -+ if let Some(path) = changed_path { -+ match fresh.changes.get_mut(path).unwrap() { -+ FileChange::Delete { content } => content.push('!'), -+ FileChange::Update { unified_diff, .. } => unified_diff.push('!'), -+ _ => unreachable!(), -+ } -+ } -+ assert_eq!( -+ compare_request(&turn, &mut EventMsg::ApplyPatchApprovalRequest(fresh)), -+ changed_path.is_none() -+ ); -+ } -+ } -+ -+ #[test] -+ fn reuses_patch_snapshot_only_for_the_live_event_and_original_changes() { -+ let path = PathBuf::from("/workspace/new"); -+ let changes = HashMap::from([( -+ path.clone(), -+ FileChange::Add { -+ content: "first".into(), -+ }, -+ )]); -+ let mut request = codex_protocol::approvals::ApplyPatchApprovalRequestEvent { -+ call_id: "original".into(), -+ turn_id: "first-turn".into(), -+ started_at_ms: 1, -+ changes: changes.clone(), -+ reason: None, -+ grant_root: None, -+ }; -+ let original = EventMsg::ApplyPatchApprovalRequest(request.clone()); -+ let snapshot = json!({"version":1,"files":{"/workspace/new":null}}); -+ let mut lease = CapturableApproval { -+ event: Some(original.clone()), -+ permissions: json!("test-fingerprint"), -+ patch: Some(CapturedPatch { -+ changes: Arc::new(changes), -+ preconditions: snapshot.clone(), -+ }), -+ }; -+ assert_eq!(lease.checkpoint(&original).unwrap()["patch"], snapshot); -+ request.started_at_ms += 1; -+ assert!( -+ lease -+ .checkpoint(&EventMsg::ApplyPatchApprovalRequest(request.clone())) -+ .is_err() -+ ); -+ request.changes.insert( -+ path, -+ FileChange::Add { -+ content: "second".into(), -+ }, -+ ); -+ let changed = EventMsg::ApplyPatchApprovalRequest(request); -+ lease.event = Some(changed.clone()); -+ assert!(lease.checkpoint(&changed).is_err()); -+ } -+ -+ #[test] -+ fn rejects_unrelated_calls_and_noninitial_approvals() { -+ let mut call = command(); -+ if let ResponseItem::FunctionCall { call_id, .. } = &mut call { -+ *call_id = "sibling".into(); -+ } -+ assert!(RestoreToolCall::new(call, approval(), Some(checkpoint())).is_err()); -+ let mut request = approval(); -+ if let EventMsg::ExecApprovalRequest(event) = &mut request { -+ event.approval_id = Some("subcommand".into()); -+ } -+ assert!(RestoreToolCall::new(command(), request, Some(checkpoint())).is_err()); -+ assert!(RestoreToolCall::new(command(), approval(), None).is_err()); -+ } -+ -+ #[tokio::test] -+ async fn requires_exact_fresh_approval_and_consumes_execution_once() { -+ let (session, turn) = crate::session::tests::make_session_and_context().await; -+ let session = Arc::new(session); -+ let turn = Arc::new(turn); -+ let step = session -+ .capture_step_context(Arc::clone(&turn), &CancellationToken::new()) -+ .await -+ .unwrap(); -+ let mut preconditions = checkpoint(); -+ preconditions["permissions"] = permission_snapshot( -+ &turn, -+ &step.environments, -+ step.settings.approval_policy(), -+ step.settings.approvals_reviewer(), -+ ); -+ let restore = RestoreToolCall::new(command(), approval(), Some(preconditions)).unwrap(); -+ turn.extension_data.insert(restore); -+ let restore = turn.extension_data.get::().unwrap(); -+ assert!(restore.claim(&turn)); -+ assert!(!restore.claim(&turn)); -+ assert!(validate_before_execution(&session, &step).await.is_err()); -+ assert!(compare_request(&turn, &mut approval())); -+ assert_eq!( -+ record_decision(&turn, ReviewDecision::Approved), -+ ReviewDecision::Approved -+ ); -+ assert!(validate_before_execution(&session, &step).await.is_ok()); -+ assert!(validate_before_execution(&session, &step).await.is_err()); -+ assert!(!compare_request(&turn, &mut approval())); -+ } -+ -+ #[tokio::test] -+ async fn refuses_changed_command_before_approval() { -+ let (_, turn) = crate::session::tests::make_session_and_context().await; -+ turn.extension_data -+ .insert(RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap()); -+ assert!( -+ turn.extension_data -+ .get::() -+ .unwrap() -+ .claim(&turn) -+ ); -+ let mut changed = approval(); -+ if let EventMsg::ExecApprovalRequest(event) = &mut changed { -+ event.command.push("unexpected".into()); -+ } -+ assert!(!compare_request(&turn, &mut changed)); -+ assert!(!compare_request(&turn, &mut approval())); -+ assert!(!active(&turn).unwrap().approved.load(Ordering::SeqCst)); -+ } -+ -+ #[tokio::test] -+ async fn refuses_changed_authority_even_with_matching_approval() { -+ let (session, turn) = crate::session::tests::make_session_and_context().await; -+ let session = Arc::new(session); -+ let turn = Arc::new(turn); -+ let step = session -+ .capture_step_context(Arc::clone(&turn), &CancellationToken::new()) -+ .await -+ .unwrap(); -+ assert_ne!( -+ permission_snapshot( -+ &turn, -+ &step.environments, -+ AskForApproval::OnRequest, -+ ApprovalsReviewer::User -+ ), -+ permission_snapshot( -+ &turn, -+ &step.environments, -+ AskForApproval::Never, -+ ApprovalsReviewer::User -+ ), -+ ); -+ turn.extension_data -+ .insert(RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap()); -+ assert!( -+ turn.extension_data -+ .get::() -+ .unwrap() -+ .claim(&turn) -+ ); -+ assert!(compare_request(&turn, &mut approval())); -+ record_decision(&turn, ReviewDecision::Approved); -+ assert!(validate_before_execution(&session, &step).await.is_err()); -+ } -+ #[tokio::test] -+ async fn capture_is_opt_in_and_excludes_execution_retries() { -+ let (_, turn) = crate::session::tests::make_session_and_context().await; -+ let turn = Arc::new(turn); -+ let action = ApprovalAction::ExecCommand { -+ id: "original".into(), -+ environment_id: "local".into(), -+ command: vec!["printf".into(), "original".into()], -+ hook_command: "printf original".into(), -+ cwd: turn.cwd.clone().into(), -+ sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, -+ additional_permissions: None, -+ justification: None, -+ tty: false, -+ proposed_execpolicy_amendment: None, -+ }; -+ let mut ctx = ApprovalContext { -+ review_context: crate::guardian::GuardianReviewContext::from(&turn), -+ cancellation_token: None, -+ call_id: "original".into(), -+ tool_name: codex_tools::ToolName { -+ name: "exec_command".into(), -+ namespace: None, -+ }, -+ strict_auto_review: false, -+ approval_reason: None, -+ retry_reason: None, -+ network_approval_context: None, -+ }; -+ assert!(capture_lease(&action, &ctx).await.is_none()); -+ let restore = RestoreToolCall::new(command(), approval(), Some(checkpoint())).unwrap(); -+ assert!(restore.accepts_approval(&action, &ctx)); -+ ctx.network_approval_context = Some(codex_protocol::approvals::NetworkApprovalContext { -+ host: "unexpected.invalid".into(), -+ protocol: codex_protocol::approvals::NetworkApprovalProtocol::Http, -+ }); -+ assert!(!restore.accepts_approval(&action, &ctx)); -+ ctx.network_approval_context = None; -+ turn.extension_data.insert(CapturableApprovals::default()); -+ ctx.retry_reason = Some("sandbox retry".into()); -+ assert!(!restore.accepts_approval(&action, &ctx)); -+ assert!(capture_lease(&action, &ctx).await.is_none()); -+ ctx.retry_reason = None; -+ let lease = capture_lease(&action, &ctx).await.unwrap(); -+ record_capture_event(&turn, &approval()); -+ let state = turn.extension_data.get::().unwrap(); -+ assert!( -+ state -+ .0 -+ .lock() -+ .unwrap() -+ .get("original") -+ .unwrap() -+ .event -+ .is_some() -+ ); -+ drop(lease); -+ assert!(state.0.lock().unwrap().is_empty()); -+ } -+} -diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs -index 66476ae..6bfe356 100644 ---- a/codex-rs/core/src/session/mod.rs -+++ b/codex-rs/core/src/session/mod.rs -@@ -2930,7 +2930,7 @@ impl Session { - .map(|task| Arc::clone(&task.turn_context)) - } - -- async fn active_turn_context_and_cancellation_token( -+ pub(crate) async fn active_turn_context_and_cancellation_token( - &self, - ) -> Option<(Arc, CancellationToken)> { - let active = self.active_turn.lock().await; -@@ -3155,7 +3155,7 @@ impl Session { - .as_ref() - .map(PluginCommandAttribution::serialized_fields) - .unzip(); -- let event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { -+ let mut event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { - model_context: Some(model_context), - kind, - call_id, -@@ -3175,9 +3175,15 @@ impl Session { - available_decisions: Some(available_decisions), - parsed_cmd, - }); -+ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { -+ return ReviewDecision::Abort; -+ } - let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); - self.send_event(turn_context, event).await; -- rx_approve.await.unwrap_or(ReviewDecision::Abort) -+ crate::restore_tool_call::record_decision( -+ turn_context, -+ rx_approve.await.unwrap_or(ReviewDecision::Abort), -+ ) - } - - #[expect( -@@ -3211,7 +3217,7 @@ impl Session { - warn!("Overwriting existing pending approval for call_id: {approval_id}"); - } - -- let event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { -+ let mut event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { - call_id, - turn_id: turn_context.sub_id.clone(), - started_at_ms: now_unix_timestamp_ms(), -@@ -3219,9 +3225,15 @@ impl Session { - reason, - grant_root, - }); -+ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { -+ return ReviewDecision::Abort; -+ } - let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); - self.send_event(turn_context, event).await; -- rx_approve.await.unwrap_or(ReviewDecision::Abort) -+ crate::restore_tool_call::record_decision( -+ turn_context, -+ rx_approve.await.unwrap_or(ReviewDecision::Abort), -+ ) - } - - #[expect( -diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs -index 6c6f3d1..1309ac1 100644 ---- a/codex-rs/core/src/tasks/regular.rs -+++ b/codex-rs/core/src/tasks/regular.rs -@@ -44,6 +44,14 @@ impl SessionTask for RegularTask { - input: Vec, - cancellation_token: CancellationToken, - ) -> SessionTaskResult { -+ if let Some(restore) = ctx -+ .extension_data -+ .get::() -+ { -+ if restore.claim(&ctx) { -+ return crate::restore_tool_call::run(sess, ctx, restore, cancellation_token).await; -+ } -+ } - if ctx - .config - .features -diff --git a/codex-rs/core/src/tools/approvals.rs b/codex-rs/core/src/tools/approvals.rs -index 74082aa..ebaa126 100644 ---- a/codex-rs/core/src/tools/approvals.rs -+++ b/codex-rs/core/src/tools/approvals.rs -@@ -499,6 +499,18 @@ impl Session { - action: ApprovalAction, - ctx: ApprovalContext, - ) -> ApprovalResult { -+ if let Some(restore) = crate::restore_tool_call::active(ctx.review_context.turn()) { -+ if !restore.accepts_approval(&action, &ctx) { -+ return ApprovalResult { -+ result: Err(ToolError::Rejected( -+ "native restoration cannot request another approval or escalation".into(), -+ )), -+ source: None, -+ cancellation_reason: None, -+ }; -+ } -+ } -+ let _capture_lease = crate::restore_tool_call::capture_lease(&action, &ctx).await; - // Retained authority outside current grants needs a new sandbox approval. - // Current grants permit review of the input; they never approve the input itself. - let policy = ctx.review_context.turn().approval_policy(); -@@ -723,7 +735,7 @@ impl Session { - .into_iter() - .map(|key| (key, &policy_fingerprint)) - .collect(); -- with_cached_approval(&self.services, tool_name, cache_keys, || async { -+ let request = || async { - self.request_command_approval( - ctx.review_context.turn(), - ExecApprovalKind::Command, -@@ -742,8 +754,12 @@ impl Session { - /*plugin_attribution_override*/ None, - ) - .await -- }) -- .await -+ }; -+ if crate::restore_tool_call::active(ctx.review_context.turn()).is_some() { -+ request().await -+ } else { -+ with_cached_approval(&self.services, tool_name, cache_keys, request).await -+ } - } - ApprovalAction::WriteStdin { - id, -@@ -788,9 +804,11 @@ impl Session { - .retry_reason - .clone() - .or_else(|| ctx.approval_reason.clone()); -- if *permissions_preapproved && reason.is_none() { -+ let restoring = -+ crate::restore_tool_call::active(ctx.review_context.turn()).is_some(); -+ if *permissions_preapproved && reason.is_none() && !restoring { - ReviewDecision::Approved -- } else if reason.is_some() { -+ } else if reason.is_some() || restoring { - self.request_patch_approval( - ctx.review_context.turn(), - ctx.call_id.clone(), -diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs -index d9afeb6..32a5820 100644 ---- a/codex-rs/core/src/tools/orchestrator.rs -+++ b/codex-rs/core/src/tools/orchestrator.rs -@@ -61,6 +61,14 @@ impl ToolOrchestrator { - where - T: ToolRuntime, - { -+ if let Err(error) = crate::restore_tool_call::validate_before_execution( -+ &tool_ctx.session, -+ &tool_ctx.step_context, -+ ) -+ .await -+ { -+ return (Err(ToolError::Codex(error)), None); -+ } - let network_approval = match begin_network_approval( - &tool_ctx.session, - &tool_ctx.step_context.turn, -@@ -165,6 +173,13 @@ impl ToolOrchestrator { - let requirement = tool.exec_approval_requirement(req).unwrap_or_else(|| { - default_exec_approval_requirement(approval_policy, &file_system_sandbox_policy) - }); -+ if crate::restore_tool_call::active(turn_ctx).is_some() -+ && !matches!(requirement, ExecApprovalRequirement::NeedsApproval { .. }) -+ { -+ return Err(ToolError::Rejected( -+ "native restoration requires a fresh human approval".into(), -+ )); -+ } - match &requirement { - ExecApprovalRequirement::Skip { .. } => { - if strict_auto_review { -@@ -228,12 +243,24 @@ impl ToolOrchestrator { - - // 2) First attempt under the selected sandbox. - let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); -- let sandbox_override = sandbox_override_for_first_attempt( -- tool.sandbox_permissions(req), -- &requirement, -- &file_system_sandbox_policy, -- /*already_approved*/ initial_approval_source.is_some(), -- ); -+ // Restored patches get one attempt. Use only authority the normal approved retry already grants. -+ let approved_restored_patch = crate::restore_tool_call::active(turn_ctx) -+ .is_some_and(|restore| restore.is_patch()) -+ && initial_approval_source == Some(ApprovalResolutionSource::User) -+ && !strict_auto_review -+ && unsandboxed_allowed -+ && tool.wants_no_sandbox_approval(approval_policy) -+ && tool.should_bypass_approval(approval_policy, true); -+ let sandbox_override = if approved_restored_patch { -+ SandboxOverride::BypassSandboxFirstAttempt -+ } else { -+ sandbox_override_for_first_attempt( -+ tool.sandbox_permissions(req), -+ &requirement, -+ &file_system_sandbox_policy, -+ /*already_approved*/ initial_approval_source.is_some(), -+ ) -+ }; - let (escalated_profile, writable_root_metadata) = if sandbox_override - == SandboxOverride::EscalatedSandboxWithRestrictions - { -diff --git a/codex-rs/exec-server/src/fs_helper.rs b/codex-rs/exec-server/src/fs_helper.rs ---- a/codex-rs/exec-server/src/fs_helper.rs -+++ b/codex-rs/exec-server/src/fs_helper.rs -@@ -71,6 +71,12 @@ - Open(FsHelperOpenParams), - #[serde(rename = "fs/readFile")] - ReadFile(FsReadFileParams), -+ #[serde(rename = "fs/readFileBounded")] -+ ReadFileBounded { -+ path: PathUri, -+ follow_symlinks: bool, -+ max_bytes: u64, -+ }, - #[serde(rename = "fs/writeFile")] - WriteFile(FsWriteFileParams), - #[serde(rename = "fs/createDirectory")] -@@ -311,6 +317,19 @@ - data_base64: STANDARD.encode(data), - })) - } -+ FsHelperRequest::ReadFileBounded { -+ path, -+ follow_symlinks, -+ max_bytes, -+ } => { -+ let data = file_system -+ .read_file_bounded(&path, ReadFileOptions { follow_symlinks }, max_bytes, None) -+ .await -+ .map_err(map_fs_error)?; -+ Ok(FsHelperPayload::ReadFile(FsReadFileResponse { -+ data_base64: STANDARD.encode(data), -+ })) -+ } - FsHelperRequest::WriteFile(params) => { - let bytes = STANDARD.decode(params.data_base64).map_err(|err| { - invalid_request(format!( -diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs ---- a/codex-rs/exec-server/src/local_file_system.rs -+++ b/codex-rs/exec-server/src/local_file_system.rs -@@ -304,6 +304,20 @@ - Box::pin(LocalFileSystem::read_file(self, path, options, sandbox)) - } - -+ fn read_file_bounded<'a>( -+ &'a self, -+ path: &'a PathUri, -+ options: ReadFileOptions, -+ max_bytes: u64, -+ sandbox: Option<&'a FileSystemSandboxContext>, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(async move { -+ let (fs, sandbox) = self.file_system_for_reads(sandbox)?; -+ fs.read_file_bounded(path, options, max_bytes, sandbox) -+ .await -+ }) -+ } -+ - fn read_file_stream<'a>( - &'a self, - path: &'a PathUri, -@@ -517,6 +531,21 @@ - Box::pin(UnsandboxedFileSystem::read_file( - self, path, options, sandbox, - )) -+ } -+ -+ fn read_file_bounded<'a>( -+ &'a self, -+ path: &'a PathUri, -+ options: ReadFileOptions, -+ max_bytes: u64, -+ sandbox: Option<&'a FileSystemSandboxContext>, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(async move { -+ reject_platform_sandbox_context(sandbox)?; -+ self.file_system -+ .read_file_bounded(path, options, max_bytes, None) -+ .await -+ }) - } - - fn read_file_stream<'a>( -@@ -998,6 +1027,24 @@ - Box::pin(DirectFileSystem::read_file(self, path, options, sandbox)) - } - -+ fn read_file_bounded<'a>( -+ &'a self, -+ path: &'a PathUri, -+ options: ReadFileOptions, -+ max_bytes: u64, -+ sandbox: Option<&'a FileSystemSandboxContext>, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(async move { -+ reject_sandbox_context(sandbox)?; -+ crate::local_file_system_read::read_file_bounded( -+ path.to_abs_path()?, -+ options, -+ max_bytes, -+ ) -+ .await -+ }) -+ } -+ - fn read_file_stream<'a>( - &'a self, - path: &'a PathUri, -diff --git a/codex-rs/exec-server/src/local_file_system_read.rs b/codex-rs/exec-server/src/local_file_system_read.rs ---- a/codex-rs/exec-server/src/local_file_system_read.rs -+++ b/codex-rs/exec-server/src/local_file_system_read.rs -@@ -16,6 +16,15 @@ - path: AbsolutePathBuf, - options: ReadFileOptions, - ) -> io::Result> { -+ read_file_bounded(path, options, MAX_READ_FILE_BYTES).await -+} -+ -+pub(super) async fn read_file_bounded( -+ path: AbsolutePathBuf, -+ options: ReadFileOptions, -+ max_bytes: u64, -+) -> io::Result> { -+ let max_bytes = max_bytes.min(MAX_READ_FILE_BYTES); - let cancelled = CancellationToken::new(); - let _cancel_on_drop = cancelled.clone().drop_guard(); - tokio::task::spawn_blocking(move || { -@@ -28,22 +37,31 @@ - no_follow::open_file(path.as_path())? - }; - let metadata = file.metadata()?; -- if metadata.len() > MAX_READ_FILE_BYTES { -- return Err(file_too_large_error()); -- } -- let mut bytes = Vec::with_capacity(metadata.len() as usize); -- CancellableReader { -- reader: file.take(MAX_READ_FILE_BYTES + 1), -- cancelled, -- } -- .read_to_end(&mut bytes)?; -- if bytes.len() as u64 > MAX_READ_FILE_BYTES { -- return Err(file_too_large_error()); -- } -- Ok(bytes) -+ read_with_limit(file, metadata.len(), max_bytes, cancelled) - }) - .await - .map_err(|error| io::Error::other(format!("filesystem task failed: {error}")))? -+} -+ -+fn read_with_limit( -+ reader: impl Read, -+ initial_len: u64, -+ max_bytes: u64, -+ cancelled: CancellationToken, -+) -> io::Result> { -+ if initial_len > max_bytes { -+ return Err(file_too_large_error(max_bytes)); -+ } -+ let mut bytes = Vec::with_capacity(initial_len as usize); -+ CancellableReader { -+ reader: reader.take(max_bytes + 1), -+ cancelled, -+ } -+ .read_to_end(&mut bytes)?; -+ if bytes.len() as u64 > max_bytes { -+ return Err(file_too_large_error(max_bytes)); -+ } -+ Ok(bytes) - } - - struct CancellableReader { -@@ -62,10 +80,10 @@ - } - } - --fn file_too_large_error() -> io::Error { -+fn file_too_large_error(max_bytes: u64) -> io::Error { - io::Error::new( - io::ErrorKind::InvalidInput, -- format!("file is too large to read: limit is {MAX_READ_FILE_BYTES} bytes"), -+ format!("file is too large to read: limit is {max_bytes} bytes"), - ) - } - -diff --git a/codex-rs/exec-server/src/local_file_system_read_tests.rs b/codex-rs/exec-server/src/local_file_system_read_tests.rs ---- a/codex-rs/exec-server/src/local_file_system_read_tests.rs -+++ b/codex-rs/exec-server/src/local_file_system_read_tests.rs -@@ -89,3 +89,64 @@ - assert_eq!((first, error.kind()), (*b"x", std::io::ErrorKind::Other)); - Ok(()) - } -+ -+#[tokio::test] -+async fn caller_limit_preserves_regular_read_and_rejects_larger_preimages() -> anyhow::Result<()> { -+ let directory = TempDir::new()?; -+ let path = directory.path().canonicalize()?.join("preimage"); -+ let uri = PathUri::from_host_native_path(&path)?; -+ let contents = vec![0x5a; 16]; -+ std::fs::write(&path, &contents)?; -+ assert_eq!( -+ LOCAL_FS -+ .read_file_bounded( -+ &uri, -+ ReadFileOptions { -+ follow_symlinks: false -+ }, -+ 16, -+ None -+ ) -+ .await?, -+ contents -+ ); -+ assert!( -+ LOCAL_FS -+ .read_file_bounded( -+ &uri, -+ ReadFileOptions { -+ follow_symlinks: false -+ }, -+ 15, -+ None -+ ) -+ .await -+ .is_err() -+ ); -+ assert_eq!( -+ LOCAL_FS -+ .read_file( -+ &uri, -+ ReadFileOptions { -+ follow_symlinks: false -+ }, -+ None -+ ) -+ .await?, -+ contents -+ ); -+ Ok(()) -+} -+ -+#[test] -+fn growth_or_misleading_metadata_cannot_read_past_limit_plus_one() { -+ use std::io::Cursor; -+ use tokio_util::sync::CancellationToken; -+ for initial_len in [0, 8, 16] { -+ let mut reader = Cursor::new(vec![0x5a; 1024]); -+ let error = super::read_with_limit(&mut reader, initial_len, 16, CancellationToken::new()) -+ .unwrap_err(); -+ assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); -+ assert_eq!(reader.position(), 17); -+ } -+} -diff --git a/codex-rs/exec-server/src/sandboxed_file_system.rs b/codex-rs/exec-server/src/sandboxed_file_system.rs ---- a/codex-rs/exec-server/src/sandboxed_file_system.rs -+++ b/codex-rs/exec-server/src/sandboxed_file_system.rs -@@ -145,16 +145,34 @@ - options: ReadFileOptions, - sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult> { -+ self.read_file_with_limit(path, options, None, sandbox) -+ .await -+ } -+ -+ async fn read_file_with_limit( -+ &self, -+ path: &PathUri, -+ options: ReadFileOptions, -+ max_bytes: Option, -+ sandbox: Option<&FileSystemSandboxContext>, -+ ) -> FileSystemResult> { - let sandbox = require_platform_sandbox(sandbox)?; - validate_native_path(path)?; - let response = self - .run_sandboxed( - sandbox, -- FsHelperRequest::ReadFile(FsReadFileParams { -- path: path.clone(), -- follow_symlinks: (!options.follow_symlinks).then_some(false), -- sandbox: None, -- }), -+ match max_bytes { -+ Some(max_bytes) => FsHelperRequest::ReadFileBounded { -+ path: path.clone(), -+ follow_symlinks: options.follow_symlinks, -+ max_bytes, -+ }, -+ None => FsHelperRequest::ReadFile(FsReadFileParams { -+ path: path.clone(), -+ follow_symlinks: (!options.follow_symlinks).then_some(false), -+ sandbox: None, -+ }), -+ }, - ) - .await? - .expect_read_file() -@@ -364,6 +382,16 @@ - Box::pin(SandboxedFileSystem::read_file(self, path, options, sandbox)) - } - -+ fn read_file_bounded<'a>( -+ &'a self, -+ path: &'a PathUri, -+ options: ReadFileOptions, -+ max_bytes: u64, -+ sandbox: Option<&'a FileSystemSandboxContext>, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(self.read_file_with_limit(path, options, Some(max_bytes), sandbox)) -+ } -+ - fn read_file_stream<'a>( - &'a self, - path: &'a PathUri, -diff --git a/codex-rs/exec-server/tests/file_system_unix.rs b/codex-rs/exec-server/tests/file_system_unix.rs ---- a/codex-rs/exec-server/tests/file_system_unix.rs -+++ b/codex-rs/exec-server/tests/file_system_unix.rs -@@ -1571,3 +1571,71 @@ - - Ok(()) - } -+ -+#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -+async fn bounded_file_reads_preserve_sandbox_and_no_follow() -> Result<()> { -+ let context = create_file_system_context(FileSystemImplementation::Local).await?; -+ let temp = TempDir::new()?; -+ let root = temp.path().canonicalize()?; -+ let allowed = root.join("allowed"); -+ std::fs::create_dir(&allowed)?; -+ let file = allowed.join("file"); -+ std::fs::write(&file, b"hello")?; -+ let secret = root.join("secret"); -+ std::fs::write(&secret, b"secret")?; -+ let link = allowed.join("link"); -+ symlink(&file, &link)?; -+ let directory_link = allowed.join("directory-link"); -+ symlink(&allowed, &directory_link)?; -+ let sandbox = read_only_sandbox(allowed); -+ let options = ReadFileOptions { -+ follow_symlinks: false, -+ }; -+ for permissions in [None, Some(&sandbox)] { -+ let fs = &context.file_system; -+ assert_eq!( -+ fs.read_file_bounded( -+ &PathUri::from_host_native_path(&file)?, -+ options, -+ 5, -+ permissions -+ ) -+ .await?, -+ b"hello" -+ ); -+ let oversized = fs -+ .read_file_bounded( -+ &PathUri::from_host_native_path(&file)?, -+ options, -+ 4, -+ permissions, -+ ) -+ .await -+ .unwrap_err(); -+ assert!(oversized.to_string().contains("limit is 4 bytes")); -+ for path in [&link, &directory_link.join("file")] { -+ assert!( -+ fs.read_file_bounded( -+ &PathUri::from_host_native_path(path)?, -+ options, -+ 16, -+ permissions -+ ) -+ .await -+ .is_err() -+ ); -+ } -+ } -+ let error = context -+ .file_system -+ .read_file_bounded( -+ &PathUri::from_host_native_path(&secret)?, -+ options, -+ 16, -+ Some(&sandbox), -+ ) -+ .await -+ .unwrap_err(); -+ assert_sandbox_denied(&error); -+ Ok(()) -+} -diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs ---- a/codex-rs/file-system/src/environment_accessor.rs -+++ b/codex-rs/file-system/src/environment_accessor.rs -@@ -40,6 +40,21 @@ - options: ReadFileOptions, - ) -> ExecutorFileSystemFuture<'a, Vec>; - -+ /// Reads a file with a hard byte limit, preserving this accessor's permissions. -+ fn read_file_bounded<'a>( -+ &'a self, -+ _path: &'a PathUri, -+ _options: ReadFileOptions, -+ _max_bytes: u64, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(async { -+ Err(std::io::Error::new( -+ std::io::ErrorKind::Unsupported, -+ "bounded file reads are unsupported", -+ )) -+ }) -+ } -+ - /// Opens an owned stream; subsequent reads do not borrow or reauthorize through the accessor. - fn read_file_stream<'a>( - &'a self, -@@ -212,6 +227,16 @@ - .read_file(path, options, self.key.sandbox.as_deref()) - } - -+ fn read_file_bounded<'a>( -+ &'a self, -+ path: &'a PathUri, -+ options: ReadFileOptions, -+ max_bytes: u64, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ self.file_system -+ .read_file_bounded(path, options, max_bytes, self.key.sandbox.as_deref()) -+ } -+ - fn read_file_stream<'a>( - &'a self, - path: &'a PathUri, -diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs ---- a/codex-rs/file-system/src/lib.rs -+++ b/codex-rs/file-system/src/lib.rs -@@ -667,6 +667,23 @@ - sandbox: Option<&'a FileSystemSandboxContext>, - ) -> ExecutorFileSystemFuture<'a, Vec>; - -+ /// Reads at most `max_bytes + 1` bytes, rejecting files larger than `max_bytes`. -+ /// Backends must enforce the bound during I/O rather than truncate a full read. -+ fn read_file_bounded<'a>( -+ &'a self, -+ _path: &'a PathUri, -+ _options: ReadFileOptions, -+ _max_bytes: u64, -+ _sandbox: Option<&'a FileSystemSandboxContext>, -+ ) -> ExecutorFileSystemFuture<'a, Vec> { -+ Box::pin(async { -+ Err(io::Error::new( -+ io::ErrorKind::Unsupported, -+ "bounded file reads are unsupported", -+ )) -+ }) -+ } -+ - /// Reads a file as a stream of chunks no larger than [`FILE_READ_CHUNK_SIZE`]. - fn read_file_stream<'a>( - &'a self, diff --git a/patches/codex/README.md b/patches/codex/README.md deleted file mode 100644 index 07876f9771..0000000000 --- a/patches/codex/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# Codex Core operation observations - -The Agents SDK has been embedding Codex Core `0.166.0` at commit `6ff1e1ca3b30fc8b914720681fcf5fe62e95f856`. This patch has been exposing model and native tool observations through the embedded Rust binding. Python has been turning those observations into SDK spans for registered trace processors, including OpenTelemetry. - -## Why the patch has been needed - -Existing Core interfaces have been leaving gaps in operation tracing. - -| Existing interface | Missing observation | -| --- | --- | -| Tool lifecycle hooks | General tool results and timing limited to handler execution after authorization | -| Session events | The final assembled inference request and its exact stream completion | -| File-backed inference traces | In-memory capture and one logical inference across identical-input transport retries | - -The patch has been adding observations at those source boundaries. The maintained changes have been covering Core model requests, tool execution, bounded content capture, and response-body model metadata in `codex-api`. The SDK has been attaching an observer to each accepted turn through `WithTurnExtensionData`. Disabled tracing has been replacing the previous observer with an empty attachment. - -## Build and verify the wheel - -The commands below have been targeting a POSIX development host with Python 3.12, Git, `uv`, and `rustup`. Linux builds have been requiring a C/C++ compiler, CMake, `pkg-config`, Perl, and OpenSSL development libraries. The [preview workflow](../../.github/workflows/codex-preview.yml) has been supplying platform-specific prerequisites and release-compatible builds for the supported wheel targets. - -Start at the SDK checkout root. Install the pinned build tools into a separate environment. - -```sh -export SDK_ROOT="$PWD" -rustup toolchain install 1.95.0 --profile minimal --component rustfmt -uv venv --python 3.12 "$SDK_ROOT/.venv-codex-build" -uv pip install --python "$SDK_ROOT/.venv-codex-build" \ - --require-hashes --only-binary=:all: \ - --requirements native/codex/build-requirements.txt -``` - -On Linux, install `patchelf` into that environment for wheel repair. - -```sh -uv pip install --python "$SDK_ROOT/.venv-codex-build" patchelf -``` - -Build the combined SDK wheel and run the native binding tests against the patched dependency graph. - -```sh -export VIRTUAL_ENV="$SDK_ROOT/.venv-codex-build" -export PATH="$VIRTUAL_ENV/bin:$PATH" -export RUSTUP_TOOLCHAIN=1.95.0 -export UV_NO_BUILD_ISOLATION=1 -export CARGO_PROFILE_DEV_DEBUG=0 -export CARGO_PROFILE_TEST_DEBUG=0 -export RUST_MIN_STACK=8388608 -python scripts/build_codex_wheel.py --profile dev --test-native --out "$SDK_ROOT/dist" -``` - -The script has been copying the exact pinned upstream source into a temporary build directory and applying the checked-in patches in filename order there. Cargo source overrides have been selecting that patched workspace for the extension and execution helper. The wheel has been including the helper assets, licenses, source revision, patches, and this guide. For a release build, omit `--profile dev`. - -Set `WHEEL` to the absolute wheel path printed by the build. Create a clean environment and run the installed-package verifier outside the checkout. - -```sh -WHEEL=/absolute/path/to/built-wheel.whl -export VERIFY_ENV="$SDK_ROOT/.venv-codex-verify" -uv venv --python 3.12 "$VERIFY_ENV" -uv pip install --python "$VERIFY_ENV" "$WHEEL" -VERIFY_DIR=$(mktemp -d) -cd "$VERIFY_DIR" -"$VERIFY_ENV/bin/python" -I "$SDK_ROOT/native/codex/verify_installed.py" \ - --output "$VERIFY_DIR/installed.json" -``` - -The verifier has been checking the installed package, native imports, bundled helpers, authentication isolation, and real Runner behavior against a loopback model service. Python's `-I` option has been excluding checkout imports and ambient Python paths. - -Install the test dependencies and run the live operation-tracing matrix with that same wheel. - -```sh -uv pip install --python "$VERIFY_ENV" --only-binary=:all: \ - 'pytest>=9.0.3' pytest-asyncio 'opentelemetry-sdk>=1.45.0,<2' -CODEX_TRACING_NATIVE_TEST=1 "$VERIFY_ENV/bin/python" -I -m pytest \ - "$SDK_ROOT/tests/models/test_operation_tracing_parity.py" \ - --import-mode=importlib -q -``` - -The matrix has been exercising both Runner entry points, capture permissions, model retries, SDK and native tools, source timings, and interruption through the compiled extension. Its model service has been using loopback HTTP with fixture responses. The installed-wheel matrix has been passing 46 cases without skips against the pinned Core revision. API credentials have been remaining unnecessary for these checks. - -## Check the patch independently - -Use a second, clean Codex checkout to check source application independently of wheel assembly. Set `CODEX_CHECKOUT` to a new absolute directory. - -```sh -CODEX_CHECKOUT=/absolute/path/to/fresh-codex -PATCH_FILE="$SDK_ROOT/patches/codex/0001-operation-tracing.patch" -git clone --branch rust-v0.166.0 --single-branch \ - https://github.com/openai/codex.git "$CODEX_CHECKOUT" -cd "$CODEX_CHECKOUT" -test "$(git rev-parse HEAD)" = 6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 -test -z "$(git status --porcelain)" -git apply --check "$PATCH_FILE" -git apply "$PATCH_FILE" -git apply --check "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" -git apply "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" -git diff --check -cd codex-rs -cargo +1.95.0 test -p codex-core --lib operation_observation -cargo +1.95.0 test -p codex-core -p codex-tools --lib bounded_capture -cargo +1.95.0 test -p codex-core --lib mcp_tool_call_ids_are_added_to_request_meta -cargo +1.95.0 test -p codex-api --lib process_sse_reports_body_model_without_routing_authority -cargo +1.95.0 fmt --all -- --check -``` - -## Observation and capture contract - -The `codex_core::operation_observation` module has been exposing the observer, event, phase, attachment, and capture-mode types. Core has been calling `emit(OperationEvent)` synchronously and awaiting `turn_finished` before the ordinary turn terminal. The binding has been owning private delivery, bounded queues, loss reporting, and ordering into Python. - -The [binding observer](../../native/codex/src/operation_events.rs) has been providing a host integration example. Started and completed events have been carrying operation, thread, and turn identifiers with source timestamps. Available fields have been including parent identifiers, outcome, model metadata, usage, and tool identity. The Rust-only `phase` field has been selecting the Python envelopes `turn/operation/started` and `turn/operation/completed`. - -| Mode | Captured content | -| --- | --- | -| Metadata | Model and tool facts without input or output | -| Content | Permitted input and output with a separate 32 KiB UTF-8 JSON budget for each direction | -| Disabled | No observer | - -Raw strings that resemble JSON have been receiving a byte check before parsing. Typed values have been passing through capped serialization before capture has been allocating JSON. Oversized text has been retaining a bounded UTF-8 prefix with truncation markers. Structured values and media have been receiving omission markers. Request identity hashing has been borrowing prepared fields. Tool-result capture has been using the opt-in `ToolOutput::bounded_code_mode_result` path before any owned result conversion. JSON and public MCP fields have been passing through capped serialization while borrowed; MCP `_meta` has been remaining outside that serializer. Common function, search, shell, aborted, patch, and wrapper outputs have been bounding their source before cloning or rendering. Oversized source has been producing `capture_budget` omission markers. Other output implementations have been producing `unsupported_output` omission markers without invoking their general output converter. Metadata mode has been skipping output conversion entirely. Tool errors have been receiving a source-byte check before error formatting. - -Content has been staying in memory, outside rollout storage and diagnostic logs. HTTP and WebSocket observations have been including the final submitted input and continuation fields. Identical-input retries have been sharing one model operation. Changed semantic input has been starting another. Response-body model names have been supplying observation metadata without changing routing authority. Stream completion has been ending the model observation before session code has been draining tool futures. - -Native tool observations have been measuring handler execution after authorization, including nested code-mode results. Python has been owning SDK function execution spans. Observer panics have been leaving model and tool execution unaffected, and operation guards have been producing one completion on success, error, or interruption. - -## Prepare an upstream change - -Keep maintained Rust source and tests in the patch. Leave local Cargo lock changes out. Refresh the patch from the pinned checkout after reviewing source changes. The refresh below has been temporarily removing `0002-native-approval-recovery.patch` so the tracing patch has been containing only its own changes. - -```sh -cd "$CODEX_CHECKOUT" -base=6ff1e1ca3b30fc8b914720681fcf5fe62e95f856 -git apply -R "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" -git add -N codex-rs/core/src/operation_observation.rs \ - codex-rs/core/src/tools/operation_observation.rs \ - codex-rs/core/src/tools/operation_observation_tests.rs -git diff --check "$base" -candidate=$(mktemp "${PATCH_FILE}.XXXXXX") -git -c diff.suppressBlankEmpty=true diff --binary "$base" -- \ - codex-rs/core codex-rs/codex-api codex-rs/otel codex-rs/tools > "$candidate" -test -s "$candidate" -git apply --stat "$candidate" -mv "$candidate" "$PATCH_FILE" -git apply --check "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" -git apply "$SDK_ROOT/patches/codex/0002-native-approval-recovery.patch" -``` - -Before opening an upstream PR, apply the refreshed patch in another clean checkout and rerun the Rust checks above. Include the pinned revision, observer API example, capture contract, source hashes, and actual source and installed-wheel test results in the PR. Follow the upstream repository's contribution instructions. - -## Retire the patch - -Wait for an upstream release with equivalent source facts, capture controls, and observer isolation. Run the installed-wheel matrix against that release and compare SDK and OpenTelemetry output for retries, native tools, interruption, concurrent capture permissions, and content canaries. - -After those checks pass, update the Core pin and remove the patch and its build-time application together. Rerun source checks, native binding tests, installed-package verification, and live parity for the resulting wheel. diff --git a/scripts/build_codex_wheel.py b/scripts/build_codex_wheel.py index d39c0050f9..a476e83952 100644 --- a/scripts/build_codex_wheel.py +++ b/scripts/build_codex_wheel.py @@ -53,52 +53,11 @@ def stage_asset(asset: dict[str, str], destination: Path, cache: Path) -> None: destination.chmod(0o755) -def patch_codex_source(source: Path, upstream: Path, metadata: dict) -> Path: - """Apply the SDK's checked-in Core patch to the exact resolved upstream pin.""" - patched = source / "native/codex-upstream" - shutil.copytree(upstream, patched, ignore=shutil.ignore_patterns(".git")) - patches = sorted((source / "patches/codex").glob("*.patch")) - if not patches: - raise ValueError("Missing bundled Codex Core patch") - patch_environment = os.environ | {"GIT_CEILING_DIRECTORIES": str(patched.parent.resolve())} - for patch in patches: - subprocess.run( - ["git", "apply", "--check", str(patch.resolve())], - cwd=patched, - env=patch_environment, - check=True, - ) - subprocess.run( - ["git", "apply", str(patch.resolve())], - cwd=patched, - env=patch_environment, - check=True, - ) - # Redirect the entire upstream workspace, including transitive crates, so Core's - # API types retain one identity. Cargo keeps all other locked dependency pins. - packages = [ - package - for package in metadata["packages"] - if (package.get("source") or "").startswith("git+https://github.com/openai/codex?") - ] - config = source / ".cargo/config.toml" - config.parent.mkdir(exist_ok=True) - lines = ['[patch."https://github.com/openai/codex"]'] - for package in sorted(packages, key=lambda package: package["name"]): - relative = Path(package["manifest_path"]).parent.relative_to(upstream) - lines.append( - f"{json.dumps(package['name'])} = {{ path = {json.dumps(str(patched / relative))} }}" - ) - config.write_text("\n".join(lines) + "\n") - return patched - - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, default=ROOT / "dist") parser.add_argument("--target", choices=TARGETS) parser.add_argument("--compatibility") - parser.add_argument("--test-native", action="store_true") parser.add_argument("--profile", choices=("dev", "release"), default="release") args = parser.parse_args() environment = os.environ.copy() @@ -189,38 +148,6 @@ def build_native( "--target", target, ] - upstream = patch_codex_source(source, upstream, metadata) - subprocess.run( - [ - "cargo", - "metadata", - "--offline", - "--format-version", - "1", - "--manifest-path", - str(manifest), - ], - cwd=source, - env=environment, - stdout=subprocess.DEVNULL, - check=True, - ) - if args.test_native: - subprocess.run( - [ - "cargo", - "test", - "--locked", - "--manifest-path", - str(manifest), - *target_flags, - "--lib", - *profile, - ], - cwd=source, - env=environment, - check=True, - ) subprocess.run( [ "cargo", @@ -260,13 +187,8 @@ def build_native( ), }.items(): shutil.copy2(path, licenses / name) - for patch in (source / "patches/codex").glob("*"): - shutil.copy2(patch, licenses / f"CODEX_SDK_{patch.name}") if environment.get("OPENSSL_STATIC") == "1" and environment.get("OPENSSL_DIR"): - shutil.copy2( - Path(environment["OPENSSL_DIR"]) / "LICENSE.txt", - licenses / "OPENSSL_LICENSE", - ) + shutil.copy2(Path(environment["OPENSSL_DIR"]) / "LICENSE.txt", licenses / "OPENSSL_LICENSE") for license_file in (native / "licenses").glob("*"): shutil.copy2(license_file, licenses) with tempfile.TemporaryDirectory(dir=target_dir, prefix="codex-wheel-") as directory: diff --git a/src/agents/memory/resumable_session.py b/src/agents/memory/resumable_session.py index abb58b9b85..a9f1314edd 100644 --- a/src/agents/memory/resumable_session.py +++ b/src/agents/memory/resumable_session.py @@ -283,7 +283,6 @@ async def load_state( return state def _validate_run(self, input: Any) -> None: - from ..run_internal.native_approvals import native_receipts_from_snapshot from ..run_state import RunState if self._checkpoint.status == "active": @@ -301,22 +300,6 @@ def _validate_run(self, input: Any) -> None: or binding.revision != self._checkpoint.revision ): raise ResumableSessionError("invalid_state") - invalid_native_state = False - try: - assert self._checkpoint.run_state is not None - original = json.loads(self._checkpoint.run_state) - original_receipts = native_receipts_from_snapshot(original) - processed = input._last_processed_response - if original_receipts is not None or ( - processed is not None and processed.native_approvals - ): - invalid_native_state = ( - native_receipts_from_snapshot(input.to_json()) != original_receipts - ) - except Exception: - invalid_native_state = True - if invalid_native_state: - raise ResumableSessionError("invalid_state") def _get_history(self) -> list[TResponseInputItem]: return copy.deepcopy(self._checkpoint.history) diff --git a/src/agents/models/_codex_items.py b/src/agents/models/_codex_items.py index f555164166..66ee061da9 100644 --- a/src/agents/models/_codex_items.py +++ b/src/agents/models/_codex_items.py @@ -1,18 +1,9 @@ -"""Internal output types attesting to work observed directly by the Codex adapter.""" - -import json -from pathlib import Path -from typing import Any, cast +"""Internal output types attesting to work already completed by the Codex adapter.""" from openai.types.responses import ( - ResponseCustomToolCall, ResponseCustomToolCallOutputItem, - ResponseFunctionToolCall, ResponseFunctionToolCallOutputItem, ) -from pydantic import Field - -from ..exceptions import UserError class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): @@ -21,68 +12,3 @@ class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): class CodexCustomToolCallOutput(ResponseCustomToolCallOutputItem): """A matching native custom call has already completed in Codex.""" - - -class CodexPendingFunctionCall(ResponseFunctionToolCall): - """A native function call has paused at Codex's own approval boundary.""" - - checkpoint: dict[str, Any] = Field(exclude=True, repr=False) - - -class CodexPendingCustomCall(ResponseCustomToolCall): - """A native custom call has paused at Codex's own approval boundary.""" - - checkpoint: dict[str, Any] = Field(exclude=True, repr=False) - - -def native_approval_checkpoint(raw_item: Any) -> dict[str, Any] | None: - """Validate an adapter-owned receipt against the call shown for approval.""" - # An unrelated approval may define __getattribute__("__class__"). Don't run it - # while taking a detached copy of a pending call or a session checkpoint. - raw_type = type(raw_item) - if raw_type is not CodexPendingFunctionCall and raw_type is not CodexPendingCustomCall: - return None - checkpoint = raw_item.checkpoint - required = {"version", "cwd", "call", "request", "model"} - if ( - not isinstance(checkpoint, dict) - or not required <= checkpoint.keys() - or set(checkpoint) - required - {"preconditions"} - or type(checkpoint["version"]) is not int - or checkpoint["version"] != 1 - or not isinstance(checkpoint["cwd"], str) - or not Path(checkpoint["cwd"]).is_absolute() - or not isinstance(checkpoint["model"], str) - or not checkpoint["model"] - or not isinstance(checkpoint["request"], dict) - or not isinstance(checkpoint["call"], dict) - or ("preconditions" in checkpoint and not isinstance(checkpoint["preconditions"], dict)) - ): - raise UserError("Invalid Codex native approval checkpoint.") - model = ( - ResponseFunctionToolCall - if isinstance(raw_item, CodexPendingFunctionCall) - else ResponseCustomToolCall - ) - display_metadata = {"codex_reasoning_summary"} - displayed = model.model_validate(raw_item.model_dump()).model_dump(exclude=display_metadata) - stored = model.model_validate(checkpoint["call"]).model_dump(exclude=display_metadata) - if displayed != stored or raw_item.name not in {"exec_command", "apply_patch"}: - raise UserError("Codex native approval no longer matches its original call.") - return cast(dict[str, Any], json.loads(json.dumps(checkpoint, allow_nan=False))) - - -def restore_native_approval_call(raw_item: Any, checkpoint: Any) -> Any: - """Restore only explicit SDK checkpoint metadata, never provider output fields.""" - if not isinstance(raw_item, dict) or not isinstance(checkpoint, dict): - raise UserError("Invalid Codex native approval checkpoint.") - item: CodexPendingFunctionCall | CodexPendingCustomCall - item_type = raw_item.get("type") - if item_type == "function_call": - item = CodexPendingFunctionCall(**raw_item, checkpoint=checkpoint) - elif item_type == "custom_tool_call": - item = CodexPendingCustomCall(**raw_item, checkpoint=checkpoint) - else: - raise UserError("Invalid Codex native approval call type.") - native_approval_checkpoint(item) - return item diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py index b18ed5e0b9..323b5d7478 100644 --- a/src/agents/models/_codex_tracing.py +++ b/src/agents/models/_codex_tracing.py @@ -1,10 +1,11 @@ +"""Trace stock Codex request and tool callbacks without copying native payloads.""" + from __future__ import annotations -from dataclasses import dataclass +import json +from datetime import datetime, timezone from typing import Any -from openai.types.responses import Response - from ..tracing import ( Span, custom_span, @@ -13,247 +14,124 @@ get_current_trace, response_span, ) -from ..tracing._content import _json, bound_content -from ..tracing._model import acquire_model_span, finish_model_span, record_model_input +from ..tracing._model import acquire_model_span, finish_model_span from ..tracing.config import get_operation_detail -from ..tracing.provider import _is_noop_span, _is_noop_trace -from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData -from ..tracing.util import _parse_iso_timestamp, time_iso +from ..tracing.provider import _is_noop_span from ..usage import _response_usage_to_usage, model_usage_to_span_usage from ._codex_usage import _usage from .interface import ModelTracing -@dataclass -class _Operation: - span: Span[Any] - kind: str - ended_at: str | None = None - - class CodexOperationTracing: - """Translate turn-private operation facts into the current SDK trace.""" - - def __init__(self, tracing: ModelTracing, input: Any, model: str | None) -> None: - self._tracing = tracing - self._detail = get_operation_detail(include_data=tracing.include_data()) - self._input = input - self._model = model - self._parent = get_current_span() - active_trace = get_current_trace() - disabled = ( - tracing.is_disabled() - or active_trace is None - or _is_noop_trace(active_trace) - or _is_noop_span(self._parent) - ) - self.mode = None if disabled else self._detail or "metadata" - self._thread_id: str | None = None - self._turn_id: str | None = None - self._operations: dict[str, _Operation] = {} - self._completed: dict[str, str] = {} - self._pending: _Operation | None = None - self._first_model = True - self._coverage_reason: str | None = None - self._coverage_emitted = False - - def request_mode(self) -> str | None: - # A hook may disable the provider after the active trace and parent were created. - if self.mode is not None and _is_noop_span(response_span(parent=self._parent)): - self.mode = None - return self.mode - - def bind(self, thread_id: str, turn_id: str) -> None: - self._thread_id, self._turn_id = thread_id, turn_id + """Own observations for one ephemeral Codex thread and one SDK model attempt.""" + + def __init__(self, sdk_tools: set[tuple[str | None, str]]) -> None: + self._sdk_tools = sdk_tools + self._parent = get_current_span() or get_current_trace() + self._thread: Any = None + self._active: dict[tuple[str, str], Span[Any]] = {} + self._reason: str | None = None + self._saw_model = False + + def bind(self, thread: Any, tracing: ModelTracing) -> None: + if tracing.is_disabled() or self._parent is None: + return + # A hook can disable the provider after the parent starts. + if _is_noop_span(response_span(parent=self._parent)): + return + if get_operation_detail(include_data=tracing.include_data()) == "content": + self._reason = "metadata_only" + try: + thread.enable_tracing() + self._thread = thread + except Exception: + self._reason = "unavailable" - def observe(self, method: str, event: Any) -> None: - if self.mode is None: + def drain(self) -> None: + if self._thread is None: return try: - if method == "turn/operation/dropped": - if event.get("threadId") != self._thread_id: - raise ValueError("Unexpected observation thread.") - self._coverage_reason = "dropped_operations" + encoded = self._thread.drain_operations() + if not encoded or encoded == "[]": return - self._observe(method, event) + for event in json.loads(encoded): + self._observe(event) except Exception: - # Telemetry must never replay a model request or interrupt tool execution. - self._coverage_reason = "telemetry_error" + # Observation failure must never change the result or cause a retry. + self._reason = "telemetry_error" - def _observe(self, method: str, event: dict[str, Any]) -> None: - if event.get("threadId") != self._thread_id or event.get("turnId") != self._turn_id: - self._coverage_reason = "telemetry_error" + def _observe(self, event: dict[str, Any]) -> None: + kind = event["kind"] + if kind == "dropped": + self._reason = "dropped_operations" + return + if kind == "tool" and (event.get("namespace"), event["name"]) in self._sdk_tools: + # Runner owns the approval and execution spans for SDK tools. return - operation_id = event["operationId"] - if not isinstance(operation_id, str) or not operation_id: - raise ValueError("Missing operation identity.") - if method == "turn/operation/started": - if operation_id in self._operations or operation_id in self._completed: - raise ValueError("Duplicate operation start.") - kind = event["kind"] - started_at = self._timestamp(event["startedAt"]) - parent_id = event.get("parentOperationId") - parent = self._parent - if parent_id is not None: - if parent_id in self._operations: - parent = self._operations[parent_id].span - else: - # Only identity is needed when a later child refers to a completed parent. - parent = function_span( - "codex.operation.parent", - span_id=self._completed[parent_id], - parent=self._parent, - ) + key = kind, event["id"] + at = datetime.fromtimestamp(event["at"] / 1000, tz=timezone.utc).isoformat() + if event["phase"] == "start": + if key in self._active or len(self._active) >= 128: + self._reason = "dropped_operations" + return span: Span[Any] if kind == "model": - self._finish_pending() span = acquire_model_span( - self._tracing, - event.get("requestedModel") or self._model, - parent=parent, - started_at=started_at, + ModelTracing.ENABLED_WITHOUT_DATA, + event["model"], + parent=self._parent, + started_at=at, ) - if isinstance(span.span_data, ResponseSpanData) and self._first_model: - if self._tracing.include_data(): - span.span_data.input = self._input - self._first_model = False + self._saw_model = True elif kind == "tool": - span = function_span(event["toolName"], parent=parent) - span.span_data.operation.detail = self._detail + name = event["name"] + if event.get("namespace"): + name = f"{event['namespace']}.{name}" + span = function_span(name, parent=self._parent) + span.span_data.operation.detail = get_operation_detail(include_data=False) + span.span_data.operation.tool_call_id = event["id"] + span.start_at(at) else: - raise ValueError("Unknown operation kind.") - operation = _Operation(span=span, kind=kind) - self._operations[operation_id] = operation - self._metadata(operation, event) - self._content(operation, event) - if kind == "tool": - span.start_at(started_at) + raise ValueError("Unknown Codex operation.") + self._active[key] = span return - operation = self._operations[operation_id] - if operation.ended_at is not None or event.get("kind") != operation.kind: - raise ValueError("Invalid operation terminal.") - ended_at = self._timestamp(event["completedAt"]) - outcome = event["outcome"] - if outcome not in ("success", "error", "interrupted"): - raise ValueError("Unknown operation outcome.") - self._metadata(operation, event) - self._content(operation, event) - operation.ended_at = ended_at - operation.span.span_data.operation.outcome = { - "success": "completed", - "error": "failed", - }.get(outcome, outcome) - if outcome != "success": - operation.span.set_error({"message": f"Codex operation {outcome}.", "data": None}) - if operation.kind == "model": - self._finish_pending() - self._pending = operation - else: - operation.span.finish_at(operation.ended_at) - self._completed[operation_id] = operation.span.span_id - self._operations.pop(operation_id) - - @staticmethod - def _timestamp(value: Any) -> str: - if not isinstance(value, str): - raise ValueError("Missing source timestamp.") - _parse_iso_timestamp(value) - return value - - def _metadata(self, operation: _Operation, event: dict[str, Any]) -> None: - data = operation.span.span_data - metadata = data.operation - metadata.operation_id = event["operationId"] - metadata.model = event.get("requestedModel") or metadata.model or self._model - metadata.response_model = event.get("observedModel") or metadata.response_model - metadata.tool_call_id = event.get("callId") or metadata.tool_call_id - if self._tracing.include_data() or event.get("responseIdTrusted") is True: - metadata.response_id = event.get("responseId") or metadata.response_id - else: - metadata.response_id = None - usage = event.get("usage") - if operation.kind == "model": - if isinstance(data, GenerationSpanData): - data.model = metadata.model - if isinstance(usage, dict): - data.usage = model_usage_to_span_usage(_response_usage_to_usage(_usage(usage))) - - def _content(self, operation: _Operation, event: dict[str, Any]) -> None: - if self._detail != "content": - return - data = operation.span.span_data - for direction in ("input", "output"): - projection = event.get(direction) - if not isinstance(projection, dict): - continue - value = projection.get("value") - marker: dict[str, Any] = { - key: True for key in ("truncated", "omitted") if projection.get(key) is True - } - if marker and isinstance(projection.get("reason"), str): - marker["reason"] = projection["reason"] - if isinstance(data, GenerationSpanData) and direction == "input": - request = dict(value) if isinstance(value, dict) else {"input": value} - request.update(marker) - record_model_input(data, request) - continue - value = bound_content(value) - if value is None and marker: - value = marker - if isinstance(data, FunctionSpanData): - setattr(data, direction, value if isinstance(value, str) else _json(value)) - elif isinstance(data, GenerationSpanData): - items = value if isinstance(value, list) else [{"content": value}] - if marker and marker not in items: - items = [*items, marker] - data.output = bound_content(items) - def _finish_pending(self) -> None: - if self._pending is not None: - pending, self._pending = self._pending, None - finish_model_span(pending.span, ended_at=pending.ended_at) - - def close(self, *, interrupted: bool) -> None: - if self.mode is None: - return - estimated_end = time_iso() - for operation_id, operation in self._operations.items(): - if operation.ended_at is not None: - continue - operation.ended_at = estimated_end - operation.span.span_data.operation.outcome = "interrupted" - operation.span.span_data.operation.estimated_end = True - operation.span.set_error({"message": "Codex operation interrupted.", "data": None}) - if operation.kind == "model": - finish_model_span(operation.span, ended_at=operation.ended_at) - else: - operation.span.finish_at(operation.ended_at) - self._completed[operation_id] = operation.span.span_id - self._coverage_reason = "interrupted" - self._operations.clear() - if interrupted: - if not self._completed: - self._coverage_reason = self._coverage_reason or "interrupted" - self._finish_pending() - self._emit_coverage() - - def finish_response(self, response: Response) -> None: - if self.mode is not None and self._first_model: - self._coverage_reason = "missing_operations" - if self._pending is not None: - data = self._pending.span.span_data - if isinstance(data, ResponseSpanData) and self._tracing.include_data(): - data.response = response - self._finish_pending() - self.close(interrupted=False) - - def _emit_coverage(self) -> None: - if self._coverage_reason is None or self._coverage_emitted: + if key not in self._active: + # Core can block a call before on_tool_start. Never invent an execution. + self._reason = "missing_start" return - self._coverage_emitted = True - with custom_span( - "codex.tracing.coverage", - {"coverage": "partial", "provider": "codex", "reason": self._coverage_reason}, - parent=self._parent, - ): - pass + if kind == "model" and event.get("usage") is not None: + self._active[key].span_data.usage = model_usage_to_span_usage( + _response_usage_to_usage(_usage(event["usage"])) + ) + self._finish(key, event["outcome"], at) + + def _finish(self, key: tuple[str, str], outcome: str, at: str | None = None) -> None: + span = self._active.pop(key) + span.span_data.operation.outcome = outcome + span.span_data.operation.estimated_end = at is None + if outcome != "completed": + span.set_error( + {"message": "Codex operation did not complete successfully.", "data": None} + ) + if key[0] == "model": + finish_model_span(span, ended_at=at) + elif at is None: + span.finish() + else: + span.finish_at(at) + + def close(self) -> None: + self.drain() + if self._thread is not None and not self._saw_model: + self._reason = self._reason or "missing_operations" + for key in list(self._active): + self._reason = self._reason or "interrupted" + self._finish(key, "interrupted") + if self._reason is not None: + with custom_span( + "codex.tracing.coverage", + {"coverage": "partial", "provider": "codex", "reason": self._reason}, + parent=self._parent, + ): + pass diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 933eaa0402..61386c14b3 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import json import os import time @@ -50,24 +49,21 @@ _response_usage_to_usage, ) from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call -from ._codex_items import ( - CodexCustomToolCallOutput, - CodexFunctionToolCallOutput, - CodexPendingCustomCall, - CodexPendingFunctionCall, -) +from ._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput from ._codex_tool_hooks import CodexToolHooks from ._codex_tracing import CodexOperationTracing from ._codex_usage import _add_sample_usage, _usage -from ._run_context import ModelToolHooks, get_model_tool_hooks +from ._run_context import get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing class CodexProvider(ModelProvider): """Run opt-in Agents SDK requests through embedded Codex Core. - Codex owns workspace tools and authentication; the Runner owns SDK function tools, - handoffs, approvals, and conversation history. Each request uses an ephemeral thread. + Each request uses a fresh Codex thread. Native request and tool spans contain + metadata only. Interactive Codex approvals are unsupported. For resumable + workspace approvals, supply a local ShellTool or ApplyPatchTool with + needs_approval=True and an executor or editor that enforces your policy. """ _supports_stored_response_id = False @@ -116,127 +112,6 @@ async def aclose(self) -> None: def get_model(self, model_name: str | None) -> Model: return _CodexModel(model_name, self) - async def _run_approved_native_call( - self, checkpoint: dict[str, Any], tracing: ModelTracing - ) -> str: - """Run only the saved invocation, without a model turn or its old Codex home.""" - if checkpoint.get("version") != 1 or checkpoint.get("cwd") != self._cwd: - raise UserError("This Codex approval belongs to a different workspace or SDK version.") - - settings = {"model": checkpoint["model"]} - observations = CodexOperationTracing(tracing, None, checkpoint["model"]) - thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) - interrupted = True - cancellation: asyncio.CancelledError | None = None - execution: asyncio.Future[Any] | None = None - observed_turn: str | None = None - observations.bind(thread.thread_id, "") - - def observe(encoded: str) -> None: - nonlocal observed_turn - event = json.loads(encoded) - params = event.get("params") - if observed_turn is None and isinstance(params, dict): - turn = params.get("turnId") - if params.get("threadId") == thread.thread_id and isinstance(turn, str): - observed_turn = turn - observations.bind(thread.thread_id, turn) - observations.observe(event["method"], params) - - try: - execution = asyncio.ensure_future( - thread.resolve_approval( - json.dumps(checkpoint["call"]), - json.dumps(checkpoint["request"]), - json.dumps(checkpoint.get("preconditions")), - observations.request_mode(), - ) - ) - while (encoded := await thread.next_operation_event()) is not None: - observe(encoded) - output, turn_id, notifications, failure = await execution - observations.bind(thread.thread_id, turn_id) - for encoded in notifications: - observe(encoded) - if failure is not None: - raise RuntimeError(failure) - interrupted = False - return str(output) - except asyncio.CancelledError as error: - cancellation = error - raise - finally: - if execution is not None: - execution.cancel() - - async def close() -> None: - try: - await thread.close() - finally: - if execution is not None: - await asyncio.gather(execution, return_exceptions=True) - - try: - observations.close(interrupted=interrupted) - finally: - try: - await _join_close(close()) - except asyncio.CancelledError: - if cancellation is not None: - raise cancellation from None - raise - except Exception: - if not interrupted: - raise - - -def _pending_native_call( - call: Any, event: dict[str, Any], cwd: str, model: str -) -> CodexPendingFunctionCall | CodexPendingCustomCall: - request = event["msg"] - wire = call.model_dump(exclude_none=True) - if event.get("preconditions") is None: - raise UserError("Codex could not confirm that this tool is still waiting to run.") - checkpoint = { - "version": 1, - "cwd": cwd, - "model": model, - "call": wire, - "request": request, - "preconditions": event["preconditions"], - } - if getattr(call, "namespace", None) not in (None, "functions"): - raise UserError("Only built-in Codex tools support native approval recovery.") - if ( - isinstance(call, ResponseFunctionToolCall) - and call.name == "exec_command" - and request["type"] == "exec_approval_request" - and request.get("kind", "command") == "command" - and request.get("approval_id") is None - and request.get("network_approval_context") is None - and request.get("environmentId") in (None, "local") - ): - return CodexPendingFunctionCall(**wire, checkpoint=checkpoint) - if ( - isinstance(call, ResponseCustomToolCall) - and call.name == "apply_patch" - and request["type"] == "apply_patch_approval_request" - ): - return CodexPendingCustomCall(**wire, checkpoint=checkpoint) - raise UserError("This Codex approval cannot safely be resumed on another worker.") - - -def _uses_sdk_workspace(tools: list[Tool], bindings: ModelToolHooks | None) -> bool: - settings = bindings.settings if bindings else None - return bool( - settings - and settings.sdk_only_tool_hooks - and (settings.prepare_tool_input or settings.prepare_tool_output) - ) or any( - getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) - for tool in tools - ) - def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: supported = { @@ -288,16 +163,6 @@ def __init__(self, name: str | None, provider: CodexProvider) -> None: def get_default_model_settings(self) -> ModelSettings: return ModelSettings() - async def _run_approved_native_call( - self, checkpoint: dict[str, Any], *, tools: list[Tool], tracing: ModelTracing - ) -> str: - bindings = get_model_tool_hooks() - if _uses_sdk_workspace(tools, bindings) or CodexToolHooks.for_run(bindings, set()): - raise UserError( - "The current workspace or native tool hooks do not allow restoring this call." - ) - return await self._provider._run_approved_native_call(checkpoint, tracing) - def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice: # Native Codex tools may have changed the workspace before a failed response. return ModelRetryAdvice(replay_safety="unsafe") @@ -396,7 +261,19 @@ async def stream_response( ) provider = self._provider bindings = get_model_tool_hooks() - sdk_sandbox = _uses_sdk_workspace(tools, bindings) + tool_execution = bindings.settings if bindings is not None else None + classic_preparation = ( + tool_execution is not None + and tool_execution.sdk_only_tool_hooks + and ( + tool_execution.prepare_tool_input is not None + or tool_execution.prepare_tool_output is not None + ) + ) + sdk_sandbox = classic_preparation or any( + getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) + for tool in tools + ) native_hooks = None if sdk_sandbox else CodexToolHooks.for_run(bindings, identities) if sdk_sandbox: system_instructions = "\n\n".join( @@ -465,7 +342,7 @@ async def stream_response( and output_schema.is_strict_json_schema() else None ) - observations = CodexOperationTracing(tracing, input, self._name) + observations = CodexOperationTracing(identities) resources = AsyncExitStack() output: dict[str, Any] = {} streamed: set[str] = set() @@ -475,7 +352,6 @@ async def stream_response( sdk_calls: dict[str, Any] = {} native_pending: dict[str, tuple[str, str]] = {} native_completed: set[str] = set() - pending_approval: dict[str, Any] | None = None sampling_pending = False interrupt_requested = False sequence = 0 @@ -483,7 +359,6 @@ async def stream_response( reasoning_summaries: list[str] = [] usage = None failure: BaseException | None = None - done_events: list[ResponseOutputItemDoneEvent] = [] async def interrupt_when_ready() -> None: nonlocal interrupt_requested @@ -502,9 +377,8 @@ async def interrupt_when_ready() -> None: native_hooks.verify(thread) if history: await thread.inject_items(json.dumps(history)) - capture = observations.request_mode() - turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema), capture) - observations.bind(thread.thread_id, turn_id) + observations.bind(thread, tracing) + turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema)) response = Response( id=turn_id, created_at=time.time(), @@ -521,29 +395,11 @@ async def interrupt_when_ready() -> None: ) sequence += 1 while True: - if pending_approval is not None and not sampling_pending: - # All calls from this sampling response are now known. A sibling - # still in flight has an unknown outcome and cannot be dropped. - call_id = pending_approval["msg"]["call_id"] - if set(native_pending) != {call_id} or sdk_calls: - raise UserError("Codex can only pause one outstanding native call.") - item_id, _ = native_pending.pop(call_id) - output[item_id] = _pending_native_call( - output[item_id], pending_approval, provider._cwd, thread.model - ) - # Close the waiting thread before handing the pause to the Runner. - break encoded = await thread.next_event() + observations.drain() if encoded is None: raise RuntimeError("Codex finished without a completed response.") event = json.loads(encoded) - if event.get("method") in ( - "turn/operation/started", - "turn/operation/completed", - "turn/operation/dropped", - ): - observations.observe(event["method"], event.get("params")) - continue message = event["msg"] kind = message["type"] if kind == "hook_completed" and native_hooks is not None: @@ -559,20 +415,6 @@ async def interrupt_when_ready() -> None: ) if event.get("id") not in ("", turn_id): continue - if kind in ("exec_approval_request", "apply_patch_approval_request"): - if ( - message.get("turn_id") != turn_id - or pending_approval is not None - or native_hooks is not None - or sdk_sandbox - or require_sdk_call - ): - raise UserError( - "Codex can only pause a single built-in tool call before execution, " - "with native workspace tools enabled and without custom tool hooks." - ) - pending_approval = event - continue if kind == "dynamic_tool_call_request": if message.get("turnId") != turn_id: continue @@ -642,8 +484,7 @@ async def interrupt_when_ready() -> None: raw_usage["token_count"] = info continue if kind == "raw_response_completed": - # An approval can pause before Core sends the cumulative token_count. - # Include each completed sample now. The next count replaces this total. + # An SDK tool can interrupt before the next cumulative token count. if message.get("token_usage") is not None: usage = _add_sample_usage(usage, message["token_usage"]) if ( @@ -847,72 +688,65 @@ async def interrupt_when_ready() -> None: type="response.output_item.done", ) sequence += 1 - if native_pending: - if not interrupt_requested: - raise RuntimeError("Codex finished with an incomplete native tool call.") - # Interrupted code-mode parents have no completed result to replay. - for item_id, _ in native_pending.values(): - output.pop(item_id, None) - if native_hooks is not None: - for item in output.values(): - native_hooks.prepare_history_item(item) - if require_sdk_call and not sdk_calls: - raise UserError( - "CodexProvider tool_choice='required' did not return " - "an SDK tool or handoff call." - ) - if not sdk_calls and not output: - raise RuntimeError("Codex finished without a completed response.") - items = list(output.values()) + list(sdk_calls.values()) - if reasoning_summaries: - items[-1].codex_reasoning_summary = reasoning_summaries - for native_item in output.values(): - if native_item.type not in ( - "function_call", - "custom_tool_call", - "function_call_output", - "custom_tool_call_output", - ): - continue - done_events.append( - ResponseOutputItemDoneEvent( - item=native_item, - output_index=indices[native_item.id], - sequence_number=sequence, - type="response.output_item.done", - ) - ) - sequence += 1 - for call_id, call in sdk_calls.items(): - done_events.append( - ResponseOutputItemDoneEvent( - item=call, - output_index=indices[call_id], - sequence_number=sequence, - type="response.output_item.done", - ) - ) - sequence += 1 - items.sort(key=lambda item: indices[item.id]) - response = response.model_copy( - update={"output": items, "status": "completed", "usage": usage} - ) - if raw_usage is not None: - _attach_raw_usage_snapshot(response, raw_usage) - observations.finish_response(response) except BaseException as error: failure = error raise finally: - observations.close(interrupted=True) try: await _join_close(resources.aclose()) except Exception: # Keep the attempt failure primary when joined cleanup also fails. if failure is None: raise - for event in done_events: - yield event + finally: + observations.close() + if native_pending: + if not interrupt_requested: + raise RuntimeError("Codex finished with an incomplete native tool call.") + # Interrupted code-mode parents have no completed result to replay. + for item_id, _ in native_pending.values(): + output.pop(item_id, None) + if native_hooks is not None: + for item in output.values(): + native_hooks.prepare_history_item(item) + if require_sdk_call and not sdk_calls: + raise UserError( + "CodexProvider tool_choice='required' did not return an SDK tool or handoff call." + ) + if not sdk_calls and not output: + raise RuntimeError("Codex finished without a completed response.") + items = list(output.values()) + list(sdk_calls.values()) + if reasoning_summaries: + items[-1].codex_reasoning_summary = reasoning_summaries + for native_item in output.values(): + if native_item.type not in ( + "function_call", + "custom_tool_call", + "function_call_output", + "custom_tool_call_output", + ): + continue + yield ResponseOutputItemDoneEvent( + item=native_item, + output_index=indices[native_item.id], + sequence_number=sequence, + type="response.output_item.done", + ) + sequence += 1 + for call_id, call in sdk_calls.items(): + yield ResponseOutputItemDoneEvent( + item=call, + output_index=indices[call_id], + sequence_number=sequence, + type="response.output_item.done", + ) + sequence += 1 + items.sort(key=lambda item: indices[item.id]) + response = response.model_copy( + update={"output": items, "status": "completed", "usage": usage} + ) + if raw_usage is not None: + _attach_raw_usage_snapshot(response, raw_usage) yield ResponseCompletedEvent( response=response, sequence_number=sequence, type="response.completed" ) diff --git a/src/agents/run.py b/src/agents/run.py index 715b5b4f24..57ca88bbe2 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -871,9 +871,6 @@ async def _run_impl( run_state._reasoning_item_id_policy = resolved_reasoning_item_id_policy run_state.set_trace(get_current_trace()) - if isinstance(session, ResumableRun): - session.bind_state(run_state) - use_task_and_turn_spans = include_task_and_turn_spans(run_config.tracing) current_task_span: Span[TaskSpanData] | None = ( task_span(name=trace_workflow_name) if use_task_and_turn_spans else None @@ -2915,7 +2912,6 @@ async def run_loop() -> None: if isinstance(session, ResumableRun): try: await session.begin(input) - session.bind_state(run_state) except BaseException: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 6ddaa62170..77ae747a8e 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -1189,10 +1189,6 @@ def _apply_approval_decision( rejection_message: str | None = None, ) -> None: """Record an approval or rejection decision.""" - from .models._codex_items import native_approval_checkpoint - - if native_approval_checkpoint(approval_item.raw_item) is not None and always: - raise UserError("Native approvals require a decision for the original call only.") hosted_request = get_hosted_mcp_approval_request_identity(approval_item) if hosted_request is not None: call_id = hosted_request.request_id diff --git a/src/agents/run_internal/native_approvals.py b/src/agents/run_internal/native_approvals.py deleted file mode 100644 index 2e9b379d19..0000000000 --- a/src/agents/run_internal/native_approvals.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Keep native approval state and execution separate from SDK tool dispatch.""" - -from __future__ import annotations - -from typing import Any, cast - -from ..exceptions import UserError -from ..items import ToolCallOutputItem, ToolCallOutputTypes -from ..memory.resumable_session import _ResumableSessionBinding -from ..models._codex_items import native_approval_checkpoint -from ..models._run_context import ModelToolHooks, model_run_context -from .run_steps import NextStepInterruption, NextStepRunAgain, SingleStepResult -from .tool_execution import pending_tool_span -from .turn_preparation import get_all_tools, get_model - - -def native_receipts_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any] | None: - """Select the immutable native proposals while leaving decisions mutable.""" - processed = snapshot.get("last_processed_response") or {} - approvals = processed.get("native_approvals", []) - if not approvals: - return None - call_ids = {item["raw_item"]["call_id"] for item in approvals} - step = snapshot.get("current_step") or {} - pending = (step.get("data") or {}).get("interruptions", []) - return { - "plans": approvals, - "pending": [ - item for item in pending if item.get("raw_item", {}).get("call_id") in call_ids - ], - "generated": [ - item - for item in snapshot.get("generated_items", []) - if item.get("raw_item", {}).get("call_id") in call_ids - ], - } - - -def _validate_native_plan(processed: Any) -> None: - if len(processed.native_approvals) != 1 or any( - ( - processed.functions, - processed.custom_tool_calls, - processed.handoffs, - processed.computer_actions, - processed.local_shell_calls, - processed.shell_calls, - processed.apply_patch_calls, - processed.function_tools_not_found, - processed.mcp_approval_requests, - ) - ): - raise UserError( - "Native approvals require a single pending native call without SDK actions." - ) - item = processed.native_approvals[0] - if native_approval_checkpoint(item.raw_item) is None or item.tool_name != item.raw_item.name: - raise UserError("Invalid native approval plan.") - - -def native_pending_step( - *, - bindings: Any, - original_input: Any, - pre_step_items: list[Any], - new_response: Any, - processed_response: Any, - run_state: Any, - run_config: Any, -) -> SingleStepResult: - _validate_native_plan(processed_response) - binding = run_state._resumable_binding if run_state is not None else None - if ( - not isinstance(binding, _ResumableSessionBinding) - or binding.session._checkpoint.status != "active" - ): - raise UserError( - "Start the run with session=ResumableSession to recover native tool approvals." - ) - pending = list(processed_response.native_approvals) - processed_response.interruptions = pending - with pending_tool_span(run_config, pending[0].tool_name, pending[0].raw_item.call_id) as span: - span.span_data.operation.outcome = "paused" - return SingleStepResult( - original_input=original_input, - model_response=new_response, - pre_step_items=pre_step_items, - new_step_items=[*processed_response.new_items, *pending], - next_step=NextStepInterruption(interruptions=pending), - tool_input_guardrail_results=[], - tool_output_guardrail_results=[], - processed_response=processed_response, - ) - - -async def resolve_native_approval_step( - *, - bindings: Any, - original_input: Any, - original_pre_step_items: list[Any], - new_response: Any, - processed_response: Any, - context_wrapper: Any, - hooks: Any, - run_config: Any, - run_state: Any, -) -> SingleStepResult: - from ..models.codex import _CodexModel - - _validate_native_plan(processed_response) - binding = run_state._resumable_binding if run_state is not None else None - if ( - not isinstance(binding, _ResumableSessionBinding) - or binding.session._checkpoint.status != "active" - ): - raise UserError("Native approval recovery requires a claimed ResumableSession.") - pending = processed_response.native_approvals[0] - checkpoint = native_approval_checkpoint(pending.raw_item) - assert checkpoint is not None - call = pending.raw_item - key = context_wrapper._resolve_approval_key(pending) - key = context_wrapper._function_approval_key(key, pending) or key - decision = context_wrapper._approvals.get(key) - if decision is not None and (decision.approved is True or decision.rejected is True): - raise UserError("Native approvals require a decision for the original call only.") - approved = context_wrapper.get_approval_status( - pending.tool_name, - call.call_id, - existing_pending=pending, - current_invocation=pending, - tool_lookup_key=pending.tool_lookup_key, - tool_namespace=pending.tool_namespace, - ) - if approved is None: - with pending_tool_span(run_config, pending.tool_name, call.call_id) as span: - span.span_data.operation.outcome = "paused" - return SingleStepResult( - original_input=original_input, - model_response=new_response, - pre_step_items=original_pre_step_items, - new_step_items=[], - next_step=NextStepInterruption(interruptions=[pending]), - tool_input_guardrail_results=[], - tool_output_guardrail_results=[], - processed_response=processed_response, - ) - # A new checkpoint can carry the same model call ID. Claiming that checkpoint - # doesn't authorize repeating an invocation that already used its decision. - status = context_wrapper._tool_invocation_status( - call, tool_lookup_key=pending.tool_lookup_key, tool_name=pending.tool_name - ) - if status is None: - raise UserError("Cannot match the native approval to a pending invocation.") - _, completed, executed = status - if executed or completed: - raise UserError( - "Cannot reuse a native call ID after the call has executed or completed. " - "Start a new invocation with a new call ID." - ) - if approved: - from ..tracing.model_tracing import get_model_tracing_impl - - model = bindings.resolved_model or get_model(bindings.execution_agent, run_config) - if not isinstance(model, _CodexModel): - raise UserError("A pending native approval requires its current Codex model.") - tools = await get_all_tools(bindings.execution_agent, context_wrapper) - with model_run_context( - run_state, - ModelToolHooks( - bindings.public_agent, context_wrapper, hooks, run_config.tool_execution - ), - ): - output = await model._run_approved_native_call( - checkpoint, - tools=tools, - tracing=get_model_tracing_impl( - run_config.tracing_disabled, run_config.trace_include_sensitive_data - ), - ) - else: - from .tool_execution import resolve_approval_rejection_message - - with pending_tool_span(run_config, pending.tool_name, call.call_id): - output = await resolve_approval_rejection_message( - context_wrapper=context_wrapper, - run_config=run_config, - tool_type="shell" if call.name == "exec_command" else "apply_patch", - tool_name=call.name, - call_id=call.call_id, - tool_call=call, - existing_pending=pending, - tool_lookup_key=pending.tool_lookup_key, - tool_namespace=pending.tool_namespace, - ) - result_item = ToolCallOutputItem( - agent=bindings.public_agent, - raw_item=cast( - ToolCallOutputTypes, - { - "type": "function_call_output" - if call.type == "function_call" - else "custom_tool_call_output", - "call_id": call.call_id, - "output": output, - }, - ), - output=output, - ) - context_wrapper._mark_tool_call_completed(result_item.raw_item) - processed_response.interruptions = [] - return SingleStepResult( - original_input=original_input, - model_response=new_response, - pre_step_items=original_pre_step_items, - new_step_items=[result_item], - next_step=NextStepRunAgain(), - tool_input_guardrail_results=[], - tool_output_guardrail_results=[], - processed_response=processed_response, - ) diff --git a/src/agents/run_internal/resumable_session.py b/src/agents/run_internal/resumable_session.py index 9754d31c09..a55fba5c31 100644 --- a/src/agents/run_internal/resumable_session.py +++ b/src/agents/run_internal/resumable_session.py @@ -37,13 +37,6 @@ async def begin(self, input: Any) -> None: await self._source._begin_run(input) self._items = self._source._get_history() - def bind_state(self, state: Any) -> None: - """Attach the current store claim to the internal execution state.""" - assert self._source is not None - state._resumable_binding = _ResumableSessionBinding( - self._source, self._source._checkpoint.revision - ) - async def finish(self, result: RunResult | RunResultStreaming) -> None: assert self._source is not None state = result.to_state() if result.interruptions else None diff --git a/src/agents/run_internal/run_steps.py b/src/agents/run_internal/run_steps.py index 4e786e46e4..e98177a1d4 100644 --- a/src/agents/run_internal/run_steps.py +++ b/src/agents/run_internal/run_steps.py @@ -137,7 +137,6 @@ class ProcessedResponse: # Original invocation recipients, independent of the currently available callables. tool_output_guardrail_result_start: int | None = None # First checkpointed tool output guardrail result owned by this response. - native_approvals: list[ToolApprovalItem] = dataclasses.field(default_factory=list) def has_tools_or_approvals_to_run(self) -> bool: # Handoffs, functions and computer actions need local processing @@ -153,7 +152,6 @@ def has_tools_or_approvals_to_run(self) -> bool: self.apply_patch_calls, self.mcp_approval_requests, self.function_tools_not_found, - self.native_approvals, ] ) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index e847c9bddd..04fad384e2 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -101,13 +101,7 @@ ) from ..lifecycle import RunHooks from ..logger import log_tool_action_error, logger -from ..models._codex_items import ( - CodexCustomToolCallOutput, - CodexFunctionToolCallOutput, - CodexPendingCustomCall, - CodexPendingFunctionCall, - native_approval_checkpoint, -) +from ..models._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -157,7 +151,6 @@ run_item_to_input_item, shell_rejection_item, ) -from .native_approvals import native_pending_step, resolve_native_approval_step from .run_steps import ( NOT_FINAL_OUTPUT, NextStepFinalOutput, @@ -994,18 +987,6 @@ async def execute_tools_and_side_effects( """Run one turn of the loop, coordinating tools, approvals, guardrails, and handoffs.""" public_agent = bindings.public_agent - if processed_response.native_approvals: - _register_tool_call_items(context_wrapper, processed_response.new_items) - return native_pending_step( - bindings=bindings, - original_input=original_input, - pre_step_items=pre_step_items, - new_response=new_response, - processed_response=processed_response, - run_state=run_state, - run_config=run_config, - ) - execute_final_output_call = execute_final_output execute_handoffs_call = execute_handoffs @@ -1425,18 +1406,6 @@ async def resolve_interrupted_turn( """Continue a turn that was previously interrupted waiting for tool approval.""" public_agent = bindings.public_agent execution_agent = bindings.execution_agent - if processed_response.native_approvals: - return await resolve_native_approval_step( - bindings=bindings, - original_input=original_input, - original_pre_step_items=original_pre_step_items, - new_response=new_response, - processed_response=processed_response, - context_wrapper=context_wrapper, - hooks=hooks, - run_config=run_config, - run_state=run_state, - ) output_index = _build_tool_output_index(original_pre_step_items) response_boundary = _current_response_boundary((), processed_response, run_state) generated_output_start = response_boundary.generated_start @@ -3203,7 +3172,6 @@ def process_model_response( allow_apply_patch_function_fallback: bool = True, ) -> ProcessedResponse: items: list[RunItem] = [] - native_approvals: list[ToolApprovalItem] = [] # Only the Codex adapter creates these types, after validating native call/output pairing. provider_completed = { @@ -3308,12 +3276,6 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: output.__class__.__name__ if hasattr(output, "__class__") else type(output), ) caller = get_mapping_or_attr(output, "caller") - if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): - native_approval_checkpoint(output) - items.append(ToolCallItem(raw_item=cast(Any, output), agent=agent)) - native_approvals.append(ToolApprovalItem(agent=agent, raw_item=output)) - tools_used.append(output.name) - continue is_program_owned = get_mapping_or_attr(caller, "type") == "program" if ( not is_program_owned @@ -4002,7 +3964,6 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tools_used=tools_used, mcp_approval_requests=mcp_approval_requests, interruptions=[], - native_approvals=native_approvals, function_tools_not_found=function_tools_not_found, mcp_tool_bindings={ run.tool_call.call_id: binding @@ -4038,9 +3999,6 @@ def _preflight_response_invocations( if isinstance(item, (CodexFunctionToolCallOutput, CodexCustomToolCallOutput)) } for index, output in enumerate(response.output): - if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): - native_approval_checkpoint(output) - continue if isinstance(output, (ResponseFunctionToolCall, ResponseCustomToolCall)) and ( output.call_id in provider_completed ): diff --git a/src/agents/run_state.py b/src/agents/run_state.py index d5c8f51f9b..aa565a8eaf 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -110,10 +110,6 @@ log_model_and_tool_data_warning, logger, ) -from .models._codex_items import ( - native_approval_checkpoint, - restore_native_approval_call, -) from .run_context import RunContextWrapper, _ApprovalRecord, _FunctionToolApprovalKey from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -316,7 +312,7 @@ def _default_run_state_validation_error( ), "1.23": ( "Binds prepared tool inputs to their original proposals across retries and resume, " - "records required ResumableSession ownership, and preserves native approval receipts." + "and records required ResumableSession ownership for paused runs." ), } SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES) @@ -789,11 +785,6 @@ def _restore_pydantic_fields_set(value: Any, source: Any) -> None: def _copy_tool_approval_raw_item(raw_item: Any) -> Any: """Copy a supported approval raw item through public Pydantic APIs.""" - native_checkpoint = native_approval_checkpoint(raw_item) - if native_checkpoint is not None: - return restore_native_approval_call( - raw_item.model_dump(exclude_unset=True), native_checkpoint - ) active_container_ids: set[int] = set() raw_item_mro = _static_type_mro(raw_item) for model_type, model_adapter in _TOOL_APPROVAL_MODEL_ADAPTERS: @@ -1419,12 +1410,6 @@ def approve(self, approval_item: ToolApprovalItem, always_approve: bool = False) nested_state.approve(nested_item, always_approve=always_approve) return current_approval_item = self._find_current_approval_item(approval_item) - native_checkpoint = native_approval_checkpoint(approval_item.raw_item) - if native_checkpoint is not None and ( - current_approval_item is None - or native_approval_checkpoint(current_approval_item.raw_item) != native_checkpoint - ): - raise UserError("Native approval does not match the current pending call.") self._context.approve_tool( current_approval_item or approval_item, always_approve=always_approve, @@ -1455,15 +1440,8 @@ def reject( rejection_message=rejection_message, ) return - current_approval_item = self._find_current_approval_item(approval_item) - native_checkpoint = native_approval_checkpoint(approval_item.raw_item) - if native_checkpoint is not None and ( - current_approval_item is None - or native_approval_checkpoint(current_approval_item.raw_item) != native_checkpoint - ): - raise UserError("Native approval does not match the current pending call.") self._context.reject_tool( - current_approval_item or approval_item, + self._find_current_approval_item(approval_item) or approval_item, always_reject=always_reject, rejection_message=rejection_message, ) @@ -2189,15 +2167,6 @@ def _serialize_processed_response( for call_id, binding in processed_response.mcp_tool_bindings.items() }, } - if processed_response.native_approvals: - result["native_approvals"] = [ - _serialize_tool_approval_interruption( - item, - include_tool_name=True, - agent_identity_keys_by_id=agent_identity_keys_by_id, - ) - for item in processed_response.native_approvals - ] if processed_response.tool_output_guardrail_result_start is not None: result["tool_output_guardrail_result_start"] = ( processed_response.tool_output_guardrail_result_start @@ -2257,9 +2226,6 @@ def _serialize_item( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } - native_checkpoint = native_approval_checkpoint(item.raw_item) - if native_checkpoint is not None: - result["codex_native_checkpoint"] = native_checkpoint if isinstance(item, InputItem): result["input_id"] = item.input_id @@ -2930,9 +2896,6 @@ def _serialize_tool_approval_interruption( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } - native_checkpoint = native_approval_checkpoint(interruption.raw_item) - if native_checkpoint is not None: - interruption_dict["codex_native_checkpoint"] = native_checkpoint if include_tool_name and interruption.tool_name is not None: interruption_dict["tool_name"] = interruption.tool_name if interruption.tool_namespace is not None: @@ -3680,19 +3643,6 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: if approval_item is not None: interruptions.append(approval_item) - native_approvals: list[ToolApprovalItem] = [] - for approval_data in processed_response_data.get("native_approvals", []): - approval = _deserialize_tool_approval_item( - approval_data, - agent_map=agent_map, - agent_identity_map=agent_identity_map, - fallback_agent=current_agent, - validation_error_factory=validation_error_factory, - ) - if approval is None or native_approval_checkpoint(approval.raw_item) is None: - raise validation_error_factory("Invalid native approval plan", UserError) - native_approvals.append(approval) - saved_bindings = processed_response_data.get("mcp_tool_bindings", {}) mcp_tool_bindings = { call_id: cast(tuple[str, str, int | None], tuple(binding)) @@ -3706,7 +3656,6 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: return ProcessedResponse( new_items=new_items, - native_approvals=native_approvals, handoffs=handoffs, functions=functions, computer_actions=computer_actions, @@ -3886,8 +3835,6 @@ def _deserialize_tool_approval_item( tool_lookup_key = deserialize_function_tool_lookup_key(item_data.get("tool_lookup_key")) allow_bare_name_alias = item_data.get("allow_bare_name_alias") is True raw_item = _deserialize_tool_approval_raw_item(raw_item_data) - if "codex_native_checkpoint" in item_data: - raw_item = restore_native_approval_call(raw_item_data, item_data["codex_native_checkpoint"]) return ToolApprovalItem( agent=agent, raw_item=raw_item, @@ -4288,11 +4235,14 @@ async def _build_run_state_from_json( state_json, validation_error_factory=validation_error_factory, ) + processed_response = state_json.get("last_processed_response") + if isinstance(processed_response, Mapping) and processed_response.get("native_approvals"): + raise validation_error_factory( + "Run state contains unsupported native tool approvals. " + "Start a new run using SDK tools.", + UserError, + ) schema_major, schema_minor = (int(part) for part in schema_version.split(".", maxsplit=1)) - if (schema_major, schema_minor) < (1, 23) and ( - (state_json.get("last_processed_response") or {}).get("native_approvals") - ): - raise validation_error_factory("Native approvals require RunState schema 1.23.", UserError) programmatic_major, programmatic_minor = ( int(part) for part in _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION.split(".", maxsplit=1) ) @@ -5400,10 +5350,6 @@ def _resolve_agent_info( # Tool call items can be function calls, shell calls, apply_patch calls, # MCP calls, etc. Check the type field to determine which type to deserialize as raw_item_tool = _deserialize_tool_call_raw_item(normalized_raw_item) - if "codex_native_checkpoint" in item_data: - raw_item_tool = restore_native_approval_call( - normalized_raw_item, item_data["codex_native_checkpoint"] - ) # Preserve display metadata if it was stored with the item. description = item_data.get("description") title = item_data.get("title") @@ -5766,6 +5712,7 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]: "RunState tool invocation must be a mapping.", "RunState tool invocation contains invalid lifecycle data.", "Tool input rewrites require RunState schema 1.23.", + "Run state contains unsupported native tool approvals. Start a new run using SDK tools.", "RunState tool input rewrite is invalid.", "Hosted MCP approval decisions require a non-empty request id.", ( diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py deleted file mode 100644 index 4abe3dbcaa..0000000000 --- a/tests/memory/test_native_approval_recovery.py +++ /dev/null @@ -1,529 +0,0 @@ -from __future__ import annotations - -import asyncio -import copy -import json - -import pytest - -from agents import Agent, ResumableSession, RunConfig, Runner, RunState -from agents.exceptions import AgentsException, UserError -from agents.items import ModelResponse -from agents.memory.resumable_session import ResumableSessionError -from agents.models._codex_items import CodexPendingCustomCall, CodexPendingFunctionCall -from agents.models.codex import CodexProvider -from agents.testing import ScriptedModel -from agents.usage import Usage -from tests.memory.test_resumable_session import MemoryStore, run_agent -from tests.models.test_codex_provider import FakeThread, threads as threads -from tests.test_responses import get_text_message -from tests.testing_processor import fetch_ordered_spans - - -def pending_call(tmp_path, kind): - call = { - "type": "function_call" if kind == "exec_command" else "custom_tool_call", - "id": "native-item", - "call_id": "native-call", - "name": kind, - "status": "in_progress", - } - if kind == "exec_command": - call["arguments"] = '{"cmd":"echo harmless"}' - item_type = CodexPendingFunctionCall - else: - call["input"] = "*** Begin Patch\n*** Add File: harmless.txt\n+safe\n*** End Patch\n" - item_type = CodexPendingCustomCall - checkpoint = { - "version": 1, - "cwd": str(tmp_path.resolve()), - "call": copy.deepcopy(call), - "request": {"type": "native_approval", "call_id": "native-call"}, - "model": "fixture-model", - } - item = item_type(**call, checkpoint=checkpoint) - return item.model_copy(update={"codex_reasoning_summary": "Review this effect."}) - - -async def setup_native(tmp_path, monkeypatch, kind="exec_command"): - provider = CodexProvider(cwd=tmp_path) - model = provider.get_model("fixture-model") - pending = pending_call(tmp_path, kind) - calls = [] - requests = [] - - async def respond(*args, **kwargs): - requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) - output = [pending] if len(requests) == 1 else [get_text_message("finished")] - return ModelResponse(output=output, usage=Usage(), response_id=None) - - async def execute(checkpoint, *, tools, tracing): - calls.append(checkpoint) - return "native accepted" - - monkeypatch.setattr(model, "get_response", respond) - monkeypatch.setattr(model, "_run_approved_native_call", execute, raising=False) - agent = Agent(name="worker", model=model) - store = MemoryStore() - session = await ResumableSession.create(store) - config = RunConfig(tracing_disabled=True) - return agent, store, session, config, pending, calls, requests - - -@pytest.mark.asyncio -@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"]) -@pytest.mark.parametrize("approval_after_count", [False, True], ids=["before-count", "after-count"]) -async def test_native_pause_preserves_all_sampling_usage_across_session_resume( - tmp_path, monkeypatch, threads, streamed, approval_after_count -): - from tests.models.test_codex_pending_approval import ApprovalThread, approval, raw - from tests.models.test_codex_provider import native_pair - - earlier = { - "input_tokens": 3, - "output_tokens": 2, - "total_tokens": 5, - "cached_input_tokens": 1, - "cache_write_input_tokens": 1, - "reasoning_output_tokens": 1, - } - current = { - "input_tokens": 12, - "output_tokens": 3, - "total_tokens": 15, - "cached_input_tokens": 2, - "cache_write_input_tokens": 3, - "reasoning_output_tokens": 2, - } - cumulative = {key: earlier[key] + current[key] for key in earlier} - - # The wire double preserves the completion/count ordering around a real Runner pause. - class SamplingApprovalThread(ApprovalThread): - async def events(self): - yield { - "type": "raw_response_completed", - "response_id": "earlier", - "token_usage": earlier, - } - yield {"type": "token_count", "info": {"total_token_usage": earlier}} - call, _ = native_pair(call_id="pending") - yield raw(call) - if not approval_after_count: - yield approval() - yield { - "type": "raw_response_completed", - "response_id": "current", - "token_usage": current, - } - yield {"type": "token_count", "info": {"total_token_usage": cumulative}} - if approval_after_count: - yield approval() - - def tokens(usage): - return ( - usage.input_tokens, - usage.output_tokens, - usage.total_tokens, - usage.input_tokens_details.cached_tokens, - usage.input_tokens_details.cache_write_tokens, - usage.output_tokens_details.reasoning_tokens, - ) - - model = CodexProvider(cwd=tmp_path).get_model("fixture-model") - agent = Agent(name="worker", model=model) - first = SamplingApprovalThread() - threads[0].extend([first, FakeThread(text="finished")]) - calls = [] - - async def execute(checkpoint, *, tools, tracing): - calls.append(checkpoint["call"]["call_id"]) - return "native accepted" - - monkeypatch.setattr(model, "_run_approved_native_call", execute) - store = MemoryStore() - session = await ResumableSession.create(store) - paused = await run_agent(agent, "start", session, streamed) - assert first.closed and len(paused.interruptions) == 1 and calls == [] - assert tokens(paused.context_wrapper.usage) == (15, 5, 20, 3, 4, 3) - assert store.checkpoint(session.session_id)["status"] == "paused" - - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - state.approve(state.get_interruptions()[0]) - completed = await run_agent(agent, state, None, streamed) - assert completed.final_output == "finished" and calls == ["pending"] - # The fresh native thread adds its own 10 input and 7 output tokens exactly once. - assert tokens(completed.context_wrapper.usage) == (25, 12, 37, 6, 4, 5) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) -@pytest.mark.parametrize("approved", [True, False]) -async def test_native_approval_round_trip_uses_original_runtime_only( - tmp_path, monkeypatch, kind, approved -): - agent, store, session, config, pending, calls, requests = await setup_native( - tmp_path, monkeypatch, kind - ) - first = await Runner.run(agent, "do the native operation", session=session, run_config=config) - assert len(first.interruptions) == 1 - assert calls == [] - assert store.checkpoint(session.session_id)["status"] == "paused" - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - interruption = state.get_interruptions()[0] - assert interruption.arguments == ( - pending.arguments if kind == "exec_command" else pending.input - ) - if approved: - state.approve(interruption) - else: - state.reject(interruption, rejection_message="Leave the release branch alone.") - result = await Runner.run(agent, state, session=recovered, run_config=config) - assert result.final_output == "finished" - assert calls == ([pending.checkpoint] if approved else []) - assert "checkpoint" not in repr(requests[-1]) - expected_message = "native accepted" if approved else "Leave the release branch alone." - assert any( - item.get("call_id") == interruption.call_id and item.get("output") == expected_message - for item in requests[-1] - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("streamed", [False, True]) -async def test_plain_runner_rejects_unrecoverable_native_pause(tmp_path, monkeypatch, streamed): - agent, _, _, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) - if streamed: - scripted = ScriptedModel([[pending]]) - monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) - with pytest.raises(UserError, match="session=ResumableSession"): - if streamed: - result = Runner.run_streamed(agent, "start", run_config=config) - async for _ in result.stream_events(): - pass - else: - await Runner.run(agent, "start", run_config=config) - assert not calls - - -@pytest.mark.asyncio -@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) -@pytest.mark.parametrize("field", ["name", "call_id", "id", "payload"]) -async def test_display_mutation_cannot_be_approved(tmp_path, monkeypatch, kind, field): - agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch, kind) - await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - item = state.get_interruptions()[0] - field = ("arguments" if kind == "exec_command" else "input") if field == "payload" else field - setattr(item.raw_item, field, "changed") - with pytest.raises(UserError): - state.approve(item) - assert not calls - assert store.checkpoint(session.session_id)["status"] == "paused" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("field", ["payload", "cwd", "request", "model"]) -async def test_coordinated_receipt_mutation_fails_before_claim(tmp_path, monkeypatch, field): - agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) - await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - state.approve(state.get_interruptions()[0]) - # Rewrite every serialized copy consistently; the store's original still binds authority. - snapshot = state.to_json() - - def rewrite(value): - if isinstance(value, dict): - if field == "payload" and value.get("type") == "function_call": - value["arguments"] = '{"cmd":"changed"}' - if value.get("version") == 1 and "cwd" in value and "request" in value: - if field == "cwd": - value["cwd"] = str(tmp_path / "other") - elif field == "request": - value["request"] = {"type": "changed"} - elif field == "model": - value["model"] = "changed" - for child in value.values(): - rewrite(child) - elif isinstance(value, list): - for child in value: - rewrite(child) - - rewrite(snapshot) - changed = await RunState.from_string(agent, json.dumps(snapshot)) - changed._resumable_binding = state._resumable_binding - original = store.records[session.session_id] - with pytest.raises(ResumableSessionError): - await Runner.run(agent, changed, session=recovered, run_config=config) - assert store.records[session.session_id] == original - assert not calls - - -@pytest.mark.asyncio -async def test_no_decision_repauses_and_unbound_copy_never_dispatches(tmp_path, monkeypatch): - agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) - first = await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - result = await Runner.run(agent, state, session=recovered, run_config=config) - assert len(result.interruptions) == 1 - assert not calls - copied = await RunState.from_string(agent, first.to_state().to_string()) - copied.approve(copied.get_interruptions()[0]) - with pytest.raises(UserError): - await Runner.run(agent, copied, run_config=config) - assert not calls - - -@pytest.mark.asyncio -@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) -async def test_completed_native_call_id_cannot_reuse_a_previous_decision( - tmp_path, monkeypatch, kind -): - agent, store, session, config, pending, calls, requests = await setup_native( - tmp_path, monkeypatch, kind - ) - - async def repeat_native_call(*args, **kwargs): - requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) - if len(requests) <= 2: - # The adapter can attest a new native request with a repeated model call ID. - repeated = pending.model_copy(deep=True) - repeated.checkpoint["request"]["turn_id"] = f"native-turn-{len(requests)}" - output = [repeated] - else: - output = [get_text_message("finished")] - return ModelResponse(output=output, usage=Usage(), response_id=None) - - monkeypatch.setattr(agent.model, "get_response", repeat_native_call) - await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - state.approve(state.get_interruptions()[0]) - - # Native execution stays at the adapter seam while public resume owns approval state. - with pytest.raises(UserError, match="completed"): - repeated_result = await Runner.run(agent, state, session=recovered, run_config=config) - assert len(repeated_result.interruptions) == 1 - assert len(calls) == 1 - replacement = await ResumableSession.resume(store, session.session_id) - repeated_state = await replacement.load_state(agent) - await Runner.run(agent, repeated_state, session=replacement, run_config=config) - assert len(calls) == 1, "The previous decision authorized another native dispatch." - - assert len(calls) == 1 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("approved", [True, False]) -async def test_sticky_native_decisions_are_rejected(tmp_path, monkeypatch, approved): - agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) - await Runner.run(agent, "start", session=session, run_config=config) - state = await session.load_state(agent) - item = state.get_interruptions()[0] - with pytest.raises(UserError, match="original call only"): - if approved: - state.approve(item, always_approve=True) - else: - state.reject(item, always_reject=True) - assert not calls - - -@pytest.mark.asyncio -@pytest.mark.parametrize("streamed", [False, True]) -async def test_native_pause_waits_for_checkpoint_ack(tmp_path, monkeypatch, streamed): - agent, store, session, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) - store.block_checkpoint = True - events = [] - if streamed: - scripted = ScriptedModel([[pending]]) - monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) - result = Runner.run_streamed(agent, "start", session=session, run_config=config) - - async def consume(): - async for event in result.stream_events(): - events.append(event) - return result - - task = asyncio.create_task(consume()) - else: - task = asyncio.create_task(Runner.run(agent, "start", session=session, run_config=config)) - try: - await asyncio.wait_for(store.saving.wait(), 5) - assert not task.done() - assert store.checkpoint(session.session_id)["status"] == "active" - assert not calls - if streamed: - assert not result.interruptions - assert not any( - getattr(event, "name", None) == "tool_approval_requested" for event in events - ) - with pytest.raises(AgentsException): - result.to_state() - finally: - store.release.set() - finished = await asyncio.wait_for(task, 5) - assert len(finished.interruptions) == 1 - assert store.checkpoint(session.session_id)["status"] == "paused" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("include_data", [False, True]) -async def test_resume_exports_the_original_native_operation_under_current_trace_policy( - tmp_path, monkeypatch, threads, include_data -): - class RestoredThread(FakeThread): - async def next_operation_event(self): - return None - - async def resolve_approval(self, call_json, request, preconditions, capture): - assert capture == ("content" if include_data else "metadata") - operation = { - "threadId": self.thread_id, - "turnId": "replacement-turn", - "operationId": "from-native-dispatch", - "kind": "tool", - "toolName": "exec_command", - "callId": json.loads(call_json)["call_id"], - "startedAt": "2026-11-20T06:00:00.000Z", - "input": {"value": "private commit message"}, - } - completed = dict(operation, completedAt="2026-11-20T06:00:02.000Z", outcome="success") - events = [ - json.dumps({"method": f"turn/operation/{kind}", "params": params}) - for kind, params in [("started", operation), ("completed", completed)] - ] - return "native accepted", "replacement-turn", events, None - - agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) - await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - pending = state.get_interruptions()[0] - state.approve(pending) - monkeypatch.delattr(agent.model, "_run_approved_native_call") - threads[0].append(RestoredThread()) - result = await Runner.run( - agent, - state, - run_config=RunConfig( - tracing={"operation_detail": "content"}, trace_include_sensitive_data=include_data - ), - ) - assert result.final_output == "finished" - (operation_span,) = [ - span.span_data - for span in fetch_ordered_spans() - if span.span_data.type == "function" - and span.span_data.operation.operation_id == "from-native-dispatch" - ] - assert operation_span.operation.outcome == "completed" - assert operation_span.operation.tool_call_id == pending.call_id - assert ("private commit message" in repr(operation_span.input)) == include_data - - -@pytest.mark.asyncio -@pytest.mark.parametrize("include_data,cancel", [(False, False), (True, False), (False, True)]) -async def test_restored_native_execution_observes_live_start_and_interruption( - tmp_path, monkeypatch, threads, include_data, cancel -): - from agents.tracing import get_current_span - from tests.testing_processor import SPAN_PROCESSOR_TESTING - - started = asyncio.Event() - release = asyncio.Event() - closed = asyncio.Event() - notifications = asyncio.Queue() - calls = [] - original_start = SPAN_PROCESSOR_TESTING.on_span_start - caller_spans = [] - - def on_start(span): - original_start(span) - if span.span_data.type == "function": - caller_spans.append(get_current_span()) - started.set() - - monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_start", on_start) - - # The FFI double preserves suspension and cancellation at the native execution boundary. - class RestoredThread(FakeThread): - async def next_operation_event(self): - return await notifications.get() - - async def resolve_approval(self, call_json, request, preconditions, capture): - calls.append(capture) - operation = { - "threadId": self.thread_id, - "turnId": "restored-turn", - "operationId": "live-native-operation", - "kind": "tool", - "toolName": "exec_command", - "callId": json.loads(call_json)["call_id"], - "startedAt": "2026-11-20T06:00:00.000Z", - } - if capture == "content": - operation["input"] = {"value": "private command"} - await notifications.put( - json.dumps({"method": "turn/operation/started", "params": operation}) - ) - await release.wait() - completed = dict(operation, completedAt="2026-11-20T06:00:02.000Z", outcome="success") - await notifications.put( - json.dumps({"method": "turn/operation/completed", "params": completed}) - ) - await notifications.put(None) - return "native accepted", "restored-turn", [], None - - async def close(self): - await super().close() - closed.set() - - agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) - await Runner.run(agent, "start", session=session, run_config=config) - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - state.approve(state.get_interruptions()[0]) - monkeypatch.delattr(agent.model, "_run_approved_native_call") - threads[0].append(RestoredThread()) - task = asyncio.create_task( - Runner.run( - agent, - state, - run_config=RunConfig( - tracing={"operation_detail": "content"}, - trace_include_sensitive_data=include_data, - ), - ) - ) - try: - await asyncio.wait_for(started.wait(), 2) - assert not task.done() - assert calls == ["content" if include_data else "metadata"] - assert caller_spans[0] is not None - assert caller_spans[0].span_data.type == "agent" - if cancel: - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - else: - release.set() - result = await asyncio.wait_for(task, 2) - assert result.final_output == "finished" - finally: - task.cancel() - await asyncio.gather(task, return_exceptions=True) - assert closed.is_set() - (span,) = [ - span - for span in fetch_ordered_spans() - if span.span_data.type == "function" - and span.span_data.operation.operation_id == "live-native-operation" - ] - assert span.span_data.operation.outcome == ("interrupted" if cancel else "completed") - assert span.started_at == "2026-11-20T06:00:00.000Z" - assert ("private command" in repr(span.span_data.input)) == include_data - assert span.parent_id == caller_spans[0].span_id diff --git a/tests/memory/test_native_approval_tracing.py b/tests/memory/test_native_approval_tracing.py deleted file mode 100644 index 2a563b0ba9..0000000000 --- a/tests/memory/test_native_approval_tracing.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -import json - -import pytest -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - -from agents import ResumableSession, RunConfig, Runner -from agents.extensions.tracing import OpenTelemetryTracingProcessor -from agents.testing import ScriptedModel -from agents.tracing import set_trace_processors -from tests.memory.test_native_approval_recovery import setup_native -from tests.test_responses import get_text_message -from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans - - -@pytest.mark.asyncio -@pytest.mark.parametrize("mode", ["default", "metadata", "content", "veto", "disabled"]) -@pytest.mark.parametrize("streamed", [False, True]) -async def test_native_pause_and_denial_emit_only_approval_metadata( - tmp_path, monkeypatch, mode, streamed -): - agent, store, session, _, pending, calls, _ = await setup_native(tmp_path, monkeypatch) - if streamed: - script = ScriptedModel([[pending], [get_text_message("finished")]]) - monkeypatch.setattr(agent.model, "stream_response", script.stream_response) - config = RunConfig( - tracing={} - if mode == "default" - else {"operation_detail": "metadata" if mode == "metadata" else "content"}, - tracing_disabled=mode == "disabled", - trace_include_sensitive_data=mode != "veto", - ) - provider = TracerProvider() - exporter = InMemorySpanExporter() - provider.add_span_processor(SimpleSpanProcessor(exporter)) - processor = OpenTelemetryTracingProcessor(tracer_provider=provider) - set_trace_processors([SPAN_PROCESSOR_TESTING, processor]) - - async def run(input, current): - if streamed: - result = Runner.run_streamed(agent, input, session=current, run_config=config) - async for _ in result.stream_events(): - pass - return result - return await Runner.run(agent, input, session=current, run_config=config) - - try: - first = await run("start", session) - assert len(first.interruptions) == 1 - recovered = await ResumableSession.resume(store, session.session_id) - second = await run(await recovered.load_state(agent), recovered) - assert len(second.interruptions) == 1 - recovered = await ResumableSession.resume(store, session.session_id) - state = await recovered.load_state(agent) - state.reject(state.get_interruptions()[0], rejection_message="DENIAL_CANARY") - result = await run(state, recovered) - assert result.final_output == "finished" - assert not calls - spans = fetch_ordered_spans() - if mode == "default": - decisions = [s.span_data for s in spans if s.span_data.type == "function"] - assert [s.operation.outcome for s in decisions] == ["paused", "paused", "rejected"] - assert all(s.operation.tool_call_id == pending.call_id for s in decisions) - assert all(s.input is None and s.output is None for s in decisions) - else: - assert not [s for s in spans if s.span_data.type == "function"] - decisions = [ - s.span_data.data - for s in spans - if getattr(s.span_data, "name", None) == "tool_approval" - ] - assert decisions == ( - [] - if mode == "disabled" - else [ - {"tool_name": pending.name, "tool_call_id": pending.call_id, "outcome": outcome} - for outcome in ["paused", "paused", "rejected"] - ] - ) - otel = exporter.get_finished_spans() - approvals = [s for s in otel if s.name == "openai.agents.tool_approval exec_command"] - assert len(approvals) == (0 if mode == "disabled" else 3) - assert not [s for s in otel if s.name.startswith("execute_tool")] - assert "DENIAL_CANARY" not in json.dumps([dict(s.attributes or {}) for s in approvals]) - finally: - set_trace_processors([SPAN_PROCESSOR_TESTING]) - provider.shutdown() diff --git a/tests/memory/test_resumable_session.py b/tests/memory/test_resumable_session.py index 0b8a7b60d2..deed22f801 100644 --- a/tests/memory/test_resumable_session.py +++ b/tests/memory/test_resumable_session.py @@ -11,7 +11,7 @@ from agents import Agent, RunConfig, Runner, RunState, Session, SQLiteSession, ToolExecutionConfig from agents.decorators import tool -from agents.exceptions import AgentsException +from agents.exceptions import AgentsException, UserError from agents.memory.resumable_session import ResumableSession, ResumableSessionError from agents.result import RunResult, RunResultStreaming from agents.run import AgentRunner @@ -52,6 +52,45 @@ def checkpoint(self, session_id: str) -> dict[str, Any]: CONFIG = RunConfig(tracing_disabled=True) +@pytest.mark.asyncio +async def test_saved_native_approval_cannot_become_an_sdk_tool_approval() -> None: + store = MemoryStore() + session = await ResumableSession.create(store) + + @tool(needs_approval=True) + def exec_command(command: str) -> str: + pytest.fail("A native approval cannot authorize a matching SDK tool.") + + agent = Agent( + name="assistant", + model=ScriptedModel([[get_function_tool_call("exec_command", '{"command":"original"}')]]), + tools=[exec_command], + ) + await Runner.run( + agent, "Run command", session=session, context={"test": True}, run_config=CONFIG + ) + checkpoint = store.checkpoint(session.session_id) + snapshot = json.loads(checkpoint["run_state"]) + processed = snapshot["last_processed_response"] + # An earlier preview stored native actions outside the SDK function queue. + processed["native_approvals"] = copy.deepcopy(processed["interruptions"]) + processed["functions"] = [] + checkpoint["run_state"] = json.dumps(snapshot) + stored = json.dumps(checkpoint) + store.records[session.session_id] = stored + + def restore_context(value: dict[str, Any]) -> dict[str, Any]: + pytest.fail("Unsupported checkpoints must fail before restoring application context.") + + with pytest.raises(UserError, match="unsupported native tool approvals"): + await RunState.from_json(agent, snapshot, context_deserializer=restore_context) + resumed = await ResumableSession.resume(store, session.session_id) + with pytest.raises(ResumableSessionError) as failure: + await resumed.load_state(agent, context_deserializer=restore_context) + assert failure.value.reason == "invalid_state" + assert store.records[session.session_id] == stored + + async def run_agent( agent: Agent[Any], input: Any, session: Session | ResumableSession | None, streamed: bool ) -> RunResult | RunResultStreaming: diff --git a/tests/models/test_codex_operation_retry_boundary.py b/tests/models/test_codex_operation_retry_boundary.py deleted file mode 100644 index 7c6c4f5342..0000000000 --- a/tests/models/test_codex_operation_retry_boundary.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import pytest - -from agents import Agent, ModelSettings, RunConfig, Runner -from agents.models.codex import CodexProvider -from agents.retry import ModelRetrySettings, RetryDecision -from tests.models.test_codex_provider import FakeThread, OperationThread, threads as threads -from tests.testing_processor import fetch_ordered_spans - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "scenario", ["same_attempt", "multiple_before_retry", "single_before_retry"] -) -async def test_distinct_codex_inferences_do_not_merge_across_provider_or_runner_attempts( - tmp_path: Path, threads: Any, scenario: str -) -> None: - # Native observations preserve inference boundaries that ScriptedModel cannot emit. - class InferenceThread(OperationThread): - def __init__(self, outcomes: list[str], complete: bool) -> None: - super().__init__(capture="content") - self.outcomes = outcomes - self.complete = complete - - async def events(self): - for index, outcome in enumerate(self.outcomes): - operation = { - "threadId": "thread_1", - "turnId": "turn_1", - "operationId": f"inference-{index}", - "kind": "model", - "requestedModel": "test-model", - "startedAt": f"2026-11-20T00:00:0{index}.000Z", - "input": {"value": {"input": [{"role": "user", "content": f"input-{index}"}]}}, - } - yield {"method": "turn/operation/started", "params": operation} - terminal = { - **operation, - "completedAt": f"2026-11-20T00:00:0{index}.250Z", - "outcome": outcome, - "usage": {"input_tokens": 10, "output_tokens": 5}, - } - if not self.complete: - terminal.update( - observedModel="stale-model", - responseId="stale-response", - output={"value": [{"role": "assistant", "content": "stale-output"}]}, - ) - yield {"method": "turn/operation/completed", "params": terminal} - if self.complete: - async for event in FakeThread.events(self): - yield event - - queue, _ = threads - if scenario == "single_before_retry": - queue.extend([InferenceThread(["error"], False), InferenceThread(["success"], True)]) - elif scenario == "multiple_before_retry": - queue.extend( - [InferenceThread(["success", "error"], False), InferenceThread(["success"], True)] - ) - else: - queue.append(InferenceThread(["error", "success"], True)) - agent = Agent( - name="boundaries", - model_settings=ModelSettings( - retry=ModelRetrySettings( - max_retries=1, - policy=lambda _: RetryDecision(retry=True, approve_unsafe_replay=True), - backoff={"initial_delay": 0}, - ) - ), - ) - result = await Runner.run( - agent, - "initial input", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "content"} - ), - ) - assert result.final_output == "A Codex response" - models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] - assert ( - len(models) - == {"same_attempt": 2, "multiple_before_retry": 3, "single_before_retry": 1}[scenario] - ) - assert len({span.span_id for span in models}) == len(models) - assert [span.span_data.operation.outcome for span in models].count("failed") == ( - 0 if scenario == "single_before_retry" else 1 - ) - assert all(span.ended_at is not None for span in models) - assert all( - span.span_data.usage["input_tokens"] == (20 if scenario == "single_before_retry" else 10) - for span in models - ) - if scenario == "single_before_retry": - assert models[0].error is None - assert models[0].span_data.output is None - assert models[0].span_data.operation.response_model is None - assert models[0].span_data.operation.response_id is None diff --git a/tests/models/test_codex_pending_approval.py b/tests/models/test_codex_pending_approval.py deleted file mode 100644 index 396404ce47..0000000000 --- a/tests/models/test_codex_pending_approval.py +++ /dev/null @@ -1,166 +0,0 @@ -import json - -import pytest - -from agents import ModelSettings, UserError -from agents.models._codex_items import ( - CodexPendingCustomCall, - CodexPendingFunctionCall, - native_approval_checkpoint, -) -from agents.models.codex import CodexProvider -from agents.models.interface import ModelTracing - -from .test_codex_provider import FakeThread, native_pair, threads as threads - - -class ApprovalThread(FakeThread): - async def next_event(self): - encoded = await super().next_event() - if encoded is None: - return None - event = json.loads(encoded) - if event["msg"]["type"] in ( - "exec_approval_request", - "apply_patch_approval_request", - ): - event["preconditions"] = {"version": 1, "first_approval": True} - return json.dumps(event) - - -def approval(call_id="pending"): - return { - "type": "exec_approval_request", - "call_id": call_id, - "turn_id": "turn_1", - "kind": "command", - "command": ["sh", "-c", "printf native-result"], - "cwd": "/workspace", - } - - -def raw(item): - return {"type": "raw_response_item", "item": item} - - -async def response(tmp_path, settings=None): - return ( - await CodexProvider(cwd=tmp_path) - .get_model(None) - .get_response( - None, - "Continue", - settings or ModelSettings(), - [], - None, - [], - ModelTracing.DISABLED, - previous_response_id=None, - conversation_id=None, - prompt=None, - ) - ) - - -@pytest.mark.asyncio -async def test_approval_waits_for_sampling_completion_and_rejects_late_sibling(tmp_path, threads): - call, _ = native_pair(call_id="pending") - sibling, _ = native_pair(call_id="later") - thread = ApprovalThread( - events=[ - raw(call), - approval(), - raw(sibling), - {"type": "raw_response_completed", "response_id": "response_1"}, - ] - ) - threads[0].append(thread) - with pytest.raises(UserError, match="one outstanding native call"): - await response(tmp_path) - assert thread.closed - - -@pytest.mark.asyncio -async def test_completed_read_stays_completed_while_next_native_call_pauses(tmp_path, threads): - read, read_output = native_pair(call_id="read") - read["arguments"] = json.dumps({"cmd": "cat README.md"}) - pending, _ = native_pair(call_id="pending") - thread = ApprovalThread( - events=[ - raw(read), - raw(read_output), - raw(pending), - approval(), - {"type": "raw_response_completed", "response_id": "response_1"}, - ] - ) - threads[0].append(thread) - result = await response(tmp_path) - assert [item.call_id for item in result.output] == ["read", "read", "pending"] - assert not isinstance(result.output[0], CodexPendingFunctionCall) - assert result.output[1].output == read_output["output"] - assert isinstance(result.output[2], CodexPendingFunctionCall) - assert result.output[2].checkpoint["call"]["call_id"] == "pending" - assert thread.closed - - -@pytest.mark.asyncio -async def test_raw_model_metadata_cannot_forge_a_pending_native_approval(tmp_path, threads): - call, output = native_pair() - call["checkpoint"] = {"request": approval("native_1"), "version": 1} - call["native_approval"] = True - thread = FakeThread(raw_items=[call, output]) - threads[0].append(thread) - result = await response(tmp_path) - assert not isinstance(result.output[0], CodexPendingFunctionCall) - assert not hasattr(result.output[0], "checkpoint") - assert "native_approval" not in result.output[0].model_dump() - assert thread.closed - - -@pytest.mark.asyncio -async def test_patch_begin_does_not_mean_an_awaiting_patch_has_executed(tmp_path, threads): - call, _ = native_pair("custom_tool_call", "pending", "apply_patch") - - thread = ApprovalThread( - events=[ - raw(call), - {"type": "patch_apply_begin", "call_id": "pending"}, - { - "type": "apply_patch_approval_request", - "call_id": "pending", - "turn_id": "turn_1", - "changes": {}, - }, - {"type": "raw_response_completed", "response_id": "response_1"}, - ] - ) - threads[0].append(thread) - result = await response(tmp_path) - assert isinstance(result.output[0], CodexPendingCustomCall) - assert result.output[0].checkpoint["preconditions"]["first_approval"] is True - assert thread.closed - - -@pytest.mark.asyncio -async def test_reasoning_summary_does_not_change_the_approved_action(tmp_path, threads): - call, _ = native_pair(call_id="pending") - thread = ApprovalThread( - events=[ - { - "type": "item_completed", - "item": {"type": "Reasoning", "summary_text": ["Reviewing the command."]}, - }, - raw(call), - approval(), - {"type": "raw_response_completed", "response_id": "response_1"}, - ] - ) - threads[0].append(thread) - result = await response(tmp_path, ModelSettings(reasoning={"summary": "auto"})) - pending = result.output[-1] - assert pending.codex_reasoning_summary == ["Reviewing the command."] - checkpoint = native_approval_checkpoint(pending) - assert checkpoint is not None - assert checkpoint["call"]["call_id"] == "pending" - assert thread.closed diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 7e38eefcd0..6e0d80e5d7 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -20,8 +20,6 @@ class FakeThread: - thread_id = "thread_1" - # This double preserves Core events and controlled suspension through the real Runner. def __init__(self, text="A Codex response", call=None, gate=None, raw_items=(), events=None): self.text = text @@ -41,8 +39,7 @@ def __init__(self, text="A Codex response", call=None, gate=None, raw_items=(), async def inject_items(self, items): self.history = json.loads(items) - async def start_turn(self, inputs, schema, capture=None): - self.operation_capture = capture + async def start_turn(self, inputs, schema): self.inputs = json.loads(inputs) self.schema = json.loads(schema) self.started.set() @@ -53,7 +50,7 @@ async def next_event(self): msg = await anext(self._events) except StopAsyncIteration: return None - return json.dumps(msg if "method" in msg else {"id": "turn_1", "msg": msg}) + return json.dumps({"id": "turn_1", "msg": msg}) async def interrupt(self): self.interrupted = True @@ -1208,30 +1205,15 @@ async def close(self): @pytest.mark.asyncio -@pytest.mark.parametrize("restored", [False, True]) @pytest.mark.parametrize("primary", ["success", "failure", "cancel"]) @pytest.mark.parametrize("late_cancel", [False, True]) async def test_native_cleanup_preserves_cancellation_and_primary_failure( - tmp_path, threads, restored, primary, late_cancel + tmp_path, threads, primary, late_cancel ): - from agents.models.interface import ModelTracing - close_started, release_close = asyncio.Event(), asyncio.Event() closed = [] class HeldCloseThread(FakeThread): - async def next_operation_event(self): - await self.started.wait() - return None - - async def resolve_approval(self, *args): - self.started.set() - if primary == "cancel": - await asyncio.Event().wait() - if primary == "failure": - raise ValueError("primary failure") - return "accepted", "turn_1", [], None - async def close(self): close_started.set() try: @@ -1247,11 +1229,8 @@ async def close(self): ) threads[0].append(thread) provider = CodexProvider(cwd=tmp_path) - checkpoint = {"version": 1, "cwd": str(tmp_path), "model": "fixture", "call": {}, "request": {}} - operation = ( - provider._run_approved_native_call(checkpoint, ModelTracing.DISABLED) - if restored - else Runner.run(Agent(name="test"), "Answer", run_config=RunConfig(model_provider=provider)) + operation = Runner.run( + Agent(name="test"), "Answer", run_config=RunConfig(model_provider=provider) ) task = asyncio.create_task(operation) try: @@ -1268,11 +1247,10 @@ async def close(self): release_close.set() error, reason = RuntimeError, "cleanup failure" if primary == "failure": - error = ValueError if restored else RuntimeError - reason = "primary failure" if restored else "Codex model turn failed" + reason = "Codex model turn failed" if primary == "cancel" or late_cancel: error = asyncio.CancelledError - first_cancel = primary == "cancel" and (restored or not late_cancel) + first_cancel = primary == "cancel" and not late_cancel reason = "primary cancellation" if first_cancel else "cleanup cancellation" # Python 3.10 has not been forwarding cancellation reasons across task boundaries. match = None if error is asyncio.CancelledError and sys.version_info < (3, 11) else reason @@ -1428,7 +1406,8 @@ async def events(self): "output": "Invalid arguments", }, } - yield {"type": "token_count", "info": {"total_token_usage": {"total_tokens": 20}}} + if first_callback_after_completion: + yield {"type": "token_count", "info": {"total_token_usage": {"total_tokens": 20}}} yield {"type": "turn_aborted"} queue, _ = threads @@ -1565,761 +1544,6 @@ def lookup() -> str: assert result.raw_responses[0].output[0].call_id == "nested_lookup" -class OperationThread(FakeThread): - # ASCII escapes would push this UTF-8 file over the capture budget. - file_text = "native_result_secret" + "🙂" * 3000 - - # This double preserves the negotiated wire and source timing at the real Runner boundary. - def __init__(self, *, capture="metadata", malformed=False, active=False): - super().__init__() - self.capture = capture - self.malformed = malformed - self.active = active - - async def events(self): - common = {"threadId": "thread_1", "turnId": "turn_1"} - for index, kind in enumerate(["model", "tool", "model"], 1): - operation = { - **common, - "operationId": f"operation_{index}", - "kind": kind, - "requestedModel": "requested-model", - "startedAt": f"2026-11-20T00:00:0{index}.000000000Z", - } - if kind == "tool": - operation.update(toolName="read_file", callId="native_call") - if self.capture == "content": - operation["input"] = { - "value": {"input": [{"role": "user", "content": "native_input_secret"}]} - if kind == "model" - else {"path": "native_arg_secret"}, - "truncated": False, - "omitted": False, - } - yield {"method": "turn/operation/started", "params": operation} - if self.active: - async for event in super().events(): - yield event - return - terminal = { - **operation, - "completedAt": f"2026-11-20T00:00:0{index}.250Z", - "durationMs": 250, - "outcome": "success", - } - if kind == "model": - terminal.update( - responseId=f"resp_source_{index}", - responseIdTrusted=False, - observedModel="observed-model", - usage={"input_tokens": index * 10, "output_tokens": index * 5}, - ) - if self.capture == "content": - terminal["output"] = { - "value": [{"role": "assistant", "content": "native_result_secret"}] - if kind == "model" - else {"text": self.file_text}, - "truncated": False, - "omitted": False, - } - yield {"method": "turn/operation/completed", "params": terminal} - if self.malformed: - yield {"method": "turn/operation/completed", "params": terminal} - yield { - "method": "turn/operation/started", - "params": {**operation, "threadId": "another_thread"}, - } - async for event in super().events(): - yield event - - -@pytest.mark.asyncio -@pytest.mark.parametrize("streamed", [False, True]) -@pytest.mark.parametrize("truncated", [False, True]) -async def test_codex_interrupted_visible_text_reaches_otel(tmp_path, threads, streamed, truncated): - import json - - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - - from agents.extensions.tracing import OpenTelemetryTracingProcessor - from agents.tracing import set_trace_processors - from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans - - # This wire double preserves the visible partial item produced before OutputItemDone. - partial = { - "type": "output_text", - "text": "visible partial answer", - "partial": True, - "truncated": truncated, - } - - class InterruptedThread(OperationThread): - async def events(self): - operation = { - "threadId": "thread_1", - "turnId": "turn_1", - "operationId": "model-1", - "kind": "model", - "requestedModel": "fixture", - "startedAt": "2026-11-20T00:00:00Z", - } - yield {"method": "turn/operation/started", "params": operation} - yield { - "method": "turn/operation/completed", - "params": { - **operation, - "completedAt": "2026-11-20T00:00:01Z", - "outcome": "interrupted", - "output": {"value": [partial], "truncated": False, "omitted": False}, - }, - } - raise RuntimeError("synthetic interrupted turn") - - queue, _ = threads - thread = InterruptedThread(capture="content") - queue.append(thread) - provider = TracerProvider() - exporter = InMemorySpanExporter() - provider.add_span_processor(SimpleSpanProcessor(exporter)) - set_trace_processors( - [SPAN_PROCESSOR_TESTING, OpenTelemetryTracingProcessor(tracer_provider=provider)] - ) - try: - config = RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tracing={"operation_detail": "content"}, - ) - with pytest.raises(RuntimeError, match="synthetic interrupted turn"): - if streamed: - run = Runner.run_streamed(Agent(name="worker"), "hello", run_config=config) - async for _ in run.stream_events(): - pass - else: - await Runner.run(Agent(name="worker"), "hello", run_config=config) - native = next(span for span in fetch_ordered_spans() if span.span_data.type == "generation") - otel = next( - span - for span in exporter.get_finished_spans() - if span.attributes is not None - and span.attributes.get("gen_ai.operation.name") == "chat" - ) - assert native.span_data.output == [partial] - assert otel.attributes is not None - output = json.loads(json.dumps(otel.attributes["gen_ai.output.messages"])) - assert output[0]["role"] == "assistant" - assert output[0]["parts"][0] == { - "type": "text", - "content": partial["text"], - "partial": True, - "truncated": truncated, - } - assert native.span_data.operation.outcome == "interrupted" - assert otel.attributes["openai.agents.operation.outcome"] == "interrupted" - assert thread.closed - finally: - set_trace_processors([SPAN_PROCESSOR_TESTING]) - provider.shutdown() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("detail", [None, "metadata", "content"]) -@pytest.mark.parametrize("sensitive", [False, True]) -async def test_codex_operations_preserve_source_spans_and_capture_policy( - tmp_path, threads, detail, sensitive -): - import json - - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - effective = "content" if detail == "content" and sensitive else "metadata" - thread = OperationThread(capture=effective) - queue.append(thread) - result = await Runner.run( - Agent(name="worker"), - "sdk_input_secret", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - trace_include_sensitive_data=sensitive, - tracing={} if detail is None else {"operation_detail": detail}, - ), - ) - assert result.final_output == "A Codex response" - assert thread.operation_capture == effective - operations = [span for span in fetch_ordered_spans() if hasattr(span.span_data, "operation")] - models = [span for span in operations if span.span_data.type in {"response", "generation"}] - tools = [span for span in operations if span.span_data.type == "function"] - assert len(models) == 2 and len(tools) == 1 - assert {span.parent_id for span in operations} == {models[0].parent_id} - assert models[0].started_at == "2026-11-20T00:00:01.000000000Z" - assert models[0].ended_at == "2026-11-20T00:00:01.250Z" - assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" - for index, span in [(1, models[0]), (3, models[-1])]: - usage = span.span_data.usage - assert { - key: usage[key] for key in ("requests", "input_tokens", "output_tokens", "total_tokens") - } == { - "requests": 1, - "input_tokens": 10 * index, - "output_tokens": 5 * index, - "total_tokens": 15 * index, - } - assert usage["input_tokens_details"]["cached_tokens"] == 0 - assert usage["input_tokens_details"]["cache_write_tokens"] == 0 - assert usage["output_tokens_details"]["reasoning_tokens"] == 0 - assert models[-1].span_data.operation.response_model == "observed-model" - assert models[-1].span_data.operation.outcome == "completed" - if detail is None: - assert all(span.span_data.type == "response" for span in models) - assert models[0].span_data.input == ( - [{"role": "user", "content": "sdk_input_secret"}] if sensitive else None - ) - assert models[-1].span_data.input is None - assert (models[-1].span_data.response is not None) == sensitive - if sensitive: - assert models[-1].span_data.response.id == "turn_1" - assert models[-1].span_data.response.usage.input_tokens == 10 - assert tools[0].span_data.input is None and tools[0].span_data.output is None - else: - assert all(span.span_data.type == "generation" for span in models) - assert (models[0].span_data.input is not None) == (effective == "content") - assert (tools[0].span_data.input is not None) == (effective == "content") - exported = json.dumps([span.export() for span in operations]) - assert ("native_result_secret" in exported) == (effective == "content") - if effective == "content": - assert json.loads(tools[0].span_data.output) == {"text": OperationThread.file_text} - if not sensitive: - assert all(span.span_data.operation.response_id is None for span in models) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("detail", [None, "metadata", "content"]) -async def test_missing_native_operations_has_one_partial_coverage_marker_without_model_replay( - tmp_path, threads, detail -): - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - queue.append(FakeThread()) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tracing={} if detail is None else {"operation_detail": detail}, - ), - ) - assert result.final_output == "A Codex response" - spans = fetch_ordered_spans() - markers = [ - span for span in spans if getattr(span.span_data, "name", None) == "codex.tracing.coverage" - ] - assert len(markers) == 1 - assert markers[0].span_data.data["coverage"] == "partial" - assert not any(span.span_data.type in {"response", "generation"} for span in spans) - - -@pytest.mark.asyncio -async def test_codex_duplicate_and_foreign_telemetry_preserve_model_result(tmp_path, threads): - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - thread = OperationThread(malformed=True) - queue.append(thread) - run = Runner.run_streamed( - Agent(name="worker"), - "Hello", - run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), - ) - async for _ in run.stream_events(): - pass - assert run.final_output == "A Codex response" - spans = fetch_ordered_spans() - assert ( - sum(getattr(span.span_data, "name", None) == "codex.tracing.coverage" for span in spans) - == 1 - ) - assert sum(span.span_data.type == "response" for span in spans) == 2 - - -@pytest.mark.asyncio -async def test_codex_early_stream_close_finishes_observed_operation(tmp_path, threads): - from agents import ModelSettings - from agents.models.interface import ModelTracing - from agents.tracing import trace - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - thread = OperationThread(active=True) - queue.append(thread) - model = CodexProvider(cwd=tmp_path).get_model("test-model") - with trace("test"): - stream = model.stream_response( - None, - "Hello", - ModelSettings(), - [], - None, - [], - ModelTracing.ENABLED, - previous_response_id=None, - conversation_id=None, - prompt=None, - ) - assert (await anext(stream)).type == "response.created" - assert (await anext(stream)).type == "response.output_item.added" - await stream.aclose() - assert thread.closed - models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] - assert len(models) == 1 - assert models[0].ended_at is not None - assert models[0].span_data.operation.outcome == "interrupted" - assert models[0].span_data.operation.estimated_end - - -@pytest.mark.asyncio -async def test_codex_nested_and_overlapping_tools_keep_explicit_parents(tmp_path, threads): - from tests.testing_processor import fetch_ordered_spans - - class NestedThread(OperationThread): - async def events(self): - common = {"threadId": "thread_1", "turnId": "turn_1"} - model = { - **common, - "operationId": "model", - "kind": "model", - "requestedModel": "test-model", - "startedAt": "2026-11-20T00:00:01Z", - } - yield {"method": "turn/operation/started", "params": model} - yield { - "method": "turn/operation/completed", - "params": {**model, "completedAt": "2026-11-20T00:00:02Z", "outcome": "success"}, - } - calls = [] - for name, parent in [("outer", None), ("nested", "outer"), ("parallel", None)]: - event = { - **common, - "operationId": name, - "kind": "tool", - "toolName": name, - "startedAt": "2026-11-20T00:00:02Z", - } - if parent: - event["parentOperationId"] = parent - calls.append(event) - yield {"method": "turn/operation/started", "params": event} - for event in reversed(calls): - yield { - "method": "turn/operation/completed", - "params": { - **event, - "completedAt": "2026-11-20T00:00:03Z", - "outcome": "success", - }, - } - async for event in FakeThread.events(self): - yield event - - queue, _ = threads - queue.append(NestedThread()) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), - ) - assert result.final_output == "A Codex response" - tools = { - span.span_data.name: span - for span in fetch_ordered_spans() - if span.span_data.type == "function" - } - assert tools["nested"].parent_id == tools["outer"].span_id - assert tools["parallel"].parent_id == tools["outer"].parent_id - assert tools["parallel"].started_at == tools["outer"].started_at - assert tools["parallel"].ended_at == tools["outer"].ended_at - - -@pytest.mark.asyncio -async def test_disabled_codex_tracing_omits_negotiation_and_operations(tmp_path, threads): - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - thread = OperationThread(capture="content") - queue.append(thread) - result = await Runner.run(Agent(name="worker"), "Hello", run_config=config(tmp_path)) - assert result.final_output == "A Codex response" - assert thread.operation_capture is None - assert fetch_ordered_spans() == [] - - -@pytest.mark.asyncio -async def test_codex_default_end_callback_receives_final_sdk_response_and_source_time( - tmp_path, threads, monkeypatch -): - from tests.testing_processor import SPAN_PROCESSOR_TESTING - - snapshots = [] - original = SPAN_PROCESSOR_TESTING.on_span_end - - def observe_end(span): - if span.span_data.type == "response": - snapshots.append((span.span_data.response, span.ended_at, span.span_data.usage)) - original(span) - - monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_end", observe_end) - queue, _ = threads - queue.append(OperationThread()) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), - ) - assert result.final_output == "A Codex response" - assert len(snapshots) == 2 - assert snapshots[0][0] is None - response, ended_at, usage = snapshots[-1] - assert response.id == "turn_1" - assert response.output[0].content[0].text == "A Codex response" - assert ended_at == "2026-11-20T00:00:03.250Z" - assert usage["requests"] == 1 - assert usage["input_tokens"] == 30 - assert usage["output_tokens"] == 15 - assert usage["total_tokens"] == 45 - assert usage["input_tokens_details"]["cached_tokens"] == 0 - assert usage["input_tokens_details"]["cache_write_tokens"] == 0 - assert usage["output_tokens_details"]["reasoning_tokens"] == 0 - - -@pytest.mark.asyncio -async def test_codex_content_retains_wire_omission_and_truncation_markers(tmp_path, threads): - import json - - from tests.testing_processor import fetch_ordered_spans - - class BoundedThread(OperationThread): - async def events(self): - async for event in super().events(): - params = event.get("params", {}) - if event.get("method") == "turn/operation/started" and params["kind"] == "model": - params["input"]["truncated"] = True - params["input"]["reason"] = "capture_budget" - if event.get("method") == "turn/operation/completed" and params["kind"] == "tool": - params["output"] = { - "value": None, - "omitted": True, - "truncated": False, - "reason": "unsupported_content", - } - yield event - - queue, _ = threads - queue.append(BoundedThread(capture="content")) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tracing={"operation_detail": "content"}, - ), - ) - assert result.final_output == "A Codex response" - models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] - assert all(any(item.get("truncated") for item in span.span_data.input) for span in models) - native = next(span for span in fetch_ordered_spans() if span.span_data.type == "function") - assert json.loads(native.span_data.output) == {"omitted": True, "reason": "unsupported_content"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize("disabled_parent", [False, True]) -async def test_codex_suppresses_capture_when_trace_provider_or_parent_is_disabled( - tmp_path, threads, disabled_parent -): - from contextlib import nullcontext - - from agents.tracing import set_tracing_disabled, trace - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - thread = OperationThread(capture="content") - queue.append(thread) - parent = trace("disabled", disabled=True) if disabled_parent else nullcontext() - if not disabled_parent: - set_tracing_disabled(True) - try: - with parent: - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tracing={"operation_detail": "content"}, - ), - ) - finally: - if not disabled_parent: - set_tracing_disabled(False) - assert result.final_output == "A Codex response" - assert thread.operation_capture is None - assert fetch_ordered_spans() == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("cancel", [False, True]) -async def test_codex_teardown_failure_finishes_pending_source_span(tmp_path, threads, cancel): - import asyncio - - from tests.testing_processor import fetch_ordered_spans - - closing = asyncio.Event() - release = asyncio.Event() - - class TeardownThread(OperationThread): - async def close(self): - self.closed = True - closing.set() - if cancel: - await release.wait() - return - raise RuntimeError("teardown failed") - - queue, _ = threads - queue.append(TeardownThread()) - task = asyncio.create_task( - Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), - ) - ) - await closing.wait() - if cancel: - task.cancel() - release.set() - with pytest.raises(asyncio.CancelledError if cancel else RuntimeError): - await task - models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] - assert len(models) == 2 - assert models[-1].ended_at == "2026-11-20T00:00:03.250Z" - - -@pytest.mark.asyncio -async def test_codex_releases_completed_tool_content_and_preserves_late_child_parent( - tmp_path, threads, monkeypatch -): - import gc - import weakref - - from tests.testing_processor import SPAN_PROCESSOR_TESTING - - completed_tools = {} - tool_edges = {} - original = SPAN_PROCESSOR_TESTING.on_span_end - - def release_completed(span): - if span.span_data.type == "function": - completed_tools[span.span_data.name] = weakref.ref(span) - tool_edges[span.span_data.name] = (span.span_id, span.parent_id) - else: - original(span) - - monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_end", release_completed) - - class LateChildThread(OperationThread): - async def events(self): - common = {"threadId": "thread_1", "turnId": "turn_1"} - for name, parent in [("outer", None), ("late_child", "outer")]: - if parent is not None: - gc.collect() - assert completed_tools["outer"]() is None - params = { - **common, - "operationId": name, - "kind": "tool", - "toolName": name, - "startedAt": "2026-11-20T00:00:01Z", - "input": {"value": "synthetic_content" * 1000}, - } - if parent is not None: - params["parentOperationId"] = parent - yield {"method": "turn/operation/started", "params": params} - yield { - "method": "turn/operation/completed", - "params": { - **params, - "completedAt": "2026-11-20T00:00:02Z", - "outcome": "success", - }, - } - async for event in FakeThread.events(self): - yield event - - queue, _ = threads - queue.append(LateChildThread(capture="content")) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tracing={"operation_detail": "content"}, - ), - ) - assert result.final_output == "A Codex response" - assert tool_edges["late_child"][1] == tool_edges["outer"][0] - - -@pytest.mark.asyncio -async def test_codex_default_span_retains_exact_sdk_visible_response(tmp_path, threads): - from agents import ModelSettings - from agents.models.interface import ModelTracing - from agents.tracing import trace - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - queue.append(OperationThread()) - with trace("test"): - events = [ - event - async for event in CodexProvider(cwd=tmp_path) - .get_model(None) - .stream_response( - None, - "Hello", - ModelSettings(), - [], - None, - [], - ModelTracing.ENABLED, - previous_response_id=None, - conversation_id=None, - prompt=None, - ) - ] - final = events[-1].response - models = [span for span in fetch_ordered_spans() if span.span_data.type == "response"] - assert models[-1].span_data.response is final - assert final.id == "turn_1" - assert final.usage is not None - assert final.usage.input_tokens == 10 - assert models[-1].span_data.usage["input_tokens"] == 30 - assert models[-1].span_data.operation.response_id == "resp_source_3" - - -@pytest.mark.asyncio -async def test_codex_direct_model_sensitive_veto_downgrades_child_capture(tmp_path, threads): - from agents import ModelSettings - from agents.models.interface import ModelTracing - from agents.tracing import trace - from agents.tracing.config import operation_tracing_scope - from tests.testing_processor import fetch_ordered_spans - - queue, _ = threads - thread = OperationThread(capture="metadata") - queue.append(thread) - with trace("test"), operation_tracing_scope({"operation_detail": "content"}, True): - async for _ in ( - CodexProvider(cwd=tmp_path) - .get_model(None) - .stream_response( - None, - "Hello", - ModelSettings(), - [], - None, - [], - ModelTracing.ENABLED_WITHOUT_DATA, - previous_response_id=None, - conversation_id=None, - prompt=None, - ) - ): - pass - assert thread.operation_capture == "metadata" - models = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"] - assert len(models) == 2 - assert all(span.span_data.operation.detail == "metadata" for span in models) - assert all(span.span_data.input is None and span.span_data.output is None for span in models) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("streamed", [False, True]) -async def test_codex_llm_hook_global_disable_prevents_child_capture(tmp_path, threads, streamed): - from agents import RunHooks - from agents.tracing import get_current_span, get_current_trace, set_tracing_disabled - from tests.testing_processor import fetch_ordered_spans - - class DisableCapture(RunHooks): - async def on_llm_start(self, context, agent, system_prompt, input_items): - active_trace, active_span = get_current_trace(), get_current_span() - assert active_trace is not None and active_trace.trace_id != "no-op" - assert active_span is not None and active_span.span_id != "no-op" - set_tracing_disabled(True) - - queue, _ = threads - thread = OperationThread(capture="content") - queue.append(thread) - run_config = RunConfig( - model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "content"} - ) - try: - if streamed: - result = Runner.run_streamed( - Agent(name="worker"), - "Hello", - hooks=DisableCapture(), - run_config=run_config, - ) - async for _ in result.stream_events(): - pass - else: - result = await Runner.run( - Agent(name="worker"), - "Hello", - hooks=DisableCapture(), - run_config=run_config, - ) - finally: - set_tracing_disabled(False) - assert result.final_output == "A Codex response" - assert thread.operation_capture is None - assert not any( - span.span_data.type in {"generation", "response"} for span in fetch_ordered_spans() - ) - - -@pytest.mark.asyncio -async def test_codex_dropped_native_observations_mark_partial_without_replaying(tmp_path, threads): - from tests.testing_processor import fetch_ordered_spans - - class DroppedThread(OperationThread): - async def events(self): - yield { - "method": "turn/operation/dropped", - "params": {"threadId": "thread_1", "droppedCount": 2}, - } - async for event in super().events(): - yield event - - queue, opened = threads - queue.append(DroppedThread()) - result = await Runner.run( - Agent(name="worker"), - "Hello", - run_config=RunConfig(model_provider=CodexProvider(cwd=tmp_path)), - ) - assert result.final_output == "A Codex response" - assert len(opened) == 1 and not queue - markers = [ - span - for span in fetch_ordered_spans() - if getattr(span.span_data, "name", None) == "codex.tracing.coverage" - ] - assert len(markers) == 1 - assert markers[0].span_data.data["reason"] == "dropped_operations" - - def native_pair(kind="function_call", call_id="native_1", name="exec_command"): call = { "type": kind, @@ -2755,101 +1979,102 @@ async def events(self): } -@pytest.mark.asyncio -@pytest.mark.parametrize("kind", ["function_call", "custom_tool_call"]) -async def test_prepared_native_history_reaches_tracing_before_stream_completion( - tmp_path, threads, monkeypatch, kind -): - from agents.models._codex_tool_hooks import CodexToolHooks - from tests.testing_processor import SPAN_PROCESSOR_TESTING +class TracedThread(FakeThread): + # The native binding delivers passive, source-timed observations outside the response stream. + def __init__(self, *observations, **kwargs): + super().__init__(**kwargs) + self.observations = list(observations) + self.tracing = False - configured = [] - snapshots = [] - original_end = SPAN_PROCESSOR_TESTING.on_span_end + def enable_tracing(self): + self.tracing = True - def on_span_end(span): - if span.span_data.type == "response" and span.span_data.response is not None: - snapshots.append(span.span_data.response.model_dump_json()) - original_end(span) + def drain_operations(self): + assert self.tracing + observations, self.observations = self.observations, [] + return json.dumps(observations) - async def configure(self, resources): - configured.append(self) - return {} - monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_end", on_span_end) - monkeypatch.setattr(CodexToolHooks, "configure", configure) - monkeypatch.setattr(CodexToolHooks, "verify", lambda self, thread: None) +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_stock_operations_stay_metadata_only_and_keep_source_times( + tmp_path, threads, streamed +): + from tests.testing_processor import fetch_ordered_spans - async def before(call): - return {"command": "redacted-input"} + model = {"kind": "model", "id": "1", "model": "native-default"} + tool = {"kind": "tool", "id": "c1", "name": "view_image"} + thread = TracedThread( + {**model, "phase": "start", "at": 1000}, + {**tool, "phase": "start", "at": 1010}, + { + **model, + "phase": "end", + "at": 1015, + "outcome": "completed", + "usage": {"input_tokens": 11, "output_tokens": 5}, + }, + {**tool, "phase": "end", "at": 1020, "outcome": "failed"}, + ) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=True, + ) + if streamed: + stream = Runner.run_streamed(Agent(name="test"), "Private input", run_config=cfg) + async for _ in stream.stream_events(): + pass + assert stream.final_output == thread.text + else: + result = await Runner.run(Agent(name="test"), "Private input", run_config=cfg) + assert result.final_output == thread.text + spans = fetch_ordered_spans() + (turn,) = [s for s in spans if s.span_data.type == "turn"] + (request,) = [s for s in spans if s.span_data.type == "generation"] + (execution,) = [s for s in spans if s.span_data.type == "function"] + assert request.parent_id == execution.parent_id == turn.span_id + assert request.started_at and request.ended_at + assert execution.started_at and execution.ended_at + assert request.started_at < execution.started_at < request.ended_at < execution.ended_at + assert request.span_data.usage["input_tokens"] == 11 + assert request.span_data.operation.model == "native-default" + assert request.span_data.operation.outcome == "completed" + assert execution.span_data.operation.tool_call_id == "c1" + assert execution.span_data.operation.outcome == "failed" + for span in (request, execution): + assert span.span_data.operation.detail == "metadata" + assert span.span_data.input is span.span_data.output is None + (notice,) = [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] + assert notice.span_data.data["reason"] == "metadata_only" - async def after(call, output): - return "redacted-output" - class PreparedThread(OperationThread): - async def events(self): - call, output = native_pair( - kind, name="exec_command" if kind == "function_call" else "apply_patch" - ) - if kind == "function_call": - call["arguments"] = json.dumps({"cmd": "private-input"}) - else: - call["input"] = "private-input" - output["output"] = "private-output" - async for event in super().events(): - yield event - params = event.get("params", {}) - if event.get("method") != "turn/operation/completed": - continue - hook = configured[0] - if params["operationId"] == "operation_1": - yield {"type": "raw_response_item", "item": call} - await hook.decision( - { - "event": "PreToolUse", - "name": "Bash", - "call_id": "native_1", - "arguments": {"command": "private-input"}, - } - ) - elif params["operationId"] == "operation_2": - await hook.decision( - { - "event": "PostToolUse", - "name": "Bash", - "call_id": "native_1", - "arguments": {"command": "redacted-input"}, - "result": "private-output", - } - ) - yield {"type": "raw_response_item", "item": output} +@pytest.mark.asyncio +async def test_native_operation_loss_does_not_block_runner_or_invent_success(tmp_path, threads): + from tests.testing_processor import fetch_ordered_spans - queue, _ = threads - thread = PreparedThread() - queue.append(thread) - run = Runner.run_streamed( - Agent(name="test"), - "Start", - run_config=RunConfig( - model_provider=CodexProvider(cwd=tmp_path), - tool_execution=ToolExecutionConfig( - prepare_tool_input=before, prepare_tool_output=after - ), - ), + thread = TracedThread( + {"kind": "model", "phase": "start", "id": "1", "model": "native-default", "at": 1000}, + {"kind": "dropped"}, ) - native_done = [] - async for event in run.stream_events(): - if event.type != "raw_response_event" or event.data.type != "response.output_item.done": - continue - item = event.data.item - if getattr(item, "call_id", None) != "native_1": - continue - native_done.append(item.type) - assert thread.closed - assert len(snapshots) == 1 - assert "redacted-input" in snapshots[0] and "redacted-output" in snapshots[0] - assert "private-input" not in snapshots[0] and "private-output" not in snapshots[0] - assert native_done == [kind, kind + "_output"] - assert run.final_output == "A Codex response" - assert "private-input" not in json.dumps(run.to_input_list()) - assert "private-output" not in json.dumps(run.to_input_list()) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "metadata"} + ) + result = await Runner.run(Agent(name="test"), "Hello", run_config=cfg) + assert result.final_output == thread.text + spans = fetch_ordered_spans() + (request,) = [s for s in spans if s.span_data.type == "generation"] + assert request.span_data.operation.outcome == "interrupted" + assert request.span_data.operation.estimated_end + (notice,) = [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] + assert notice.span_data.data["reason"] == "dropped_operations" + + +@pytest.mark.asyncio +async def test_disabled_tracing_never_subscribes_to_native_operations(tmp_path, threads): + thread = TracedThread() + threads[0].append(thread) + await Runner.run(Agent(name="test"), "Hello", run_config=config(tmp_path)) + assert not thread.tracing diff --git a/tests/models/test_operation_tracing_parity.py b/tests/models/test_operation_tracing_parity.py deleted file mode 100644 index 7cae606914..0000000000 --- a/tests/models/test_operation_tracing_parity.py +++ /dev/null @@ -1,761 +0,0 @@ -"""Exercise provider-wire conversion and real embedded Codex tools over loopback.""" - -from __future__ import annotations - -import asyncio -import json -import os -import sys -import threading -import time -from collections.abc import Iterator -from datetime import datetime -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from socketserver import TCPServer -from typing import Any - -import pytest - -from agents import Agent, RunConfig, Runner -from agents.decorators import tool -from agents.models.codex import CodexProvider -from agents.models.interface import ModelProvider -from agents.models.openai_provider import OpenAIProvider -from agents.tracing import TracingConfig, set_trace_processors - -from ..testing_processor import SPAN_PROCESSOR_TESTING, SpanProcessorForTests - -pytestmark = pytest.mark.allow_call_model_methods - -MODEL = "fixture-model" -PROMPT = "parity_prompt_canary" -INSTRUCTIONS = "Answer using the supplied tools. parity_instructions_canary" -ARGUMENT = "parity_argument_canary" -TOOL_RESULT = "parity_tool_result_canary" -ANSWER = "parity_answer_canary" -FILE_CONTENT = "parity_native_file_canary" -PAYLOADS = (PROMPT, INSTRUCTIONS, ARGUMENT, TOOL_RESULT, ANSWER) - - -class _LoopbackHTTPServer(ThreadingHTTPServer): - def server_bind(self) -> None: - # This fixture has been using numeric loopback addresses without reverse DNS. - TCPServer.server_bind(self) - self.server_name, self.server_port = self.server_address[:2] - - -class _ResponsesFixture: - def __init__( - self, - *, - mixed: bool = False, - model_header: bool = True, - held: bool = False, - retry_once: bool = False, - observed_model: str = MODEL, - ) -> None: - self.mixed = mixed - self.model_header = model_header - self.held = held - self.retry_once = retry_once - self.observed_model = observed_model - self.request_started = threading.Event() - self.release = threading.Event() - self.requests: list[dict[str, Any]] = [] - self.intervals: list[tuple[int, int]] = [] - fixture = self - - class Handler(BaseHTTPRequestHandler): - def log_message(self, *args: Any) -> None: - pass - - def do_GET(self) -> None: - self._reply(b'{"data":[]}', "application/json") - - def _reply(self, body: bytes, content_type: str, status: int = 200) -> None: - self.send_response(status) - self.send_header("Content-Type", content_type) - if status == 500: - self.send_header("Retry-After", "0") - if fixture.model_header: - self.send_header("openai-model", fixture.observed_model) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - try: - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - pass - - def do_POST(self) -> None: - started = time.time_ns() - request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) - fixture.requests.append(request) - fixture.request_started.set() - if fixture.held: - fixture.release.wait(timeout=10) - number = len(fixture.requests) - if fixture.retry_once and number == 1: - fixture.intervals.append((started, time.time_ns())) - self._reply( - b'{"error":{"message":"Fixture retry","type":"server_error"}}', - "application/json", - 500, - ) - return - if fixture.retry_once: - number -= 1 - response = fixture.response(number) - if request.get("stream"): - item = response["output"][0] - events = [ - { - "type": "response.created", - "response": dict(response, output=[], status="in_progress"), - }, - { - "type": "response.output_item.added", - "output_index": 0, - "item": dict(item, status="in_progress"), - }, - {"type": "response.output_item.done", "output_index": 0, "item": item}, - {"type": "response.completed", "response": response}, - ] - body = "".join( - f"event: {event['type']}\ndata: " - + json.dumps(dict(event, sequence_number=index)) - + "\n\n" - for index, event in enumerate(events) - ).encode() - content_type = "text/event-stream" - else: - body = json.dumps(response).encode() - content_type = "application/json" - fixture.intervals.append((started, time.time_ns())) - self._reply(body, content_type) - - self.server = _LoopbackHTTPServer(("127.0.0.1", 0), Handler) - self.thread = threading.Thread( - target=self.server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True - ) - self.thread.start() - - @property - def base_url(self) -> str: - return f"http://127.0.0.1:{self.server.server_port}" - - def response(self, number: int) -> dict[str, Any]: - if self.mixed and number == 1: - item: dict[str, Any] = { - "type": "function_call", - "name": "read_file", - "arguments": "{}", - "call_id": "native_read", - "id": "fc_native_read", - "status": "completed", - } - if any( - tool.get("type") == "namespace" and tool.get("name") == "mcp__parity" - for tool in self.requests[0].get("tools", []) - ): - item["namespace"] = "mcp__parity" - else: - item["name"] = "mcp__parity__read_file" - elif number == (2 if self.mixed else 1) and not self.retry_once: - item = { - "type": "function_call", - "name": "lookup", - "arguments": json.dumps({"key": ARGUMENT}), - "call_id": "python_lookup", - "id": "fc_python_lookup", - "status": "completed", - } - else: - item = { - "type": "message", - "id": "msg_answer", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": ANSWER, "annotations": []}], - } - return { - "id": f"resp_parity_{number}", - "object": "response", - "created_at": 1, - "model": self.observed_model, - "status": "completed", - "output": [item], - "usage": { - "input_tokens": 10 * number, - "output_tokens": 5 * number, - "total_tokens": 15 * number, - "input_tokens_details": {"cached_tokens": number}, - "output_tokens_details": {"reasoning_tokens": number}, - }, - "parallel_tool_calls": True, - "tool_choice": "auto", - "tools": [], - } - - def close(self) -> None: - self.release.set() - self.server.shutdown() - self.server.server_close() - self.thread.join(timeout=2) - assert not self.thread.is_alive() - - -class _LiveProcessor(SpanProcessorForTests): - def __init__(self) -> None: - super().__init__() - self.started_models: list[Any] = [] - self.model_started = asyncio.Event() - - def on_span_start(self, span: Any) -> None: - super().on_span_start(span) - if span.span_data.type in {"response", "generation"}: - self.started_models.append(span) - self.model_started.set() - - -@pytest.fixture -def telemetry( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> Iterator[tuple[Any, Any, SpanProcessorForTests, Any]]: - monkeypatch.delenv("OPENAI_AGENTS_TRACE_INCLUDE_SENSITIVE_DATA", raising=False) - monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home")) - if os.environ.get("CODEX_TRACING_NATIVE_TEST") != "1": - pytest.importorskip("opentelemetry.sdk.trace") - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - - from agents.extensions.tracing import OpenTelemetryTracingProcessor - - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(exporter)) - native = _LiveProcessor() - otel_processor = OpenTelemetryTracingProcessor(tracer_provider=provider) - set_trace_processors([native, otel_processor]) - try: - yield provider.get_tracer("operation-parity"), exporter, native, otel_processor - finally: - set_trace_processors([SPAN_PROCESSOR_TESTING]) - provider.shutdown() - - -def _codex_provider(tmp_path: Path, server: _ResponsesFixture) -> CodexProvider: - if os.environ.get("CODEX_TRACING_NATIVE_TEST") != "1": - pytest.skip("Set CODEX_TRACING_NATIVE_TEST=1 to test the installed Codex native wheel.") - home = tmp_path / "codex-home" - home.mkdir(mode=0o700) - config = f'''model = "{MODEL}" -model_provider = "fixture" -approval_policy = "never" -sandbox_mode = "read-only" -[analytics] -enabled = false -[features] -apps = false -hooks = false -plugins = false -[model_providers.fixture] -name = "Synthetic fixture" -base_url = "{server.base_url}" -wire_api = "responses" -requires_openai_auth = false -supports_websockets = false -[otel] -metrics_exporter = "none" -exporter = "none" -trace_exporter = "none" -''' - if server.mixed: - mcp_server = tmp_path / "parity_mcp.py" - mcp_server.write_text( - "import sys\n" - "from pathlib import Path\n" - "try:\n" - " from mcp.server.mcpserver import MCPServer\n" - "except ImportError:\n" - " from mcp.server.fastmcp import FastMCP as MCPServer\n" - "from mcp.types import ToolAnnotations\n" - 'server = MCPServer("parity")\n' - "@server.tool(annotations=ToolAnnotations(readOnlyHint=True, destructiveHint=False))\n" - "def read_file() -> str:\n" - ' """Read the synthetic fixture file."""\n' - " return Path(sys.argv[1]).read_text()\n" - 'server.run(transport="stdio")\n' - ) - config += ( - "\n[mcp_servers.parity]\ncommand = " - + json.dumps(sys.executable) - + "\nargs = " - + json.dumps([str(mcp_server), str(tmp_path / "parity-native.txt")]) - + "\nrequired = true\nstartup_timeout_sec = 10\ntool_timeout_sec = 5\n" - + 'default_tools_approval_mode = "approve"\n' - ) - (home / "config.toml").write_text(config) - return CodexProvider(cwd=tmp_path) - - -def _ns(timestamp: str | None) -> int: - assert timestamp is not None - return int(datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp() * 1_000_000_000) - - -def _model_provider(name: str, tmp_path: Path, server: _ResponsesFixture) -> ModelProvider: - if name == "codex": - return _codex_provider(tmp_path, server) - return OpenAIProvider( - api_key="loopback-test-placeholder", base_url=server.base_url, use_responses=True - ) - - -async def _exercise( - *, - tmp_path: Path, - telemetry: tuple[Any, Any, SpanProcessorForTests, Any], - provider_name: str, - streamed: bool, - capture: str, - mixed: bool = False, - consumer: str = "both", - omit_task_spans: bool = False, - model_header: bool = True, - observed_model: str = MODEL, -) -> None: - tracer, exporter, native, otel_processor = telemetry - set_trace_processors( - [native] - if consumer == "sdk" - else [otel_processor] - if consumer == "otel" - else [native, otel_processor] - ) - server = _ResponsesFixture( - mixed=mixed, - model_header=model_header, - observed_model=observed_model, - ) - callback_intervals: list[tuple[int, int]] = [] - model_provider = None - try: - (tmp_path / "parity-native.txt").write_text(FILE_CONTENT) - model_provider = _model_provider(provider_name, tmp_path, server) - - @tool - async def lookup(key: str) -> str: - assert key == ARGUMENT - started = time.time_ns() - with tracer.start_as_current_span("database.query"): - pass - callback_intervals.append((started, time.time_ns())) - return TOOL_RESULT - - agent = Agent(name="worker", model=MODEL, instructions=INSTRUCTIONS, tools=[lookup]) - tracing: TracingConfig = {} - if omit_task_spans: - tracing["include_task_and_turn_spans"] = False - if capture in {"metadata", "content", "veto"}: - tracing["operation_detail"] = "content" if capture == "veto" else capture - config = RunConfig( - model_provider=model_provider, - workflow_name="parity-workflow", - tracing=tracing, - tracing_disabled=capture == "disabled", - ) - if capture in {"veto", "default-veto"}: - config.trace_include_sensitive_data = False - with tracer.start_as_current_span("host_job") as host: - if streamed: - result = Runner.run_streamed(agent, PROMPT, run_config=config) - async for _ in result.stream_events(): - pass - else: - result = await Runner.run(agent, PROMPT, run_config=config) - assert result.final_output == ANSWER - assert len(server.requests) == (3 if mixed else 2) - assert all(request["model"] == MODEL for request in server.requests) - assert len(callback_intervals) == 1 - if mixed: - native_results = [ - item.get("output") - for item in server.requests[1]["input"] - if item.get("call_id") == "native_read" and "output" in item - ] - assert FILE_CONTENT in json.dumps(native_results), { - "result": native_results, - "mcp_tools": [ - tool - for tool in server.requests[0].get("tools", []) - if "parity" in json.dumps(tool) - ], - } - spans = native.get_ordered_spans() - otel = exporter.get_finished_spans() - database = next(span for span in otel if span.name == "database.query") - if capture == "disabled": - assert not spans - assert len(otel) == 2 - assert database.parent.span_id == host.get_span_context().span_id - return - - models = [span for span in spans if span.span_data.type in {"response", "generation"}] - tools = [span for span in spans if span.span_data.type == "function"] - turns = [span for span in spans if span.span_data.type == "turn"] - if consumer != "otel": - assert len(models) == len(server.requests) - assert len(tools) == (2 if mixed else 1) - assert len(turns) == (0 if omit_task_spans else 2) - assert {span.span_data.type for span in models} == { - "generation" if capture in {"metadata", "content", "veto"} else "response" - } - native_index = {span.span_id: span for span in spans} - assert all( - native_index[span.parent_id].span_data.type - == ("agent" if omit_task_spans else "turn") - for span in models + tools - ) - assert all(span.error is None for span in models + tools) - for index, span in enumerate(models, start=1): - assert span.span_data.operation.model == MODEL - assert span.span_data.operation.response_model == observed_model - assert span.span_data.operation.outcome == "completed" - if capture in {"metadata", "veto", "default-veto"}: - assert span.span_data.operation.response_id is None - else: - assert span.span_data.operation.response_id == f"resp_parity_{index}" - assert span.span_data.usage["input_tokens"] == 10 * index - assert span.span_data.usage["output_tokens"] == 5 * index - assert span.span_data.usage["requests"] == 1 - assert span.span_data.usage["input_tokens_details"]["cached_tokens"] == index - assert ( - span.span_data.usage["input_tokens_details"].get("cache_write_tokens", 0) == 0 - ) - assert span.span_data.usage["output_tokens_details"] == { - "reasoning_tokens": index, - } - started, ended = server.intervals[index - 1] - assert _ns(span.started_at) <= started + 1_000_000 - assert _ns(span.ended_at) >= ended - 1_000_000 - if not omit_task_spans: - expected_turn_usage = [(30, 15), (30, 15)] if mixed else [(10, 5), (20, 10)] - assert [ - (span.span_data.usage["input_tokens"], span.span_data.usage["output_tokens"]) - for span in turns - ] == expected_turn_usage - if mixed: - native_tool = next(span for span in tools if span.span_data.name != "lookup") - assert _ns(models[0].started_at) <= _ns(native_tool.started_at) - assert _ns(native_tool.ended_at) <= _ns(models[1].started_at) - operations = [ - span for span in otel if span.attributes.get("gen_ai.operation.name") == "chat" - ] - executions = [ - span for span in otel if span.attributes.get("gen_ai.operation.name") == "execute_tool" - ] - if consumer == "sdk": - assert not operations and not executions - assert database.parent.span_id == host.get_span_context().span_id - return - assert len(operations) == len(server.requests) - assert len(executions) == (2 if mixed else 1) - otel_index = {span.context.span_id: span for span in otel} - for span in operations + executions: - assert span.context.trace_id == host.get_span_context().trace_id - assert otel_index[span.parent.span_id].attributes["gen_ai.operation.name"] == ( - "invoke_agent" if omit_task_spans else "openai.agents.turn" - ) - for index, span in enumerate( - sorted(operations, key=lambda value: value.start_time), start=1 - ): - assert span.attributes["gen_ai.request.model"] == MODEL - assert span.attributes["gen_ai.response.model"] == observed_model - assert span.attributes["gen_ai.usage.input_tokens"] == 10 * index - assert span.attributes["gen_ai.usage.output_tokens"] == 5 * index - assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == index - assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == index - assert "gen_ai.usage.cache_write.input_tokens" not in span.attributes - assert span.attributes["openai.api.type"] == "responses" - assert span.kind.name == "CLIENT" - started, ended = server.intervals[index - 1] - assert span.start_time <= started + 1_000_000 - assert span.end_time >= ended - 1_000_000 - assert otel_index[database.parent.span_id] in executions - python_execution = otel_index[database.parent.span_id] - callback_started, callback_ended = callback_intervals[0] - # The SDK records the tool interval with datetime's microsecond precision. - assert python_execution.start_time // 1_000 <= callback_started // 1_000 - assert callback_started <= database.start_time <= database.end_time <= callback_ended - assert callback_ended // 1_000 <= python_execution.end_time // 1_000 - if models: - assert _ns(models[0].ended_at) <= callback_started + 1_000_000 - for span in otel: - current = span - while current.parent is not None: - current = otel_index[current.parent.span_id] - assert current.context.span_id == host.get_span_context().span_id - - exported = json.dumps( - [ - { - "attributes": dict(span.attributes), - "events": [ - {"name": event.name, "attributes": dict(event.attributes or {})} - for event in span.events - ], - "status": span.status.description, - } - for span in otel - ] - ) - if capture == "content": - assert all(marker in exported for marker in PAYLOADS) - for span, request in zip(models, server.requests, strict=True): - submitted = {"input": request["input"], "instructions": request.get("instructions")} - assert len(json.dumps(submitted, ensure_ascii=False).encode()) < 32 * 1024 - assert span.span_data.input == request["input"] - if isinstance(request.get("instructions"), str): - assert span.span_data.model_config["instructions"] == request["instructions"] - else: - assert all(marker not in exported for marker in PAYLOADS) - if capture in {"metadata", "veto", "default-veto"}: - assert all(span.span_data.input is None for span in models + tools) - assert all(getattr(span.span_data, "output", None) is None for span in models + tools) - assert all(getattr(span.span_data, "response", None) is None for span in models) - if capture == "default" and consumer != "otel": - assert PROMPT in json.dumps(models[0].span_data.input) - assert ANSWER in models[-1].span_data.response.model_dump_json() - python_tool = next(span for span in tools if span.span_data.name == "lookup") - assert ARGUMENT in python_tool.span_data.input - assert TOOL_RESULT in str(python_tool.span_data.output) - if mixed: - native_payloads = json.dumps([span.export() for span in spans], default=str) - if capture == "content": - assert FILE_CONTENT in exported - else: - assert FILE_CONTENT not in exported - assert FILE_CONTENT not in native_payloads - finally: - if model_provider is not None: - await model_provider.aclose() - server.close() - - -@pytest.mark.parametrize("provider_name", ["default", "codex"]) -@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) -@pytest.mark.parametrize( - "capture", ["default", "default-veto", "metadata", "content", "veto", "disabled"] -) -async def test_operation_tracing_parity( - tmp_path: Path, telemetry: Any, provider_name: str, streamed: bool, capture: str -) -> None: - await asyncio.wait_for( - _exercise( - tmp_path=tmp_path, - telemetry=telemetry, - provider_name=provider_name, - streamed=streamed, - capture=capture, - ), - timeout=30, - ) - - -@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) -@pytest.mark.parametrize("capture", ["default", "metadata", "content", "veto"]) -async def test_codex_native_and_python_tool_parity( - tmp_path: Path, telemetry: Any, streamed: bool, capture: str -) -> None: - await asyncio.wait_for( - _exercise( - tmp_path=tmp_path, - telemetry=telemetry, - provider_name="codex", - streamed=streamed, - capture=capture, - mixed=True, - ), - timeout=30, - ) - - -@pytest.mark.parametrize("provider_name", ["default", "codex"]) -@pytest.mark.parametrize("consumer", ["sdk", "otel"]) -async def test_operation_tracing_individual_consumers( - tmp_path: Path, telemetry: Any, provider_name: str, consumer: str -) -> None: - await asyncio.wait_for( - _exercise( - tmp_path=tmp_path, - telemetry=telemetry, - provider_name=provider_name, - streamed=False, - capture="default", - consumer=consumer, - ), - timeout=30, - ) - - -async def test_operation_tracing_without_task_and_turn_spans( - tmp_path: Path, telemetry: Any -) -> None: - await asyncio.wait_for( - _exercise( - tmp_path=tmp_path, - telemetry=telemetry, - provider_name="default", - streamed=False, - capture="metadata", - omit_task_spans=True, - ), - timeout=30, - ) - - -@pytest.mark.parametrize("provider_name", ["default", "codex"]) -async def test_operation_tracing_observed_model_without_header( - tmp_path: Path, telemetry: Any, provider_name: str -) -> None: - await asyncio.wait_for( - _exercise( - tmp_path=tmp_path, - telemetry=telemetry, - provider_name=provider_name, - streamed=False, - capture="metadata", - model_header=False, - observed_model=f"{MODEL}-observed", - ), - timeout=30, - ) - - -@pytest.mark.parametrize( - ("provider_name", "termination"), - [("default", "cancel"), ("codex", "cancel"), ("codex", "provider-close")], -) -async def test_operation_tracing_live_start_and_cancel( - tmp_path: Path, telemetry: Any, provider_name: str, termination: str -) -> None: - tracer, exporter, native, _ = telemetry - server = _ResponsesFixture(held=True) - model_provider = None - task = None - try: - model_provider = _model_provider(provider_name, tmp_path, server) - with tracer.start_as_current_span("host_job") as host: - task = asyncio.create_task( - Runner.run( - Agent(name="worker", model=MODEL), - PROMPT, - run_config=RunConfig( - model_provider=model_provider, tracing={"operation_detail": "metadata"} - ), - ) - ) - assert await asyncio.to_thread(server.request_started.wait, 5) - await asyncio.wait_for(native.model_started.wait(), 5) - assert len(native.started_models) == 1 - assert native.started_models[0].ended_at is None - if termination == "provider-close": - await asyncio.wait_for(model_provider.aclose(), 10) - with pytest.raises(RuntimeError): - await asyncio.wait_for(task, 10) - else: - task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(task, 10) - assert host.get_span_context().is_valid - models = [ - span for span in native.get_ordered_spans() if span.span_data.type == "generation" - ] - assert len(models) == 1 - assert models[0] is native.started_models[0] - assert models[0].ended_at is not None - assert models[0].span_data.operation.outcome == "interrupted" - chats = [ - span - for span in exporter.get_finished_spans() - if span.attributes.get("gen_ai.operation.name") == "chat" - ] - assert len(chats) == 1 - assert chats[0].end_time is not None - if models[0].span_data.operation.estimated_end: - assert chats[0].attributes["openai.agents.operation.estimated_end"] is True - assert chats[0].attributes["openai.agents.operation.outcome"] == "interrupted" - coverage = [ - span - for span in native.get_ordered_spans() - if span.span_data.type == "custom" - and span.span_data.name == "codex.tracing.coverage" - ] - assert len(coverage) == 1 - assert coverage[0].span_data.data["coverage"] == "partial" - exported_coverage = [ - span - for span in exporter.get_finished_spans() - if span.name == "codex.tracing.coverage" - ] - assert len(exported_coverage) == 1 - assert exported_coverage[0].attributes["openai.agents.operation.coverage"] == "partial" - assert len(server.requests) == 1 - finally: - server.release.set() - if task is not None and not task.done(): - task.cancel() - await asyncio.gather(task, return_exceptions=True) - if model_provider is not None: - await model_provider.aclose() - server.close() - - -@pytest.mark.parametrize("provider_name", ["default", "codex"]) -@pytest.mark.parametrize("streamed", [False, True], ids=["run", "stream"]) -async def test_operation_tracing_transport_retry( - tmp_path: Path, telemetry: Any, provider_name: str, streamed: bool -) -> None: - tracer, exporter, native, _ = telemetry - server = _ResponsesFixture(retry_once=True) - model_provider = None - try: - model_provider = _model_provider(provider_name, tmp_path, server) - with tracer.start_as_current_span("host_job"): - config = RunConfig( - model_provider=model_provider, tracing={"operation_detail": "metadata"} - ) - agent = Agent(name="worker", model=MODEL) - if streamed: - result = Runner.run_streamed(agent, PROMPT, run_config=config) - async for _ in result.stream_events(): - pass - else: - result = await asyncio.wait_for(Runner.run(agent, PROMPT, run_config=config), 15) - assert result.final_output == ANSWER - assert len(server.requests) == 2 - assert server.requests[0]["input"] == server.requests[1]["input"] - models = [ - span for span in native.get_ordered_spans() if span.span_data.type == "generation" - ] - assert len(models) == 1 - assert models[0].error is None - assert models[0].span_data.operation.outcome == "completed" - assert models[0].span_data.usage["input_tokens"] == 10 - assert models[0].span_data.usage["output_tokens"] == 5 - assert _ns(models[0].started_at) <= server.intervals[0][0] + 1_000_000 - assert _ns(models[0].ended_at) >= server.intervals[-1][1] - 1_000_000 - chats = [ - span - for span in exporter.get_finished_spans() - if span.attributes.get("gen_ai.operation.name") == "chat" - ] - assert len(chats) == 1 - assert chats[0].attributes["gen_ai.usage.input_tokens"] == 10 - assert chats[0].attributes["gen_ai.usage.output_tokens"] == 5 - finally: - if model_provider is not None: - await model_provider.aclose() - server.close() diff --git a/tests/test_build_codex_wheel.py b/tests/test_build_codex_wheel.py deleted file mode 100644 index 400756af14..0000000000 --- a/tests/test_build_codex_wheel.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Regression coverage for the native wheel's local Core patch boundary.""" - -import importlib.util -import json -import subprocess -from pathlib import Path - - -def test_core_patch_applies_inside_an_ancestor_git_checkout(tmp_path: Path) -> None: - script = Path(__file__).resolve().parents[1] / "scripts/build_codex_wheel.py" - spec = importlib.util.spec_from_file_location("build_codex_wheel", script) - assert spec is not None and spec.loader is not None - build = importlib.util.module_from_spec(spec) - spec.loader.exec_module(build) - subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True) - upstream = tmp_path / "upstream" - crate = upstream / "codex-rs/core" - crate.mkdir(parents=True) - (crate / "Cargo.toml").write_text('[package]\nname = "codex-core"\n') - (crate / "value.txt").write_bytes(b"stock\n") - source = tmp_path / "target/staged" - patches = source / "patches/codex" - patches.mkdir(parents=True) - (source / "native").mkdir() - (patches / "0001.patch").write_text( - "diff --git a/codex-rs/core/value.txt b/codex-rs/core/value.txt\n" - "--- a/codex-rs/core/value.txt\n" - "+++ b/codex-rs/core/value.txt\n" - "@@ -1 +1 @@\n-stock\n+patched\n", - newline="\n", - ) - metadata = { - "packages": [ - { - "name": "codex-core", - "source": "git+https://github.com/openai/codex?rev=pinned#pinned", - "manifest_path": str(crate / "Cargo.toml"), - }, - {"name": "sdk", "source": None}, - ] - } - - patched = build.patch_codex_source(source, upstream, metadata) - - assert (patched / "codex-rs/core/value.txt").read_text() == "patched\n" - assert (crate / "value.txt").read_text() == "stock\n" - assert json.dumps(str(patched / "codex-rs/core")) in (source / ".cargo/config.toml").read_text() From edf541c555ed6466490b87930f05b3f784681f0a Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 05:33:46 -0800 Subject: [PATCH 30/33] fix: reject native approvals in nested run state before restoring context --- src/agents/run_state.py | 33 +++++++++++++---- tests/memory/test_resumable_session.py | 51 +++++++++++++++++++++----- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/agents/run_state.py b/src/agents/run_state.py index aa565a8eaf..a5a1ea69a0 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -2539,6 +2539,8 @@ def validation_error_factory( state_json, validation_error_factory=validation_error_factory, ) + if _has_native_approvals(state_json): + raise validation_error_factory(_UNSUPPORTED_NATIVE_APPROVALS, UserError) from .sandbox._mount_security import sanitize_run_state_sandbox_mount_authority @@ -2590,6 +2592,28 @@ def validation_error_factory( # -------------------------- +_UNSUPPORTED_NATIVE_APPROVALS = ( + "Run state contains unsupported native tool approvals. " + "Check for prior effects before starting a new run with SDK tools." +) + + +def _has_native_approvals(state: dict[str, Any]) -> bool: + """Check nested agent runs before best-effort context restoration can skip a child.""" + response = state.get("last_processed_response") + if not isinstance(response, dict): + return False + if response.get("native_approvals"): + return True + functions = response.get("functions") + if isinstance(functions, list): + for entry in functions: + child = entry.get("agent_run_state") if isinstance(entry, dict) else None + if isinstance(child, dict) and _has_native_approvals(child): + return True + return False + + def _validate_run_state_json_value(value: object) -> None: """Validate the exact built-in JSON tree without invoking caller-defined protocols.""" if type(value) is dict: @@ -4235,13 +4259,6 @@ async def _build_run_state_from_json( state_json, validation_error_factory=validation_error_factory, ) - processed_response = state_json.get("last_processed_response") - if isinstance(processed_response, Mapping) and processed_response.get("native_approvals"): - raise validation_error_factory( - "Run state contains unsupported native tool approvals. " - "Start a new run using SDK tools.", - UserError, - ) schema_major, schema_minor = (int(part) for part in schema_version.split(".", maxsplit=1)) programmatic_major, programmatic_minor = ( int(part) for part in _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION.split(".", maxsplit=1) @@ -5712,7 +5729,7 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]: "RunState tool invocation must be a mapping.", "RunState tool invocation contains invalid lifecycle data.", "Tool input rewrites require RunState schema 1.23.", - "Run state contains unsupported native tool approvals. Start a new run using SDK tools.", + _UNSUPPORTED_NATIVE_APPROVALS, "RunState tool input rewrite is invalid.", "Hosted MCP approval decisions require a non-empty request id.", ( diff --git a/tests/memory/test_resumable_session.py b/tests/memory/test_resumable_session.py index deed22f801..f9aaa7b500 100644 --- a/tests/memory/test_resumable_session.py +++ b/tests/memory/test_resumable_session.py @@ -5,6 +5,7 @@ import asyncio import copy import json +from pathlib import Path from typing import Any import pytest @@ -53,12 +54,22 @@ def checkpoint(self, session_id: str) -> dict[str, Any]: @pytest.mark.asyncio -async def test_saved_native_approval_cannot_become_an_sdk_tool_approval() -> None: +@pytest.mark.parametrize("nested", [False, True], ids=["direct", "agent-as-tool"]) +@pytest.mark.parametrize( + "with_context_callback", [False, True], ids=["default", "context-callback"] +) +@pytest.mark.parametrize("loader", ["run-state", "session"]) +async def test_saved_native_approval_cannot_become_an_sdk_tool_approval( + nested: bool, with_context_callback: bool, loader: str, tmp_path: Path +) -> None: store = MemoryStore() session = await ResumableSession.create(store) + workspace = tmp_path / "unchanged.txt" + workspace.write_text("pre-existing work") @tool(needs_approval=True) def exec_command(command: str) -> str: + workspace.write_text(command) pytest.fail("A native approval cannot authorize a matching SDK tool.") agent = Agent( @@ -66,12 +77,23 @@ def exec_command(command: str) -> str: model=ScriptedModel([[get_function_tool_call("exec_command", '{"command":"original"}')]]), tools=[exec_command], ) + if nested: + agent = Agent( + name="coordinator", + model=ScriptedModel([[get_function_tool_call("delegate", '{"input":"Run command"}')]]), + tools=[agent.as_tool("delegate", "Delegate the command")], + ) await Runner.run( agent, "Run command", session=session, context={"test": True}, run_config=CONFIG ) checkpoint = store.checkpoint(session.session_id) snapshot = json.loads(checkpoint["run_state"]) - processed = snapshot["last_processed_response"] + pending_state = ( + snapshot["last_processed_response"]["functions"][0]["agent_run_state"] + if nested + else snapshot + ) + processed = pending_state["last_processed_response"] # An earlier preview stored native actions outside the SDK function queue. processed["native_approvals"] = copy.deepcopy(processed["interruptions"]) processed["functions"] = [] @@ -79,16 +101,25 @@ def exec_command(command: str) -> str: stored = json.dumps(checkpoint) store.records[session.session_id] = stored + context_callbacks: list[str] = [] + def restore_context(value: dict[str, Any]) -> dict[str, Any]: - pytest.fail("Unsupported checkpoints must fail before restoring application context.") - - with pytest.raises(UserError, match="unsupported native tool approvals"): - await RunState.from_json(agent, snapshot, context_deserializer=restore_context) - resumed = await ResumableSession.resume(store, session.session_id) - with pytest.raises(ResumableSessionError) as failure: - await resumed.load_state(agent, context_deserializer=restore_context) - assert failure.value.reason == "invalid_state" + context_callbacks.append("restored") + return value + + context_deserializer = restore_context if with_context_callback else None + if loader == "run-state": + with pytest.raises(UserError, match="unsupported native tool approvals"): + await RunState.from_json(agent, snapshot, context_deserializer=context_deserializer) + else: + resumed = await ResumableSession.resume(store, session.session_id) + with pytest.raises(ResumableSessionError) as failure: + await resumed.load_state(agent, context_deserializer=context_deserializer) + assert failure.value.reason == "invalid_state" assert store.records[session.session_id] == stored + assert store.checkpoint(session.session_id)["status"] == "paused" + assert workspace.read_text() == "pre-existing work" + assert not context_callbacks async def run_agent( From f9f9235fc550b8709b90c750ab6e997cc1936e22 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 06:54:19 -0800 Subject: [PATCH 31/33] feat(codex): recover native approvals and trace native calls --- .github/workflows/codex-preview.yml | 5 +- docs/models/codex.md | 4 +- native/codex/Cargo.lock | 1 + native/codex/Cargo.toml | 1 + native/codex/patches/01-operation-tracing.md | 24 + .../codex/patches/01-operation-tracing.patch | 147 ++ native/codex/patches/02-approval-recovery.md | 25 + .../codex/patches/02-approval-recovery.patch | 1436 +++++++++++++++++ native/codex/patches/README.md | 34 + native/codex/src/lib.rs | 93 +- native/codex/src/native_approval.rs | 134 ++ native/codex/src/operation_tracing.rs | 261 ++- native/codex/src/operation_tracing_content.rs | 203 +++ native/codex/verify_installed.py | 126 +- scripts/build_codex_wheel.py | 55 +- .../extensions/tracing/opentelemetry.py | 14 +- src/agents/memory/resumable_session.py | 17 + src/agents/models/_codex_items.py | 74 +- src/agents/models/_codex_tracing.py | 145 +- src/agents/models/_run_context.py | 1 + src/agents/models/codex.py | 171 +- src/agents/run.py | 4 + src/agents/run_context.py | 4 + src/agents/run_internal/native_approvals.py | 220 +++ src/agents/run_internal/resumable_session.py | 7 + src/agents/run_internal/run_loop.py | 18 +- src/agents/run_internal/run_steps.py | 2 + src/agents/run_internal/turn_resolution.py | 44 +- src/agents/run_state.py | 80 +- tests/memory/test_native_approval_recovery.py | 475 ++++++ tests/memory/test_native_approval_tracing.py | 90 ++ tests/memory/test_resumable_session.py | 10 + tests/models/test_codex_pending_approval.py | 166 ++ tests/models/test_codex_provider.py | 87 +- tests/test_build_codex_wheel.py | 57 + tests/tracing/test_opentelemetry.py | 18 +- 36 files changed, 4115 insertions(+), 138 deletions(-) create mode 100644 native/codex/patches/01-operation-tracing.md create mode 100644 native/codex/patches/01-operation-tracing.patch create mode 100644 native/codex/patches/02-approval-recovery.md create mode 100644 native/codex/patches/02-approval-recovery.patch create mode 100644 native/codex/patches/README.md create mode 100644 native/codex/src/native_approval.rs create mode 100644 native/codex/src/operation_tracing_content.rs create mode 100644 src/agents/run_internal/native_approvals.py create mode 100644 tests/memory/test_native_approval_recovery.py create mode 100644 tests/memory/test_native_approval_tracing.py create mode 100644 tests/models/test_codex_pending_approval.py create mode 100644 tests/test_build_codex_wheel.py diff --git a/.github/workflows/codex-preview.yml b/.github/workflows/codex-preview.yml index 385a1475f4..63974c46c4 100644 --- a/.github/workflows/codex-preview.yml +++ b/.github/workflows/codex-preview.yml @@ -70,6 +70,7 @@ jobs: include: - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + test_native: true platform: manylinux_2_28_x86_64 image: quay.io/pypa/manylinux_2_28_x86_64@sha256:9555ba6c479fa69a2aa80ad1dc57154a60d562b19df159361d9379a02beb1e36 - runner: ubuntu-24.04-arm @@ -161,11 +162,9 @@ jobs: run: | options=() if [[ "$RUNNER_OS" == Linux ]]; then options+=(--compatibility manylinux_2_28); fi + if [[ "${{ matrix.test_native }}" == true ]]; then options+=(--test-native); fi uv run --no-sync python scripts/build_codex_wheel.py \ --out "$GITHUB_WORKSPACE/dist" --target "$WHEEL_TARGET" "${options[@]}" - - name: Test native policy and configuration - if: matrix.platform == 'manylinux_2_28_x86_64' - run: cargo test --locked --release --lib --manifest-path native/codex/Cargo.toml - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: codex-wheel-${{ matrix.platform }} diff --git a/docs/models/codex.md b/docs/models/codex.md index 037c6562f6..e0b8ef6def 100644 --- a/docs/models/codex.md +++ b/docs/models/codex.md @@ -134,7 +134,9 @@ The workspace sandbox protects the directory that contains the installed helpers On Windows, the preview rejects a local Codex executor configured with the legacy elevated sandbox. That sandbox needs a setup helper that Codex cannot securely locate from a Python process. If your organization requires this sandbox, use an SDK-managed executor or configure a remote executor with `include_local = false` in `environments.toml`. -Native Codex approval prompts are separate from SDK tool approvals. This provider cannot answer native approval requests and fails the model call when one arrives. For a non-interactive read-only workflow, configure Codex to avoid approval prompts: +Pass a `ResumableSession` to `Runner.run(..., session=session)` to pause a built-in `exec_command` or `apply_patch` for approval. A new worker can load that session and approve or reject the original call. Recovery requires the same workspace and permissions, one local environment, and no tool hooks. Other native approval requests fail the model call. + +For a non-interactive read-only workflow, disable approval prompts. ```python provider = CodexProvider( diff --git a/native/codex/Cargo.lock b/native/codex/Cargo.lock index 41cb579aa3..122fc2dac0 100644 --- a/native/codex/Cargo.lock +++ b/native/codex/Cargo.lock @@ -8059,6 +8059,7 @@ dependencies = [ "futures", "pyo3", "pyo3-async-runtimes", + "serde", "serde_json", "tempfile", "tokio", diff --git a/native/codex/Cargo.toml b/native/codex/Cargo.toml index 2721505279..853509a154 100644 --- a/native/codex/Cargo.toml +++ b/native/codex/Cargo.toml @@ -18,6 +18,7 @@ pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] } pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } tokio = { version = "1", features = ["rt-multi-thread", "sync", "time"] } futures = "0.3" +serde = "1" serde_json = "1" toml = "0.9.5" codex-arg0 = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } diff --git a/native/codex/patches/01-operation-tracing.md b/native/codex/patches/01-operation-tracing.md new file mode 100644 index 0000000000..7e03cd59d5 --- /dev/null +++ b/native/codex/patches/01-operation-tracing.md @@ -0,0 +1,24 @@ +# Trace the request Codex sends and the time a tool runs + +Codex can change the model input just before sending it. Its normal tool timer can also include time spent waiting for approval. Those details matter when debugging a slow or unexpected turn. + +[This patch](01-operation-tracing.patch) lets the SDK observe the final HTTP or WebSocket request, distinguish a failed connection from cancellation, and start timing a command or edit after approval. These callbacks cannot change the request or grant permission to run a tool. + +Native spans contain metadata by default. To include text in model and tool spans, pass this configuration to the Runner. + +```python +from agents import RunConfig +from agents.models.codex import CodexProvider + +config = RunConfig( + model_provider=CodexProvider(), + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=True, +) +``` + +The SDK uses existing Codex events for model responses, usage and tool results. It excludes request headers, encrypted reasoning, media and values over 32 KiB. Captured text can still contain secrets. Setting `trace_include_sensitive_data=False` always disables content capture. + +## Check and remove + +Build and test the wheel as described in the [patch guide](README.md). Check both metadata and content modes. Failed requests and cancellations should have distinct outcomes. Denied commands should have no execution span. Remove the patch and guide once upstream Codex exposes the same events. diff --git a/native/codex/patches/01-operation-tracing.patch b/native/codex/patches/01-operation-tracing.patch new file mode 100644 index 0000000000..53f8027f17 --- /dev/null +++ b/native/codex/patches/01-operation-tracing.patch @@ -0,0 +1,147 @@ +diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs +index 2ac9b93..7257291 100644 +--- a/codex-rs/core/src/client.rs ++++ b/codex-rs/core/src/client.rs +@@ -2017,7 +2017,7 @@ impl ResponsesHttpExecutor<'_> { + let request_session_telemetry = + session_telemetry_for_request(session_telemetry, &request); + options.extra_headers.extend(responses_headers); +- let interceptors = crate::model_request::prepare( ++ let mut interceptors = crate::model_request::prepare( + &self.client.request_contributors, + &self.client.state.thread_id.to_string(), + &model_info.slug, +@@ -2032,6 +2032,11 @@ impl ResponsesHttpExecutor<'_> { + } + request.input = input; + } ++ for interceptor in &mut interceptors { ++ interceptor.observe_request(codex_extension_api::ModelRequestContent::Responses( ++ &request, ++ )); ++ } + inference_trace_attempt.record_started(&request); + let client = ApiResponsesClient::new( + transport, +@@ -2040,6 +2045,11 @@ impl ResponsesHttpExecutor<'_> { + ) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); + let stream_result = client.stream_request(request, options).await; ++ if stream_result.is_err() { ++ for interceptor in &mut interceptors { ++ interceptor.request_failed(); ++ } ++ } + + match stream_result { + Ok(stream) => { +@@ -2315,7 +2325,7 @@ impl ModelClientSession { + client_setup.auth.as_ref(), + &responses_headers, + ); +- let interceptors = crate::model_request::prepare( ++ let mut interceptors = crate::model_request::prepare( + &self.client.request_contributors, + &self.client.state.thread_id.to_string(), + &model_info.slug, +@@ -2337,6 +2347,11 @@ impl ModelClientSession { + let ResponsesWsRequest::ResponseCreate(payload) = &mut ws_request; + payload.input = input; + } ++ let ResponsesWsRequest::ResponseCreate(payload) = &ws_request; ++ for interceptor in &mut interceptors { ++ interceptor ++ .observe_request(codex_extension_api::ModelRequestContent::WebSocket(payload)); ++ } + if !previous_response_id_from_untraced_warmup { + inference_trace_attempt.record_started(&ws_request); + } +@@ -2377,6 +2392,9 @@ impl ModelClientSession { + self.websocket_session.last_request = Some(request); + self.websocket_session.last_response_from_untraced_warmup = warmup; + let stream_result = stream_result.map_err(|err| { ++ for interceptor in &mut interceptors { ++ interceptor.request_failed(); ++ } + let response_debug_context = extract_response_debug_context_from_api_error(&err); + let err = self.client.state.provider.map_api_error(err); + inference_trace_attempt.record_failed( +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index a1a88e4..c5f8077 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -102,6 +102,18 @@ impl ToolOrchestrator { + .as_ref() + .map(ActiveNetworkApproval::execution_proxy), + }; ++ for contributor in tool_ctx ++ .session ++ .services ++ .extensions ++ .tool_lifecycle_contributors() ++ { ++ contributor.on_tool_execution_start( ++ &tool_ctx.session.thread_id.to_string(), ++ &tool_ctx.step_context.turn.sub_id, ++ &tool_ctx.call_id, ++ ); ++ } + let run_result = tool + .run(req, &attempt_with_network_approval, &attempt_tool_ctx) + .await; +diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs +index dc95f99..e985a9a 100644 +--- a/codex-rs/ext/extension-api/src/contributors.rs ++++ b/codex-rs/ext/extension-api/src/contributors.rs +@@ -413,6 +413,10 @@ pub trait ToolContributor: Send + Sync { + /// rewriting the invocation. Use `ToolContributor` for owning a tool implementation + /// and hooks for policy that changes tool payloads. + pub trait ToolLifecycleContributor: Send + Sync { ++ /// One admitted native runtime attempt, after approval and network admission. ++ /// May repeat on sandbox retry; denied attempts never reach this callback. ++ fn on_tool_execution_start(&self, _thread_id: &str, _turn_id: &str, _call_id: &str) {} ++ + /// Observe direct calls before readiness, dispatch waiting, and hooks, including blocked calls. + /// Excludes nested code-mode calls. Observers must return promptly. + fn on_tool_dispatch(&self, _input: ToolDispatchInput<'_>) {} +diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs +index 147b52f..bd94076 100644 +--- a/codex-rs/ext/extension-api/src/lib.rs ++++ b/codex-rs/ext/extension-api/src/lib.rs +@@ -128,6 +128,7 @@ pub use contributors::ToolDispatchInput; + mod model_request; + pub use model_request::GuardianAssessment; + pub use model_request::ModelRequestContributor; ++pub use model_request::ModelRequestContent; + pub use model_request::ModelRequestInput; + pub use model_request::ModelRequestKind; + pub use model_request::ModelResponseError; +diff --git a/codex-rs/ext/extension-api/src/model_request.rs b/codex-rs/ext/extension-api/src/model_request.rs +index 0f477aa..6514694 100644 +--- a/codex-rs/ext/extension-api/src/model_request.rs ++++ b/codex-rs/ext/extension-api/src/model_request.rs +@@ -35,10 +35,24 @@ pub trait ModelRequestContributor: Send + Sync + std::fmt::Debug { + fn request(&self, input: ModelRequestInput<'_>) -> Option>; + } + ++/// Borrowed final request body, after transport-specific input adjustment. ++/// Headers and authentication are deliberately outside this view. ++pub enum ModelRequestContent<'a> { ++ Responses(&'a codex_api::ResponsesApiRequest), ++ WebSocket(&'a codex_api::ResponseCreateWsRequest<'a>), ++} ++ + /// Owns request state until consumed by the response stream or dropped on failure. + /// Interceptors compose in registration order; dropping their stream must release unfinished work. + /// Trusted host implementations must bound any model-visible content they synthesize. + pub trait ModelResponseInterceptor: Send + Sync { ++ /// Observe the final request without modifying it. Implementations must opt in, ++ /// bound any retained content, and return promptly; the default retains nothing. ++ fn observe_request(&mut self, _request: ModelRequestContent<'_>) {} ++ ++ /// The transport returned an error before opening a response stream. ++ fn request_failed(&mut self) {} ++ + /// Wrap the response stream, optionally buffering, inspecting, or transforming events. + /// The returned stream must own request state and propagate cancellation by dropping upstream. + fn intercept(self: Box, stream: ModelResponseStream) -> ModelResponseStream; diff --git a/native/codex/patches/02-approval-recovery.md b/native/codex/patches/02-approval-recovery.md new file mode 100644 index 0000000000..a2732c54d8 --- /dev/null +++ b/native/codex/patches/02-approval-recovery.md @@ -0,0 +1,25 @@ +# Resume a built-in command or edit after a worker stops + +Codex normally keeps a pending approval in memory. When the process stops, restoring conversation history cannot finish that action. Asking the model to propose it again could run a different command or repeat a write. + +[This patch](02-approval-recovery.patch) lets the SDK save one local `exec_command` or `apply_patch` before it runs. A replacement worker can submit that call directly to Codex without asking the model again. + +1. Create a session with `session = await ResumableSession.create(store)`. Pass it to `Runner.run(..., session=session)` and keep `session.session_id`. The SDK saves the pending call before returning the approval request. +2. On the replacement worker, load the same session with `session = await ResumableSession.resume(store, session_id)`. Then use `state = await session.load_state(agent)`. +3. Match the user's decision to the call in `state.get_interruptions()`. Use `state.approve(call)` or `state.reject(call)`, then `await Runner.run(agent, state, session=session, run_config=config)`. + +Only one worker can claim that session revision. Denial stops the call. On approval, Codex checks the workspace, permissions and newly prepared action. A mismatch stops execution. + +## Safety limits + +Recovery requires direct tools in one local environment, human review and no tool hooks. The SDK saves the model's submitted command or patch, so protect the session store. Codex replaces prepared update diffs with hashes and removes the original contents of deleted files. + +The patch includes a bounded file read because the existing stream can follow symbolic links. Before saving and again before execution, Codex checks file hashes and confirms that new destinations are absent. Each check covers at most 32 paths, 8 MiB per file, 16 MiB in total and 10 seconds. An unreadable or oversized file stops recovery. + +A restored call gets one attempt. When Codex policy already allows an approved edit to run outside the sandbox on retry, recovery uses that permission on the first attempt. + +The application must restore the same workspace and prevent concurrent edits. If the worker stops after a write and before saving the result, the session stays active. Check the external system before retrying. + +## Check and remove + +Follow the [patch guide](README.md) and run the approval tests. On another worker, approve and deny a saved command and edit. Check that a stale file, changed permissions or a duplicate approval causes no extra write. Remove the patch when upstream Codex can resume the same action with these checks. diff --git a/native/codex/patches/02-approval-recovery.patch b/native/codex/patches/02-approval-recovery.patch new file mode 100644 index 0000000000..da8d69d155 --- /dev/null +++ b/native/codex/patches/02-approval-recovery.patch @@ -0,0 +1,1436 @@ +diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock +index 1d27abb..f8972a1 100644 +--- a/codex-rs/Cargo.lock ++++ b/codex-rs/Cargo.lock +@@ -3051,6 +3051,7 @@ dependencies = [ + "serde_json", + "serial_test", + "sha1 0.10.6", ++ "sha2 0.10.9", + "shlex", + "similar", + "tempfile", +diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml +index 542abf9..5ffc5cf 100644 +--- a/codex-rs/core/Cargo.toml ++++ b/codex-rs/core/Cargo.toml +@@ -114,6 +114,7 @@ rmcp = { workspace = true, default-features = false, features = [ + serde = { workspace = true, features = ["derive"] } + serde_json = { workspace = true } + sha1 = { workspace = true } ++sha2 = { workspace = true } + shlex = { workspace = true } + similar = { workspace = true } + tempfile = { workspace = true } +diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs +index 968d331..a6a24d6 100644 +--- a/codex-rs/core/src/codex_thread.rs ++++ b/codex-rs/core/src/codex_thread.rs +@@ -45,6 +45,7 @@ use codex_protocol::protocol::AskForApproval; + use codex_protocol::protocol::EnvironmentConfig; + use codex_protocol::protocol::EnvironmentConfigState; + use codex_protocol::protocol::Event; ++use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::MultiAgentVersion; + use codex_protocol::protocol::Op; + use codex_protocol::protocol::SandboxPolicy; +@@ -474,6 +475,33 @@ impl CodexThread { + .await + } + ++ /// Replays one native call without model sampling, requiring the original human approval. ++ pub async fn restore_tool_call_if_idle( ++ &self, ++ call: ResponseItem, ++ expected_approval: EventMsg, ++ preconditions: serde_json::Value, ++ ) -> CodexResult { ++ let restore = crate::restore_tool_call::NativeApprovals::Restore(Arc::new( ++ crate::restore_tool_call::RestoreToolCall::new(call, expected_approval, preconditions)?, ++ )); ++ let mut init = self.config_snapshot().await.turn_extension_init; ++ init.insert(restore); ++ self.start_turn_if_idle(crate::WithTurnExtensionData::new( ++ TurnInputRequest::user_input(Vec::new()), ++ init, ++ )) ++ .await ++ } ++ ++ /// Returns the live approval without file preimages, paired with its verified preconditions. ++ pub async fn capture_tool_approval( ++ &self, ++ approval: EventMsg, ++ ) -> CodexResult<(EventMsg, serde_json::Value)> { ++ crate::restore_tool_call::capture(&self.session, approval).await ++ } ++ + /// Starts a regular turn only when the thread is idle. + /// + /// Core declines the input without recording or enqueueing it when idle +diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs +index 61400a1..925df52 100644 +--- a/codex-rs/core/src/lib.rs ++++ b/codex-rs/core/src/lib.rs +@@ -31,6 +31,8 @@ pub use codex_protocol::turn_input::TurnStartOptions; + pub use responses_metadata::CodexResponsesMetadata; + pub use turn_metadata::detached_memory_responses_metadata; + mod codex_thread; ++mod restore_tool_call; ++mod patch_preconditions; + mod turn_extension_data; + pub use turn_extension_data::WithTurnExtensionData; + mod compact_model_fallback; +diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs +new file mode 100644 +index 0000000..cfeb59d +--- /dev/null ++++ b/codex-rs/core/src/patch_preconditions.rs +@@ -0,0 +1,272 @@ ++//! Filesystem preconditions for restoring an interrupted native patch approval. ++ ++use crate::session::step_context::StepContext; ++use crate::tools::ApprovalContext; ++use codex_file_system::EnvironmentAccess; ++use codex_file_system::GetMetadataOptions; ++use codex_file_system::ReadFileOptions; ++use codex_protocol::error::CodexErr; ++use codex_protocol::error::Result as CodexResult; ++use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; ++use codex_protocol::protocol::FileChange; ++use codex_utils_path_uri::PathUri; ++use serde_json::Value; ++use sha2::Digest; ++use sha2::Sha256; ++use std::collections::BTreeMap; ++use std::collections::HashMap; ++use std::io::ErrorKind; ++use std::path::Component; ++use std::path::Path; ++use std::path::PathBuf; ++use std::time::Duration; ++ ++// Approval recovery fails closed for larger preimages instead of retaining large reads. ++pub(crate) const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; ++const MAX_PATCH_PATHS: usize = 32; ++pub(crate) const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; ++const MAX_PATCH_SCAN_DURATION: Duration = Duration::from_secs(10); ++ ++fn invalid(message: impl Into) -> CodexErr { ++ CodexErr::InvalidRequest(message.into()) ++} ++ ++pub(crate) async fn capture_patch_approval_preconditions( ++ context: &ApprovalContext, ++ changes: &HashMap, ++) -> CodexResult { ++ let review = &context.review_context; ++ let environment = review ++ .environments() ++ .single_local_environment() ++ .ok_or_else(|| invalid("Patch recovery requires one local environment"))?; ++ let grants = review ++ .turn() ++ .granted_permissions(&environment.selection.environment_id); ++ capture(&environment.fs_accessor(grants), changes, true).await ++} ++ ++pub(crate) async fn validate_patch_preconditions( ++ step: &StepContext, ++ approval: &ApplyPatchApprovalRequestEvent, ++ checkpoint: &Value, ++) -> CodexResult<()> { ++ let expected = validate_checkpoint(&approval.changes, checkpoint, true)?; ++ let check_deleted_content = checkpoint.get("version").and_then(Value::as_u64) == Some(1); ++ let environment = step ++ .environments ++ .single_local_environment() ++ .ok_or_else(|| invalid("Patch recovery requires one local environment"))?; ++ let observed = capture( ++ &step.fs_accessor(environment), ++ &approval.changes, ++ check_deleted_content, ++ ) ++ .await?; ++ if observed.get("files").and_then(Value::as_object) != Some(expected) { ++ return Err(invalid("Pending patch files have changed")); ++ } ++ Ok(()) ++} ++ ++fn patch_paths(changes: &HashMap) -> CodexResult> { ++ if changes.is_empty() { ++ return Err(invalid("A pending patch must include file changes")); ++ } ++ if changes.len() > MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } ++ let mut paths: BTreeMap<_, _> = changes ++ .iter() ++ .map(|(path, change)| (path, matches!(change, FileChange::Add { .. }))) ++ .collect(); ++ for change in changes.values() { ++ if let FileChange::Update { ++ move_path: Some(destination), ++ .. ++ } = change ++ { ++ if paths.len() == MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } ++ if paths.insert(destination, true).is_some() { ++ return Err(invalid("Patch move destinations must be distinct")); ++ } ++ } ++ } ++ Ok(paths) ++} ++ ++fn valid_digest(value: &str) -> bool { ++ value.len() == 64 ++ && value ++ .bytes() ++ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) ++} ++ ++/// Binds a live native deletion to its saved digest before redacting its content. ++/// No file I/O or content cloning occurs here; callers bound any other live fields separately. ++pub(crate) fn matches_delete_preimages( ++ changes: &HashMap, ++ checkpoint: &Value, ++) -> bool { ++ let Ok(files) = validate_checkpoint(changes, checkpoint, false) else { ++ return false; ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for change in changes.values() { ++ if let FileChange::Delete { content } = change { ++ let size = content.len() as u64; ++ if size > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return false; ++ } ++ remaining -= size; ++ } ++ } ++ changes.iter().all(|(path, change)| { ++ let FileChange::Delete { content } = change else { ++ return true; ++ }; ++ path.to_str() ++ .and_then(|path| files.get(path)) ++ .and_then(Value::as_str) ++ .is_some_and(|expected| digest(content.as_bytes()) == expected) ++ }) ++} ++ ++fn validate_checkpoint<'a>( ++ changes: &HashMap, ++ checkpoint: &'a Value, ++ require_redaction: bool, ++) -> CodexResult<&'a serde_json::Map> { ++ let paths = patch_paths(changes)?; ++ let root = checkpoint ++ .as_object() ++ .ok_or_else(|| invalid("Invalid patch approval preconditions"))?; ++ let files = root ++ .get("files") ++ .and_then(Value::as_object) ++ .ok_or_else(|| invalid("Invalid patch approval file map"))?; ++ if root.len() != 2 ++ || !matches!(root.get("version").and_then(Value::as_u64), Some(1 | 2)) ++ || files.len() > MAX_PATCH_PATHS ++ || files.len() != paths.len() ++ { ++ return Err(invalid("Invalid patch approval version or file count")); ++ } ++ for (path, absent) in paths { ++ let value = path ++ .to_str() ++ .and_then(|path| files.get(path)) ++ .ok_or_else(|| invalid("Patch approval file paths do not match the pending patch"))?; ++ if absent { ++ if !value.is_null() { ++ return Err(invalid( ++ "Pending patch destinations must have absent preimages", ++ )); ++ } ++ } else if !value.as_str().is_some_and(valid_digest) { ++ return Err(invalid("Invalid patch approval preimage digest")); ++ } ++ } ++ if require_redaction ++ && checkpoint.get("version").and_then(Value::as_u64) == Some(2) ++ && changes.values().any(|change| match change { ++ FileChange::Delete { content } => !content.is_empty(), ++ FileChange::Update { unified_diff, .. } => !valid_digest(unified_diff), ++ FileChange::Add { .. } => false, ++ }) ++ { ++ return Err(invalid("Invalid redacted patch changes")); ++ } ++ Ok(files) ++} ++ ++async fn capture( ++ fs: &impl EnvironmentAccess, ++ changes: &HashMap, ++ check_deleted_content: bool, ++) -> CodexResult { ++ let paths = patch_paths(changes)?; ++ tokio::time::timeout(MAX_PATCH_SCAN_DURATION, async { ++ let mut files = serde_json::Map::new(); ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for (path, absent) in paths { ++ let deleted = match changes.get(path) { ++ Some(FileChange::Delete { content }) if check_deleted_content => { ++ Some(content.as_bytes()) ++ } ++ _ => None, ++ }; ++ files.insert( ++ path.to_str() ++ .ok_or_else(|| invalid("Patch approval paths require UTF-8"))? ++ .to_owned(), ++ observe_file(fs, path, absent, &mut remaining, deleted) ++ .await? ++ .map_or(Value::Null, Value::String), ++ ); ++ } ++ Ok(serde_json::json!({"version": 1, "files": files})) ++ }) ++ .await ++ .map_err(|_| invalid("Pending patch scan exceeded the 10 second limit"))? ++} ++ ++fn digest(bytes: &[u8]) -> String { ++ format!("{:x}", Sha256::digest(bytes)) ++} ++ ++async fn observe_file( ++ fs: &impl EnvironmentAccess, ++ path: &Path, ++ must_be_absent: bool, ++ remaining: &mut u64, ++ deleted: Option<&[u8]>, ++) -> CodexResult> { ++ if !path.is_absolute() ++ || path ++ .components() ++ .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) ++ { ++ return Err(invalid("Patch paths must be absolute and normalized")); ++ } ++ // The selected local filesystem's no-follow operations reject symlinks in ++ // every component, including directory ancestors, without opening targets. ++ let uri = PathUri::from_host_native_path(path).map_err(|error| invalid(error.to_string()))?; ++ match fs ++ .get_metadata( ++ &uri, ++ GetMetadataOptions { ++ follow_symlinks: false, ++ }, ++ ) ++ .await ++ { ++ Err(error) if must_be_absent && error.kind() == ErrorKind::NotFound => Ok(None), ++ Err(error) => Err(invalid(format!( ++ "Cannot inspect pending patch file: {error}" ++ ))), ++ Ok(_) if must_be_absent => Err(invalid("A pending patch destination already exists")), ++ Ok(metadata) if !metadata.is_file || metadata.is_symlink => { ++ Err(invalid("Pending patch sources must be regular files")) ++ } ++ Ok(_) => { ++ let bytes = fs ++ .read_file_bounded( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false, ++ }, ++ MAX_PATCH_PREIMAGE_BYTES.min(*remaining), ++ ) ++ .await ++ .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; ++ *remaining -= bytes.len() as u64; ++ if deleted.is_some_and(|expected| expected != bytes.as_slice()) { ++ return Err(invalid("The pending deletion no longer matches the file")); ++ } ++ Ok(Some(digest(&bytes))) ++ } ++ } ++} +diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs +new file mode 100644 +index 0000000..a20cd34 +--- /dev/null ++++ b/codex-rs/core/src/restore_tool_call.rs +@@ -0,0 +1,516 @@ ++//! Model-free replay of one native call, requiring the same fresh human approval. ++ ++use std::collections::HashMap; ++use std::sync::{Arc, Mutex, OnceLock, Weak}; ++ ++use codex_protocol::approvals::ExecApprovalKind; ++use codex_protocol::config_types::ApprovalsReviewer; ++use codex_protocol::error::{CodexErr, Result as CodexResult}; ++use codex_protocol::models::ResponseItem; ++use codex_protocol::openai_models::ToolMode; ++use codex_protocol::protocol::{EventMsg, FileChange, ReviewDecision}; ++use serde_json::{Value, json}; ++use sha2::{Digest, Sha256}; ++use tokio_util::sync::CancellationToken; ++ ++use crate::patch_preconditions::{MAX_PATCH_PREIMAGE_BYTES, MAX_PATCH_SCAN_BYTES}; ++use crate::session::session::Session; ++use crate::session::step_context::StepContext; ++use crate::session::turn_context::TurnContext; ++use crate::state::TaskKind; ++use crate::tools::ApprovalContext; ++use crate::tools::parallel::ToolCallRuntime; ++use crate::tools::router::ToolRouter; ++use crate::tools::sandboxing::ApprovalAction; ++ ++fn refusal(message: &str) -> CodexErr { ++ CodexErr::InvalidRequest(format!("Native tool restoration refused: {message}")) ++} ++ ++#[derive(Debug)] ++pub(crate) struct CapturableApproval { ++ event: OnceLock, ++ permissions: Value, ++ patch: Option, ++} ++ ++/// Capture and replay are mutually exclusive modes of a turn, never separate authorities. ++#[derive(Debug)] ++pub(crate) enum NativeApprovals { ++ Capture(Mutex>>), ++ Restore(Arc), ++} ++ ++impl NativeApprovals { ++ pub(crate) fn restoration(&self) -> Option> { ++ match self { ++ NativeApprovals::Restore(restore) => Some(Arc::clone(restore)), ++ NativeApprovals::Capture(_) => None, ++ } ++ } ++} ++ ++impl crate::WithTurnExtensionData { ++ /// Enables pre-execution native approval checkpoint capture for SDK-owned turns. ++ pub fn capture_native_approvals(mut self) -> Self { ++ self.turn_extension_init ++ .get_or_insert_with(Default::default) ++ .insert(NativeApprovals::Capture(Default::default())); ++ self ++ } ++} ++ ++fn permission_snapshot( ++ turn: &TurnContext, ++ environments: &crate::environment_selection::TurnEnvironmentSnapshot, ++ policy: codex_protocol::protocol::AskForApproval, ++ reviewer: ApprovalsReviewer, ++) -> Value { ++ let environments: Vec<_> = environments ++ .turn_environments() ++ .map(|environment| { ++ json!({ ++ "id": environment.selection.environment_id, ++ "profile": environment.permission_profile_with_workspace_roots(), ++ "symbolic_profile": environment.permission_profile(), ++ "network": environment.config().network_policy, ++ "grants": turn.granted_permissions(&environment.selection.environment_id), ++ }) ++ }) ++ .collect(); ++ let mut snapshot = json!({ ++ "approval_policy": policy, "reviewer": reviewer, "environments": environments, ++ "managed_network": turn.config.permissions.network.as_ref().map(|network| network.environment_policy()), ++ }); ++ snapshot.sort_all_objects(); ++ json!(format!( ++ "{:x}", ++ Sha256::digest(snapshot.to_string().as_bytes()) ++ )) ++} ++ ++pub(crate) async fn capture_lease( ++ action: &ApprovalAction, ++ ctx: &ApprovalContext, ++) -> Option> { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || !matches!( ++ action, ++ ApprovalAction::ExecCommand { .. } | ApprovalAction::ApplyPatch { .. } ++ ) ++ { ++ return None; ++ } ++ let state = ctx ++ .review_context ++ .turn() ++ .extension_data ++ .get::()?; ++ let NativeApprovals::Capture(pending) = state.as_ref() else { ++ return None; ++ }; ++ let patch = if let ApprovalAction::ApplyPatch { changes, .. } = action { ++ Some( ++ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, changes) ++ .await ++ .ok()?, ++ ) ++ } else { ++ None ++ }; ++ let lease = Arc::new(CapturableApproval { ++ event: OnceLock::new(), ++ patch, ++ permissions: permission_snapshot( ++ ctx.review_context.turn(), ++ ctx.review_context.environments(), ++ ctx.review_context.approval_policy, ++ ctx.review_context.approvals_reviewer, ++ ), ++ }); ++ // Only the live request owns the lease. Reused IDs stay invalid for the whole turn. ++ let mut pending = pending.lock().unwrap(); ++ if let Some(previous) = pending.get_mut(&ctx.call_id) { ++ *previous = Weak::new(); ++ return None; ++ } ++ pending.insert(ctx.call_id.clone(), Arc::downgrade(&lease)); ++ Some(lease) ++} ++ ++fn approval_id(event: &EventMsg) -> Option<&str> { ++ match event { ++ EventMsg::ExecApprovalRequest(event) ++ if event.kind == ExecApprovalKind::Command ++ && event.approval_id.is_none() ++ && event.network_approval_context.is_none() => ++ { ++ Some(&event.call_id) ++ } ++ EventMsg::ApplyPatchApprovalRequest(event) if event.grant_root.is_none() => { ++ Some(&event.call_id) ++ } ++ _ => None, ++ } ++} ++ ++fn comparable(event: &EventMsg) -> CodexResult { ++ let mut value = serde_json::to_value(event).map_err(|_| refusal("invalid approval event"))?; ++ let object = value ++ .as_object_mut() ++ .ok_or_else(|| refusal("invalid approval event"))?; ++ object.remove("turn_id"); ++ object.remove("started_at_ms"); ++ Ok(value) ++} ++ ++fn record_capture_event(turn: &TurnContext, event: &EventMsg) { ++ let Some(id) = approval_id(event) else { return }; ++ let Some(state) = turn.extension_data.get::() else { ++ return; ++ }; ++ if let NativeApprovals::Capture(pending) = state.as_ref() { ++ if let Some(slot) = pending.lock().unwrap().get_mut(id) ++ && let Some(approval) = slot.upgrade() ++ && !serde_json::to_value(event).is_ok_and(|value| approval.event.set(value).is_ok()) ++ { ++ *slot = Weak::new(); ++ } ++ } ++} ++ ++#[derive(Debug, Default, PartialEq)] ++enum Phase { ++ #[default] ++ Ready, ++ Running, ++ Requested, ++ Approved, ++ Finished, ++} ++ ++#[derive(Debug, Default)] ++struct Progress { ++ owner: Option, ++ phase: Phase, ++} ++ ++#[derive(Debug)] ++pub(crate) struct RestoreToolCall { ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Value, ++ progress: Mutex, ++} ++ ++impl RestoreToolCall { ++ pub(crate) fn new( ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Value, ++ ) -> CodexResult { ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let (namespace, call_id) = match (&call, &expected) { ++ ( ++ ResponseItem::FunctionCall { ++ name, ++ namespace, ++ call_id, ++ .. ++ }, ++ EventMsg::ExecApprovalRequest(_), ++ ) if name == "exec_command" => (namespace, call_id), ++ ( ++ ResponseItem::CustomToolCall { ++ name, ++ namespace, ++ call_id, ++ .. ++ }, ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ) if name == "apply_patch" => (namespace, call_id), ++ _ => return Err(refusal("unsupported native action")), ++ }; ++ if call_id != id || namespace.as_deref().is_some_and(|ns| ns != "functions") { ++ return Err(refusal( ++ "only one direct exec_command or apply_patch call is supported", ++ )); ++ } ++ if checkpoint.get("version") != Some(&json!(1)) ++ || checkpoint.get("first_approval") != Some(&json!(true)) ++ { ++ return Err(refusal("checkpoint does not match the original approval")); ++ } ++ Ok(Self { ++ call, ++ expected, ++ checkpoint, ++ progress: Mutex::new(Progress::default()), ++ }) ++ } ++ ++ pub(crate) fn accepts_approval(&self, action: &ApprovalAction, ctx: &ApprovalContext) -> bool { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || ctx.strict_auto_review ++ || self.progress.lock().unwrap().phase != Phase::Running ++ || approval_id(&self.expected) != Some(ctx.call_id.as_str()) ++ { ++ return false; ++ } ++ matches!( ++ (&self.expected, action), ++ ( ++ EventMsg::ExecApprovalRequest(_), ++ ApprovalAction::ExecCommand { .. } ++ ) | ( ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ApprovalAction::ApplyPatch { .. } ++ ) ++ ) ++ } ++ ++ pub(crate) fn is_patch(&self) -> bool { ++ matches!(self.expected, EventMsg::ApplyPatchApprovalRequest(_)) ++ } ++ ++ pub(crate) fn claim(&self, turn: &TurnContext) -> bool { ++ let mut progress = self.progress.lock().unwrap(); ++ if progress.phase != Phase::Ready { ++ return false; ++ } ++ progress.owner = Some(turn.sub_id.clone()); ++ progress.phase = Phase::Running; ++ true ++ } ++} ++ ++pub(crate) fn active(turn: &TurnContext) -> Option> { ++ turn.extension_data ++ .get::() ++ .and_then(|state| state.restoration()) ++ .filter(|restore| restore.progress.lock().unwrap().owner.as_ref() == Some(&turn.sub_id)) ++} ++ ++async fn validate_step(session: &Arc, step: &StepContext) -> CodexResult<()> { ++ let config = session.get_config().await; ++ let requirements = config.config_layer_stack.requirements(); ++ if requirements.auto_review_required_for_model(&step.settings.model_info.slug) ++ || requirements ++ .approvals_reviewer ++ .can_set(&ApprovalsReviewer::User) ++ .is_err() ++ { ++ return Err(refusal("managed automatic review is unsupported")); ++ } ++ if step.tool_router.tool_mode() != ToolMode::Direct ++ || step.environments.single_local_environment().is_none() ++ { ++ return Err(refusal( ++ "only direct tools in one local environment are supported", ++ )); ++ } ++ if step.settings.approvals_reviewer() != ApprovalsReviewer::User ++ || step.turn.strict_auto_review_enabled() ++ { ++ return Err(refusal("automatic approval review is unsupported")); ++ } ++ let hooks = session.hooks().list_with_environment_hooks( ++ crate::hook_environment_executor::environment_hook_sources(&step.environments, &[]), ++ ); ++ if !hooks.hooks.is_empty() ++ || !hooks.warnings.is_empty() ++ || !step.executor_hook_sources.is_empty() ++ { ++ return Err(refusal("configured hooks are unsupported")); ++ } ++ if let Some(restore) = active(&step.turn) { ++ if restore.checkpoint.get("permissions") ++ != Some(&permission_snapshot( ++ &step.turn, ++ &step.environments, ++ step.settings.approval_policy(), ++ step.settings.approvals_reviewer(), ++ )) ++ { ++ return Err(refusal("permissions or approval policy changed")); ++ } ++ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { ++ crate::patch_preconditions::validate_patch_preconditions( ++ step, ++ event, ++ &restore.checkpoint["patch"], ++ ) ++ .await?; ++ } ++ } ++ Ok(()) ++} ++ ++// Prepared deletes include the complete file. Prepared diffs can include neighboring ++// lines absent from the submitted patch. Keep those bytes out of the SDK checkpoint. ++fn compact_patch(event: &mut EventMsg) -> CodexResult { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return Ok(false); ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ let mut compacted = false; ++ for change in request.changes.values_mut() { ++ match change { ++ FileChange::Add { .. } => continue, ++ FileChange::Delete { content } => { ++ // The trusted file snapshot already binds the entire deletion. ++ *content = String::new(); ++ } ++ FileChange::Update { unified_diff, .. } => { ++ let bytes = unified_diff.len() as u64; ++ if bytes > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return Err(refusal("prepared patch diff exceeds the recovery limit")); ++ } ++ remaining -= bytes; ++ *unified_diff = format!("{:x}", Sha256::digest(unified_diff.as_bytes())); ++ } ++ } ++ compacted = true; ++ } ++ Ok(compacted) ++} ++ ++pub(crate) async fn capture( ++ session: &Arc, ++ mut expected: EventMsg, ++) -> CodexResult<(EventMsg, Value)> { ++ let (turn, cancellation) = session ++ .active_turn_context_and_cancellation_token() ++ .await ++ .ok_or_else(|| refusal("no active approval"))?; ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let state = turn ++ .extension_data ++ .get::() ++ .ok_or_else(|| refusal("not a first pre-execution approval"))?; ++ let step = session.capture_step_context(turn, &cancellation).await?; ++ validate_step(session, &step).await?; ++ let NativeApprovals::Capture(pending) = state.as_ref() else { ++ return Err(refusal("the current turn is restoring an action")); ++ }; ++ let approvals = pending.lock().unwrap(); ++ let approval = approvals ++ .get(id) ++ .and_then(Weak::upgrade) ++ .ok_or_else(|| refusal("not a pending first pre-execution approval"))?; ++ // The live event is emitted by the same request_approval invocation holding this lease. ++ if approval.event.get() ++ != Some(&serde_json::to_value(&expected).map_err(|_| refusal("invalid approval event"))?) ++ { ++ return Err(refusal("approval does not match the live request")); ++ } ++ let mut checkpoint = json!({"version":1,"first_approval":true,"permissions":approval.permissions,"patch":approval.patch}); ++ if compact_patch(&mut expected)? { ++ checkpoint["patch"]["version"] = json!(2); ++ } ++ Ok((expected, checkpoint)) ++} ++ ++pub(crate) fn compare_request(turn: &TurnContext, event: &mut EventMsg) -> bool { ++ record_capture_event(turn, event); ++ let Some(restore) = active(turn) else { ++ return true; ++ }; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Running { ++ return false; ++ } ++ // A mismatch consumes the request as well: it cannot be corrected and retried. ++ progress.phase = Phase::Finished; ++ if approval_id(event) != approval_id(&restore.expected) { ++ return false; ++ } ++ // Version 1 keeps comparing the original request verbatim, including any ++ // digest-looking text. Only v2 uses a compact request and file fingerprints. ++ let patch = &restore.checkpoint["patch"]; ++ if patch["version"] == 2 { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return false; ++ }; ++ if !crate::patch_preconditions::matches_delete_preimages(&request.changes, patch) ++ || !matches!(compact_patch(event), Ok(true)) ++ { ++ return false; ++ } ++ } ++ let matches = matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected); ++ if matches { ++ progress.phase = Phase::Requested; ++ } ++ matches ++} ++ ++pub(crate) fn record_decision(turn: &TurnContext, decision: ReviewDecision) -> ReviewDecision { ++ let Some(restore) = active(turn) else { ++ return decision; ++ }; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Requested { ++ return ReviewDecision::Abort; ++ } ++ progress.phase = if decision == ReviewDecision::Approved { ++ Phase::Approved ++ } else { ++ Phase::Finished ++ }; ++ match decision { ++ ReviewDecision::Approved | ReviewDecision::Denied { .. } | ReviewDecision::Abort => { ++ decision ++ } ++ _ => ReviewDecision::Abort, ++ } ++} ++ ++pub(crate) async fn validate_before_execution( ++ session: &Arc, ++ step: &StepContext, ++) -> CodexResult<()> { ++ let Some(restore) = active(&step.turn) else { ++ return Ok(()); ++ }; ++ validate_step(session, step).await?; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Approved { ++ return Err(refusal( ++ "a matching approval permits only one execution attempt", ++ )); ++ } ++ progress.phase = Phase::Finished; ++ Ok(()) ++} ++ ++pub(crate) async fn run( ++ session: Arc, ++ turn: Arc, ++ restore: Arc, ++ cancellation: CancellationToken, ++) -> crate::tasks::SessionTaskResult { ++ if !restore.claim(&turn) { ++ return Err(refusal("the native action was already claimed")); ++ } ++ session.emit_turn_started(&turn, TaskKind::Regular).await; ++ let step = session ++ .capture_step_context(Arc::clone(&turn), &cancellation) ++ .await?; ++ validate_step(&session, &step).await?; ++ let call = ToolRouter::build_tool_call(restore.call.clone()) ++ .map_err(|_| refusal("invalid native call"))? ++ .ok_or_else(|| refusal("unsupported native call"))?; ++ if !step.tool_router.exposes_tool(&call.tool_name) { ++ return Err(refusal("tool is unavailable in the replacement worker")); ++ } ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &restore.call) ++ .await; ++ let result = ToolCallRuntime::new(Arc::clone(&session), Arc::clone(&step), Default::default()) ++ .handle_tool_call(call, cancellation) ++ .await?; ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &result.item).await; ++ Ok(None) ++} +diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs +index cc418ca..b767210 100644 +--- a/codex-rs/core/src/session/mod.rs ++++ b/codex-rs/core/src/session/mod.rs +@@ -2921,7 +2921,7 @@ impl Session { + .map(|task| Arc::clone(&task.turn_context)) + } + +- async fn active_turn_context_and_cancellation_token( ++ pub(crate) async fn active_turn_context_and_cancellation_token( + &self, + ) -> Option<(Arc, CancellationToken)> { + let active = self.active_turn.lock().await; +@@ -3146,7 +3146,7 @@ impl Session { + .as_ref() + .map(PluginCommandAttribution::serialized_fields) + .unzip(); +- let event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { ++ let mut event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + model_context: Some(model_context), + kind, + call_id, +@@ -3166,9 +3166,15 @@ impl Session { + available_decisions: Some(available_decisions), + parsed_cmd, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +@@ -3202,7 +3208,7 @@ impl Session { + warn!("Overwriting existing pending approval for call_id: {approval_id}"); + } + +- let event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { ++ let mut event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { + call_id, + turn_id: turn_context.sub_id.clone(), + started_at_ms: now_unix_timestamp_ms(), +@@ -3210,9 +3216,15 @@ impl Session { + reason, + grant_root, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs +index 6c6f3d1..bc3c888 100644 +--- a/codex-rs/core/src/tasks/regular.rs ++++ b/codex-rs/core/src/tasks/regular.rs +@@ -44,6 +44,13 @@ impl SessionTask for RegularTask { + input: Vec, + cancellation_token: CancellationToken, + ) -> SessionTaskResult { ++ if let Some(restore) = ctx ++ .extension_data ++ .get::() ++ .and_then(|state| state.restoration()) ++ { ++ return crate::restore_tool_call::run(sess, ctx, restore, cancellation_token).await; ++ } + if ctx + .config + .features +diff --git a/codex-rs/core/src/tools/approvals.rs b/codex-rs/core/src/tools/approvals.rs +index 74082aa..ebaa126 100644 +--- a/codex-rs/core/src/tools/approvals.rs ++++ b/codex-rs/core/src/tools/approvals.rs +@@ -499,6 +499,18 @@ impl Session { + action: ApprovalAction, + ctx: ApprovalContext, + ) -> ApprovalResult { ++ if let Some(restore) = crate::restore_tool_call::active(ctx.review_context.turn()) { ++ if !restore.accepts_approval(&action, &ctx) { ++ return ApprovalResult { ++ result: Err(ToolError::Rejected( ++ "native restoration cannot request another approval or escalation".into(), ++ )), ++ source: None, ++ cancellation_reason: None, ++ }; ++ } ++ } ++ let _capture_lease = crate::restore_tool_call::capture_lease(&action, &ctx).await; + // Retained authority outside current grants needs a new sandbox approval. + // Current grants permit review of the input; they never approve the input itself. + let policy = ctx.review_context.turn().approval_policy(); +@@ -723,7 +735,7 @@ impl Session { + .into_iter() + .map(|key| (key, &policy_fingerprint)) + .collect(); +- with_cached_approval(&self.services, tool_name, cache_keys, || async { ++ let request = || async { + self.request_command_approval( + ctx.review_context.turn(), + ExecApprovalKind::Command, +@@ -742,8 +754,12 @@ impl Session { + /*plugin_attribution_override*/ None, + ) + .await +- }) +- .await ++ }; ++ if crate::restore_tool_call::active(ctx.review_context.turn()).is_some() { ++ request().await ++ } else { ++ with_cached_approval(&self.services, tool_name, cache_keys, request).await ++ } + } + ApprovalAction::WriteStdin { + id, +@@ -788,9 +804,11 @@ impl Session { + .retry_reason + .clone() + .or_else(|| ctx.approval_reason.clone()); +- if *permissions_preapproved && reason.is_none() { ++ let restoring = ++ crate::restore_tool_call::active(ctx.review_context.turn()).is_some(); ++ if *permissions_preapproved && reason.is_none() && !restoring { + ReviewDecision::Approved +- } else if reason.is_some() { ++ } else if reason.is_some() || restoring { + self.request_patch_approval( + ctx.review_context.turn(), + ctx.call_id.clone(), +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index a1a88e4..c26c85d 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -61,6 +61,14 @@ impl ToolOrchestrator { + where + T: ToolRuntime, + { ++ if let Err(error) = crate::restore_tool_call::validate_before_execution( ++ &tool_ctx.session, ++ &tool_ctx.step_context, ++ ) ++ .await ++ { ++ return (Err(ToolError::Codex(error)), None); ++ } + let network_approval = match begin_network_approval( + &tool_ctx.session, + &tool_ctx.step_context.turn, +@@ -164,6 +172,13 @@ impl ToolOrchestrator { + let requirement = tool.exec_approval_requirement(req).unwrap_or_else(|| { + default_exec_approval_requirement(approval_policy, &file_system_sandbox_policy) + }); ++ if crate::restore_tool_call::active(turn_ctx).is_some() ++ && !matches!(requirement, ExecApprovalRequirement::NeedsApproval { .. }) ++ { ++ return Err(ToolError::Rejected( ++ "native restoration requires a fresh human approval".into(), ++ )); ++ } + match &requirement { + ExecApprovalRequirement::Skip { .. } => { + if strict_auto_review { +@@ -227,12 +242,24 @@ impl ToolOrchestrator { + + // 2) First attempt under the selected sandbox. + let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); +- let sandbox_override = sandbox_override_for_first_attempt( +- tool.sandbox_permissions(req), +- &requirement, +- &file_system_sandbox_policy, +- /*already_approved*/ initial_approval_source.is_some(), +- ); ++ // Restored patches get one attempt. Use only authority the normal approved retry already grants. ++ let approved_restored_patch = crate::restore_tool_call::active(turn_ctx) ++ .is_some_and(|restore| restore.is_patch()) ++ && initial_approval_source == Some(ApprovalResolutionSource::User) ++ && !strict_auto_review ++ && unsandboxed_allowed ++ && tool.wants_no_sandbox_approval(approval_policy) ++ && tool.should_bypass_approval(approval_policy, true); ++ let sandbox_override = if approved_restored_patch { ++ SandboxOverride::BypassSandboxFirstAttempt ++ } else { ++ sandbox_override_for_first_attempt( ++ tool.sandbox_permissions(req), ++ &requirement, ++ &file_system_sandbox_policy, ++ /*already_approved*/ initial_approval_source.is_some(), ++ ) ++ }; + let (escalated_profile, writable_root_metadata) = if sandbox_override + == SandboxOverride::EscalatedSandboxWithRestrictions + { +diff --git a/codex-rs/exec-server/src/fs_helper.rs b/codex-rs/exec-server/src/fs_helper.rs +index d307663..e98a391 100644 +--- a/codex-rs/exec-server/src/fs_helper.rs ++++ b/codex-rs/exec-server/src/fs_helper.rs +@@ -71,6 +71,12 @@ pub(crate) enum FsHelperRequest { + Open(FsHelperOpenParams), + #[serde(rename = "fs/readFile")] + ReadFile(FsReadFileParams), ++ #[serde(rename = "fs/readFileBounded")] ++ ReadFileBounded { ++ path: PathUri, ++ follow_symlinks: bool, ++ max_bytes: u64, ++ }, + #[serde(rename = "fs/writeFile")] + WriteFile(FsWriteFileParams), + #[serde(rename = "fs/createDirectory")] +@@ -311,6 +317,19 @@ pub(crate) async fn run_direct_request( + data_base64: STANDARD.encode(data), + })) + } ++ FsHelperRequest::ReadFileBounded { ++ path, ++ follow_symlinks, ++ max_bytes, ++ } => { ++ let data = file_system ++ .read_file_bounded(&path, ReadFileOptions { follow_symlinks }, max_bytes, None) ++ .await ++ .map_err(map_fs_error)?; ++ Ok(FsHelperPayload::ReadFile(FsReadFileResponse { ++ data_base64: STANDARD.encode(data), ++ })) ++ } + FsHelperRequest::WriteFile(params) => { + let bytes = STANDARD.decode(params.data_base64).map_err(|err| { + invalid_request(format!( +diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs +index abe2a0a..b721a69 100644 +--- a/codex-rs/exec-server/src/local_file_system.rs ++++ b/codex-rs/exec-server/src/local_file_system.rs +@@ -304,6 +304,20 @@ impl ExecutorFileSystem for LocalFileSystem { + Box::pin(LocalFileSystem::read_file(self, path, options, sandbox)) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ let (fs, sandbox) = self.file_system_for_reads(sandbox)?; ++ fs.read_file_bounded(path, options, max_bytes, sandbox) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +@@ -519,6 +533,21 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { + )) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ reject_platform_sandbox_context(sandbox)?; ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, None) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +@@ -630,15 +659,16 @@ impl DirectFileSystem { + Ok(PathUri::from_abs_path(&canonicalized)) + } + +- async fn read_file( ++ async fn read_bytes( + &self, + path: &PathUri, + options: ReadFileOptions, ++ max_bytes: Option, + sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; +- crate::local_file_system_read::read_file(path, options).await ++ crate::local_file_system_read::read_file(path, options, max_bytes).await + } + + async fn read_file_stream( +@@ -995,7 +1025,25 @@ impl ExecutorFileSystem for DirectFileSystem { + options: ReadFileOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { +- Box::pin(DirectFileSystem::read_file(self, path, options, sandbox)) ++ Box::pin(DirectFileSystem::read_bytes( ++ self, path, options, None, sandbox, ++ )) ++ } ++ ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(DirectFileSystem::read_bytes( ++ self, ++ path, ++ options, ++ Some(max_bytes), ++ sandbox, ++ )) + } + + fn read_file_stream<'a>( +diff --git a/codex-rs/exec-server/src/local_file_system_read.rs b/codex-rs/exec-server/src/local_file_system_read.rs +index e9bd05e..80ce9ae 100644 +--- a/codex-rs/exec-server/src/local_file_system_read.rs ++++ b/codex-rs/exec-server/src/local_file_system_read.rs +@@ -15,7 +15,11 @@ const MAX_READ_FILE_BYTES: u64 = 512 * 1024 * 1024; + pub(super) async fn read_file( + path: AbsolutePathBuf, + options: ReadFileOptions, ++ max_bytes: Option, + ) -> io::Result> { ++ let max_bytes = max_bytes ++ .unwrap_or(MAX_READ_FILE_BYTES) ++ .min(MAX_READ_FILE_BYTES); + let cancelled = CancellationToken::new(); + let _cancel_on_drop = cancelled.clone().drop_guard(); + tokio::task::spawn_blocking(move || { +@@ -28,24 +32,33 @@ pub(super) async fn read_file( + no_follow::open_file(path.as_path())? + }; + let metadata = file.metadata()?; +- if metadata.len() > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- let mut bytes = Vec::with_capacity(metadata.len() as usize); +- CancellableReader { +- reader: file.take(MAX_READ_FILE_BYTES + 1), +- cancelled, +- } +- .read_to_end(&mut bytes)?; +- if bytes.len() as u64 > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- Ok(bytes) ++ read_with_limit(file, metadata.len(), max_bytes, cancelled) + }) + .await + .map_err(|error| io::Error::other(format!("filesystem task failed: {error}")))? + } + ++fn read_with_limit( ++ reader: impl Read, ++ initial_len: u64, ++ max_bytes: u64, ++ cancelled: CancellationToken, ++) -> io::Result> { ++ if initial_len > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ let mut bytes = Vec::with_capacity(initial_len as usize); ++ CancellableReader { ++ reader: reader.take(max_bytes + 1), ++ cancelled, ++ } ++ .read_to_end(&mut bytes)?; ++ if bytes.len() as u64 > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ Ok(bytes) ++} ++ + struct CancellableReader { + reader: R, + cancelled: CancellationToken, +@@ -62,10 +75,10 @@ impl Read for CancellableReader { + } + } + +-fn file_too_large_error() -> io::Error { ++fn file_too_large_error(max_bytes: u64) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, +- format!("file is too large to read: limit is {MAX_READ_FILE_BYTES} bytes"), ++ format!("file is too large to read: limit is {max_bytes} bytes"), + ) + } + +diff --git a/codex-rs/exec-server/src/sandboxed_file_system.rs b/codex-rs/exec-server/src/sandboxed_file_system.rs +index f592edf..7538822 100644 +--- a/codex-rs/exec-server/src/sandboxed_file_system.rs ++++ b/codex-rs/exec-server/src/sandboxed_file_system.rs +@@ -139,10 +139,11 @@ impl SandboxedFileSystem { + Ok(response.path) + } + +- async fn read_file( ++ async fn read_bytes( + &self, + path: &PathUri, + options: ReadFileOptions, ++ max_bytes: Option, + sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + let sandbox = require_platform_sandbox(sandbox)?; +@@ -150,11 +151,18 @@ impl SandboxedFileSystem { + let response = self + .run_sandboxed( + sandbox, +- FsHelperRequest::ReadFile(FsReadFileParams { +- path: path.clone(), +- follow_symlinks: (!options.follow_symlinks).then_some(false), +- sandbox: None, +- }), ++ match max_bytes { ++ Some(max_bytes) => FsHelperRequest::ReadFileBounded { ++ path: path.clone(), ++ follow_symlinks: options.follow_symlinks, ++ max_bytes, ++ }, ++ None => FsHelperRequest::ReadFile(FsReadFileParams { ++ path: path.clone(), ++ follow_symlinks: (!options.follow_symlinks).then_some(false), ++ sandbox: None, ++ }), ++ }, + ) + .await? + .expect_read_file() +@@ -361,7 +369,25 @@ impl ExecutorFileSystem for SandboxedFileSystem { + options: ReadFileOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { +- Box::pin(SandboxedFileSystem::read_file(self, path, options, sandbox)) ++ Box::pin(SandboxedFileSystem::read_bytes( ++ self, path, options, None, sandbox, ++ )) ++ } ++ ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(SandboxedFileSystem::read_bytes( ++ self, ++ path, ++ options, ++ Some(max_bytes), ++ sandbox, ++ )) + } + + fn read_file_stream<'a>( +diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs +index 06f63ed..a1f7159 100644 +--- a/codex-rs/file-system/src/environment_accessor.rs ++++ b/codex-rs/file-system/src/environment_accessor.rs +@@ -40,6 +40,21 @@ pub trait EnvironmentAccess: Send + Sync { + options: ReadFileOptions, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads a file with a hard byte limit, preserving this accessor's permissions. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(std::io::Error::new( ++ std::io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Opens an owned stream; subsequent reads do not borrow or reauthorize through the accessor. + fn read_file_stream<'a>( + &'a self, +@@ -212,6 +227,16 @@ impl EnvironmentAccess for FileSystemEnvironmentAccessor<'_> { + .read_file(path, options, self.key.sandbox.as_deref()) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, self.key.sandbox.as_deref()) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs +index 9e032e5..be5af03 100644 +--- a/codex-rs/file-system/src/lib.rs ++++ b/codex-rs/file-system/src/lib.rs +@@ -667,6 +667,23 @@ pub trait ExecutorFileSystem: Send + Sync { + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads at most `max_bytes + 1` bytes, rejecting files larger than `max_bytes`. ++ /// Backends must enforce the bound during I/O rather than truncate a full read. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ _sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(io::Error::new( ++ io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Reads a file as a stream of chunks no larger than [`FILE_READ_CHUNK_SIZE`]. + fn read_file_stream<'a>( + &'a self, diff --git a/native/codex/patches/README.md b/native/codex/patches/README.md new file mode 100644 index 0000000000..9c8725c12f --- /dev/null +++ b/native/codex/patches/README.md @@ -0,0 +1,34 @@ +# Changes to Codex in the native SDK wheel + +The SDK pins Codex in [Cargo.toml](../Cargo.toml). The wheel builder copies that revision into a temporary directory and applies these changes in filename order. It leaves the Cargo cache and any installed Codex CLI alone. + +- [Detailed tracing](01-operation-tracing.md) observes the final model request and the start of tool execution. +- [Approval recovery](02-approval-recovery.md) lets a new worker resume a built-in command or edit after a human decision. + +## Rules for every patch + +Use a patch only for a capability the pinned Codex APIs cannot provide. Keep it as small as possible. Put SDK behavior and tests in ordinary source and test files. + +Pair each `.patch` with a Markdown guide of the same name. Explain why the change is necessary, what it does, what can go wrong, how to test it and when to remove it. The build rejects missing or empty guides. The wheel includes each patch and guide in `agents/models/_codex_bin/licenses/codex-patches/`. + +## Test a change + +From the SDK repository root, use Python 3.12, uv, Git and the [native build prerequisites](../../../.github/workflows/codex-preview.yml). Build on the target machine. On Linux, install `patchelf` or add `--compatibility linux` for a local build. + +```sh +uv sync --python 3.12 +uv pip install --require-hashes -r native/codex/build-requirements.txt +uv run --no-sync python scripts/build_codex_wheel.py --profile dev --test-native --out dist/local +``` + +`--test-native` runs the SDK's Rust tests against the same source as the wheel. CI enables it for matrix entries with `test_native: true`. Every platform also runs the [installed-wheel checks](../verify_installed.py). + +To check a local wheel on Linux or macOS, start with an empty `dist/local` and install the result in a fresh environment. + +```sh +uv venv --python 3.12 .venv-wheel +uv pip install --python .venv-wheel/bin/python dist/local/*.whl +.venv-wheel/bin/python -I native/codex/verify_installed.py +``` + +When updating the Codex pin, remove each patch whose behavior is now available upstream. Rerun the same checks. diff --git a/native/codex/src/lib.rs b/native/codex/src/lib.rs index 553efea382..5cb2d6dd6b 100644 --- a/native/codex/src/lib.rs +++ b/native/codex/src/lib.rs @@ -1,5 +1,6 @@ //! Private, asynchronous Core bindings for the opt-in Codex model provider. mod config; +mod native_approval; mod operation_tracing; mod tool_hooks; @@ -9,6 +10,7 @@ use std::sync::{Arc, Once, OnceLock}; use codex_core_api::{ CodexThread, StartIfIdleSubmission, StartThreadOptions, TurnInputRequest, TurnStartOptions, + WithTurnExtensionData, }; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::normalize_dynamic_tool_specs; @@ -399,12 +401,21 @@ struct Thread { #[pymethods] impl Thread { - fn enable_tracing(&self) -> PyResult<()> { + #[pyo3(signature=(mode=None))] + fn enable_tracing(&self, mode: Option<&str>) -> PyResult<()> { self.lease.ensure_open().map_err(py_error)?; self.lease .operations - .enable(self.lease.thread.thread_extension_data()); - Ok(()) + .enable(self.lease.thread.thread_extension_data(), mode) + .map_err(|message| py_error(message.to_owned())) + } + + fn wait_for_operations<'py>(&self, py: Python<'py>) -> PyResult> { + let buffer = self.lease.operations.buffer.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + buffer.updates.notified().await; + Ok(()) + }) } fn drain_operations(&self) -> Option { @@ -444,11 +455,13 @@ impl Thread { }) } + #[pyo3(signature=(input_json, output_schema_json, recover_approvals=false))] fn start_turn<'py>( &self, py: Python<'py>, input_json: String, output_schema_json: String, + recover_approvals: bool, ) -> PyResult> { let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { @@ -463,6 +476,10 @@ impl Thread { final_output_json_schema: schema, ..Default::default() }); + let mut request = WithTurnExtensionData::from(request); + if recover_approvals { + request = request.capture_native_approvals(); + } match Box::pin(lease.thread.start_turn_if_idle(request)) .await .map_err(|_| "Unable to start the Codex turn.".to_owned())? @@ -481,22 +498,62 @@ impl Thread { fn next_event<'py>(&self, py: Python<'py>) -> PyResult> { let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = async { - lease.ensure_open()?; - let event = lease.thread.next_event().await.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; - match &event.msg { - EventMsg::ExecApprovalRequest(_) - | EventMsg::ApplyPatchApprovalRequest(_) - | EventMsg::RequestPermissions(_) - | EventMsg::RequestUserInput(_) - | EventMsg::ElicitationRequest(_) => { - return Err("Codex requested interactive approval. This provider supports non-interactive execution only.".to_owned()); - } - _ => {} + lease.ensure_open().map_err(py_error)?; + let event = lease + .thread + .next_event() + .await + .map_err(|_| py_error("The Codex event stream ended unexpectedly."))?; + match &event.msg { + EventMsg::ExecApprovalRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) => { + let (request, preconditions) = lease.thread.capture_tool_approval(event.msg) + .await.map_err(|_| py_error( + "Cannot save this Codex approval. Use session=ResumableSession with a supported local command or file edit." + ))?; + return Ok(serde_json::json!({ + "id": event.id, "msg": request, "preconditions": preconditions + }) + .to_string()); } - serde_json::to_string(&event).map_err(|_| "Unable to decode the Codex event.".to_owned()) - }.await; - result.map_err(py_error) + EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) => { + return Err(py_error( + "Codex requested an unsupported kind of interactive approval.", + )); + } + EventMsg::RawResponseItem(raw) => lease.operations.observe_item(&raw.item), + _ => {} + } + serde_json::to_string(&event).map_err(|_| py_error("Unable to decode the Codex event.")) + }) + } + + fn resolve_approval<'py>( + &self, + py: Python<'py>, + call_json: String, + request_json: String, + preconditions_json: String, + ) -> PyResult> { + let call = + serde_json::from_str(&call_json).map_err(|_| py_error("Invalid saved Codex call."))?; + let request = serde_json::from_str(&request_json) + .map_err(|_| py_error("Invalid saved Codex request."))?; + let preconditions = serde_json::from_str(&preconditions_json) + .map_err(|_| py_error("Invalid saved Codex preconditions."))?; + let lease = self.lease.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + lease.ensure_open().map_err(py_error)?; + native_approval::resolve( + &lease.thread, + call, + request, + preconditions, + &lease.operations, + ) + .await + .map_err(py_error) }) } diff --git a/native/codex/src/native_approval.rs b/native/codex/src/native_approval.rs new file mode 100644 index 0000000000..1b1a0f583b --- /dev/null +++ b/native/codex/src/native_approval.rs @@ -0,0 +1,134 @@ +//! Resume one native call after the SDK has atomically claimed its saved approval. + +use codex_core_api::{CodexThread, StartIfIdleSubmission}; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::{EventMsg, Op, ReviewDecision}; + +use crate::NativeResult; + +pub(super) async fn resolve( + thread: &CodexThread, + call: ResponseItem, + expected: EventMsg, + preconditions: serde_json::Value, + operations: &crate::operation_tracing::Root, +) -> NativeResult { + let (call_id, patch) = match &expected { + EventMsg::ExecApprovalRequest(request) => (request.call_id.clone(), false), + EventMsg::ApplyPatchApprovalRequest(request) => (request.call_id.clone(), true), + _ => return Err("Unsupported Codex approval request.".into()), + }; + // Restoration dispatches this call through Core. Core checks the saved request + // against its freshly prepared action before it can ask us for the decision. + let StartIfIdleSubmission::Started { turn_id, .. } = + Box::pin(thread.restore_tool_call_if_idle(call, expected, preconditions)) + .await + .map_err(|_| "Could not restore the pending Codex approval.")? + else { + return Err("The Codex thread is busy.".into()); + }; + + let mut answered = false; + let mut finished = false; + let mut result = None; + loop { + let event = thread + .next_event() + .await + .map_err(|_| "The restored Codex turn ended unexpectedly.")?; + if event.id != turn_id { + continue; + } + let decision = match event.msg { + EventMsg::ExecApprovalRequest(request) + if !patch + && !answered + && request.call_id == call_id + && request.approval_id.is_none() + && request.turn_id == turn_id + && request + .effective_available_decisions() + .contains(&ReviewDecision::Approved) => + { + Op::ExecApproval { + id: request.call_id, + turn_id: Some(turn_id.clone()), + decision: ReviewDecision::Approved, + } + } + EventMsg::ApplyPatchApprovalRequest(request) + if patch + && !answered + && request.call_id == call_id + && request.turn_id == turn_id => + { + Op::PatchApproval { + id: request.call_id, + decision: ReviewDecision::Approved, + } + } + EventMsg::ExecApprovalRequest(_) + | EventMsg::ApplyPatchApprovalRequest(_) + | EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) + | EventMsg::DynamicToolCallRequest(_) => { + return Err("Codex requested another decision while restoring the call.".into()); + } + EventMsg::ExecCommandEnd(end) + if answered && !patch && end.call_id == call_id && end.turn_id == turn_id => + { + finished = true; + continue; + } + EventMsg::PatchApplyEnd(end) + if answered && patch && end.call_id == call_id && end.turn_id == turn_id => + { + finished = true; + continue; + } + EventMsg::RawResponseItem(item) => { + operations.observe_item(&item.item); + let output = match item.item { + ResponseItem::FunctionCallOutput { + call_id: Some(id), + output, + .. + } if !patch && id == call_id => output, + ResponseItem::CustomToolCallOutput { + call_id: id, + output, + .. + } if patch && id == call_id => output, + _ => continue, + }; + if !answered || result.is_some() { + return Err( + "Codex could not confirm the saved approval before dispatch.".into(), + ); + } + result = output.body.to_text(); + continue; + } + EventMsg::TurnComplete(_) => { + if !finished { + // An exec can yield a handle to a still-running process. That handle + // dies with this thread. Don't checkpoint its uncertain effect as done. + return Err("Codex could not confirm that the approved call finished.".into()); + } + return result.ok_or_else(|| "Codex did not complete the approved call.".into()); + } + EventMsg::TurnAborted(_) | EventMsg::Error(_) | EventMsg::ShutdownComplete => { + return Err( + "Codex could not complete the restored approval under current policy.".into(), + ); + } + _ => continue, + }; + thread + .submit(decision) + .await + .map_err(|_| "Could not submit the saved approval to Codex.")?; + answered = true; + } +} diff --git a/native/codex/src/operation_tracing.rs b/native/codex/src/operation_tracing.rs index 638e5a7f92..93bbe1b54f 100644 --- a/native/codex/src/operation_tracing.rs +++ b/native/codex/src/operation_tracing.rs @@ -1,19 +1,22 @@ -//! Optional, passive metadata observations for SDK-owned ephemeral root threads. -//! Stock request streams and tool execution remain unchanged. Warmups, child threads, -//! content capture, transport-level timing, and generic tool outputs are out of scope. +//! Model and tool spans for the root thread. Content capture requires explicit opt-in. use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; use std::time::{SystemTime, UNIX_EPOCH}; +#[path = "operation_tracing_content.rs"] +mod content; + +use codex_extension_api::ToolPayload; use codex_extension_api::{ - ExtensionData, ModelRequestContributor, ModelRequestInput, ModelRequestKind, - ModelResponseError, ModelResponseInterceptor, ModelResponseStream, ResponseEvent, - ToolCallOutcome, ToolFinishInput, ToolLifecycleContributor, ToolLifecycleFuture, - ToolStartInput, + CommandStartInput, ExtensionData, ModelRequestContent, ModelRequestContributor, + ModelRequestInput, ModelRequestKind, ModelResponseError, ModelResponseInterceptor, + ModelResponseStream, ResponseEvent, ToolCallOutcome, ToolFinishInput, ToolLifecycleContributor, + ToolLifecycleFuture, ToolStartInput, }; use codex_protocol::ToolName; +use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TokenUsage; use futures::StreamExt; use serde_json::{Value, json}; @@ -32,12 +35,16 @@ fn now() -> u64 { pub(crate) struct Buffer { events: Mutex, sequence: AtomicU64, + content: AtomicBool, + pending: Mutex, bool)>>, + pub(crate) updates: tokio::sync::Notify, } #[derive(Debug, Default)] struct Events { items: Vec, dropped: bool, + bytes: usize, } impl Buffer { @@ -46,11 +53,16 @@ impl Buffer { .events .lock() .unwrap_or_else(|error| error.into_inner()); - if events.items.len() < MAX_EVENTS { + let size = serde_json::to_vec(&event) + .map(|v| v.len()) + .unwrap_or(usize::MAX); + if events.items.len() < MAX_EVENTS && events.bytes.saturating_add(size) <= 1024 * 1024 { + events.bytes += size; events.items.push(event); } else { events.dropped = true; } + self.updates.notify_one(); } fn bounded(&self, value: &str) -> bool { @@ -70,6 +82,7 @@ impl Buffer { .events .lock() .unwrap_or_else(|error| error.into_inner()); + buffered.bytes = 0; let mut events = std::mem::take(&mut buffered.items); if std::mem::take(&mut buffered.dropped) { events.push(json!({"kind": "dropped"})); @@ -82,14 +95,17 @@ impl Buffer { Some(serde_json::to_string(&events).unwrap_or_else(|_| "[{\"kind\":\"dropped\"}]".into())) } - fn tool(&self, phase: &str, call_id: &str, name: &ToolName, outcome: Option<&str>) { - if !self.bounded(call_id) - || !self.bounded(&name.name) - || name + fn bounded_tool(&self, call_id: &str, name: &ToolName) -> bool { + self.bounded(call_id) + && self.bounded(&name.name) + && name .namespace .as_deref() - .is_some_and(|value| !self.bounded(value)) - { + .is_none_or(|value| self.bounded(value)) + } + + fn tool(&self, phase: &str, call_id: &str, name: &ToolName, outcome: Option<&str>) { + if !self.bounded_tool(call_id, name) { return; } let mut event = json!({ @@ -135,7 +151,17 @@ pub(crate) struct Root { } impl Root { - pub(crate) fn enable(&self, store: &ExtensionData) { + pub(crate) fn enable( + &self, + store: &ExtensionData, + mode: Option<&str>, + ) -> Result<(), &'static str> { + let capture = match mode { + None | Some("metadata") => false, + Some("content") => true, + _ => return Err("Invalid Codex operation capture mode."), + }; + self.buffer.content.store(capture, Ordering::Release); if !self.enabled.swap(true, Ordering::AcqRel) { store.insert_arc(self.buffer.clone()); self.observers @@ -144,6 +170,20 @@ impl Root { .unwrap_or_else(|error| error.into_inner()) .insert(self.id.clone(), Arc::downgrade(&self.buffer)); } + Ok(()) + } + + pub(crate) fn observe_item(&self, item: &ResponseItem) { + if !self.enabled.load(Ordering::Acquire) || !self.buffer.content.load(Ordering::Acquire) { + return; + } + match item { + ResponseItem::FunctionCallOutput { call_id: Some(id), output, .. } if self.buffer.bounded(id) => + self.buffer.push(json!({"kind":"tool", "phase":"output", "id":id, "output":content::capture(output)})), + ResponseItem::CustomToolCallOutput { call_id: id, output, .. } if self.buffer.bounded(id) => + self.buffer.push(json!({"kind":"tool", "phase":"output", "id":id, "output":content::capture(output)})), + _ => {} + } } pub(crate) fn close(&self) { @@ -171,6 +211,9 @@ struct ModelObservation { buffer: Arc, id: String, finished: bool, + output: Vec, + output_bytes: usize, + response_id: Option, } impl ModelObservation { @@ -183,6 +226,10 @@ impl ModelObservation { "kind": "model", "phase": "end", "id": self.id, "at": now(), "outcome": outcome, }); + if self.buffer.content.load(Ordering::Acquire) { + event["response_id"] = json!(self.response_id); + event["output"] = content::capture(&self.output); + } if let Some(usage) = usage { event["usage"] = json!(usage); } @@ -197,11 +244,65 @@ impl Drop for ModelObservation { } impl ModelResponseInterceptor for ModelObservation { + fn request_failed(&mut self) { + self.finish("failed", None); + } + + fn observe_request(&mut self, request: ModelRequestContent<'_>) { + if !self.buffer.content.load(Ordering::Acquire) { + return; + } + let value = match request { + ModelRequestContent::Responses(value) => content::request( + &value.input, + value.tools.as_ref(), + None, + &value.tool_choice, + value.parallel_tool_calls, + ), + ModelRequestContent::WebSocket(value) => content::request( + value.input, + value.tools, + value.previous_response_id.as_deref(), + value.tool_choice, + value.parallel_tool_calls, + ), + }; + self.buffer + .push(json!({"kind":"model", "phase":"input", "id":self.id, "input":value})); + } + fn intercept(self: Box, stream: ModelResponseStream) -> ModelResponseStream { let mut observation = *self; // Inspect borrows each event and yields the original event without buffering or rewriting. Box::pin(stream.inspect(move |event| match event { - Ok(ResponseEvent::Completed { token_usage, .. }) => { + Ok(ResponseEvent::OutputItemDone(item)) + if observation.buffer.content.load(Ordering::Acquire) => + { + let value = content::capture(item); + let size = serde_json::to_vec(&value) + .map(|v| v.len()) + .unwrap_or(content::LIMIT); + if observation.output_bytes.saturating_add(size) <= content::LIMIT + && observation.output.len() < 128 + { + observation.output_bytes += size; + observation.output.push(value); + } else if observation.output.last() != Some(&content::omitted()) { + observation.output.push(content::omitted()); + observation.output_bytes = content::LIMIT; + } + } + Ok(ResponseEvent::Completed { + token_usage, + response_id, + .. + }) => { + if observation.buffer.content.load(Ordering::Acquire) + && response_id.len() <= MAX_STRING_BYTES + { + observation.response_id = Some(response_id.clone()); + } observation.finish("completed", token_usage.as_ref()); } Err(ModelResponseError::ResponseIncomplete(error)) => { @@ -231,19 +332,71 @@ impl ModelRequestContributor for Observers { buffer, id, finished: false, + output: Vec::new(), + output_bytes: 0, + response_id: None, })) } } impl ToolLifecycleContributor for Observers { fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() + && buffer.bounded_tool(input.call_id, input.tool_name) + { + let capture = buffer + .content + .load(Ordering::Acquire) + .then(|| match input.payload { + ToolPayload::Function { arguments } => content::capture(arguments), + ToolPayload::Custom { input } => content::capture(input), + _ => content::omitted(), + }); + let delayed = matches!( + input.tool_name.name.as_str(), + "exec_command" | "apply_patch" | "shell" | "shell_command" + ); + let mut pending = buffer.pending.lock().unwrap_or_else(|e| e.into_inner()); + if pending.len() < 128 { + pending.insert( + input.call_id.to_owned(), + (input.tool_name.clone(), capture, false), + ); + } else { + buffer.push(json!({"kind": "dropped"})); + } + drop(pending); + if !delayed { + start_tool(&buffer, input.call_id); + } + } + Box::pin(std::future::ready(())) + } + + fn on_command_start<'a>(&'a self, input: CommandStartInput<'a>) -> ToolLifecycleFuture<'a> { if let Some(buffer) = input.thread_store.get::() { - // A handler start can still encounter an approval denial before any effect occurs. - buffer.tool("start", input.call_id, input.tool_name, None); + if buffer.content.load(Ordering::Acquire) { + if let Some((_, payload, _)) = buffer + .pending + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get_mut(input.call_id) + { + *payload = Some(content::capture( + &json!({"command":content::capture(input.command), "cwd":content::capture(input.cwd)}), + )); + } + } } Box::pin(std::future::ready(())) } + fn on_tool_execution_start(&self, thread_id: &str, _turn_id: &str, call_id: &str) { + if let Some(buffer) = self.buffer(thread_id) { + start_tool(&buffer, call_id); + } + } + fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> { if let Some(buffer) = input.thread_store.get::() { let outcome = match input.outcome { @@ -251,13 +404,32 @@ impl ToolLifecycleContributor for Observers { ToolCallOutcome::Aborted => "interrupted", _ => "failed", }; - // Stock finish callbacks may arrive without start when policy or cancellation wins. + buffer + .pending + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(input.call_id); buffer.tool("end", input.call_id, input.tool_name, Some(outcome)); } Box::pin(std::future::ready(())) } } +fn start_tool(buffer: &Buffer, call_id: &str) { + let mut pending = buffer.pending.lock().unwrap_or_else(|e| e.into_inner()); + let Some((name, payload, started)) = pending.get_mut(call_id) else { + return; + }; + if *started { + return; + } + *started = true; + buffer.tool("start", call_id, name, None); + if let Some(payload) = payload { + buffer.push(json!({"kind":"tool", "phase":"input", "id":call_id, "input":payload})); + } +} + #[cfg(test)] mod tests { use super::*; @@ -288,7 +460,7 @@ mod tests { let observers = Arc::new(Observers::default()); let root = observers.root("root".into()); assert!(request(&observers, "root", ModelRequestKind::Generation).is_none()); - root.enable(&ExtensionData::new("root")); + root.enable(&ExtensionData::new("root"), None).unwrap(); assert!(request(&observers, "child", ModelRequestKind::Generation).is_none()); assert!(request(&observers, "root", ModelRequestKind::Warmup).is_none()); let interceptor = request(&observers, "root", ModelRequestKind::Generation).unwrap(); @@ -333,9 +505,15 @@ mod tests { let observers = Arc::new(Observers::default()); let first = observers.root("first".into()); let second = observers.root("second".into()); - first.enable(&ExtensionData::new("first")); - second.enable(&ExtensionData::new("second")); + first.enable(&ExtensionData::new("first"), None).unwrap(); + second.enable(&ExtensionData::new("second"), None).unwrap(); let pending = request(&observers, "second", ModelRequestKind::Generation).unwrap(); + let mut failed = request(&observers, "first", ModelRequestKind::Generation).unwrap(); + failed.request_failed(); + drop(failed); + let events = drain(&first.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[1]["outcome"], "failed"); let interceptor = request(&observers, "first", ModelRequestKind::Generation).unwrap(); let mut observed = interceptor.intercept(Box::pin(stream::iter([Err( ModelResponseError::Stream("private failure detail".into()), @@ -454,3 +632,42 @@ mod tests { assert!(received.iter().all(|event| event["kind"] == "tool")); } } + +#[cfg(test)] +mod content_tests { + use super::*; + + #[test] + fn native_result_capture_is_explicit_and_runtime_start_is_not_repeated() { + let observers = Arc::new(Observers::default()); + let root = observers.root("root".into()); + let store = ExtensionData::new("root"); + assert!(root.enable(&store, Some("unknown")).is_err()); + let item: ResponseItem = serde_json::from_value(json!({ + "type":"function_call_output", "call_id":"call", "output":"accepted" })) + .unwrap(); + root.enable(&store, None).unwrap(); + root.observe_item(&item); + assert!(root.buffer.drain().is_none()); + root.enable(&store, Some("content")).unwrap(); + root.observe_item(&item); + let output = root.buffer.drain().unwrap(); + assert!(output.contains("accepted")); + root.buffer.pending.lock().unwrap().insert( + "call".into(), + ( + ToolName { + name: "exec_command".into(), + namespace: None, + }, + None, + false, + ), + ); + assert!(root.buffer.drain().is_none()); + observers.on_tool_execution_start("root", "turn", "call"); + observers.on_tool_execution_start("root", "turn", "call"); + let events: Vec = serde_json::from_str(&root.buffer.drain().unwrap()).unwrap(); + assert_eq!(events.iter().filter(|e| e["phase"] == "start").count(), 1); + } +} diff --git a/native/codex/src/operation_tracing_content.rs b/native/codex/src/operation_tracing_content.rs new file mode 100644 index 0000000000..fdceedef0f --- /dev/null +++ b/native/codex/src/operation_tracing_content.rs @@ -0,0 +1,203 @@ +//! Bounded, opt-in projections; no transport headers or binary/reasoning payloads. +use serde::Serialize; +use serde_json::{Value, json}; +use std::io::{self, Write}; + +pub(super) const LIMIT: usize = 32 * 1024; + +struct Bounded(Vec); +impl Write for Bounded { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.0.len().saturating_add(bytes.len()) > LIMIT { + return Err(io::Error::other("capture budget")); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(super) fn omitted() -> Value { + json!({"omitted": true, "reason": "unsupported_or_oversized_content"}) +} + +pub(super) fn capture(value: &T) -> Value { + let mut bytes = Bounded(Vec::new()); + if serde_json::to_writer(&mut bytes, value).is_err() { + return omitted(); + } + let projected = serde_json::from_slice(&bytes.0) + .map(|value| project(value, 0)) + .unwrap_or_else(|_| omitted()); + let mut check = Bounded(Vec::new()); + if serde_json::to_writer(&mut check, &projected).is_err() { + omitted() + } else { + projected + } +} + +/// Prioritize actual messages; optional tool schemas share the retained budget. +pub(super) fn request( + input: &[codex_protocol::models::ResponseItem], + tools: Option<&T>, + previous: Option<&str>, + tool_choice: &str, + parallel_tool_calls: bool, +) -> Value { + // Newest messages and accepted results take priority over old instructions. + let mut items = Vec::new(); + let mut remaining = LIMIT - 2048; + let mut omitted_input = input.len() > 128; + for item in input.iter().rev().take(128) { + let item = capture(item); + let size = serde_json::to_vec(&item) + .map(|v| v.len() + 1) + .unwrap_or(LIMIT); + if item.get("omitted") == Some(&Value::Bool(true)) || size > remaining { + omitted_input = true; + continue; + } + remaining -= size; + items.push(item); + } + if omitted_input { + items.push(omitted()); + } + items.reverse(); + let mut value = json!({"input":items, "tool_choice":capture(tool_choice), "parallel_tool_calls":parallel_tool_calls}); + if let Some(previous) = previous.filter(|id| id.len() <= 256) { + value["previous_response_id"] = json!(previous); + } + if let Some(tools) = tools { + value["tools"] = capture(tools); + let mut check = Bounded(Vec::new()); + if serde_json::to_writer(&mut check, &value).is_err() { + value["tools"] = json!([omitted()]); + } + } + capture(&value) +} + +fn project(value: Value, depth: usize) -> Value { + if depth > 24 { + return omitted(); + } + match value { + Value::Object(mut fields) => { + if fields + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| { + matches!( + kind, + "reasoning" + | "reasoning_text" + | "thinking" + | "compaction" + | "input_image" + | "output_image" + | "image" + | "input_audio" + | "output_audio" + | "input_file" + | "file" + | "audio" + | "video" + | "image_generation_call" + ) + }) + { + return omitted(); + } + fields.retain(|key, _| { + !matches!( + key.to_ascii_lowercase().replace('_', "").as_str(), + "apikey" + | "authorization" + | "accesstoken" + | "refreshtoken" + | "password" + | "headers" + | "clientmetadata" + | "encryptedcontent" + | "reasoning" + | "reasoningcontent" + | "reasoningdetails" + | "thinking" + | "imageurl" + | "filedata" + | "imagedata" + | "audiodata" + | "b64json" + | "xapikey" + ) + }); + Value::Object( + fields + .into_iter() + .map(|(k, v)| (k, project(v, depth + 1))) + .collect(), + ) + } + Value::Array(items) => { + Value::Array(items.into_iter().map(|v| project(v, depth + 1)).collect()) + } + Value::String(text) => { + if text.to_ascii_lowercase().contains("data:") { + return omitted(); + } + if matches!(text.trim_start().as_bytes().first(), Some(b'{' | b'[')) { + return match serde_json::from_str::(&text) { + Ok(value) => Value::String(project(value, depth + 1).to_string()), + Err(_) => omitted(), + }; + } + Value::String(text) + } + value => value, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_projection_omits_nested_credentials_media_and_oversized_values() { + let value = capture(&json!({ + "input": [{"type":"message", "content":[{"type":"input_text", "text":"keep"}, + {"type":"input_image", "image_url":"hidden-image"}]}], + "arguments": "{\"password\":\"hidden-password\",\"query\":\"keep\"}", + "client_metadata": {"secret":"hidden-metadata"}, + "encrypted_content":"hidden-reasoning" + })); + let encoded = value.to_string(); + assert!(encoded.contains("keep")); + assert!(!encoded.contains("hidden")); + assert_eq!(capture(&"x".repeat(LIMIT + 1)), omitted()); + } + #[test] + fn request_prioritizes_recent_prompt_and_accepted_result_over_large_instructions() { + let input: Vec = serde_json::from_value(json!([ + {"type":"message", "role":"developer", "content":[{"type":"input_text", "text":"x".repeat(LIMIT + 1)}]}, + {"type":"message", "role":"user", "content":[{"type":"input_text", "text":"recent user prompt"}]}, + {"type":"function_call_output", "call_id":"call", "output":"recent accepted result"} + ])).unwrap(); + let tools = serde_json::value::RawValue::from_string( + json!([{"name":"large", "description":"x".repeat(LIMIT)}]).to_string(), + ) + .unwrap(); + let value = request(&input, Some(&tools), Some("previous"), "auto", true); + let encoded = value.to_string(); + assert!(encoded.len() <= LIMIT); + assert!( + encoded.contains("recent user prompt") && encoded.contains("recent accepted result") + ); + assert_eq!(value["input"][0]["omitted"], true); + assert!(value["tools"].to_string().contains("omitted")); + assert_eq!(value["previous_response_id"], "previous"); + } +} diff --git a/native/codex/verify_installed.py b/native/codex/verify_installed.py index 637e2543a9..43181b4da2 100644 --- a/native/codex/verify_installed.py +++ b/native/codex/verify_installed.py @@ -6,6 +6,7 @@ import argparse import asyncio +import base64 import importlib import importlib.abc import importlib.machinery @@ -141,13 +142,16 @@ async def native_route(folder): from agents import Agent, RunConfig, Runner, RunState from agents.decorators import tool + from agents.memory.resumable_session import ResumableSession from agents.models.codex import CodexProvider from agents.tracing import TracingProcessor, set_trace_processors, set_tracing_disabled class Answer(BaseModel): label: str + image_path = folder / "workspace" / "trace-image.png" requests = [] + approval_item = None errors = [] tool_calls = [] live_delta = threading.Event() @@ -171,7 +175,7 @@ def do_POST(self): ) body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) requests.append(body) - assert len(requests) <= 7, "Unexpected extra inference request." + assert len(requests) <= 13, "Unexpected extra inference request." number = len(requests) history = json.dumps(body["input"]) response = {"id": f"response-{number}"} @@ -179,7 +183,10 @@ def do_POST(self): self.send_header("Content-Type", "text/event-stream") self.end_headers() self.send_event({"type": "response.created", "response": response}) - if number == 1: + if number > 9 and number % 2 == 0: + assert approval_item is not None + item = approval_item + elif number == 1: item = { "type": "function_call", "call_id": "route-call", @@ -193,8 +200,19 @@ def do_POST(self): "name": "lookup", "arguments": '{"value":"record"}', } + elif number == 8: + item = { + "type": "function_call", + "call_id": "native-image-call", + "name": "view_image", + "arguments": json.dumps({"path": str(image_path)}), + } else: - if "blocked-input" in history: + if "approval-input" in history: + text = "approval resolved" + elif "content-input" in history: + text = "native-content" + elif "blocked-input" in history: assert ( "survivor-input" not in history and "after-cancel-input" not in history ) @@ -374,16 +392,118 @@ async def consume_blocked(): await asyncio.gather(consumer, return_exceptions=True) reused = await Runner.run(plain, "after-cancel-input", run_config=config) assert reused.final_output == "reused" and len(requests) == 7 + image_path.write_bytes( + base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNoaGgAAAMEAYFL09IQAAAAAElFTkSuQmCC" + ) + ) + before = len(processor.on_span_end.call_args_list) + content_config = RunConfig( + model_provider=provider, + trace_include_sensitive_data=True, + tracing={"operation_detail": "content"}, + ) + content_result = await Runner.run(plain, "content-input", run_config=content_config) + assert content_result.final_output == "native-content" and len(requests) == 9 + captured = [ + call.args[0].span_data for call in processor.on_span_end.call_args_list[before:] + ] + native_models = [span for span in captured if span.type == "generation"] + native_tools = [ + span + for span in captured + if span.type == "function" and span.name in {"view_image", "functions.view_image"} + ] + request_sizes = [len(json.dumps(body).encode()) for body in requests[-2:]] + assert len(native_models) == 2 and len(native_tools) == 1 + assert "content-input" in json.dumps(native_models[0].input), ( + f"Native content omitted; request sizes: {request_sizes}" + ) + assert "native-content" in json.dumps(native_models[-1].output) + assert str(image_path) in native_tools[0].input + assert "omitted" in native_tools[0].output + rendered = json.dumps([span.export() for span in captured]) + assert ( + "synthetic-installed-key" not in rendered + and "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" not in rendered + ) + assert "opaque-late-reasoning" not in rendered + + class ApprovalStore: + value = None + + async def load(self, _session_id): + return self.value + + async def save(self, _session_id, value, *, expected): + if self.value != expected: + return False + self.value = value + return True + + def fresh_provider(): + return CodexProvider( + cwd=folder / "workspace", + api_key="synthetic-installed-key", + config_overrides=[ + 'approval_policy="on-request"', + 'approvals_reviewer="user"', + ], + ) + + await provider.aclose() + approval_checks = [] + for approved in (False, True): + target = folder / "workspace" / f"builtin-command-{approved}.txt" + call_id = f"builtin-command-{approved}" + approval_item = { + "type": "function_call", + "name": "exec_command", + "call_id": call_id, + "arguments": json.dumps( + { + "cmd": f'echo builtin-approved > "{target}"', + "login": False, + "sandbox_permissions": "require_escalated", + } + ), + } + provider = fresh_provider() + options = RunConfig(model_provider=provider, tracing_disabled=True) + store = ApprovalStore() + session = await ResumableSession.create(store) + paused = await Runner.run(plain, "approval-input", session=session, run_config=options) + assert len(paused.interruptions) == 1 and not target.exists() + stored = store.value + await provider.aclose() + provider = fresh_provider() + options.model_provider = provider + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(plain) + assert store.value == stored and not target.exists() + (pending,) = state.get_interruptions() + assert pending.raw_item.call_id == call_id + state.approve(pending) if approved else state.reject(pending) + result = await Runner.run(plain, state, run_config=options) + assert result.final_output == "approval resolved" + assert target.exists() == approved + if approved: + assert b"builtin-approved" in target.read_bytes().replace(b"\x00", b"") + approval_checks.append("approve" if approved else "deny") + await provider.aclose() assert "agents.models._codex_native" in sys.modules assert "agents.models._codex_app_server" not in sys.modules assert not config_path.with_name("auth.json").exists(), "The API key persisted to disk." assert not errors, errors return { "provider_requests": len(requests), + "native_content_request_bytes": request_sizes, + "native_content_roundtrip": True, "tool_effects": len(tool_calls), "resumed_tokens": usage.total_tokens, "live_streaming": True, "concurrent_survivor_and_reuse_after_cancel": True, + "native_approval_checks": approval_checks, } finally: dynamic_delta.set() diff --git a/scripts/build_codex_wheel.py b/scripts/build_codex_wheel.py index a476e83952..2d82651933 100644 --- a/scripts/build_codex_wheel.py +++ b/scripts/build_codex_wheel.py @@ -53,11 +53,41 @@ def stage_asset(asset: dict[str, str], destination: Path, cache: Path) -> None: destination.chmod(0o755) +def patch_codex_source(source: Path, upstream: Path, metadata: dict) -> None: + """Stage the pinned Codex workspace and apply its documented changes in order.""" + patches = sorted((source / "native/codex/patches").glob("*.patch")) + if not patches: + raise ValueError("The Codex wheel is missing its patches.") + for patch in patches: + guide = patch.with_suffix(".md") + if not guide.is_file() or not guide.read_text(encoding="utf-8").strip(): + raise ValueError(f"Document {patch.name} in {guide.name} before building.") + + staged = source / "native/codex-upstream" + shutil.copytree(upstream, staged, ignore=shutil.ignore_patterns(".git")) + subprocess.run( + ["git", "apply", "--whitespace=error", *(str(p.resolve()) for p in patches)], + cwd=staged, + env=os.environ | {"GIT_CEILING_DIRECTORIES": str(staged.parent.resolve())}, + check=True, + ) + # All Codex crates must share the same types. Redirect only the resolved Codex workspace. + lines = ['[patch."https://github.com/openai/codex"]'] + for package in metadata["packages"]: + if (package["source"] or "").startswith("git+https://github.com/openai/codex?"): + relative = Path(package["manifest_path"]).parent.relative_to(upstream) + lines.append(f"{package['name']} = {{ path = {json.dumps(str(staged / relative))} }}") + config = source / ".cargo/config.toml" + config.parent.mkdir(exist_ok=True) + config.write_text("\n".join(lines) + "\n", encoding="utf-8") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, default=ROOT / "dist") parser.add_argument("--target", choices=TARGETS) parser.add_argument("--compatibility") + parser.add_argument("--test-native", action="store_true", help="Test the staged SDK binding.") parser.add_argument("--profile", choices=("dev", "release"), default="release") args = parser.parse_args() environment = os.environ.copy() @@ -148,18 +178,20 @@ def build_native( "--target", target, ] + patch_codex_source(source, upstream, metadata) + # Resolve local paths in the temporary lockfile without fetching different dependencies. subprocess.run( - [ - "cargo", - "build", - "--locked", - "--manifest-path", - str(manifest), - *target_flags, - "--bin", - "codex-sdk-runtime", - *profile, - ], + ["cargo", "metadata", "--offline", "--format-version=1", "--manifest-path", manifest], + cwd=source, + env=environment, + stdout=subprocess.DEVNULL, + check=True, + ) + flags = ["--locked", "--manifest-path", str(manifest), *target_flags, *profile] + if args.test_native: + subprocess.run(["cargo", "test", "--lib", *flags], cwd=source, env=environment, check=True) + subprocess.run( + ["cargo", "build", "--bin", "codex-sdk-runtime", *flags], cwd=source, env=environment, check=True, @@ -177,6 +209,7 @@ def build_native( stage_asset(asset, path, cache) licenses = staged / "licenses" licenses.mkdir() + shutil.copytree(native / "patches", licenses / "codex-patches") for name, path in { "CODEX_LICENSE": upstream / "LICENSE", "CODEX_NOTICE": upstream / "NOTICE", diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 08057cdfc3..7824698337 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -82,8 +82,18 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: if isinstance(item.get(flag), bool): part[flag] = item[flag] parts.append(part) - elif item.get("truncated") is True and "content" not in item and "tool_calls" not in item: - parts.append({"type": "openai.omitted", "reason": "truncated"}) + elif ( + (item.get("truncated") is True or item.get("omitted") is True) + and "content" not in item + and "tool_calls" not in item + ): + reason = item.get("reason") + parts.append( + { + "type": "openai.omitted", + "reason": reason if isinstance(reason, str) else "truncated", + } + ) elif kind in ("function_call", "custom_tool_call"): role = "assistant" parts.append( diff --git a/src/agents/memory/resumable_session.py b/src/agents/memory/resumable_session.py index a9f1314edd..abb58b9b85 100644 --- a/src/agents/memory/resumable_session.py +++ b/src/agents/memory/resumable_session.py @@ -283,6 +283,7 @@ async def load_state( return state def _validate_run(self, input: Any) -> None: + from ..run_internal.native_approvals import native_receipts_from_snapshot from ..run_state import RunState if self._checkpoint.status == "active": @@ -300,6 +301,22 @@ def _validate_run(self, input: Any) -> None: or binding.revision != self._checkpoint.revision ): raise ResumableSessionError("invalid_state") + invalid_native_state = False + try: + assert self._checkpoint.run_state is not None + original = json.loads(self._checkpoint.run_state) + original_receipts = native_receipts_from_snapshot(original) + processed = input._last_processed_response + if original_receipts is not None or ( + processed is not None and processed.native_approvals + ): + invalid_native_state = ( + native_receipts_from_snapshot(input.to_json()) != original_receipts + ) + except Exception: + invalid_native_state = True + if invalid_native_state: + raise ResumableSessionError("invalid_state") def _get_history(self) -> list[TResponseInputItem]: return copy.deepcopy(self._checkpoint.history) diff --git a/src/agents/models/_codex_items.py b/src/agents/models/_codex_items.py index 66ee061da9..1fbac165a3 100644 --- a/src/agents/models/_codex_items.py +++ b/src/agents/models/_codex_items.py @@ -1,9 +1,18 @@ -"""Internal output types attesting to work already completed by the Codex adapter.""" +"""Internal output types attesting to work observed directly by the Codex adapter.""" + +import json +from pathlib import Path +from typing import Any, cast from openai.types.responses import ( + ResponseCustomToolCall, ResponseCustomToolCallOutputItem, + ResponseFunctionToolCall, ResponseFunctionToolCallOutputItem, ) +from pydantic import Field + +from ..exceptions import UserError class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): @@ -12,3 +21,66 @@ class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): class CodexCustomToolCallOutput(ResponseCustomToolCallOutputItem): """A matching native custom call has already completed in Codex.""" + + +class CodexPendingFunctionCall(ResponseFunctionToolCall): + """A native function call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +class CodexPendingCustomCall(ResponseCustomToolCall): + """A native custom call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +def native_approval_checkpoint(raw_item: Any) -> dict[str, Any] | None: + """Validate an adapter-owned receipt against the call shown for approval.""" + # An unrelated approval may define __getattribute__("__class__"). Don't run it + # while taking a detached copy of a pending call or a session checkpoint. + raw_type = type(raw_item) + if raw_type is not CodexPendingFunctionCall and raw_type is not CodexPendingCustomCall: + return None + checkpoint = raw_item.checkpoint + if ( + not isinstance(checkpoint, dict) + or set(checkpoint) != {"version", "cwd", "call", "request", "model", "preconditions"} + or type(checkpoint["version"]) is not int + or checkpoint["version"] != 1 + or not isinstance(checkpoint["cwd"], str) + or not Path(checkpoint["cwd"]).is_absolute() + or not isinstance(checkpoint["model"], str) + or not checkpoint["model"] + or not isinstance(checkpoint["request"], dict) + or not isinstance(checkpoint["call"], dict) + or not isinstance(checkpoint["preconditions"], dict) + ): + raise UserError("Invalid Codex native approval checkpoint.") + model = ( + ResponseFunctionToolCall + if isinstance(raw_item, CodexPendingFunctionCall) + else ResponseCustomToolCall + ) + display_metadata = {"codex_reasoning_summary"} + displayed = model.model_validate(raw_item.model_dump()).model_dump(exclude=display_metadata) + stored = model.model_validate(checkpoint["call"]).model_dump(exclude=display_metadata) + if displayed != stored or raw_item.name not in {"exec_command", "apply_patch"}: + raise UserError("Codex native approval no longer matches its original call.") + return cast(dict[str, Any], json.loads(json.dumps(checkpoint, allow_nan=False))) + + +def restore_native_approval_call(raw_item: Any, checkpoint: Any) -> Any: + """Restore only explicit SDK checkpoint metadata, never provider output fields.""" + if not isinstance(raw_item, dict) or not isinstance(checkpoint, dict): + raise UserError("Invalid Codex native approval checkpoint.") + item: CodexPendingFunctionCall | CodexPendingCustomCall + item_type = raw_item.get("type") + if item_type == "function_call": + item = CodexPendingFunctionCall(**raw_item, checkpoint=checkpoint) + elif item_type == "custom_tool_call": + item = CodexPendingCustomCall(**raw_item, checkpoint=checkpoint) + else: + raise UserError("Invalid Codex native approval call type.") + native_approval_checkpoint(item) + return item diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py index 323b5d7478..59557de152 100644 --- a/src/agents/models/_codex_tracing.py +++ b/src/agents/models/_codex_tracing.py @@ -1,4 +1,4 @@ -"""Trace stock Codex request and tool callbacks without copying native payloads.""" +"""Translate bounded native observations without affecting model or tool execution.""" from __future__ import annotations @@ -6,6 +6,8 @@ from datetime import datetime, timezone from typing import Any +from openai.types.responses import Response + from ..tracing import ( Span, custom_span, @@ -14,39 +16,62 @@ get_current_trace, response_span, ) -from ..tracing._model import acquire_model_span, finish_model_span +from ..tracing._content import _json, bound_content +from ..tracing._model import acquire_model_span, finish_model_span, record_model_input from ..tracing.config import get_operation_detail from ..tracing.provider import _is_noop_span +from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData from ..usage import _response_usage_to_usage, model_usage_to_span_usage from ._codex_usage import _usage from .interface import ModelTracing class CodexOperationTracing: - """Own observations for one ephemeral Codex thread and one SDK model attempt.""" + """Own observations for one ephemeral native thread, including tool restoration.""" - def __init__(self, sdk_tools: set[tuple[str | None, str]]) -> None: + def __init__( + self, sdk_tools: set[tuple[str | None, str]], *, expect_model: bool = True + ) -> None: self._sdk_tools = sdk_tools + self._expect_model = expect_model self._parent = get_current_span() or get_current_trace() self._thread: Any = None self._active: dict[tuple[str, str], Span[Any]] = {} + self._ends: dict[tuple[str, str], tuple[str, str]] = {} + self._ignored: set[str] = set() self._reason: str | None = None self._saw_model = False + self._tracing = ModelTracing.ENABLED_WITHOUT_DATA + self._capture = False def bind(self, thread: Any, tracing: ModelTracing) -> None: if tracing.is_disabled() or self._parent is None: return - # A hook can disable the provider after the parent starts. if _is_noop_span(response_span(parent=self._parent)): return - if get_operation_detail(include_data=tracing.include_data()) == "content": - self._reason = "metadata_only" + self._capture = ( + tracing.include_data() and get_operation_detail(include_data=True) == "content" + ) + self._tracing = tracing if self._capture else ModelTracing.ENABLED_WITHOUT_DATA try: - thread.enable_tracing() + if self._capture: + thread.enable_tracing("content") + else: + thread.enable_tracing() self._thread = thread except Exception: self._reason = "unavailable" + async def watch(self) -> None: + if self._thread is None: + return + try: + while True: + await self._thread.wait_for_operations() + self.drain() + except Exception: + self._reason = "telemetry_error" + def drain(self) -> None: if self._thread is None: return @@ -57,7 +82,6 @@ def drain(self) -> None: for event in json.loads(encoded): self._observe(event) except Exception: - # Observation failure must never change the result or cause a retry. self._reason = "telemetry_error" def _observe(self, event: dict[str, Any]) -> None: @@ -65,22 +89,29 @@ def _observe(self, event: dict[str, Any]) -> None: if kind == "dropped": self._reason = "dropped_operations" return - if kind == "tool" and (event.get("namespace"), event["name"]) in self._sdk_tools: - # Runner owns the approval and execution spans for SDK tools. - return + phase = event["phase"] + if kind == "tool": + if phase == "start" and (event.get("namespace"), event["name"]) in self._sdk_tools: + self._ignored.add(event["id"]) + if event["id"] in self._ignored: + return key = kind, event["id"] + if phase in ("input", "output"): + span = self._active.get(key) + if span is not None: + self._content(span, kind, event) + if phase == "output" and key in self._ends: + outcome, at = self._ends.pop(key) + self._finish(key, outcome, at) + return at = datetime.fromtimestamp(event["at"] / 1000, tz=timezone.utc).isoformat() - if event["phase"] == "start": + if phase == "start": if key in self._active or len(self._active) >= 128: self._reason = "dropped_operations" return - span: Span[Any] if kind == "model": span = acquire_model_span( - ModelTracing.ENABLED_WITHOUT_DATA, - event["model"], - parent=self._parent, - started_at=at, + self._tracing, event["model"], parent=self._parent, started_at=at ) self._saw_model = True elif kind == "tool": @@ -88,23 +119,63 @@ def _observe(self, event: dict[str, Any]) -> None: if event.get("namespace"): name = f"{event['namespace']}.{name}" span = function_span(name, parent=self._parent) - span.span_data.operation.detail = get_operation_detail(include_data=False) + span.span_data.operation.detail = get_operation_detail(include_data=self._capture) span.span_data.operation.tool_call_id = event["id"] span.start_at(at) else: - raise ValueError("Unknown Codex operation.") + raise ValueError("Unknown native operation.") self._active[key] = span return - if key not in self._active: - # Core can block a call before on_tool_start. Never invent an execution. - self._reason = "missing_start" + # Rejected calls never reached the post-authorization execution boundary. return - if kind == "model" and event.get("usage") is not None: - self._active[key].span_data.usage = model_usage_to_span_usage( - _response_usage_to_usage(_usage(event["usage"])) - ) - self._finish(key, event["outcome"], at) + span = self._active[key] + if kind == "model": + if event.get("usage") is not None: + span.span_data.usage = model_usage_to_span_usage( + _response_usage_to_usage(_usage(event["usage"])) + ) + self._content(span, kind, event) + if kind == "tool" and self._capture: + # Core publishes the accepted model-visible output after the lifecycle callback. + self._ends[key] = event["outcome"], at + else: + self._finish(key, event["outcome"], at) + + def _content(self, span: Span[Any], kind: str, event: dict[str, Any]) -> None: + if not self._capture: + return + data = span.span_data + if kind == "tool" and isinstance(data, FunctionSpanData): + for direction in ("input", "output"): + if direction in event: + value = bound_content(event[direction]) + setattr(data, direction, value if isinstance(value, str) else _json(value)) + return + if "input" in event: + request = bound_content(event["input"]) + if isinstance(data, GenerationSpanData): + record_model_input(data, request) + if isinstance(request, dict): + data.tools = bound_content(request.get("tools")) + config = { + key: request[key] + for key in ("tool_choice", "parallel_tool_calls", "text", "service_tier") + if key in request + } + data.model_config = {**(data.model_config or {}), **config} or None + elif isinstance(data, ResponseSpanData): + data.input = ( + request.get("input", [request]) if isinstance(request, dict) else request + ) + if "output" in event: + output = bound_content(event["output"]) + if isinstance(data, GenerationSpanData): + data.output = output + elif isinstance(data, ResponseSpanData) and isinstance(event.get("response_id"), str): + data.response = Response.model_construct(id=event["response_id"], output=output) + if isinstance(event.get("response_id"), str): + data.operation.response_id = event["response_id"] def _finish(self, key: tuple[str, str], outcome: str, at: str | None = None) -> None: span = self._active.pop(key) @@ -112,7 +183,7 @@ def _finish(self, key: tuple[str, str], outcome: str, at: str | None = None) -> span.span_data.operation.estimated_end = at is None if outcome != "completed": span.set_error( - {"message": "Codex operation did not complete successfully.", "data": None} + {"message": "Native operation did not complete successfully.", "data": None} ) if key[0] == "model": finish_model_span(span, ended_at=at) @@ -123,11 +194,21 @@ def _finish(self, key: tuple[str, str], outcome: str, at: str | None = None) -> def close(self) -> None: self.drain() - if self._thread is not None and not self._saw_model: + if self._thread is not None and self._expect_model and not self._saw_model: self._reason = self._reason or "missing_operations" for key in list(self._active): - self._reason = self._reason or "interrupted" - self._finish(key, "interrupted") + if key in self._ends: + outcome, at = self._ends.pop(key) + if self._capture: + self._content( + self._active[key], + key[0], + {"output": {"omitted": True, "reason": "output_unavailable"}}, + ) + self._finish(key, outcome, at) + else: + self._reason = self._reason or "interrupted" + self._finish(key, "interrupted") if self._reason is not None: with custom_span( "codex.tracing.coverage", diff --git a/src/agents/models/_run_context.py b/src/agents/models/_run_context.py index ba57e3f19d..9a9fccbf57 100644 --- a/src/agents/models/_run_context.py +++ b/src/agents/models/_run_context.py @@ -19,6 +19,7 @@ class ModelToolHooks: context: RunContextWrapper[Any] lifecycle: RunHooks[Any] settings: ToolExecutionConfig | None + resumable: bool = False _MODEL_RUN_CONTEXT: ContextVar[tuple[object | None, ModelToolHooks | None]] = ContextVar( diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index 61386c14b3..4f6d838fbc 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import os import time @@ -49,21 +50,25 @@ _response_usage_to_usage, ) from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call -from ._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, +) from ._codex_tool_hooks import CodexToolHooks from ._codex_tracing import CodexOperationTracing from ._codex_usage import _add_sample_usage, _usage -from ._run_context import get_model_tool_hooks +from ._run_context import ModelToolHooks, get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing class CodexProvider(ModelProvider): """Run opt-in Agents SDK requests through embedded Codex Core. - Each request uses a fresh Codex thread. Native request and tool spans contain - metadata only. Interactive Codex approvals are unsupported. For resumable - workspace approvals, supply a local ShellTool or ApplyPatchTool with - needs_approval=True and an executor or editor that enforces your policy. + Codex handles workspace tools and authentication. Each request uses a fresh + Codex thread. Pass a ResumableSession to the Runner to save a pending local + command or edit for approval. The SDK session ID stays the same across workers. """ _supports_stored_response_id = False @@ -112,6 +117,100 @@ async def aclose(self) -> None: def get_model(self, model_name: str | None) -> Model: return _CodexModel(model_name, self) + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], tracing: ModelTracing + ) -> str: + """Run only the saved invocation, without a model turn or its old Codex home.""" + if checkpoint.get("version") != 1 or checkpoint.get("cwd") != self._cwd: + raise UserError("This Codex approval belongs to a different workspace or SDK version.") + + settings = {"model": checkpoint["model"]} + observations = CodexOperationTracing(set(), expect_model=False) + thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) + watcher: asyncio.Task[None] | None = None + failure: BaseException | None = None + try: + observations.bind(thread, tracing) + watcher = asyncio.create_task(observations.watch()) + return str( + await thread.resolve_approval( + json.dumps(checkpoint["call"]), + json.dumps(checkpoint["request"]), + json.dumps(checkpoint["preconditions"]), + ) + ) + except BaseException as error: + failure = error + raise + finally: + + async def close() -> None: + try: + await thread.close() + finally: + if watcher is not None: + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + observations.close() + + try: + await _join_close(close()) + except asyncio.CancelledError: + if isinstance(failure, asyncio.CancelledError): + raise failure from None + raise + except Exception: + if failure is None: + raise + + +def _pending_native_call( + call: Any, event: dict[str, Any], cwd: str, model: str +) -> CodexPendingFunctionCall | CodexPendingCustomCall: + request = event["msg"] + wire = call.model_dump(exclude_none=True) + if event.get("preconditions") is None: + raise UserError("Codex could not confirm that this tool is still waiting to run.") + checkpoint = { + "version": 1, + "cwd": cwd, + "model": model, + "call": wire, + "request": request, + "preconditions": event["preconditions"], + } + if getattr(call, "namespace", None) not in (None, "functions"): + raise UserError("Only built-in Codex tools support native approval recovery.") + if ( + isinstance(call, ResponseFunctionToolCall) + and call.name == "exec_command" + and request["type"] == "exec_approval_request" + and request.get("kind", "command") == "command" + and request.get("approval_id") is None + and request.get("network_approval_context") is None + and request.get("environmentId") in (None, "local") + ): + return CodexPendingFunctionCall(**wire, checkpoint=checkpoint) + if ( + isinstance(call, ResponseCustomToolCall) + and call.name == "apply_patch" + and request["type"] == "apply_patch_approval_request" + ): + return CodexPendingCustomCall(**wire, checkpoint=checkpoint) + raise UserError("This Codex approval cannot safely be resumed on another worker.") + + +def _uses_sdk_workspace(tools: list[Tool], bindings: ModelToolHooks | None) -> bool: + settings = bindings.settings if bindings else None + return bool( + settings + and settings.sdk_only_tool_hooks + and (settings.prepare_tool_input or settings.prepare_tool_output) + ) or any( + getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) + for tool in tools + ) + def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: supported = { @@ -163,6 +262,16 @@ def __init__(self, name: str | None, provider: CodexProvider) -> None: def get_default_model_settings(self) -> ModelSettings: return ModelSettings() + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], *, tools: list[Tool], tracing: ModelTracing + ) -> str: + bindings = get_model_tool_hooks() + if _uses_sdk_workspace(tools, bindings) or CodexToolHooks.for_run(bindings, set()): + raise UserError( + "The current workspace or native tool hooks do not allow restoring this call." + ) + return await self._provider._run_approved_native_call(checkpoint, tracing) + def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice: # Native Codex tools may have changed the workspace before a failed response. return ModelRetryAdvice(replay_safety="unsafe") @@ -261,19 +370,7 @@ async def stream_response( ) provider = self._provider bindings = get_model_tool_hooks() - tool_execution = bindings.settings if bindings is not None else None - classic_preparation = ( - tool_execution is not None - and tool_execution.sdk_only_tool_hooks - and ( - tool_execution.prepare_tool_input is not None - or tool_execution.prepare_tool_output is not None - ) - ) - sdk_sandbox = classic_preparation or any( - getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) - for tool in tools - ) + sdk_sandbox = _uses_sdk_workspace(tools, bindings) native_hooks = None if sdk_sandbox else CodexToolHooks.for_run(bindings, identities) if sdk_sandbox: system_instructions = "\n\n".join( @@ -352,6 +449,7 @@ async def stream_response( sdk_calls: dict[str, Any] = {} native_pending: dict[str, tuple[str, str]] = {} native_completed: set[str] = set() + pending_approval: dict[str, Any] | None = None sampling_pending = False interrupt_requested = False sequence = 0 @@ -378,7 +476,14 @@ async def interrupt_when_ready() -> None: if history: await thread.inject_items(json.dumps(history)) observations.bind(thread, tracing) - turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema)) + recover = bool( + bindings + and bindings.resumable + and not native_hooks + and not sdk_sandbox + and not require_sdk_call + ) + turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema), recover) response = Response( id=turn_id, created_at=time.time(), @@ -395,6 +500,18 @@ async def interrupt_when_ready() -> None: ) sequence += 1 while True: + if pending_approval is not None and not sampling_pending: + # All calls from this sampling response are now known. A sibling + # still in flight has an unknown outcome and cannot be dropped. + call_id = pending_approval["msg"]["call_id"] + if set(native_pending) != {call_id} or sdk_calls: + raise UserError("Codex can only pause one outstanding native call.") + item_id, _ = native_pending.pop(call_id) + output[item_id] = _pending_native_call( + output[item_id], pending_approval, provider._cwd, thread.model + ) + # Close the waiting thread before handing the pause to the Runner. + break encoded = await thread.next_event() observations.drain() if encoded is None: @@ -415,6 +532,20 @@ async def interrupt_when_ready() -> None: ) if event.get("id") not in ("", turn_id): continue + if kind in ("exec_approval_request", "apply_patch_approval_request"): + if ( + message.get("turn_id") != turn_id + or pending_approval is not None + or native_hooks is not None + or sdk_sandbox + or require_sdk_call + ): + raise UserError( + "Codex can only pause a single built-in tool call before execution, " + "with native workspace tools enabled and without custom tool hooks." + ) + pending_approval = event + continue if kind == "dynamic_tool_call_request": if message.get("turnId") != turn_id: continue diff --git a/src/agents/run.py b/src/agents/run.py index 57ca88bbe2..715b5b4f24 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -871,6 +871,9 @@ async def _run_impl( run_state._reasoning_item_id_policy = resolved_reasoning_item_id_policy run_state.set_trace(get_current_trace()) + if isinstance(session, ResumableRun): + session.bind_state(run_state) + use_task_and_turn_spans = include_task_and_turn_spans(run_config.tracing) current_task_span: Span[TaskSpanData] | None = ( task_span(name=trace_workflow_name) if use_task_and_turn_spans else None @@ -2912,6 +2915,7 @@ async def run_loop() -> None: if isinstance(session, ResumableRun): try: await session.begin(input) + session.bind_state(run_state) except BaseException: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 77ae747a8e..6ddaa62170 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -1189,6 +1189,10 @@ def _apply_approval_decision( rejection_message: str | None = None, ) -> None: """Record an approval or rejection decision.""" + from .models._codex_items import native_approval_checkpoint + + if native_approval_checkpoint(approval_item.raw_item) is not None and always: + raise UserError("Native approvals require a decision for the original call only.") hosted_request = get_hosted_mcp_approval_request_identity(approval_item) if hosted_request is not None: call_id = hosted_request.request_id diff --git a/src/agents/run_internal/native_approvals.py b/src/agents/run_internal/native_approvals.py new file mode 100644 index 0000000000..2e9b379d19 --- /dev/null +++ b/src/agents/run_internal/native_approvals.py @@ -0,0 +1,220 @@ +"""Keep native approval state and execution separate from SDK tool dispatch.""" + +from __future__ import annotations + +from typing import Any, cast + +from ..exceptions import UserError +from ..items import ToolCallOutputItem, ToolCallOutputTypes +from ..memory.resumable_session import _ResumableSessionBinding +from ..models._codex_items import native_approval_checkpoint +from ..models._run_context import ModelToolHooks, model_run_context +from .run_steps import NextStepInterruption, NextStepRunAgain, SingleStepResult +from .tool_execution import pending_tool_span +from .turn_preparation import get_all_tools, get_model + + +def native_receipts_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any] | None: + """Select the immutable native proposals while leaving decisions mutable.""" + processed = snapshot.get("last_processed_response") or {} + approvals = processed.get("native_approvals", []) + if not approvals: + return None + call_ids = {item["raw_item"]["call_id"] for item in approvals} + step = snapshot.get("current_step") or {} + pending = (step.get("data") or {}).get("interruptions", []) + return { + "plans": approvals, + "pending": [ + item for item in pending if item.get("raw_item", {}).get("call_id") in call_ids + ], + "generated": [ + item + for item in snapshot.get("generated_items", []) + if item.get("raw_item", {}).get("call_id") in call_ids + ], + } + + +def _validate_native_plan(processed: Any) -> None: + if len(processed.native_approvals) != 1 or any( + ( + processed.functions, + processed.custom_tool_calls, + processed.handoffs, + processed.computer_actions, + processed.local_shell_calls, + processed.shell_calls, + processed.apply_patch_calls, + processed.function_tools_not_found, + processed.mcp_approval_requests, + ) + ): + raise UserError( + "Native approvals require a single pending native call without SDK actions." + ) + item = processed.native_approvals[0] + if native_approval_checkpoint(item.raw_item) is None or item.tool_name != item.raw_item.name: + raise UserError("Invalid native approval plan.") + + +def native_pending_step( + *, + bindings: Any, + original_input: Any, + pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + run_state: Any, + run_config: Any, +) -> SingleStepResult: + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError( + "Start the run with session=ResumableSession to recover native tool approvals." + ) + pending = list(processed_response.native_approvals) + processed_response.interruptions = pending + with pending_tool_span(run_config, pending[0].tool_name, pending[0].raw_item.call_id) as span: + span.span_data.operation.outcome = "paused" + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=pre_step_items, + new_step_items=[*processed_response.new_items, *pending], + next_step=NextStepInterruption(interruptions=pending), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + + +async def resolve_native_approval_step( + *, + bindings: Any, + original_input: Any, + original_pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + context_wrapper: Any, + hooks: Any, + run_config: Any, + run_state: Any, +) -> SingleStepResult: + from ..models.codex import _CodexModel + + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError("Native approval recovery requires a claimed ResumableSession.") + pending = processed_response.native_approvals[0] + checkpoint = native_approval_checkpoint(pending.raw_item) + assert checkpoint is not None + call = pending.raw_item + key = context_wrapper._resolve_approval_key(pending) + key = context_wrapper._function_approval_key(key, pending) or key + decision = context_wrapper._approvals.get(key) + if decision is not None and (decision.approved is True or decision.rejected is True): + raise UserError("Native approvals require a decision for the original call only.") + approved = context_wrapper.get_approval_status( + pending.tool_name, + call.call_id, + existing_pending=pending, + current_invocation=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + if approved is None: + with pending_tool_span(run_config, pending.tool_name, call.call_id) as span: + span.span_data.operation.outcome = "paused" + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[], + next_step=NextStepInterruption(interruptions=[pending]), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + # A new checkpoint can carry the same model call ID. Claiming that checkpoint + # doesn't authorize repeating an invocation that already used its decision. + status = context_wrapper._tool_invocation_status( + call, tool_lookup_key=pending.tool_lookup_key, tool_name=pending.tool_name + ) + if status is None: + raise UserError("Cannot match the native approval to a pending invocation.") + _, completed, executed = status + if executed or completed: + raise UserError( + "Cannot reuse a native call ID after the call has executed or completed. " + "Start a new invocation with a new call ID." + ) + if approved: + from ..tracing.model_tracing import get_model_tracing_impl + + model = bindings.resolved_model or get_model(bindings.execution_agent, run_config) + if not isinstance(model, _CodexModel): + raise UserError("A pending native approval requires its current Codex model.") + tools = await get_all_tools(bindings.execution_agent, context_wrapper) + with model_run_context( + run_state, + ModelToolHooks( + bindings.public_agent, context_wrapper, hooks, run_config.tool_execution + ), + ): + output = await model._run_approved_native_call( + checkpoint, + tools=tools, + tracing=get_model_tracing_impl( + run_config.tracing_disabled, run_config.trace_include_sensitive_data + ), + ) + else: + from .tool_execution import resolve_approval_rejection_message + + with pending_tool_span(run_config, pending.tool_name, call.call_id): + output = await resolve_approval_rejection_message( + context_wrapper=context_wrapper, + run_config=run_config, + tool_type="shell" if call.name == "exec_command" else "apply_patch", + tool_name=call.name, + call_id=call.call_id, + tool_call=call, + existing_pending=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + result_item = ToolCallOutputItem( + agent=bindings.public_agent, + raw_item=cast( + ToolCallOutputTypes, + { + "type": "function_call_output" + if call.type == "function_call" + else "custom_tool_call_output", + "call_id": call.call_id, + "output": output, + }, + ), + output=output, + ) + context_wrapper._mark_tool_call_completed(result_item.raw_item) + processed_response.interruptions = [] + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[result_item], + next_step=NextStepRunAgain(), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) diff --git a/src/agents/run_internal/resumable_session.py b/src/agents/run_internal/resumable_session.py index a55fba5c31..9754d31c09 100644 --- a/src/agents/run_internal/resumable_session.py +++ b/src/agents/run_internal/resumable_session.py @@ -37,6 +37,13 @@ async def begin(self, input: Any) -> None: await self._source._begin_run(input) self._items = self._source._get_history() + def bind_state(self, state: Any) -> None: + """Attach the current store claim to the internal execution state.""" + assert self._source is not None + state._resumable_binding = _ResumableSessionBinding( + self._source, self._source._checkpoint.revision + ) + async def finish(self, result: RunResult | RunResultStreaming) -> None: assert self._source is not None state = result.to_state() if result.interruptions else None diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 659642845a..7646456f04 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -164,7 +164,7 @@ ) from .oai_conversation import OpenAIServerConversationTracker from .prompt_cache_key import PromptCacheKeyResolver, model_settings_with_prompt_cache_key -from .resumable_session import checkpoint_streamed_result +from .resumable_session import ResumableRun, checkpoint_streamed_result from .run_steps import ( NextStepFinalOutput, NextStepHandoff, @@ -2692,7 +2692,13 @@ def get_model_stream() -> AsyncIterator[TResponseStreamEvent]: model_run_context_stream( retry_stream, tool_use_tracker, - ModelToolHooks(public_agent, context_wrapper, hooks, run_config.tool_execution), + ModelToolHooks( + public_agent, + context_wrapper, + hooks, + run_config.tool_execution, + resumable=isinstance(session, ResumableRun), + ), ) ) as model_events: async for event in model_events: @@ -3117,7 +3123,13 @@ async def rewind_model_request() -> None: with ( model_run_context( tool_use_tracker, - ModelToolHooks(public_agent, context_wrapper, hooks, run_config.tool_execution), + ModelToolHooks( + public_agent, + context_wrapper, + hooks, + run_config.tool_execution, + resumable=isinstance(session, ResumableRun), + ), ), record_failed_attempt_usage_on_error(context_wrapper.usage) as reported_usage, ): diff --git a/src/agents/run_internal/run_steps.py b/src/agents/run_internal/run_steps.py index e98177a1d4..4e786e46e4 100644 --- a/src/agents/run_internal/run_steps.py +++ b/src/agents/run_internal/run_steps.py @@ -137,6 +137,7 @@ class ProcessedResponse: # Original invocation recipients, independent of the currently available callables. tool_output_guardrail_result_start: int | None = None # First checkpointed tool output guardrail result owned by this response. + native_approvals: list[ToolApprovalItem] = dataclasses.field(default_factory=list) def has_tools_or_approvals_to_run(self) -> bool: # Handoffs, functions and computer actions need local processing @@ -152,6 +153,7 @@ def has_tools_or_approvals_to_run(self) -> bool: self.apply_patch_calls, self.mcp_approval_requests, self.function_tools_not_found, + self.native_approvals, ] ) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 04fad384e2..e847c9bddd 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -101,7 +101,13 @@ ) from ..lifecycle import RunHooks from ..logger import log_tool_action_error, logger -from ..models._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ..models._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -151,6 +157,7 @@ run_item_to_input_item, shell_rejection_item, ) +from .native_approvals import native_pending_step, resolve_native_approval_step from .run_steps import ( NOT_FINAL_OUTPUT, NextStepFinalOutput, @@ -987,6 +994,18 @@ async def execute_tools_and_side_effects( """Run one turn of the loop, coordinating tools, approvals, guardrails, and handoffs.""" public_agent = bindings.public_agent + if processed_response.native_approvals: + _register_tool_call_items(context_wrapper, processed_response.new_items) + return native_pending_step( + bindings=bindings, + original_input=original_input, + pre_step_items=pre_step_items, + new_response=new_response, + processed_response=processed_response, + run_state=run_state, + run_config=run_config, + ) + execute_final_output_call = execute_final_output execute_handoffs_call = execute_handoffs @@ -1406,6 +1425,18 @@ async def resolve_interrupted_turn( """Continue a turn that was previously interrupted waiting for tool approval.""" public_agent = bindings.public_agent execution_agent = bindings.execution_agent + if processed_response.native_approvals: + return await resolve_native_approval_step( + bindings=bindings, + original_input=original_input, + original_pre_step_items=original_pre_step_items, + new_response=new_response, + processed_response=processed_response, + context_wrapper=context_wrapper, + hooks=hooks, + run_config=run_config, + run_state=run_state, + ) output_index = _build_tool_output_index(original_pre_step_items) response_boundary = _current_response_boundary((), processed_response, run_state) generated_output_start = response_boundary.generated_start @@ -3172,6 +3203,7 @@ def process_model_response( allow_apply_patch_function_fallback: bool = True, ) -> ProcessedResponse: items: list[RunItem] = [] + native_approvals: list[ToolApprovalItem] = [] # Only the Codex adapter creates these types, after validating native call/output pairing. provider_completed = { @@ -3276,6 +3308,12 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: output.__class__.__name__ if hasattr(output, "__class__") else type(output), ) caller = get_mapping_or_attr(output, "caller") + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + items.append(ToolCallItem(raw_item=cast(Any, output), agent=agent)) + native_approvals.append(ToolApprovalItem(agent=agent, raw_item=output)) + tools_used.append(output.name) + continue is_program_owned = get_mapping_or_attr(caller, "type") == "program" if ( not is_program_owned @@ -3964,6 +4002,7 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tools_used=tools_used, mcp_approval_requests=mcp_approval_requests, interruptions=[], + native_approvals=native_approvals, function_tools_not_found=function_tools_not_found, mcp_tool_bindings={ run.tool_call.call_id: binding @@ -3999,6 +4038,9 @@ def _preflight_response_invocations( if isinstance(item, (CodexFunctionToolCallOutput, CodexCustomToolCallOutput)) } for index, output in enumerate(response.output): + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + continue if isinstance(output, (ResponseFunctionToolCall, ResponseCustomToolCall)) and ( output.call_id in provider_completed ): diff --git a/src/agents/run_state.py b/src/agents/run_state.py index a5a1ea69a0..29d5d6458e 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -110,6 +110,10 @@ log_model_and_tool_data_warning, logger, ) +from .models._codex_items import ( + native_approval_checkpoint, + restore_native_approval_call, +) from .run_context import RunContextWrapper, _ApprovalRecord, _FunctionToolApprovalKey from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -312,7 +316,7 @@ def _default_run_state_validation_error( ), "1.23": ( "Binds prepared tool inputs to their original proposals across retries and resume, " - "and records required ResumableSession ownership for paused runs." + "records required ResumableSession ownership, and preserves native approval receipts." ), } SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES) @@ -785,6 +789,11 @@ def _restore_pydantic_fields_set(value: Any, source: Any) -> None: def _copy_tool_approval_raw_item(raw_item: Any) -> Any: """Copy a supported approval raw item through public Pydantic APIs.""" + native_checkpoint = native_approval_checkpoint(raw_item) + if native_checkpoint is not None: + return restore_native_approval_call( + raw_item.model_dump(exclude_unset=True), native_checkpoint + ) active_container_ids: set[int] = set() raw_item_mro = _static_type_mro(raw_item) for model_type, model_adapter in _TOOL_APPROVAL_MODEL_ADAPTERS: @@ -1237,6 +1246,13 @@ def _find_current_approval_item( ) return canonical_matches[0] if canonical_matches else None + def _decision_item(self, proposal: ToolApprovalItem) -> ToolApprovalItem: + current = self._find_current_approval_item(proposal) + expected = native_approval_checkpoint(current.raw_item) if current else None + if native_approval_checkpoint(proposal.raw_item) != expected: + raise UserError("Native approval does not match the current pending call.") + return current or proposal + def _find_nested_approval_state( self, approval_item: ToolApprovalItem, @@ -1409,9 +1425,8 @@ def approve(self, approval_item: ToolApprovalItem, always_approve: bool = False) nested_state, nested_item = nested_approval nested_state.approve(nested_item, always_approve=always_approve) return - current_approval_item = self._find_current_approval_item(approval_item) self._context.approve_tool( - current_approval_item or approval_item, + self._decision_item(approval_item), always_approve=always_approve, ) @@ -1441,7 +1456,7 @@ def reject( ) return self._context.reject_tool( - self._find_current_approval_item(approval_item) or approval_item, + self._decision_item(approval_item), always_reject=always_reject, rejection_message=rejection_message, ) @@ -2167,6 +2182,15 @@ def _serialize_processed_response( for call_id, binding in processed_response.mcp_tool_bindings.items() }, } + if processed_response.native_approvals: + result["native_approvals"] = [ + _serialize_tool_approval_interruption( + item, + include_tool_name=True, + agent_identity_keys_by_id=agent_identity_keys_by_id, + ) + for item in processed_response.native_approvals + ] if processed_response.tool_output_guardrail_result_start is not None: result["tool_output_guardrail_result_start"] = ( processed_response.tool_output_guardrail_result_start @@ -2226,6 +2250,9 @@ def _serialize_item( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(item.raw_item) + if native_checkpoint is not None: + result["codex_native_checkpoint"] = native_checkpoint if isinstance(item, InputItem): result["input_id"] = item.input_id @@ -2539,7 +2566,8 @@ def validation_error_factory( state_json, validation_error_factory=validation_error_factory, ) - if _has_native_approvals(state_json): + + if _unsupported_native_approvals(state_json): raise validation_error_factory(_UNSUPPORTED_NATIVE_APPROVALS, UserError) from .sandbox._mount_security import sanitize_run_state_sandbox_mount_authority @@ -2598,18 +2626,25 @@ def validation_error_factory( ) -def _has_native_approvals(state: dict[str, Any]) -> bool: +def _unsupported_native_approvals(state: dict[str, Any], *, nested: bool = False) -> bool: """Check nested agent runs before best-effort context restoration can skip a child.""" response = state.get("last_processed_response") if not isinstance(response, dict): return False - if response.get("native_approvals"): - return True + approvals = response.get("native_approvals") + if approvals: + # Only the root session has a claimed checkpoint. A nested state cannot + # authorize an action, even when it contains an otherwise valid receipt. + if nested or not isinstance(approvals, list) or len(approvals) != 1: + return True + item = approvals[0] + if not isinstance(item, dict) or not isinstance(item.get("codex_native_checkpoint"), dict): + return True functions = response.get("functions") if isinstance(functions, list): for entry in functions: child = entry.get("agent_run_state") if isinstance(entry, dict) else None - if isinstance(child, dict) and _has_native_approvals(child): + if isinstance(child, dict) and _unsupported_native_approvals(child, nested=True): return True return False @@ -2920,6 +2955,9 @@ def _serialize_tool_approval_interruption( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(interruption.raw_item) + if native_checkpoint is not None: + interruption_dict["codex_native_checkpoint"] = native_checkpoint if include_tool_name and interruption.tool_name is not None: interruption_dict["tool_name"] = interruption.tool_name if interruption.tool_namespace is not None: @@ -3667,6 +3705,19 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: if approval_item is not None: interruptions.append(approval_item) + native_approvals: list[ToolApprovalItem] = [] + for approval_data in processed_response_data.get("native_approvals", []): + approval = _deserialize_tool_approval_item( + approval_data, + agent_map=agent_map, + agent_identity_map=agent_identity_map, + fallback_agent=current_agent, + validation_error_factory=validation_error_factory, + ) + if approval is None or native_approval_checkpoint(approval.raw_item) is None: + raise validation_error_factory("Invalid native approval plan", UserError) + native_approvals.append(approval) + saved_bindings = processed_response_data.get("mcp_tool_bindings", {}) mcp_tool_bindings = { call_id: cast(tuple[str, str, int | None], tuple(binding)) @@ -3680,6 +3731,7 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: return ProcessedResponse( new_items=new_items, + native_approvals=native_approvals, handoffs=handoffs, functions=functions, computer_actions=computer_actions, @@ -3859,6 +3911,8 @@ def _deserialize_tool_approval_item( tool_lookup_key = deserialize_function_tool_lookup_key(item_data.get("tool_lookup_key")) allow_bare_name_alias = item_data.get("allow_bare_name_alias") is True raw_item = _deserialize_tool_approval_raw_item(raw_item_data) + if "codex_native_checkpoint" in item_data: + raw_item = restore_native_approval_call(raw_item_data, item_data["codex_native_checkpoint"]) return ToolApprovalItem( agent=agent, raw_item=raw_item, @@ -4260,6 +4314,10 @@ async def _build_run_state_from_json( validation_error_factory=validation_error_factory, ) schema_major, schema_minor = (int(part) for part in schema_version.split(".", maxsplit=1)) + if (schema_major, schema_minor) < (1, 23) and ( + (state_json.get("last_processed_response") or {}).get("native_approvals") + ): + raise validation_error_factory("Native approvals require RunState schema 1.23.", UserError) programmatic_major, programmatic_minor = ( int(part) for part in _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION.split(".", maxsplit=1) ) @@ -5367,6 +5425,10 @@ def _resolve_agent_info( # Tool call items can be function calls, shell calls, apply_patch calls, # MCP calls, etc. Check the type field to determine which type to deserialize as raw_item_tool = _deserialize_tool_call_raw_item(normalized_raw_item) + if "codex_native_checkpoint" in item_data: + raw_item_tool = restore_native_approval_call( + normalized_raw_item, item_data["codex_native_checkpoint"] + ) # Preserve display metadata if it was stored with the item. description = item_data.get("description") title = item_data.get("title") diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py new file mode 100644 index 0000000000..5d9ef00a95 --- /dev/null +++ b/tests/memory/test_native_approval_recovery.py @@ -0,0 +1,475 @@ +from __future__ import annotations + +import asyncio +import copy +import json + +import pytest + +from agents import Agent, ResumableSession, RunConfig, Runner, RunState +from agents.exceptions import AgentsException, UserError +from agents.items import ModelResponse +from agents.memory.resumable_session import ResumableSessionError +from agents.models._codex_items import CodexPendingCustomCall, CodexPendingFunctionCall +from agents.models.codex import CodexProvider +from agents.testing import ScriptedModel +from agents.usage import Usage +from tests.memory.test_resumable_session import MemoryStore, run_agent +from tests.models.test_codex_provider import FakeThread, threads as threads +from tests.test_responses import get_text_message +from tests.testing_processor import fetch_ordered_spans + + +def pending_call(tmp_path, kind): + call = { + "type": "function_call" if kind == "exec_command" else "custom_tool_call", + "id": "native-item", + "call_id": "native-call", + "name": kind, + "status": "in_progress", + } + if kind == "exec_command": + call["arguments"] = '{"cmd":"echo harmless"}' + item_type = CodexPendingFunctionCall + else: + call["input"] = "*** Begin Patch\n*** Add File: harmless.txt\n+safe\n*** End Patch\n" + item_type = CodexPendingCustomCall + checkpoint = { + "version": 1, + "cwd": str(tmp_path.resolve()), + "call": copy.deepcopy(call), + "request": {"type": "native_approval", "call_id": "native-call"}, + "model": "fixture-model", + "preconditions": {"version": 1, "first_approval": True}, + } + item = item_type(**call, checkpoint=checkpoint) + return item.model_copy(update={"codex_reasoning_summary": "Review this effect."}) + + +async def setup_native(tmp_path, monkeypatch, kind="exec_command"): + provider = CodexProvider(cwd=tmp_path) + model = provider.get_model("fixture-model") + pending = pending_call(tmp_path, kind) + calls = [] + requests = [] + + async def respond(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = [pending] if len(requests) == 1 else [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint) + return "native accepted" + + monkeypatch.setattr(model, "get_response", respond) + monkeypatch.setattr(model, "_run_approved_native_call", execute, raising=False) + agent = Agent(name="worker", model=model) + store = MemoryStore() + session = await ResumableSession.create(store) + config = RunConfig(tracing_disabled=True) + return agent, store, session, config, pending, calls, requests + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"]) +@pytest.mark.parametrize("approval_after_count", [False, True], ids=["before-count", "after-count"]) +async def test_native_pause_preserves_all_sampling_usage_across_session_resume( + tmp_path, monkeypatch, threads, streamed, approval_after_count +): + from tests.models.test_codex_pending_approval import ApprovalThread, approval, raw + from tests.models.test_codex_provider import native_pair + + earlier = { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + "cached_input_tokens": 1, + "cache_write_input_tokens": 1, + "reasoning_output_tokens": 1, + } + current = { + "input_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + "cached_input_tokens": 2, + "cache_write_input_tokens": 3, + "reasoning_output_tokens": 2, + } + cumulative = {key: earlier[key] + current[key] for key in earlier} + + # The wire double preserves the completion/count ordering around a real Runner pause. + class SamplingApprovalThread(ApprovalThread): + async def events(self): + yield { + "type": "raw_response_completed", + "response_id": "earlier", + "token_usage": earlier, + } + yield {"type": "token_count", "info": {"total_token_usage": earlier}} + call, _ = native_pair(call_id="pending") + yield raw(call) + if not approval_after_count: + yield approval() + yield { + "type": "raw_response_completed", + "response_id": "current", + "token_usage": current, + } + yield {"type": "token_count", "info": {"total_token_usage": cumulative}} + if approval_after_count: + yield approval() + + def tokens(usage): + return ( + usage.input_tokens, + usage.output_tokens, + usage.total_tokens, + usage.input_tokens_details.cached_tokens, + usage.input_tokens_details.cache_write_tokens, + usage.output_tokens_details.reasoning_tokens, + ) + + model = CodexProvider(cwd=tmp_path).get_model("fixture-model") + agent = Agent(name="worker", model=model) + first = SamplingApprovalThread() + threads[0].extend([first, FakeThread(text="finished")]) + calls = [] + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint["call"]["call_id"]) + return "native accepted" + + monkeypatch.setattr(model, "_run_approved_native_call", execute) + store = MemoryStore() + session = await ResumableSession.create(store) + paused = await run_agent(agent, "start", session, streamed) + assert first.recover_approvals + assert first.closed and len(paused.interruptions) == 1 and calls == [] + assert tokens(paused.context_wrapper.usage) == (15, 5, 20, 3, 4, 3) + assert store.checkpoint(session.session_id)["status"] == "paused" + + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + completed = await run_agent(agent, state, None, streamed) + assert completed.final_output == "finished" and calls == ["pending"] + # The fresh native thread adds its own 10 input and 7 output tokens exactly once. + assert tokens(completed.context_wrapper.usage) == (25, 12, 37, 6, 4, 5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("approved", [True, False]) +async def test_native_approval_round_trip_uses_original_runtime_only( + tmp_path, monkeypatch, kind, approved +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + first = await Runner.run(agent, "do the native operation", session=session, run_config=config) + assert len(first.interruptions) == 1 + assert calls == [] + assert store.checkpoint(session.session_id)["status"] == "paused" + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + interruption = state.get_interruptions()[0] + assert interruption.arguments == ( + pending.arguments if kind == "exec_command" else pending.input + ) + if approved: + state.approve(interruption) + else: + state.reject(interruption, rejection_message="Leave the release branch alone.") + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert result.final_output == "finished" + assert calls == ([pending.checkpoint] if approved else []) + assert "checkpoint" not in repr(requests[-1]) + expected_message = "native accepted" if approved else "Leave the release branch alone." + assert any( + item.get("call_id") == interruption.call_id and item.get("output") == expected_message + for item in requests[-1] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_plain_runner_rejects_unrecoverable_native_pause(tmp_path, monkeypatch, streamed): + agent, _, _, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + with pytest.raises(UserError, match="session=ResumableSession"): + if streamed: + result = Runner.run_streamed(agent, "start", run_config=config) + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "start", run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("field", ["name", "call_id", "id", "payload"]) +async def test_display_mutation_cannot_be_approved(tmp_path, monkeypatch, kind, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch, kind) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + item = state.get_interruptions()[0] + field = ("arguments" if kind == "exec_command" else "input") if field == "payload" else field + setattr(item.raw_item, field, "changed") + with pytest.raises(UserError): + state.approve(item) + assert not calls + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["payload", "cwd", "request", "model"]) +async def test_coordinated_receipt_mutation_fails_before_claim(tmp_path, monkeypatch, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + # Rewrite every serialized copy consistently; the store's original still binds authority. + snapshot = state.to_json() + + def rewrite(value): + if isinstance(value, dict): + if field == "payload" and value.get("type") == "function_call": + value["arguments"] = '{"cmd":"changed"}' + if value.get("version") == 1 and "cwd" in value and "request" in value: + if field == "cwd": + value["cwd"] = str(tmp_path / "other") + elif field == "request": + value["request"] = {"type": "changed"} + elif field == "model": + value["model"] = "changed" + for child in value.values(): + rewrite(child) + elif isinstance(value, list): + for child in value: + rewrite(child) + + rewrite(snapshot) + changed = await RunState.from_string(agent, json.dumps(snapshot)) + changed._resumable_binding = state._resumable_binding + original = store.records[session.session_id] + with pytest.raises(ResumableSessionError): + await Runner.run(agent, changed, session=recovered, run_config=config) + assert store.records[session.session_id] == original + assert not calls + + +@pytest.mark.asyncio +async def test_no_decision_repauses_and_unbound_copy_never_dispatches(tmp_path, monkeypatch): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + first = await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(result.interruptions) == 1 + assert not calls + copied = await RunState.from_string(agent, first.to_state().to_string()) + copied.approve(copied.get_interruptions()[0]) + with pytest.raises(UserError): + await Runner.run(agent, copied, run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +async def test_completed_native_call_id_cannot_reuse_a_previous_decision( + tmp_path, monkeypatch, kind +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + + async def repeat_native_call(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + if len(requests) <= 2: + # The adapter can attest a new native request with a repeated model call ID. + repeated = pending.model_copy(deep=True) + repeated.checkpoint["request"]["turn_id"] = f"native-turn-{len(requests)}" + output = [repeated] + else: + output = [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + monkeypatch.setattr(agent.model, "get_response", repeat_native_call) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + + # Native execution stays at the adapter seam while public resume owns approval state. + with pytest.raises(UserError, match="completed"): + repeated_result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(repeated_result.interruptions) == 1 + assert len(calls) == 1 + replacement = await ResumableSession.resume(store, session.session_id) + repeated_state = await replacement.load_state(agent) + await Runner.run(agent, repeated_state, session=replacement, run_config=config) + assert len(calls) == 1, "The previous decision authorized another native dispatch." + + assert len(calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("approved", [True, False]) +async def test_sticky_native_decisions_are_rejected(tmp_path, monkeypatch, approved): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + state = await session.load_state(agent) + item = state.get_interruptions()[0] + with pytest.raises(UserError, match="original call only"): + if approved: + state.approve(item, always_approve=True) + else: + state.reject(item, always_reject=True) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_waits_for_checkpoint_ack(tmp_path, monkeypatch, streamed): + agent, store, session, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + store.block_checkpoint = True + events = [] + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + result = Runner.run_streamed(agent, "start", session=session, run_config=config) + + async def consume(): + async for event in result.stream_events(): + events.append(event) + return result + + task = asyncio.create_task(consume()) + else: + task = asyncio.create_task(Runner.run(agent, "start", session=session, run_config=config)) + try: + await asyncio.wait_for(store.saving.wait(), 5) + assert not task.done() + assert store.checkpoint(session.session_id)["status"] == "active" + assert not calls + if streamed: + assert not result.interruptions + assert not any( + getattr(event, "name", None) == "tool_approval_requested" for event in events + ) + with pytest.raises(AgentsException): + result.to_state() + finally: + store.release.set() + finished = await asyncio.wait_for(task, 5) + assert len(finished.interruptions) == 1 + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_data,cancel", [(False, False), (True, False), (False, True)]) +async def test_restored_native_execution_observes_live_start_and_interruption( + tmp_path, monkeypatch, threads, include_data, cancel +): + from agents.tracing import get_current_span + from tests.testing_processor import SPAN_PROCESSOR_TESTING + + started = asyncio.Event() + release = asyncio.Event() + closed = asyncio.Event() + updates = asyncio.Event() + events = [] + calls = [] + original_start = SPAN_PROCESSOR_TESTING.on_span_start + caller_spans = [] + + def on_start(span): + original_start(span) + if span.span_data.type == "function": + caller_spans.append(get_current_span()) + started.set() + + monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_start", on_start) + + # The FFI double preserves suspension and cancellation at the native execution boundary. + class RestoredThread(FakeThread): + def enable_tracing(self, mode="metadata"): + calls.append(mode) + + async def wait_for_operations(self): + await updates.wait() + + def drain_operations(self): + encoded = json.dumps(events) + events.clear() + updates.clear() + return encoded + + async def resolve_approval(self, call_json, request, preconditions): + key = {"kind": "tool", "name": "exec_command", "id": json.loads(call_json)["call_id"]} + events.append(dict(key, phase="start", at=1_795_154_400_000)) + if calls == ["content"]: + events.append(dict(key, phase="input", input="private command")) + updates.set() + await release.wait() + events.append(dict(key, phase="end", outcome="completed", at=1_795_154_402_000)) + if calls == ["content"]: + events.append(dict(key, phase="output", output="native accepted")) + updates.set() + return "native accepted" + + async def close(self): + await super().close() + closed.set() + + agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + monkeypatch.delattr(agent.model, "_run_approved_native_call") + threads[0].append(RestoredThread()) + task = asyncio.create_task( + Runner.run( + agent, + state, + run_config=RunConfig( + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=include_data, + ), + ) + ) + try: + await asyncio.wait_for(started.wait(), 2) + assert not task.done() + assert calls == ["content" if include_data else "metadata"] + assert caller_spans[0] is not None + assert caller_spans[0].span_data.type == "agent" + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + result = await asyncio.wait_for(task, 2) + assert result.final_output == "finished" + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert closed.is_set() + (span,) = [ + span + for span in fetch_ordered_spans() + if span.span_data.type == "function" + and span.span_data.operation.tool_call_id == "native-call" + ] + assert span.span_data.operation.outcome == ("interrupted" if cancel else "completed") + assert span.started_at == "2026-11-20T06:00:00+00:00" + assert ("private command" in repr(span.span_data.input)) == include_data + assert span.parent_id == caller_spans[0].span_id diff --git a/tests/memory/test_native_approval_tracing.py b/tests/memory/test_native_approval_tracing.py new file mode 100644 index 0000000000..2a563b0ba9 --- /dev/null +++ b/tests/memory/test_native_approval_tracing.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agents import ResumableSession, RunConfig, Runner +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.testing import ScriptedModel +from agents.tracing import set_trace_processors +from tests.memory.test_native_approval_recovery import setup_native +from tests.test_responses import get_text_message +from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["default", "metadata", "content", "veto", "disabled"]) +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_and_denial_emit_only_approval_metadata( + tmp_path, monkeypatch, mode, streamed +): + agent, store, session, _, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + script = ScriptedModel([[pending], [get_text_message("finished")]]) + monkeypatch.setattr(agent.model, "stream_response", script.stream_response) + config = RunConfig( + tracing={} + if mode == "default" + else {"operation_detail": "metadata" if mode == "metadata" else "content"}, + tracing_disabled=mode == "disabled", + trace_include_sensitive_data=mode != "veto", + ) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([SPAN_PROCESSOR_TESTING, processor]) + + async def run(input, current): + if streamed: + result = Runner.run_streamed(agent, input, session=current, run_config=config) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, input, session=current, run_config=config) + + try: + first = await run("start", session) + assert len(first.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + second = await run(await recovered.load_state(agent), recovered) + assert len(second.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.reject(state.get_interruptions()[0], rejection_message="DENIAL_CANARY") + result = await run(state, recovered) + assert result.final_output == "finished" + assert not calls + spans = fetch_ordered_spans() + if mode == "default": + decisions = [s.span_data for s in spans if s.span_data.type == "function"] + assert [s.operation.outcome for s in decisions] == ["paused", "paused", "rejected"] + assert all(s.operation.tool_call_id == pending.call_id for s in decisions) + assert all(s.input is None and s.output is None for s in decisions) + else: + assert not [s for s in spans if s.span_data.type == "function"] + decisions = [ + s.span_data.data + for s in spans + if getattr(s.span_data, "name", None) == "tool_approval" + ] + assert decisions == ( + [] + if mode == "disabled" + else [ + {"tool_name": pending.name, "tool_call_id": pending.call_id, "outcome": outcome} + for outcome in ["paused", "paused", "rejected"] + ] + ) + otel = exporter.get_finished_spans() + approvals = [s for s in otel if s.name == "openai.agents.tool_approval exec_command"] + assert len(approvals) == (0 if mode == "disabled" else 3) + assert not [s for s in otel if s.name.startswith("execute_tool")] + assert "DENIAL_CANARY" not in json.dumps([dict(s.attributes or {}) for s in approvals]) + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() diff --git a/tests/memory/test_resumable_session.py b/tests/memory/test_resumable_session.py index f9aaa7b500..69987751ca 100644 --- a/tests/memory/test_resumable_session.py +++ b/tests/memory/test_resumable_session.py @@ -96,6 +96,16 @@ def exec_command(command: str) -> str: processed = pending_state["last_processed_response"] # An earlier preview stored native actions outside the SDK function queue. processed["native_approvals"] = copy.deepcopy(processed["interruptions"]) + if nested: + item = processed["native_approvals"][0] + item["codex_native_checkpoint"] = { + "version": 1, + "cwd": str(tmp_path.resolve()), + "model": "fixture-model", + "call": item["raw_item"], + "request": {"type": "exec_approval_request", "call_id": item["raw_item"]["call_id"]}, + "preconditions": {"version": 1, "first_approval": True}, + } processed["functions"] = [] checkpoint["run_state"] = json.dumps(snapshot) stored = json.dumps(checkpoint) diff --git a/tests/models/test_codex_pending_approval.py b/tests/models/test_codex_pending_approval.py new file mode 100644 index 0000000000..396404ce47 --- /dev/null +++ b/tests/models/test_codex_pending_approval.py @@ -0,0 +1,166 @@ +import json + +import pytest + +from agents import ModelSettings, UserError +from agents.models._codex_items import ( + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) +from agents.models.codex import CodexProvider +from agents.models.interface import ModelTracing + +from .test_codex_provider import FakeThread, native_pair, threads as threads + + +class ApprovalThread(FakeThread): + async def next_event(self): + encoded = await super().next_event() + if encoded is None: + return None + event = json.loads(encoded) + if event["msg"]["type"] in ( + "exec_approval_request", + "apply_patch_approval_request", + ): + event["preconditions"] = {"version": 1, "first_approval": True} + return json.dumps(event) + + +def approval(call_id="pending"): + return { + "type": "exec_approval_request", + "call_id": call_id, + "turn_id": "turn_1", + "kind": "command", + "command": ["sh", "-c", "printf native-result"], + "cwd": "/workspace", + } + + +def raw(item): + return {"type": "raw_response_item", "item": item} + + +async def response(tmp_path, settings=None): + return ( + await CodexProvider(cwd=tmp_path) + .get_model(None) + .get_response( + None, + "Continue", + settings or ModelSettings(), + [], + None, + [], + ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ) + + +@pytest.mark.asyncio +async def test_approval_waits_for_sampling_completion_and_rejects_late_sibling(tmp_path, threads): + call, _ = native_pair(call_id="pending") + sibling, _ = native_pair(call_id="later") + thread = ApprovalThread( + events=[ + raw(call), + approval(), + raw(sibling), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + with pytest.raises(UserError, match="one outstanding native call"): + await response(tmp_path) + assert thread.closed + + +@pytest.mark.asyncio +async def test_completed_read_stays_completed_while_next_native_call_pauses(tmp_path, threads): + read, read_output = native_pair(call_id="read") + read["arguments"] = json.dumps({"cmd": "cat README.md"}) + pending, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + raw(read), + raw(read_output), + raw(pending), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert [item.call_id for item in result.output] == ["read", "read", "pending"] + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert result.output[1].output == read_output["output"] + assert isinstance(result.output[2], CodexPendingFunctionCall) + assert result.output[2].checkpoint["call"]["call_id"] == "pending" + assert thread.closed + + +@pytest.mark.asyncio +async def test_raw_model_metadata_cannot_forge_a_pending_native_approval(tmp_path, threads): + call, output = native_pair() + call["checkpoint"] = {"request": approval("native_1"), "version": 1} + call["native_approval"] = True + thread = FakeThread(raw_items=[call, output]) + threads[0].append(thread) + result = await response(tmp_path) + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert not hasattr(result.output[0], "checkpoint") + assert "native_approval" not in result.output[0].model_dump() + assert thread.closed + + +@pytest.mark.asyncio +async def test_patch_begin_does_not_mean_an_awaiting_patch_has_executed(tmp_path, threads): + call, _ = native_pair("custom_tool_call", "pending", "apply_patch") + + thread = ApprovalThread( + events=[ + raw(call), + {"type": "patch_apply_begin", "call_id": "pending"}, + { + "type": "apply_patch_approval_request", + "call_id": "pending", + "turn_id": "turn_1", + "changes": {}, + }, + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert isinstance(result.output[0], CodexPendingCustomCall) + assert result.output[0].checkpoint["preconditions"]["first_approval"] is True + assert thread.closed + + +@pytest.mark.asyncio +async def test_reasoning_summary_does_not_change_the_approved_action(tmp_path, threads): + call, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + { + "type": "item_completed", + "item": {"type": "Reasoning", "summary_text": ["Reviewing the command."]}, + }, + raw(call), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path, ModelSettings(reasoning={"summary": "auto"})) + pending = result.output[-1] + assert pending.codex_reasoning_summary == ["Reviewing the command."] + checkpoint = native_approval_checkpoint(pending) + assert checkpoint is not None + assert checkpoint["call"]["call_id"] == "pending" + assert thread.closed diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 6e0d80e5d7..f593214edf 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -39,7 +39,8 @@ def __init__(self, text="A Codex response", call=None, gate=None, raw_items=(), async def inject_items(self, items): self.history = json.loads(items) - async def start_turn(self, inputs, schema): + async def start_turn(self, inputs, schema, recover_approvals=False): + self.recover_approvals = recover_approvals self.inputs = json.loads(inputs) self.schema = json.loads(schema) self.started.set() @@ -1986,8 +1987,9 @@ def __init__(self, *observations, **kwargs): self.observations = list(observations) self.tracing = False - def enable_tracing(self): + def enable_tracing(self, mode=None): self.tracing = True + self.capture_mode = mode def drain_operations(self): assert self.tracing @@ -1997,9 +1999,7 @@ def drain_operations(self): @pytest.mark.asyncio @pytest.mark.parametrize("streamed", [False, True]) -async def test_stock_operations_stay_metadata_only_and_keep_source_times( - tmp_path, threads, streamed -): +async def test_metadata_operations_keep_source_times(tmp_path, threads, streamed): from tests.testing_processor import fetch_ordered_spans model = {"kind": "model", "id": "1", "model": "native-default"} @@ -2019,7 +2019,7 @@ async def test_stock_operations_stay_metadata_only_and_keep_source_times( threads[0].append(thread) cfg = RunConfig( model_provider=CodexProvider(cwd=tmp_path), - tracing={"operation_detail": "content"}, + tracing={"operation_detail": "metadata"}, trace_include_sensitive_data=True, ) if streamed: @@ -2046,8 +2046,8 @@ async def test_stock_operations_stay_metadata_only_and_keep_source_times( for span in (request, execution): assert span.span_data.operation.detail == "metadata" assert span.span_data.input is span.span_data.output is None - (notice,) = [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] - assert notice.span_data.data["reason"] == "metadata_only" + assert thread.capture_mode is None + assert not [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] @pytest.mark.asyncio @@ -2078,3 +2078,74 @@ async def test_disabled_tracing_never_subscribes_to_native_operations(tmp_path, threads[0].append(thread) await Runner.run(Agent(name="test"), "Hello", run_config=config(tmp_path)) assert not thread.tracing + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "detail,sensitive,captures", + [(None, True, False), ("content", False, False), ("content", True, True)], +) +async def test_native_content_requires_explicit_policy_and_correlates_final_output( + tmp_path, threads, detail, sensitive, captures +): + from tests.testing_processor import fetch_ordered_spans + + model = {"kind": "model", "id": "request", "model": "fixture"} + tool = {"kind": "tool", "id": "command", "name": "exec_command"} + thread = TracedThread( + {**model, "phase": "start", "at": 1000}, + { + **model, + "phase": "input", + "input": { + "input": [{"role": "user", "content": "request text"}], + "tools": [{"type": "function", "name": "lookup"}], + "client_metadata": {"api_key": "never-export"}, + }, + }, + {**tool, "phase": "start", "at": 1100}, + { + **tool, + "phase": "input", + "input": {"command": ["echo", "prepared"], "password": "never-export"}, + }, + {**tool, "phase": "end", "at": 1200, "outcome": "completed"}, + {"kind": "tool", "phase": "output", "id": "unrelated", "output": "wrong output"}, + {"kind": "tool", "phase": "output", "id": "command", "output": "accepted result"}, + { + **model, + "phase": "end", + "at": 1300, + "outcome": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "response text"}], + } + ], + }, + ) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": detail} if detail else None, + trace_include_sensitive_data=sensitive, + ) + await Runner.run(Agent(name="test"), "sdk boundary", run_config=cfg) + spans = fetch_ordered_spans() + (request,) = [s for s in spans if s.span_data.type in ("generation", "response")] + (execution,) = [s for s in spans if s.span_data.type == "function"] + assert thread.capture_mode == ("content" if captures else None) + if captures: + assert "request text" in json.dumps(request.span_data.input) + assert "response text" in json.dumps(request.span_data.output) + assert execution.span_data.output == "accepted result" + assert "prepared" in execution.span_data.input + assert "never-export" not in json.dumps([s.export() for s in (request, execution)]) + assert "wrong output" not in execution.span_data.output + else: + assert request.span_data.input is None + assert execution.span_data.input is execution.span_data.output is None + assert execution.ended_at is not None and request.ended_at is not None + assert execution.ended_at < request.ended_at diff --git a/tests/test_build_codex_wheel.py b/tests/test_build_codex_wheel.py new file mode 100644 index 0000000000..65190714f3 --- /dev/null +++ b/tests/test_build_codex_wheel.py @@ -0,0 +1,57 @@ +"""Regression coverage for the native wheel's local Core patch boundary.""" + +import importlib.util +import json +import subprocess +from pathlib import Path + +import pytest + + +def test_core_patch_applies_inside_an_ancestor_git_checkout(tmp_path: Path) -> None: + script = Path(__file__).resolve().parents[1] / "scripts/build_codex_wheel.py" + spec = importlib.util.spec_from_file_location("build_codex_wheel", script) + assert spec is not None and spec.loader is not None + build = importlib.util.module_from_spec(spec) + spec.loader.exec_module(build) + subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True) + upstream = tmp_path / "upstream" + crate = upstream / "codex-rs/core" + crate.mkdir(parents=True) + (crate / "Cargo.toml").write_text('[package]\nname = "codex-core"\n') + (crate / "value.txt").write_bytes(b"stock\n") + source = tmp_path / "target/staged" + patches = source / "native/codex/patches" + patches.mkdir(parents=True) + (patches / "0001.patch").write_text( + "diff --git a/codex-rs/core/value.txt b/codex-rs/core/value.txt\n" + "--- a/codex-rs/core/value.txt\n" + "+++ b/codex-rs/core/value.txt\n" + "@@ -1 +1 @@\n-stock\n+patched\n", + newline="\n", + ) + metadata = { + "packages": [ + { + "name": "codex-core", + "source": "git+https://github.com/openai/codex?rev=pinned#pinned", + "manifest_path": str(crate / "Cargo.toml"), + }, + {"name": "sdk", "source": None}, + ] + } + + guide = patches / "0001.md" + for text in (None, " \n"): + if text is not None: + guide.write_text(text) + with pytest.raises(ValueError, match=r"Document 0001\.patch in 0001\.md"): + build.patch_codex_source(source, upstream, metadata) + + guide.write_text("Replace the stock value for the build test.") + build.patch_codex_source(source, upstream, metadata) + patched = source / "native/codex-upstream" + + assert (patched / "codex-rs/core/value.txt").read_text() == "patched\n" + assert (crate / "value.txt").read_text() == "stock\n" + assert json.dumps(str(patched / "codex-rs/core")) in (source / ".cargo/config.toml").read_text() diff --git a/tests/tracing/test_opentelemetry.py b/tests/tracing/test_opentelemetry.py index 523d769872..42f2508da7 100644 --- a/tests/tracing/test_opentelemetry.py +++ b/tests/tracing/test_opentelemetry.py @@ -350,16 +350,26 @@ def test_tool_approval_mapping_and_partial_coverage(otel) -> None: ) -def test_marker_only_content_preserves_omission(otel) -> None: +@pytest.mark.parametrize( + "marker,reason", + [ + ({"truncated": True}, "truncated"), + ( + {"omitted": True, "reason": "unsupported_or_oversized_content"}, + "unsupported_or_oversized_content", + ), + ], +) +def test_marker_only_content_preserves_omission(otel, marker, reason) -> None: _, exporter = otel with trace("group"), generation_span(model="test-model") as model: model.span_data.operation.detail = "content" - model.span_data.input = [{"truncated": True}] - model.span_data.output = [{"truncated": True}] + model.span_data.input = [marker] + model.span_data.output = [marker] span = exporter.get_finished_spans()[0] for attribute in ("gen_ai.input.messages", "gen_ai.output.messages"): assert list(span.attributes[attribute][0]["parts"]) == [ - {"type": "openai.omitted", "reason": "truncated"} + {"type": "openai.omitted", "reason": reason} ] From 8ede4602d208cfdf3cb7e5d78f7e396fc562efd6 Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 07:23:08 -0800 Subject: [PATCH 32/33] Compare the decoded image path in the installed Codex wheel test --- native/codex/verify_installed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/codex/verify_installed.py b/native/codex/verify_installed.py index 43181b4da2..a3367bdeb1 100644 --- a/native/codex/verify_installed.py +++ b/native/codex/verify_installed.py @@ -420,7 +420,7 @@ async def consume_blocked(): f"Native content omitted; request sizes: {request_sizes}" ) assert "native-content" in json.dumps(native_models[-1].output) - assert str(image_path) in native_tools[0].input + assert json.loads(native_tools[0].input)["path"] == str(image_path) assert "omitted" in native_tools[0].output rendered = json.dumps([span.export() for span in captured]) assert ( From e74c1eea16b16b39bcea2433fb42320b47fc114f Mon Sep 17 00:00:00 2001 From: Mark Stuart Date: Sat, 21 Nov 2026 07:39:54 -0800 Subject: [PATCH 33/33] fix(codex): confirm restored commands from the process result --- native/codex/patches/02-approval-recovery.md | 4 +++- .../codex/patches/02-approval-recovery.patch | 24 +++++++++++++++++++ native/codex/src/native_approval.rs | 16 ++++++------- native/codex/verify_installed.py | 3 ++- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/native/codex/patches/02-approval-recovery.md b/native/codex/patches/02-approval-recovery.md index a2732c54d8..492dc24a14 100644 --- a/native/codex/patches/02-approval-recovery.md +++ b/native/codex/patches/02-approval-recovery.md @@ -16,7 +16,9 @@ Recovery requires direct tools in one local environment, human review and no too The patch includes a bounded file read because the existing stream can follow symbolic links. Before saving and again before execution, Codex checks file hashes and confirms that new destinations are absent. Each check covers at most 32 paths, 8 MiB per file, 16 MiB in total and 10 seconds. An unreadable or oversized file stops recovery. -A restored call gets one attempt. When Codex policy already allows an approved edit to run outside the sandbox on retry, recovery uses that permission on the first attempt. +A restored call gets one attempt. For a command, Codex checks the call ID and exit code and refuses a live process handle. This check works even if the exit notification arrives after the turn. A new worker cannot continue an interactive process. + +When Codex policy already allows an approved edit to run outside the sandbox on retry, recovery uses that permission on the first attempt. The application must restore the same workspace and prevent concurrent edits. If the worker stops after a write and before saving the result, the session stays active. Check the external system before retrying. diff --git a/native/codex/patches/02-approval-recovery.patch b/native/codex/patches/02-approval-recovery.patch index da8d69d155..d45f947aee 100644 --- a/native/codex/patches/02-approval-recovery.patch +++ b/native/codex/patches/02-approval-recovery.patch @@ -1025,6 +1025,30 @@ index 74082aa..ebaa126 100644 self.request_patch_approval( ctx.review_context.turn(), ctx.call_id.clone(), +diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +index 6a81b62..59e4900 100644 +--- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs ++++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +@@ -236,6 +236,19 @@ impl ExecCommandHandler { + } + None => manager.exec_command(request, &context).await, + }; ++ // The exit event can arrive after the turn. A restored command needs a terminal ++ // result here because the next worker cannot use a live process handle. ++ if crate::restore_tool_call::active(&step_context.turn).is_some() ++ && !result.as_ref().is_ok_and(|output| { ++ output.event_call_id == call_id ++ && output.process_id.is_none() ++ && output.exit_code.is_some() ++ }) ++ { ++ return Err(FunctionCallError::Fatal( ++ "Cannot confirm the restored command exited.".to_string(), ++ )); ++ } + match result { + Ok(response) => Ok(boxed_tool_output(response)), + Err(UnifiedExecError::SandboxDenied { diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index a1a88e4..c26c85d 100644 --- a/codex-rs/core/src/tools/orchestrator.rs diff --git a/native/codex/src/native_approval.rs b/native/codex/src/native_approval.rs index 1b1a0f583b..7349af3e2e 100644 --- a/native/codex/src/native_approval.rs +++ b/native/codex/src/native_approval.rs @@ -75,12 +75,6 @@ pub(super) async fn resolve( | EventMsg::DynamicToolCallRequest(_) => { return Err("Codex requested another decision while restoring the call.".into()); } - EventMsg::ExecCommandEnd(end) - if answered && !patch && end.call_id == call_id && end.turn_id == turn_id => - { - finished = true; - continue; - } EventMsg::PatchApplyEnd(end) if answered && patch && end.call_id == call_id && end.turn_id == turn_id => { @@ -107,18 +101,24 @@ pub(super) async fn resolve( "Codex could not confirm the saved approval before dispatch.".into(), ); } + if !patch { + // Core only returns a successful restored command result after exit. + // Its asynchronous ExecCommandEnd event may follow TurnComplete. + finished = output.success == Some(true); + } result = output.body.to_text(); continue; } EventMsg::TurnComplete(_) => { if !finished { - // An exec can yield a handle to a still-running process. That handle - // dies with this thread. Don't checkpoint its uncertain effect as done. return Err("Codex could not confirm that the approved call finished.".into()); } return result.ok_or_else(|| "Codex did not complete the approved call.".into()); } EventMsg::TurnAborted(_) | EventMsg::Error(_) | EventMsg::ShutdownComplete => { + if answered { + return Err("Codex could not confirm that the approved call finished.".into()); + } return Err( "Codex could not complete the restored approval under current policy.".into(), ); diff --git a/native/codex/verify_installed.py b/native/codex/verify_installed.py index a3367bdeb1..4064568129 100644 --- a/native/codex/verify_installed.py +++ b/native/codex/verify_installed.py @@ -462,7 +462,8 @@ def fresh_provider(): "call_id": call_id, "arguments": json.dumps( { - "cmd": f'echo builtin-approved > "{target}"', + # Keep the process alive long enough for Codex's async exit watcher. + "cmd": f'sleep 1; echo builtin-approved > "{target}"', "login": False, "sandbox_permissions": "require_escalated", }