diff --git a/.github/workflows/codex-preview.yml b/.github/workflows/codex-preview.yml index 385a1475f4..63974c46c4 100644 --- a/.github/workflows/codex-preview.yml +++ b/.github/workflows/codex-preview.yml @@ -70,6 +70,7 @@ jobs: include: - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + test_native: true platform: manylinux_2_28_x86_64 image: quay.io/pypa/manylinux_2_28_x86_64@sha256:9555ba6c479fa69a2aa80ad1dc57154a60d562b19df159361d9379a02beb1e36 - runner: ubuntu-24.04-arm @@ -161,11 +162,9 @@ jobs: run: | options=() if [[ "$RUNNER_OS" == Linux ]]; then options+=(--compatibility manylinux_2_28); fi + if [[ "${{ matrix.test_native }}" == true ]]; then options+=(--test-native); fi uv run --no-sync python scripts/build_codex_wheel.py \ --out "$GITHUB_WORKSPACE/dist" --target "$WHEEL_TARGET" "${options[@]}" - - name: Test native policy and configuration - if: matrix.platform == 'manylinux_2_28_x86_64' - run: cargo test --locked --release --lib --manifest-path native/codex/Cargo.toml - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: codex-wheel-${{ matrix.platform }} diff --git a/docs/models/codex.md b/docs/models/codex.md index 037c6562f6..e0b8ef6def 100644 --- a/docs/models/codex.md +++ b/docs/models/codex.md @@ -134,7 +134,9 @@ The workspace sandbox protects the directory that contains the installed helpers On Windows, the preview rejects a local Codex executor configured with the legacy elevated sandbox. That sandbox needs a setup helper that Codex cannot securely locate from a Python process. If your organization requires this sandbox, use an SDK-managed executor or configure a remote executor with `include_local = false` in `environments.toml`. -Native Codex approval prompts are separate from SDK tool approvals. This provider cannot answer native approval requests and fails the model call when one arrives. For a non-interactive read-only workflow, configure Codex to avoid approval prompts: +Pass a `ResumableSession` to `Runner.run(..., session=session)` to pause a built-in `exec_command` or `apply_patch` for approval. A new worker can load that session and approve or reject the original call. Recovery requires the same workspace and permissions, one local environment, and no tool hooks. Other native approval requests fail the model call. + +For a non-interactive read-only workflow, disable approval prompts. ```python provider = CodexProvider( diff --git a/native/codex/Cargo.lock b/native/codex/Cargo.lock index 6ebf0faa55..122fc2dac0 100644 --- a/native/codex/Cargo.lock +++ b/native/codex/Cargo.lock @@ -8041,6 +8041,7 @@ dependencies = [ "codex-core", "codex-core-api", "codex-exec-server", + "codex-extension-api", "codex-features", "codex-guardian-v2", "codex-hooks", @@ -8058,6 +8059,7 @@ dependencies = [ "futures", "pyo3", "pyo3-async-runtimes", + "serde", "serde_json", "tempfile", "tokio", diff --git a/native/codex/Cargo.toml b/native/codex/Cargo.toml index 6c1143c54b..853509a154 100644 --- a/native/codex/Cargo.toml +++ b/native/codex/Cargo.toml @@ -18,6 +18,7 @@ pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] } pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } tokio = { version = "1", features = ["rt-multi-thread", "sync", "time"] } futures = "0.3" +serde = "1" serde_json = "1" toml = "0.9.5" codex-arg0 = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } @@ -27,6 +28,7 @@ codex-config = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8 codex-core = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-core-api = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-exec-server = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } +codex-extension-api = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-features = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-http-client = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } codex-hooks = { git = "https://github.com/openai/codex", rev = "6ff1e1ca3b30fc8b914720681fcf5fe62e95f856" } diff --git a/native/codex/patches/01-operation-tracing.md b/native/codex/patches/01-operation-tracing.md new file mode 100644 index 0000000000..7e03cd59d5 --- /dev/null +++ b/native/codex/patches/01-operation-tracing.md @@ -0,0 +1,24 @@ +# Trace the request Codex sends and the time a tool runs + +Codex can change the model input just before sending it. Its normal tool timer can also include time spent waiting for approval. Those details matter when debugging a slow or unexpected turn. + +[This patch](01-operation-tracing.patch) lets the SDK observe the final HTTP or WebSocket request, distinguish a failed connection from cancellation, and start timing a command or edit after approval. These callbacks cannot change the request or grant permission to run a tool. + +Native spans contain metadata by default. To include text in model and tool spans, pass this configuration to the Runner. + +```python +from agents import RunConfig +from agents.models.codex import CodexProvider + +config = RunConfig( + model_provider=CodexProvider(), + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=True, +) +``` + +The SDK uses existing Codex events for model responses, usage and tool results. It excludes request headers, encrypted reasoning, media and values over 32 KiB. Captured text can still contain secrets. Setting `trace_include_sensitive_data=False` always disables content capture. + +## Check and remove + +Build and test the wheel as described in the [patch guide](README.md). Check both metadata and content modes. Failed requests and cancellations should have distinct outcomes. Denied commands should have no execution span. Remove the patch and guide once upstream Codex exposes the same events. diff --git a/native/codex/patches/01-operation-tracing.patch b/native/codex/patches/01-operation-tracing.patch new file mode 100644 index 0000000000..53f8027f17 --- /dev/null +++ b/native/codex/patches/01-operation-tracing.patch @@ -0,0 +1,147 @@ +diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs +index 2ac9b93..7257291 100644 +--- a/codex-rs/core/src/client.rs ++++ b/codex-rs/core/src/client.rs +@@ -2017,7 +2017,7 @@ impl ResponsesHttpExecutor<'_> { + let request_session_telemetry = + session_telemetry_for_request(session_telemetry, &request); + options.extra_headers.extend(responses_headers); +- let interceptors = crate::model_request::prepare( ++ let mut interceptors = crate::model_request::prepare( + &self.client.request_contributors, + &self.client.state.thread_id.to_string(), + &model_info.slug, +@@ -2032,6 +2032,11 @@ impl ResponsesHttpExecutor<'_> { + } + request.input = input; + } ++ for interceptor in &mut interceptors { ++ interceptor.observe_request(codex_extension_api::ModelRequestContent::Responses( ++ &request, ++ )); ++ } + inference_trace_attempt.record_started(&request); + let client = ApiResponsesClient::new( + transport, +@@ -2040,6 +2045,11 @@ impl ResponsesHttpExecutor<'_> { + ) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); + let stream_result = client.stream_request(request, options).await; ++ if stream_result.is_err() { ++ for interceptor in &mut interceptors { ++ interceptor.request_failed(); ++ } ++ } + + match stream_result { + Ok(stream) => { +@@ -2315,7 +2325,7 @@ impl ModelClientSession { + client_setup.auth.as_ref(), + &responses_headers, + ); +- let interceptors = crate::model_request::prepare( ++ let mut interceptors = crate::model_request::prepare( + &self.client.request_contributors, + &self.client.state.thread_id.to_string(), + &model_info.slug, +@@ -2337,6 +2347,11 @@ impl ModelClientSession { + let ResponsesWsRequest::ResponseCreate(payload) = &mut ws_request; + payload.input = input; + } ++ let ResponsesWsRequest::ResponseCreate(payload) = &ws_request; ++ for interceptor in &mut interceptors { ++ interceptor ++ .observe_request(codex_extension_api::ModelRequestContent::WebSocket(payload)); ++ } + if !previous_response_id_from_untraced_warmup { + inference_trace_attempt.record_started(&ws_request); + } +@@ -2377,6 +2392,9 @@ impl ModelClientSession { + self.websocket_session.last_request = Some(request); + self.websocket_session.last_response_from_untraced_warmup = warmup; + let stream_result = stream_result.map_err(|err| { ++ for interceptor in &mut interceptors { ++ interceptor.request_failed(); ++ } + let response_debug_context = extract_response_debug_context_from_api_error(&err); + let err = self.client.state.provider.map_api_error(err); + inference_trace_attempt.record_failed( +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index a1a88e4..c5f8077 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -102,6 +102,18 @@ impl ToolOrchestrator { + .as_ref() + .map(ActiveNetworkApproval::execution_proxy), + }; ++ for contributor in tool_ctx ++ .session ++ .services ++ .extensions ++ .tool_lifecycle_contributors() ++ { ++ contributor.on_tool_execution_start( ++ &tool_ctx.session.thread_id.to_string(), ++ &tool_ctx.step_context.turn.sub_id, ++ &tool_ctx.call_id, ++ ); ++ } + let run_result = tool + .run(req, &attempt_with_network_approval, &attempt_tool_ctx) + .await; +diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs +index dc95f99..e985a9a 100644 +--- a/codex-rs/ext/extension-api/src/contributors.rs ++++ b/codex-rs/ext/extension-api/src/contributors.rs +@@ -413,6 +413,10 @@ pub trait ToolContributor: Send + Sync { + /// rewriting the invocation. Use `ToolContributor` for owning a tool implementation + /// and hooks for policy that changes tool payloads. + pub trait ToolLifecycleContributor: Send + Sync { ++ /// One admitted native runtime attempt, after approval and network admission. ++ /// May repeat on sandbox retry; denied attempts never reach this callback. ++ fn on_tool_execution_start(&self, _thread_id: &str, _turn_id: &str, _call_id: &str) {} ++ + /// Observe direct calls before readiness, dispatch waiting, and hooks, including blocked calls. + /// Excludes nested code-mode calls. Observers must return promptly. + fn on_tool_dispatch(&self, _input: ToolDispatchInput<'_>) {} +diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs +index 147b52f..bd94076 100644 +--- a/codex-rs/ext/extension-api/src/lib.rs ++++ b/codex-rs/ext/extension-api/src/lib.rs +@@ -128,6 +128,7 @@ pub use contributors::ToolDispatchInput; + mod model_request; + pub use model_request::GuardianAssessment; + pub use model_request::ModelRequestContributor; ++pub use model_request::ModelRequestContent; + pub use model_request::ModelRequestInput; + pub use model_request::ModelRequestKind; + pub use model_request::ModelResponseError; +diff --git a/codex-rs/ext/extension-api/src/model_request.rs b/codex-rs/ext/extension-api/src/model_request.rs +index 0f477aa..6514694 100644 +--- a/codex-rs/ext/extension-api/src/model_request.rs ++++ b/codex-rs/ext/extension-api/src/model_request.rs +@@ -35,10 +35,24 @@ pub trait ModelRequestContributor: Send + Sync + std::fmt::Debug { + fn request(&self, input: ModelRequestInput<'_>) -> Option>; + } + ++/// Borrowed final request body, after transport-specific input adjustment. ++/// Headers and authentication are deliberately outside this view. ++pub enum ModelRequestContent<'a> { ++ Responses(&'a codex_api::ResponsesApiRequest), ++ WebSocket(&'a codex_api::ResponseCreateWsRequest<'a>), ++} ++ + /// Owns request state until consumed by the response stream or dropped on failure. + /// Interceptors compose in registration order; dropping their stream must release unfinished work. + /// Trusted host implementations must bound any model-visible content they synthesize. + pub trait ModelResponseInterceptor: Send + Sync { ++ /// Observe the final request without modifying it. Implementations must opt in, ++ /// bound any retained content, and return promptly; the default retains nothing. ++ fn observe_request(&mut self, _request: ModelRequestContent<'_>) {} ++ ++ /// The transport returned an error before opening a response stream. ++ fn request_failed(&mut self) {} ++ + /// Wrap the response stream, optionally buffering, inspecting, or transforming events. + /// The returned stream must own request state and propagate cancellation by dropping upstream. + fn intercept(self: Box, stream: ModelResponseStream) -> ModelResponseStream; diff --git a/native/codex/patches/02-approval-recovery.md b/native/codex/patches/02-approval-recovery.md new file mode 100644 index 0000000000..492dc24a14 --- /dev/null +++ b/native/codex/patches/02-approval-recovery.md @@ -0,0 +1,27 @@ +# Resume a built-in command or edit after a worker stops + +Codex normally keeps a pending approval in memory. When the process stops, restoring conversation history cannot finish that action. Asking the model to propose it again could run a different command or repeat a write. + +[This patch](02-approval-recovery.patch) lets the SDK save one local `exec_command` or `apply_patch` before it runs. A replacement worker can submit that call directly to Codex without asking the model again. + +1. Create a session with `session = await ResumableSession.create(store)`. Pass it to `Runner.run(..., session=session)` and keep `session.session_id`. The SDK saves the pending call before returning the approval request. +2. On the replacement worker, load the same session with `session = await ResumableSession.resume(store, session_id)`. Then use `state = await session.load_state(agent)`. +3. Match the user's decision to the call in `state.get_interruptions()`. Use `state.approve(call)` or `state.reject(call)`, then `await Runner.run(agent, state, session=session, run_config=config)`. + +Only one worker can claim that session revision. Denial stops the call. On approval, Codex checks the workspace, permissions and newly prepared action. A mismatch stops execution. + +## Safety limits + +Recovery requires direct tools in one local environment, human review and no tool hooks. The SDK saves the model's submitted command or patch, so protect the session store. Codex replaces prepared update diffs with hashes and removes the original contents of deleted files. + +The patch includes a bounded file read because the existing stream can follow symbolic links. Before saving and again before execution, Codex checks file hashes and confirms that new destinations are absent. Each check covers at most 32 paths, 8 MiB per file, 16 MiB in total and 10 seconds. An unreadable or oversized file stops recovery. + +A restored call gets one attempt. For a command, Codex checks the call ID and exit code and refuses a live process handle. This check works even if the exit notification arrives after the turn. A new worker cannot continue an interactive process. + +When Codex policy already allows an approved edit to run outside the sandbox on retry, recovery uses that permission on the first attempt. + +The application must restore the same workspace and prevent concurrent edits. If the worker stops after a write and before saving the result, the session stays active. Check the external system before retrying. + +## Check and remove + +Follow the [patch guide](README.md) and run the approval tests. On another worker, approve and deny a saved command and edit. Check that a stale file, changed permissions or a duplicate approval causes no extra write. Remove the patch when upstream Codex can resume the same action with these checks. diff --git a/native/codex/patches/02-approval-recovery.patch b/native/codex/patches/02-approval-recovery.patch new file mode 100644 index 0000000000..d45f947aee --- /dev/null +++ b/native/codex/patches/02-approval-recovery.patch @@ -0,0 +1,1460 @@ +diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock +index 1d27abb..f8972a1 100644 +--- a/codex-rs/Cargo.lock ++++ b/codex-rs/Cargo.lock +@@ -3051,6 +3051,7 @@ dependencies = [ + "serde_json", + "serial_test", + "sha1 0.10.6", ++ "sha2 0.10.9", + "shlex", + "similar", + "tempfile", +diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml +index 542abf9..5ffc5cf 100644 +--- a/codex-rs/core/Cargo.toml ++++ b/codex-rs/core/Cargo.toml +@@ -114,6 +114,7 @@ rmcp = { workspace = true, default-features = false, features = [ + serde = { workspace = true, features = ["derive"] } + serde_json = { workspace = true } + sha1 = { workspace = true } ++sha2 = { workspace = true } + shlex = { workspace = true } + similar = { workspace = true } + tempfile = { workspace = true } +diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs +index 968d331..a6a24d6 100644 +--- a/codex-rs/core/src/codex_thread.rs ++++ b/codex-rs/core/src/codex_thread.rs +@@ -45,6 +45,7 @@ use codex_protocol::protocol::AskForApproval; + use codex_protocol::protocol::EnvironmentConfig; + use codex_protocol::protocol::EnvironmentConfigState; + use codex_protocol::protocol::Event; ++use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::MultiAgentVersion; + use codex_protocol::protocol::Op; + use codex_protocol::protocol::SandboxPolicy; +@@ -474,6 +475,33 @@ impl CodexThread { + .await + } + ++ /// Replays one native call without model sampling, requiring the original human approval. ++ pub async fn restore_tool_call_if_idle( ++ &self, ++ call: ResponseItem, ++ expected_approval: EventMsg, ++ preconditions: serde_json::Value, ++ ) -> CodexResult { ++ let restore = crate::restore_tool_call::NativeApprovals::Restore(Arc::new( ++ crate::restore_tool_call::RestoreToolCall::new(call, expected_approval, preconditions)?, ++ )); ++ let mut init = self.config_snapshot().await.turn_extension_init; ++ init.insert(restore); ++ self.start_turn_if_idle(crate::WithTurnExtensionData::new( ++ TurnInputRequest::user_input(Vec::new()), ++ init, ++ )) ++ .await ++ } ++ ++ /// Returns the live approval without file preimages, paired with its verified preconditions. ++ pub async fn capture_tool_approval( ++ &self, ++ approval: EventMsg, ++ ) -> CodexResult<(EventMsg, serde_json::Value)> { ++ crate::restore_tool_call::capture(&self.session, approval).await ++ } ++ + /// Starts a regular turn only when the thread is idle. + /// + /// Core declines the input without recording or enqueueing it when idle +diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs +index 61400a1..925df52 100644 +--- a/codex-rs/core/src/lib.rs ++++ b/codex-rs/core/src/lib.rs +@@ -31,6 +31,8 @@ pub use codex_protocol::turn_input::TurnStartOptions; + pub use responses_metadata::CodexResponsesMetadata; + pub use turn_metadata::detached_memory_responses_metadata; + mod codex_thread; ++mod restore_tool_call; ++mod patch_preconditions; + mod turn_extension_data; + pub use turn_extension_data::WithTurnExtensionData; + mod compact_model_fallback; +diff --git a/codex-rs/core/src/patch_preconditions.rs b/codex-rs/core/src/patch_preconditions.rs +new file mode 100644 +index 0000000..cfeb59d +--- /dev/null ++++ b/codex-rs/core/src/patch_preconditions.rs +@@ -0,0 +1,272 @@ ++//! Filesystem preconditions for restoring an interrupted native patch approval. ++ ++use crate::session::step_context::StepContext; ++use crate::tools::ApprovalContext; ++use codex_file_system::EnvironmentAccess; ++use codex_file_system::GetMetadataOptions; ++use codex_file_system::ReadFileOptions; ++use codex_protocol::error::CodexErr; ++use codex_protocol::error::Result as CodexResult; ++use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; ++use codex_protocol::protocol::FileChange; ++use codex_utils_path_uri::PathUri; ++use serde_json::Value; ++use sha2::Digest; ++use sha2::Sha256; ++use std::collections::BTreeMap; ++use std::collections::HashMap; ++use std::io::ErrorKind; ++use std::path::Component; ++use std::path::Path; ++use std::path::PathBuf; ++use std::time::Duration; ++ ++// Approval recovery fails closed for larger preimages instead of retaining large reads. ++pub(crate) const MAX_PATCH_PREIMAGE_BYTES: u64 = 8 * 1024 * 1024; ++const MAX_PATCH_PATHS: usize = 32; ++pub(crate) const MAX_PATCH_SCAN_BYTES: u64 = 16 * 1024 * 1024; ++const MAX_PATCH_SCAN_DURATION: Duration = Duration::from_secs(10); ++ ++fn invalid(message: impl Into) -> CodexErr { ++ CodexErr::InvalidRequest(message.into()) ++} ++ ++pub(crate) async fn capture_patch_approval_preconditions( ++ context: &ApprovalContext, ++ changes: &HashMap, ++) -> CodexResult { ++ let review = &context.review_context; ++ let environment = review ++ .environments() ++ .single_local_environment() ++ .ok_or_else(|| invalid("Patch recovery requires one local environment"))?; ++ let grants = review ++ .turn() ++ .granted_permissions(&environment.selection.environment_id); ++ capture(&environment.fs_accessor(grants), changes, true).await ++} ++ ++pub(crate) async fn validate_patch_preconditions( ++ step: &StepContext, ++ approval: &ApplyPatchApprovalRequestEvent, ++ checkpoint: &Value, ++) -> CodexResult<()> { ++ let expected = validate_checkpoint(&approval.changes, checkpoint, true)?; ++ let check_deleted_content = checkpoint.get("version").and_then(Value::as_u64) == Some(1); ++ let environment = step ++ .environments ++ .single_local_environment() ++ .ok_or_else(|| invalid("Patch recovery requires one local environment"))?; ++ let observed = capture( ++ &step.fs_accessor(environment), ++ &approval.changes, ++ check_deleted_content, ++ ) ++ .await?; ++ if observed.get("files").and_then(Value::as_object) != Some(expected) { ++ return Err(invalid("Pending patch files have changed")); ++ } ++ Ok(()) ++} ++ ++fn patch_paths(changes: &HashMap) -> CodexResult> { ++ if changes.is_empty() { ++ return Err(invalid("A pending patch must include file changes")); ++ } ++ if changes.len() > MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } ++ let mut paths: BTreeMap<_, _> = changes ++ .iter() ++ .map(|(path, change)| (path, matches!(change, FileChange::Add { .. }))) ++ .collect(); ++ for change in changes.values() { ++ if let FileChange::Update { ++ move_path: Some(destination), ++ .. ++ } = change ++ { ++ if paths.len() == MAX_PATCH_PATHS { ++ return Err(invalid("Pending patch scans are limited to 32 paths")); ++ } ++ if paths.insert(destination, true).is_some() { ++ return Err(invalid("Patch move destinations must be distinct")); ++ } ++ } ++ } ++ Ok(paths) ++} ++ ++fn valid_digest(value: &str) -> bool { ++ value.len() == 64 ++ && value ++ .bytes() ++ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) ++} ++ ++/// Binds a live native deletion to its saved digest before redacting its content. ++/// No file I/O or content cloning occurs here; callers bound any other live fields separately. ++pub(crate) fn matches_delete_preimages( ++ changes: &HashMap, ++ checkpoint: &Value, ++) -> bool { ++ let Ok(files) = validate_checkpoint(changes, checkpoint, false) else { ++ return false; ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for change in changes.values() { ++ if let FileChange::Delete { content } = change { ++ let size = content.len() as u64; ++ if size > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return false; ++ } ++ remaining -= size; ++ } ++ } ++ changes.iter().all(|(path, change)| { ++ let FileChange::Delete { content } = change else { ++ return true; ++ }; ++ path.to_str() ++ .and_then(|path| files.get(path)) ++ .and_then(Value::as_str) ++ .is_some_and(|expected| digest(content.as_bytes()) == expected) ++ }) ++} ++ ++fn validate_checkpoint<'a>( ++ changes: &HashMap, ++ checkpoint: &'a Value, ++ require_redaction: bool, ++) -> CodexResult<&'a serde_json::Map> { ++ let paths = patch_paths(changes)?; ++ let root = checkpoint ++ .as_object() ++ .ok_or_else(|| invalid("Invalid patch approval preconditions"))?; ++ let files = root ++ .get("files") ++ .and_then(Value::as_object) ++ .ok_or_else(|| invalid("Invalid patch approval file map"))?; ++ if root.len() != 2 ++ || !matches!(root.get("version").and_then(Value::as_u64), Some(1 | 2)) ++ || files.len() > MAX_PATCH_PATHS ++ || files.len() != paths.len() ++ { ++ return Err(invalid("Invalid patch approval version or file count")); ++ } ++ for (path, absent) in paths { ++ let value = path ++ .to_str() ++ .and_then(|path| files.get(path)) ++ .ok_or_else(|| invalid("Patch approval file paths do not match the pending patch"))?; ++ if absent { ++ if !value.is_null() { ++ return Err(invalid( ++ "Pending patch destinations must have absent preimages", ++ )); ++ } ++ } else if !value.as_str().is_some_and(valid_digest) { ++ return Err(invalid("Invalid patch approval preimage digest")); ++ } ++ } ++ if require_redaction ++ && checkpoint.get("version").and_then(Value::as_u64) == Some(2) ++ && changes.values().any(|change| match change { ++ FileChange::Delete { content } => !content.is_empty(), ++ FileChange::Update { unified_diff, .. } => !valid_digest(unified_diff), ++ FileChange::Add { .. } => false, ++ }) ++ { ++ return Err(invalid("Invalid redacted patch changes")); ++ } ++ Ok(files) ++} ++ ++async fn capture( ++ fs: &impl EnvironmentAccess, ++ changes: &HashMap, ++ check_deleted_content: bool, ++) -> CodexResult { ++ let paths = patch_paths(changes)?; ++ tokio::time::timeout(MAX_PATCH_SCAN_DURATION, async { ++ let mut files = serde_json::Map::new(); ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ for (path, absent) in paths { ++ let deleted = match changes.get(path) { ++ Some(FileChange::Delete { content }) if check_deleted_content => { ++ Some(content.as_bytes()) ++ } ++ _ => None, ++ }; ++ files.insert( ++ path.to_str() ++ .ok_or_else(|| invalid("Patch approval paths require UTF-8"))? ++ .to_owned(), ++ observe_file(fs, path, absent, &mut remaining, deleted) ++ .await? ++ .map_or(Value::Null, Value::String), ++ ); ++ } ++ Ok(serde_json::json!({"version": 1, "files": files})) ++ }) ++ .await ++ .map_err(|_| invalid("Pending patch scan exceeded the 10 second limit"))? ++} ++ ++fn digest(bytes: &[u8]) -> String { ++ format!("{:x}", Sha256::digest(bytes)) ++} ++ ++async fn observe_file( ++ fs: &impl EnvironmentAccess, ++ path: &Path, ++ must_be_absent: bool, ++ remaining: &mut u64, ++ deleted: Option<&[u8]>, ++) -> CodexResult> { ++ if !path.is_absolute() ++ || path ++ .components() ++ .any(|part| matches!(part, Component::ParentDir | Component::CurDir)) ++ { ++ return Err(invalid("Patch paths must be absolute and normalized")); ++ } ++ // The selected local filesystem's no-follow operations reject symlinks in ++ // every component, including directory ancestors, without opening targets. ++ let uri = PathUri::from_host_native_path(path).map_err(|error| invalid(error.to_string()))?; ++ match fs ++ .get_metadata( ++ &uri, ++ GetMetadataOptions { ++ follow_symlinks: false, ++ }, ++ ) ++ .await ++ { ++ Err(error) if must_be_absent && error.kind() == ErrorKind::NotFound => Ok(None), ++ Err(error) => Err(invalid(format!( ++ "Cannot inspect pending patch file: {error}" ++ ))), ++ Ok(_) if must_be_absent => Err(invalid("A pending patch destination already exists")), ++ Ok(metadata) if !metadata.is_file || metadata.is_symlink => { ++ Err(invalid("Pending patch sources must be regular files")) ++ } ++ Ok(_) => { ++ let bytes = fs ++ .read_file_bounded( ++ &uri, ++ ReadFileOptions { ++ follow_symlinks: false, ++ }, ++ MAX_PATCH_PREIMAGE_BYTES.min(*remaining), ++ ) ++ .await ++ .map_err(|error| invalid(format!("Cannot read pending patch file: {error}")))?; ++ *remaining -= bytes.len() as u64; ++ if deleted.is_some_and(|expected| expected != bytes.as_slice()) { ++ return Err(invalid("The pending deletion no longer matches the file")); ++ } ++ Ok(Some(digest(&bytes))) ++ } ++ } ++} +diff --git a/codex-rs/core/src/restore_tool_call.rs b/codex-rs/core/src/restore_tool_call.rs +new file mode 100644 +index 0000000..a20cd34 +--- /dev/null ++++ b/codex-rs/core/src/restore_tool_call.rs +@@ -0,0 +1,516 @@ ++//! Model-free replay of one native call, requiring the same fresh human approval. ++ ++use std::collections::HashMap; ++use std::sync::{Arc, Mutex, OnceLock, Weak}; ++ ++use codex_protocol::approvals::ExecApprovalKind; ++use codex_protocol::config_types::ApprovalsReviewer; ++use codex_protocol::error::{CodexErr, Result as CodexResult}; ++use codex_protocol::models::ResponseItem; ++use codex_protocol::openai_models::ToolMode; ++use codex_protocol::protocol::{EventMsg, FileChange, ReviewDecision}; ++use serde_json::{Value, json}; ++use sha2::{Digest, Sha256}; ++use tokio_util::sync::CancellationToken; ++ ++use crate::patch_preconditions::{MAX_PATCH_PREIMAGE_BYTES, MAX_PATCH_SCAN_BYTES}; ++use crate::session::session::Session; ++use crate::session::step_context::StepContext; ++use crate::session::turn_context::TurnContext; ++use crate::state::TaskKind; ++use crate::tools::ApprovalContext; ++use crate::tools::parallel::ToolCallRuntime; ++use crate::tools::router::ToolRouter; ++use crate::tools::sandboxing::ApprovalAction; ++ ++fn refusal(message: &str) -> CodexErr { ++ CodexErr::InvalidRequest(format!("Native tool restoration refused: {message}")) ++} ++ ++#[derive(Debug)] ++pub(crate) struct CapturableApproval { ++ event: OnceLock, ++ permissions: Value, ++ patch: Option, ++} ++ ++/// Capture and replay are mutually exclusive modes of a turn, never separate authorities. ++#[derive(Debug)] ++pub(crate) enum NativeApprovals { ++ Capture(Mutex>>), ++ Restore(Arc), ++} ++ ++impl NativeApprovals { ++ pub(crate) fn restoration(&self) -> Option> { ++ match self { ++ NativeApprovals::Restore(restore) => Some(Arc::clone(restore)), ++ NativeApprovals::Capture(_) => None, ++ } ++ } ++} ++ ++impl crate::WithTurnExtensionData { ++ /// Enables pre-execution native approval checkpoint capture for SDK-owned turns. ++ pub fn capture_native_approvals(mut self) -> Self { ++ self.turn_extension_init ++ .get_or_insert_with(Default::default) ++ .insert(NativeApprovals::Capture(Default::default())); ++ self ++ } ++} ++ ++fn permission_snapshot( ++ turn: &TurnContext, ++ environments: &crate::environment_selection::TurnEnvironmentSnapshot, ++ policy: codex_protocol::protocol::AskForApproval, ++ reviewer: ApprovalsReviewer, ++) -> Value { ++ let environments: Vec<_> = environments ++ .turn_environments() ++ .map(|environment| { ++ json!({ ++ "id": environment.selection.environment_id, ++ "profile": environment.permission_profile_with_workspace_roots(), ++ "symbolic_profile": environment.permission_profile(), ++ "network": environment.config().network_policy, ++ "grants": turn.granted_permissions(&environment.selection.environment_id), ++ }) ++ }) ++ .collect(); ++ let mut snapshot = json!({ ++ "approval_policy": policy, "reviewer": reviewer, "environments": environments, ++ "managed_network": turn.config.permissions.network.as_ref().map(|network| network.environment_policy()), ++ }); ++ snapshot.sort_all_objects(); ++ json!(format!( ++ "{:x}", ++ Sha256::digest(snapshot.to_string().as_bytes()) ++ )) ++} ++ ++pub(crate) async fn capture_lease( ++ action: &ApprovalAction, ++ ctx: &ApprovalContext, ++) -> Option> { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || !matches!( ++ action, ++ ApprovalAction::ExecCommand { .. } | ApprovalAction::ApplyPatch { .. } ++ ) ++ { ++ return None; ++ } ++ let state = ctx ++ .review_context ++ .turn() ++ .extension_data ++ .get::()?; ++ let NativeApprovals::Capture(pending) = state.as_ref() else { ++ return None; ++ }; ++ let patch = if let ApprovalAction::ApplyPatch { changes, .. } = action { ++ Some( ++ crate::patch_preconditions::capture_patch_approval_preconditions(ctx, changes) ++ .await ++ .ok()?, ++ ) ++ } else { ++ None ++ }; ++ let lease = Arc::new(CapturableApproval { ++ event: OnceLock::new(), ++ patch, ++ permissions: permission_snapshot( ++ ctx.review_context.turn(), ++ ctx.review_context.environments(), ++ ctx.review_context.approval_policy, ++ ctx.review_context.approvals_reviewer, ++ ), ++ }); ++ // Only the live request owns the lease. Reused IDs stay invalid for the whole turn. ++ let mut pending = pending.lock().unwrap(); ++ if let Some(previous) = pending.get_mut(&ctx.call_id) { ++ *previous = Weak::new(); ++ return None; ++ } ++ pending.insert(ctx.call_id.clone(), Arc::downgrade(&lease)); ++ Some(lease) ++} ++ ++fn approval_id(event: &EventMsg) -> Option<&str> { ++ match event { ++ EventMsg::ExecApprovalRequest(event) ++ if event.kind == ExecApprovalKind::Command ++ && event.approval_id.is_none() ++ && event.network_approval_context.is_none() => ++ { ++ Some(&event.call_id) ++ } ++ EventMsg::ApplyPatchApprovalRequest(event) if event.grant_root.is_none() => { ++ Some(&event.call_id) ++ } ++ _ => None, ++ } ++} ++ ++fn comparable(event: &EventMsg) -> CodexResult { ++ let mut value = serde_json::to_value(event).map_err(|_| refusal("invalid approval event"))?; ++ let object = value ++ .as_object_mut() ++ .ok_or_else(|| refusal("invalid approval event"))?; ++ object.remove("turn_id"); ++ object.remove("started_at_ms"); ++ Ok(value) ++} ++ ++fn record_capture_event(turn: &TurnContext, event: &EventMsg) { ++ let Some(id) = approval_id(event) else { return }; ++ let Some(state) = turn.extension_data.get::() else { ++ return; ++ }; ++ if let NativeApprovals::Capture(pending) = state.as_ref() { ++ if let Some(slot) = pending.lock().unwrap().get_mut(id) ++ && let Some(approval) = slot.upgrade() ++ && !serde_json::to_value(event).is_ok_and(|value| approval.event.set(value).is_ok()) ++ { ++ *slot = Weak::new(); ++ } ++ } ++} ++ ++#[derive(Debug, Default, PartialEq)] ++enum Phase { ++ #[default] ++ Ready, ++ Running, ++ Requested, ++ Approved, ++ Finished, ++} ++ ++#[derive(Debug, Default)] ++struct Progress { ++ owner: Option, ++ phase: Phase, ++} ++ ++#[derive(Debug)] ++pub(crate) struct RestoreToolCall { ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Value, ++ progress: Mutex, ++} ++ ++impl RestoreToolCall { ++ pub(crate) fn new( ++ call: ResponseItem, ++ expected: EventMsg, ++ checkpoint: Value, ++ ) -> CodexResult { ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let (namespace, call_id) = match (&call, &expected) { ++ ( ++ ResponseItem::FunctionCall { ++ name, ++ namespace, ++ call_id, ++ .. ++ }, ++ EventMsg::ExecApprovalRequest(_), ++ ) if name == "exec_command" => (namespace, call_id), ++ ( ++ ResponseItem::CustomToolCall { ++ name, ++ namespace, ++ call_id, ++ .. ++ }, ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ) if name == "apply_patch" => (namespace, call_id), ++ _ => return Err(refusal("unsupported native action")), ++ }; ++ if call_id != id || namespace.as_deref().is_some_and(|ns| ns != "functions") { ++ return Err(refusal( ++ "only one direct exec_command or apply_patch call is supported", ++ )); ++ } ++ if checkpoint.get("version") != Some(&json!(1)) ++ || checkpoint.get("first_approval") != Some(&json!(true)) ++ { ++ return Err(refusal("checkpoint does not match the original approval")); ++ } ++ Ok(Self { ++ call, ++ expected, ++ checkpoint, ++ progress: Mutex::new(Progress::default()), ++ }) ++ } ++ ++ pub(crate) fn accepts_approval(&self, action: &ApprovalAction, ctx: &ApprovalContext) -> bool { ++ if ctx.retry_reason.is_some() ++ || ctx.network_approval_context.is_some() ++ || ctx.strict_auto_review ++ || self.progress.lock().unwrap().phase != Phase::Running ++ || approval_id(&self.expected) != Some(ctx.call_id.as_str()) ++ { ++ return false; ++ } ++ matches!( ++ (&self.expected, action), ++ ( ++ EventMsg::ExecApprovalRequest(_), ++ ApprovalAction::ExecCommand { .. } ++ ) | ( ++ EventMsg::ApplyPatchApprovalRequest(_), ++ ApprovalAction::ApplyPatch { .. } ++ ) ++ ) ++ } ++ ++ pub(crate) fn is_patch(&self) -> bool { ++ matches!(self.expected, EventMsg::ApplyPatchApprovalRequest(_)) ++ } ++ ++ pub(crate) fn claim(&self, turn: &TurnContext) -> bool { ++ let mut progress = self.progress.lock().unwrap(); ++ if progress.phase != Phase::Ready { ++ return false; ++ } ++ progress.owner = Some(turn.sub_id.clone()); ++ progress.phase = Phase::Running; ++ true ++ } ++} ++ ++pub(crate) fn active(turn: &TurnContext) -> Option> { ++ turn.extension_data ++ .get::() ++ .and_then(|state| state.restoration()) ++ .filter(|restore| restore.progress.lock().unwrap().owner.as_ref() == Some(&turn.sub_id)) ++} ++ ++async fn validate_step(session: &Arc, step: &StepContext) -> CodexResult<()> { ++ let config = session.get_config().await; ++ let requirements = config.config_layer_stack.requirements(); ++ if requirements.auto_review_required_for_model(&step.settings.model_info.slug) ++ || requirements ++ .approvals_reviewer ++ .can_set(&ApprovalsReviewer::User) ++ .is_err() ++ { ++ return Err(refusal("managed automatic review is unsupported")); ++ } ++ if step.tool_router.tool_mode() != ToolMode::Direct ++ || step.environments.single_local_environment().is_none() ++ { ++ return Err(refusal( ++ "only direct tools in one local environment are supported", ++ )); ++ } ++ if step.settings.approvals_reviewer() != ApprovalsReviewer::User ++ || step.turn.strict_auto_review_enabled() ++ { ++ return Err(refusal("automatic approval review is unsupported")); ++ } ++ let hooks = session.hooks().list_with_environment_hooks( ++ crate::hook_environment_executor::environment_hook_sources(&step.environments, &[]), ++ ); ++ if !hooks.hooks.is_empty() ++ || !hooks.warnings.is_empty() ++ || !step.executor_hook_sources.is_empty() ++ { ++ return Err(refusal("configured hooks are unsupported")); ++ } ++ if let Some(restore) = active(&step.turn) { ++ if restore.checkpoint.get("permissions") ++ != Some(&permission_snapshot( ++ &step.turn, ++ &step.environments, ++ step.settings.approval_policy(), ++ step.settings.approvals_reviewer(), ++ )) ++ { ++ return Err(refusal("permissions or approval policy changed")); ++ } ++ if let EventMsg::ApplyPatchApprovalRequest(event) = &restore.expected { ++ crate::patch_preconditions::validate_patch_preconditions( ++ step, ++ event, ++ &restore.checkpoint["patch"], ++ ) ++ .await?; ++ } ++ } ++ Ok(()) ++} ++ ++// Prepared deletes include the complete file. Prepared diffs can include neighboring ++// lines absent from the submitted patch. Keep those bytes out of the SDK checkpoint. ++fn compact_patch(event: &mut EventMsg) -> CodexResult { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return Ok(false); ++ }; ++ let mut remaining = MAX_PATCH_SCAN_BYTES; ++ let mut compacted = false; ++ for change in request.changes.values_mut() { ++ match change { ++ FileChange::Add { .. } => continue, ++ FileChange::Delete { content } => { ++ // The trusted file snapshot already binds the entire deletion. ++ *content = String::new(); ++ } ++ FileChange::Update { unified_diff, .. } => { ++ let bytes = unified_diff.len() as u64; ++ if bytes > MAX_PATCH_PREIMAGE_BYTES.min(remaining) { ++ return Err(refusal("prepared patch diff exceeds the recovery limit")); ++ } ++ remaining -= bytes; ++ *unified_diff = format!("{:x}", Sha256::digest(unified_diff.as_bytes())); ++ } ++ } ++ compacted = true; ++ } ++ Ok(compacted) ++} ++ ++pub(crate) async fn capture( ++ session: &Arc, ++ mut expected: EventMsg, ++) -> CodexResult<(EventMsg, Value)> { ++ let (turn, cancellation) = session ++ .active_turn_context_and_cancellation_token() ++ .await ++ .ok_or_else(|| refusal("no active approval"))?; ++ let id = approval_id(&expected).ok_or_else(|| refusal("unsupported approval request"))?; ++ let state = turn ++ .extension_data ++ .get::() ++ .ok_or_else(|| refusal("not a first pre-execution approval"))?; ++ let step = session.capture_step_context(turn, &cancellation).await?; ++ validate_step(session, &step).await?; ++ let NativeApprovals::Capture(pending) = state.as_ref() else { ++ return Err(refusal("the current turn is restoring an action")); ++ }; ++ let approvals = pending.lock().unwrap(); ++ let approval = approvals ++ .get(id) ++ .and_then(Weak::upgrade) ++ .ok_or_else(|| refusal("not a pending first pre-execution approval"))?; ++ // The live event is emitted by the same request_approval invocation holding this lease. ++ if approval.event.get() ++ != Some(&serde_json::to_value(&expected).map_err(|_| refusal("invalid approval event"))?) ++ { ++ return Err(refusal("approval does not match the live request")); ++ } ++ let mut checkpoint = json!({"version":1,"first_approval":true,"permissions":approval.permissions,"patch":approval.patch}); ++ if compact_patch(&mut expected)? { ++ checkpoint["patch"]["version"] = json!(2); ++ } ++ Ok((expected, checkpoint)) ++} ++ ++pub(crate) fn compare_request(turn: &TurnContext, event: &mut EventMsg) -> bool { ++ record_capture_event(turn, event); ++ let Some(restore) = active(turn) else { ++ return true; ++ }; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Running { ++ return false; ++ } ++ // A mismatch consumes the request as well: it cannot be corrected and retried. ++ progress.phase = Phase::Finished; ++ if approval_id(event) != approval_id(&restore.expected) { ++ return false; ++ } ++ // Version 1 keeps comparing the original request verbatim, including any ++ // digest-looking text. Only v2 uses a compact request and file fingerprints. ++ let patch = &restore.checkpoint["patch"]; ++ if patch["version"] == 2 { ++ let EventMsg::ApplyPatchApprovalRequest(request) = event else { ++ return false; ++ }; ++ if !crate::patch_preconditions::matches_delete_preimages(&request.changes, patch) ++ || !matches!(compact_patch(event), Ok(true)) ++ { ++ return false; ++ } ++ } ++ let matches = matches!((comparable(event), comparable(&restore.expected)), (Ok(observed), Ok(expected)) if observed == expected); ++ if matches { ++ progress.phase = Phase::Requested; ++ } ++ matches ++} ++ ++pub(crate) fn record_decision(turn: &TurnContext, decision: ReviewDecision) -> ReviewDecision { ++ let Some(restore) = active(turn) else { ++ return decision; ++ }; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Requested { ++ return ReviewDecision::Abort; ++ } ++ progress.phase = if decision == ReviewDecision::Approved { ++ Phase::Approved ++ } else { ++ Phase::Finished ++ }; ++ match decision { ++ ReviewDecision::Approved | ReviewDecision::Denied { .. } | ReviewDecision::Abort => { ++ decision ++ } ++ _ => ReviewDecision::Abort, ++ } ++} ++ ++pub(crate) async fn validate_before_execution( ++ session: &Arc, ++ step: &StepContext, ++) -> CodexResult<()> { ++ let Some(restore) = active(&step.turn) else { ++ return Ok(()); ++ }; ++ validate_step(session, step).await?; ++ let mut progress = restore.progress.lock().unwrap(); ++ if progress.phase != Phase::Approved { ++ return Err(refusal( ++ "a matching approval permits only one execution attempt", ++ )); ++ } ++ progress.phase = Phase::Finished; ++ Ok(()) ++} ++ ++pub(crate) async fn run( ++ session: Arc, ++ turn: Arc, ++ restore: Arc, ++ cancellation: CancellationToken, ++) -> crate::tasks::SessionTaskResult { ++ if !restore.claim(&turn) { ++ return Err(refusal("the native action was already claimed")); ++ } ++ session.emit_turn_started(&turn, TaskKind::Regular).await; ++ let step = session ++ .capture_step_context(Arc::clone(&turn), &cancellation) ++ .await?; ++ validate_step(&session, &step).await?; ++ let call = ToolRouter::build_tool_call(restore.call.clone()) ++ .map_err(|_| refusal("invalid native call"))? ++ .ok_or_else(|| refusal("unsupported native call"))?; ++ if !step.tool_router.exposes_tool(&call.tool_name) { ++ return Err(refusal("tool is unavailable in the replacement worker")); ++ } ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &restore.call) ++ .await; ++ let result = ToolCallRuntime::new(Arc::clone(&session), Arc::clone(&step), Default::default()) ++ .handle_tool_call(call, cancellation) ++ .await?; ++ crate::stream_events_utils::record_completed_response_item(&session, &step, &result.item).await; ++ Ok(None) ++} +diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs +index cc418ca..b767210 100644 +--- a/codex-rs/core/src/session/mod.rs ++++ b/codex-rs/core/src/session/mod.rs +@@ -2921,7 +2921,7 @@ impl Session { + .map(|task| Arc::clone(&task.turn_context)) + } + +- async fn active_turn_context_and_cancellation_token( ++ pub(crate) async fn active_turn_context_and_cancellation_token( + &self, + ) -> Option<(Arc, CancellationToken)> { + let active = self.active_turn.lock().await; +@@ -3146,7 +3146,7 @@ impl Session { + .as_ref() + .map(PluginCommandAttribution::serialized_fields) + .unzip(); +- let event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { ++ let mut event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + model_context: Some(model_context), + kind, + call_id, +@@ -3166,9 +3166,15 @@ impl Session { + available_decisions: Some(available_decisions), + parsed_cmd, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +@@ -3202,7 +3208,7 @@ impl Session { + warn!("Overwriting existing pending approval for call_id: {approval_id}"); + } + +- let event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { ++ let mut event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { + call_id, + turn_id: turn_context.sub_id.clone(), + started_at_ms: now_unix_timestamp_ms(), +@@ -3210,9 +3216,15 @@ impl Session { + reason, + grant_root, + }); ++ if !crate::restore_tool_call::compare_request(turn_context, &mut event) { ++ return ReviewDecision::Abort; ++ } + let _user_wait = self.start_user_wait_timing(turn_context, user_wait_owner.as_deref()); + self.send_event(turn_context, event).await; +- rx_approve.await.unwrap_or(ReviewDecision::Abort) ++ crate::restore_tool_call::record_decision( ++ turn_context, ++ rx_approve.await.unwrap_or(ReviewDecision::Abort), ++ ) + } + + #[expect( +diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs +index 6c6f3d1..bc3c888 100644 +--- a/codex-rs/core/src/tasks/regular.rs ++++ b/codex-rs/core/src/tasks/regular.rs +@@ -44,6 +44,13 @@ impl SessionTask for RegularTask { + input: Vec, + cancellation_token: CancellationToken, + ) -> SessionTaskResult { ++ if let Some(restore) = ctx ++ .extension_data ++ .get::() ++ .and_then(|state| state.restoration()) ++ { ++ return crate::restore_tool_call::run(sess, ctx, restore, cancellation_token).await; ++ } + if ctx + .config + .features +diff --git a/codex-rs/core/src/tools/approvals.rs b/codex-rs/core/src/tools/approvals.rs +index 74082aa..ebaa126 100644 +--- a/codex-rs/core/src/tools/approvals.rs ++++ b/codex-rs/core/src/tools/approvals.rs +@@ -499,6 +499,18 @@ impl Session { + action: ApprovalAction, + ctx: ApprovalContext, + ) -> ApprovalResult { ++ if let Some(restore) = crate::restore_tool_call::active(ctx.review_context.turn()) { ++ if !restore.accepts_approval(&action, &ctx) { ++ return ApprovalResult { ++ result: Err(ToolError::Rejected( ++ "native restoration cannot request another approval or escalation".into(), ++ )), ++ source: None, ++ cancellation_reason: None, ++ }; ++ } ++ } ++ let _capture_lease = crate::restore_tool_call::capture_lease(&action, &ctx).await; + // Retained authority outside current grants needs a new sandbox approval. + // Current grants permit review of the input; they never approve the input itself. + let policy = ctx.review_context.turn().approval_policy(); +@@ -723,7 +735,7 @@ impl Session { + .into_iter() + .map(|key| (key, &policy_fingerprint)) + .collect(); +- with_cached_approval(&self.services, tool_name, cache_keys, || async { ++ let request = || async { + self.request_command_approval( + ctx.review_context.turn(), + ExecApprovalKind::Command, +@@ -742,8 +754,12 @@ impl Session { + /*plugin_attribution_override*/ None, + ) + .await +- }) +- .await ++ }; ++ if crate::restore_tool_call::active(ctx.review_context.turn()).is_some() { ++ request().await ++ } else { ++ with_cached_approval(&self.services, tool_name, cache_keys, request).await ++ } + } + ApprovalAction::WriteStdin { + id, +@@ -788,9 +804,11 @@ impl Session { + .retry_reason + .clone() + .or_else(|| ctx.approval_reason.clone()); +- if *permissions_preapproved && reason.is_none() { ++ let restoring = ++ crate::restore_tool_call::active(ctx.review_context.turn()).is_some(); ++ if *permissions_preapproved && reason.is_none() && !restoring { + ReviewDecision::Approved +- } else if reason.is_some() { ++ } else if reason.is_some() || restoring { + self.request_patch_approval( + ctx.review_context.turn(), + ctx.call_id.clone(), +diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +index 6a81b62..59e4900 100644 +--- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs ++++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +@@ -236,6 +236,19 @@ impl ExecCommandHandler { + } + None => manager.exec_command(request, &context).await, + }; ++ // The exit event can arrive after the turn. A restored command needs a terminal ++ // result here because the next worker cannot use a live process handle. ++ if crate::restore_tool_call::active(&step_context.turn).is_some() ++ && !result.as_ref().is_ok_and(|output| { ++ output.event_call_id == call_id ++ && output.process_id.is_none() ++ && output.exit_code.is_some() ++ }) ++ { ++ return Err(FunctionCallError::Fatal( ++ "Cannot confirm the restored command exited.".to_string(), ++ )); ++ } + match result { + Ok(response) => Ok(boxed_tool_output(response)), + Err(UnifiedExecError::SandboxDenied { +diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs +index a1a88e4..c26c85d 100644 +--- a/codex-rs/core/src/tools/orchestrator.rs ++++ b/codex-rs/core/src/tools/orchestrator.rs +@@ -61,6 +61,14 @@ impl ToolOrchestrator { + where + T: ToolRuntime, + { ++ if let Err(error) = crate::restore_tool_call::validate_before_execution( ++ &tool_ctx.session, ++ &tool_ctx.step_context, ++ ) ++ .await ++ { ++ return (Err(ToolError::Codex(error)), None); ++ } + let network_approval = match begin_network_approval( + &tool_ctx.session, + &tool_ctx.step_context.turn, +@@ -164,6 +172,13 @@ impl ToolOrchestrator { + let requirement = tool.exec_approval_requirement(req).unwrap_or_else(|| { + default_exec_approval_requirement(approval_policy, &file_system_sandbox_policy) + }); ++ if crate::restore_tool_call::active(turn_ctx).is_some() ++ && !matches!(requirement, ExecApprovalRequirement::NeedsApproval { .. }) ++ { ++ return Err(ToolError::Rejected( ++ "native restoration requires a fresh human approval".into(), ++ )); ++ } + match &requirement { + ExecApprovalRequirement::Skip { .. } => { + if strict_auto_review { +@@ -227,12 +242,24 @@ impl ToolOrchestrator { + + // 2) First attempt under the selected sandbox. + let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); +- let sandbox_override = sandbox_override_for_first_attempt( +- tool.sandbox_permissions(req), +- &requirement, +- &file_system_sandbox_policy, +- /*already_approved*/ initial_approval_source.is_some(), +- ); ++ // Restored patches get one attempt. Use only authority the normal approved retry already grants. ++ let approved_restored_patch = crate::restore_tool_call::active(turn_ctx) ++ .is_some_and(|restore| restore.is_patch()) ++ && initial_approval_source == Some(ApprovalResolutionSource::User) ++ && !strict_auto_review ++ && unsandboxed_allowed ++ && tool.wants_no_sandbox_approval(approval_policy) ++ && tool.should_bypass_approval(approval_policy, true); ++ let sandbox_override = if approved_restored_patch { ++ SandboxOverride::BypassSandboxFirstAttempt ++ } else { ++ sandbox_override_for_first_attempt( ++ tool.sandbox_permissions(req), ++ &requirement, ++ &file_system_sandbox_policy, ++ /*already_approved*/ initial_approval_source.is_some(), ++ ) ++ }; + let (escalated_profile, writable_root_metadata) = if sandbox_override + == SandboxOverride::EscalatedSandboxWithRestrictions + { +diff --git a/codex-rs/exec-server/src/fs_helper.rs b/codex-rs/exec-server/src/fs_helper.rs +index d307663..e98a391 100644 +--- a/codex-rs/exec-server/src/fs_helper.rs ++++ b/codex-rs/exec-server/src/fs_helper.rs +@@ -71,6 +71,12 @@ pub(crate) enum FsHelperRequest { + Open(FsHelperOpenParams), + #[serde(rename = "fs/readFile")] + ReadFile(FsReadFileParams), ++ #[serde(rename = "fs/readFileBounded")] ++ ReadFileBounded { ++ path: PathUri, ++ follow_symlinks: bool, ++ max_bytes: u64, ++ }, + #[serde(rename = "fs/writeFile")] + WriteFile(FsWriteFileParams), + #[serde(rename = "fs/createDirectory")] +@@ -311,6 +317,19 @@ pub(crate) async fn run_direct_request( + data_base64: STANDARD.encode(data), + })) + } ++ FsHelperRequest::ReadFileBounded { ++ path, ++ follow_symlinks, ++ max_bytes, ++ } => { ++ let data = file_system ++ .read_file_bounded(&path, ReadFileOptions { follow_symlinks }, max_bytes, None) ++ .await ++ .map_err(map_fs_error)?; ++ Ok(FsHelperPayload::ReadFile(FsReadFileResponse { ++ data_base64: STANDARD.encode(data), ++ })) ++ } + FsHelperRequest::WriteFile(params) => { + let bytes = STANDARD.decode(params.data_base64).map_err(|err| { + invalid_request(format!( +diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs +index abe2a0a..b721a69 100644 +--- a/codex-rs/exec-server/src/local_file_system.rs ++++ b/codex-rs/exec-server/src/local_file_system.rs +@@ -304,6 +304,20 @@ impl ExecutorFileSystem for LocalFileSystem { + Box::pin(LocalFileSystem::read_file(self, path, options, sandbox)) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ let (fs, sandbox) = self.file_system_for_reads(sandbox)?; ++ fs.read_file_bounded(path, options, max_bytes, sandbox) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +@@ -519,6 +533,21 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { + )) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async move { ++ reject_platform_sandbox_context(sandbox)?; ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, None) ++ .await ++ }) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +@@ -630,15 +659,16 @@ impl DirectFileSystem { + Ok(PathUri::from_abs_path(&canonicalized)) + } + +- async fn read_file( ++ async fn read_bytes( + &self, + path: &PathUri, + options: ReadFileOptions, ++ max_bytes: Option, + sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; +- crate::local_file_system_read::read_file(path, options).await ++ crate::local_file_system_read::read_file(path, options, max_bytes).await + } + + async fn read_file_stream( +@@ -995,7 +1025,25 @@ impl ExecutorFileSystem for DirectFileSystem { + options: ReadFileOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { +- Box::pin(DirectFileSystem::read_file(self, path, options, sandbox)) ++ Box::pin(DirectFileSystem::read_bytes( ++ self, path, options, None, sandbox, ++ )) ++ } ++ ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(DirectFileSystem::read_bytes( ++ self, ++ path, ++ options, ++ Some(max_bytes), ++ sandbox, ++ )) + } + + fn read_file_stream<'a>( +diff --git a/codex-rs/exec-server/src/local_file_system_read.rs b/codex-rs/exec-server/src/local_file_system_read.rs +index e9bd05e..80ce9ae 100644 +--- a/codex-rs/exec-server/src/local_file_system_read.rs ++++ b/codex-rs/exec-server/src/local_file_system_read.rs +@@ -15,7 +15,11 @@ const MAX_READ_FILE_BYTES: u64 = 512 * 1024 * 1024; + pub(super) async fn read_file( + path: AbsolutePathBuf, + options: ReadFileOptions, ++ max_bytes: Option, + ) -> io::Result> { ++ let max_bytes = max_bytes ++ .unwrap_or(MAX_READ_FILE_BYTES) ++ .min(MAX_READ_FILE_BYTES); + let cancelled = CancellationToken::new(); + let _cancel_on_drop = cancelled.clone().drop_guard(); + tokio::task::spawn_blocking(move || { +@@ -28,24 +32,33 @@ pub(super) async fn read_file( + no_follow::open_file(path.as_path())? + }; + let metadata = file.metadata()?; +- if metadata.len() > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- let mut bytes = Vec::with_capacity(metadata.len() as usize); +- CancellableReader { +- reader: file.take(MAX_READ_FILE_BYTES + 1), +- cancelled, +- } +- .read_to_end(&mut bytes)?; +- if bytes.len() as u64 > MAX_READ_FILE_BYTES { +- return Err(file_too_large_error()); +- } +- Ok(bytes) ++ read_with_limit(file, metadata.len(), max_bytes, cancelled) + }) + .await + .map_err(|error| io::Error::other(format!("filesystem task failed: {error}")))? + } + ++fn read_with_limit( ++ reader: impl Read, ++ initial_len: u64, ++ max_bytes: u64, ++ cancelled: CancellationToken, ++) -> io::Result> { ++ if initial_len > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ let mut bytes = Vec::with_capacity(initial_len as usize); ++ CancellableReader { ++ reader: reader.take(max_bytes + 1), ++ cancelled, ++ } ++ .read_to_end(&mut bytes)?; ++ if bytes.len() as u64 > max_bytes { ++ return Err(file_too_large_error(max_bytes)); ++ } ++ Ok(bytes) ++} ++ + struct CancellableReader { + reader: R, + cancelled: CancellationToken, +@@ -62,10 +75,10 @@ impl Read for CancellableReader { + } + } + +-fn file_too_large_error() -> io::Error { ++fn file_too_large_error(max_bytes: u64) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, +- format!("file is too large to read: limit is {MAX_READ_FILE_BYTES} bytes"), ++ format!("file is too large to read: limit is {max_bytes} bytes"), + ) + } + +diff --git a/codex-rs/exec-server/src/sandboxed_file_system.rs b/codex-rs/exec-server/src/sandboxed_file_system.rs +index f592edf..7538822 100644 +--- a/codex-rs/exec-server/src/sandboxed_file_system.rs ++++ b/codex-rs/exec-server/src/sandboxed_file_system.rs +@@ -139,10 +139,11 @@ impl SandboxedFileSystem { + Ok(response.path) + } + +- async fn read_file( ++ async fn read_bytes( + &self, + path: &PathUri, + options: ReadFileOptions, ++ max_bytes: Option, + sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + let sandbox = require_platform_sandbox(sandbox)?; +@@ -150,11 +151,18 @@ impl SandboxedFileSystem { + let response = self + .run_sandboxed( + sandbox, +- FsHelperRequest::ReadFile(FsReadFileParams { +- path: path.clone(), +- follow_symlinks: (!options.follow_symlinks).then_some(false), +- sandbox: None, +- }), ++ match max_bytes { ++ Some(max_bytes) => FsHelperRequest::ReadFileBounded { ++ path: path.clone(), ++ follow_symlinks: options.follow_symlinks, ++ max_bytes, ++ }, ++ None => FsHelperRequest::ReadFile(FsReadFileParams { ++ path: path.clone(), ++ follow_symlinks: (!options.follow_symlinks).then_some(false), ++ sandbox: None, ++ }), ++ }, + ) + .await? + .expect_read_file() +@@ -361,7 +369,25 @@ impl ExecutorFileSystem for SandboxedFileSystem { + options: ReadFileOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { +- Box::pin(SandboxedFileSystem::read_file(self, path, options, sandbox)) ++ Box::pin(SandboxedFileSystem::read_bytes( ++ self, path, options, None, sandbox, ++ )) ++ } ++ ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(SandboxedFileSystem::read_bytes( ++ self, ++ path, ++ options, ++ Some(max_bytes), ++ sandbox, ++ )) + } + + fn read_file_stream<'a>( +diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs +index 06f63ed..a1f7159 100644 +--- a/codex-rs/file-system/src/environment_accessor.rs ++++ b/codex-rs/file-system/src/environment_accessor.rs +@@ -40,6 +40,21 @@ pub trait EnvironmentAccess: Send + Sync { + options: ReadFileOptions, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads a file with a hard byte limit, preserving this accessor's permissions. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(std::io::Error::new( ++ std::io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Opens an owned stream; subsequent reads do not borrow or reauthorize through the accessor. + fn read_file_stream<'a>( + &'a self, +@@ -212,6 +227,16 @@ impl EnvironmentAccess for FileSystemEnvironmentAccessor<'_> { + .read_file(path, options, self.key.sandbox.as_deref()) + } + ++ fn read_file_bounded<'a>( ++ &'a self, ++ path: &'a PathUri, ++ options: ReadFileOptions, ++ max_bytes: u64, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ self.file_system ++ .read_file_bounded(path, options, max_bytes, self.key.sandbox.as_deref()) ++ } ++ + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, +diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs +index 9e032e5..be5af03 100644 +--- a/codex-rs/file-system/src/lib.rs ++++ b/codex-rs/file-system/src/lib.rs +@@ -667,6 +667,23 @@ pub trait ExecutorFileSystem: Send + Sync { + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec>; + ++ /// Reads at most `max_bytes + 1` bytes, rejecting files larger than `max_bytes`. ++ /// Backends must enforce the bound during I/O rather than truncate a full read. ++ fn read_file_bounded<'a>( ++ &'a self, ++ _path: &'a PathUri, ++ _options: ReadFileOptions, ++ _max_bytes: u64, ++ _sandbox: Option<&'a FileSystemSandboxContext>, ++ ) -> ExecutorFileSystemFuture<'a, Vec> { ++ Box::pin(async { ++ Err(io::Error::new( ++ io::ErrorKind::Unsupported, ++ "bounded file reads are unsupported", ++ )) ++ }) ++ } ++ + /// Reads a file as a stream of chunks no larger than [`FILE_READ_CHUNK_SIZE`]. + fn read_file_stream<'a>( + &'a self, diff --git a/native/codex/patches/README.md b/native/codex/patches/README.md new file mode 100644 index 0000000000..9c8725c12f --- /dev/null +++ b/native/codex/patches/README.md @@ -0,0 +1,34 @@ +# Changes to Codex in the native SDK wheel + +The SDK pins Codex in [Cargo.toml](../Cargo.toml). The wheel builder copies that revision into a temporary directory and applies these changes in filename order. It leaves the Cargo cache and any installed Codex CLI alone. + +- [Detailed tracing](01-operation-tracing.md) observes the final model request and the start of tool execution. +- [Approval recovery](02-approval-recovery.md) lets a new worker resume a built-in command or edit after a human decision. + +## Rules for every patch + +Use a patch only for a capability the pinned Codex APIs cannot provide. Keep it as small as possible. Put SDK behavior and tests in ordinary source and test files. + +Pair each `.patch` with a Markdown guide of the same name. Explain why the change is necessary, what it does, what can go wrong, how to test it and when to remove it. The build rejects missing or empty guides. The wheel includes each patch and guide in `agents/models/_codex_bin/licenses/codex-patches/`. + +## Test a change + +From the SDK repository root, use Python 3.12, uv, Git and the [native build prerequisites](../../../.github/workflows/codex-preview.yml). Build on the target machine. On Linux, install `patchelf` or add `--compatibility linux` for a local build. + +```sh +uv sync --python 3.12 +uv pip install --require-hashes -r native/codex/build-requirements.txt +uv run --no-sync python scripts/build_codex_wheel.py --profile dev --test-native --out dist/local +``` + +`--test-native` runs the SDK's Rust tests against the same source as the wheel. CI enables it for matrix entries with `test_native: true`. Every platform also runs the [installed-wheel checks](../verify_installed.py). + +To check a local wheel on Linux or macOS, start with an empty `dist/local` and install the result in a fresh environment. + +```sh +uv venv --python 3.12 .venv-wheel +uv pip install --python .venv-wheel/bin/python dist/local/*.whl +.venv-wheel/bin/python -I native/codex/verify_installed.py +``` + +When updating the Codex pin, remove each patch whose behavior is now available upstream. Rerun the same checks. diff --git a/native/codex/src/config.rs b/native/codex/src/config.rs index ea77d44944..fbf7b44353 100644 --- a/native/codex/src/config.rs +++ b/native/codex/src/config.rs @@ -34,6 +34,7 @@ type Result = std::result::Result; pub struct CoreServices { pub config: Config, pub manager: Arc, + pub operations: Arc, home: PathBuf, cwd: PathBuf, overrides: Vec<(String, toml::Value)>, @@ -447,8 +448,11 @@ pub async fn build_services( } else { "codex-code-mode-host" }); + let operations = Arc::new(crate::operation_tracing::Observers::default()); let manager = Arc::new_cyclic(|weak| { let mut extensions = ExtensionRegistryBuilder::::new(); + extensions.model_request_contributor(operations.clone()); + extensions.tool_lifecycle_contributor(operations.clone()); codex_guardian_v2::install(&mut extensions, auth.clone(), weak.clone()); codex_core::install_agent_message_board(&mut extensions, weak.clone()); codex_mcp_extension::install(&mut extensions); @@ -515,6 +519,7 @@ pub async fn build_services( Ok(CoreServices { config, manager, + operations, home, cwd, overrides, diff --git a/native/codex/src/lib.rs b/native/codex/src/lib.rs index f6082ddb38..5cb2d6dd6b 100644 --- a/native/codex/src/lib.rs +++ b/native/codex/src/lib.rs @@ -1,5 +1,7 @@ //! Private, asynchronous Core bindings for the opt-in Codex model provider. mod config; +mod native_approval; +mod operation_tracing; mod tool_hooks; use std::path::PathBuf; @@ -8,6 +10,7 @@ use std::sync::{Arc, Once, OnceLock}; use codex_core_api::{ CodexThread, StartIfIdleSubmission, StartThreadOptions, TurnInputRequest, TurnStartOptions, + WithTurnExtensionData, }; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::normalize_dynamic_tool_specs; @@ -45,6 +48,7 @@ struct ThreadRecord { thread: Arc, id: ThreadId, model: String, + operations: Arc, cleanup: OnceLock, } @@ -55,33 +59,42 @@ impl ThreadRecord { let thread = self.thread.clone(); let services = self.services.clone(); let id = self.id; + let operations = self.operations.clone(); let task = runtime().spawn(async move { - // Each fresh root owns an independent local agent runtime. Fence that - // exact tree before waiting, so native subagents cannot outlive an attempt. - let shutdown = services.manager.request_agent_tree_shutdown(id).await; - let result = match shutdown { - Ok(shutdown) => shutdown.wait().await, - Err(error) => { - let _ = thread.shutdown_and_wait().await; - Err(error) - } - }; - if result.is_ok() { - let ids = services - .manager - .list_agent_subtree_thread_ids(id) - .await - .map_err(|_| "Unable to release the Codex agent tree.".to_owned())?; - for id in ids { - if let Ok(current) = services.manager.get_thread(id).await { - services - .manager - .remove_thread_if_matches(&id, ¤t) - .await; + let result = async { + // Each fresh root owns an independent local agent runtime. Fence that + // exact tree before waiting, so native subagents cannot outlive an attempt. + let shutdown = services.manager.request_agent_tree_shutdown(id).await; + let result = match shutdown { + Ok(shutdown) => shutdown.wait().await, + Err(error) => { + let _ = thread.shutdown_and_wait().await; + Err(error) + } + }; + if result.is_ok() { + let ids = services + .manager + .list_agent_subtree_thread_ids(id) + .await + .map_err(|_| { + "Unable to release the Codex agent tree.".to_owned() + })?; + for id in ids { + if let Ok(current) = services.manager.get_thread(id).await { + services + .manager + .remove_thread_if_matches(&id, ¤t) + .await; + } } } + result.map_err(|_| "Unable to shut down the Codex agent tree.".to_owned()) } - result.map_err(|_| "Unable to shut down the Codex agent tree.".to_owned()) + .await; + // Detach even when a joined shutdown returns an error; the buffer remains drainable. + operations.close(); + result }); async move { task.await @@ -188,12 +201,14 @@ async fn start_thread( let created = Box::pin(services.manager.start_thread(options)) .await .map_err(|_| "Unable to start the Codex thread.".to_owned())?; + let operations = services.operations.root(created.thread_id.to_string()); Ok(Arc::new(ThreadLease { record: Arc::new(ThreadRecord { model: created.session_configured.model, id: created.thread_id, thread: created.thread, services, + operations, cleanup: OnceLock::new(), }), })) @@ -386,6 +401,28 @@ struct Thread { #[pymethods] impl Thread { + #[pyo3(signature=(mode=None))] + fn enable_tracing(&self, mode: Option<&str>) -> PyResult<()> { + self.lease.ensure_open().map_err(py_error)?; + self.lease + .operations + .enable(self.lease.thread.thread_extension_data(), mode) + .map_err(|message| py_error(message.to_owned())) + } + + fn wait_for_operations<'py>(&self, py: Python<'py>) -> PyResult> { + let buffer = self.lease.operations.buffer.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + buffer.updates.notified().await; + Ok(()) + }) + } + + fn drain_operations(&self) -> Option { + // Draining a closed lease is intentional: shutdown can finish or cancel observations. + self.lease.operations.buffer.drain() + } + #[getter] fn model(&self) -> String { self.lease.model.clone() @@ -418,11 +455,13 @@ impl Thread { }) } + #[pyo3(signature=(input_json, output_schema_json, recover_approvals=false))] fn start_turn<'py>( &self, py: Python<'py>, input_json: String, output_schema_json: String, + recover_approvals: bool, ) -> PyResult> { let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { @@ -437,6 +476,10 @@ impl Thread { final_output_json_schema: schema, ..Default::default() }); + let mut request = WithTurnExtensionData::from(request); + if recover_approvals { + request = request.capture_native_approvals(); + } match Box::pin(lease.thread.start_turn_if_idle(request)) .await .map_err(|_| "Unable to start the Codex turn.".to_owned())? @@ -455,22 +498,62 @@ impl Thread { fn next_event<'py>(&self, py: Python<'py>) -> PyResult> { let lease = self.lease.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = async { - lease.ensure_open()?; - let event = lease.thread.next_event().await.map_err(|_| "The Codex event stream ended unexpectedly.".to_owned())?; - match &event.msg { - EventMsg::ExecApprovalRequest(_) - | EventMsg::ApplyPatchApprovalRequest(_) - | EventMsg::RequestPermissions(_) - | EventMsg::RequestUserInput(_) - | EventMsg::ElicitationRequest(_) => { - return Err("Codex requested interactive approval. This provider supports non-interactive execution only.".to_owned()); - } - _ => {} + lease.ensure_open().map_err(py_error)?; + let event = lease + .thread + .next_event() + .await + .map_err(|_| py_error("The Codex event stream ended unexpectedly."))?; + match &event.msg { + EventMsg::ExecApprovalRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) => { + let (request, preconditions) = lease.thread.capture_tool_approval(event.msg) + .await.map_err(|_| py_error( + "Cannot save this Codex approval. Use session=ResumableSession with a supported local command or file edit." + ))?; + return Ok(serde_json::json!({ + "id": event.id, "msg": request, "preconditions": preconditions + }) + .to_string()); } - serde_json::to_string(&event).map_err(|_| "Unable to decode the Codex event.".to_owned()) - }.await; - result.map_err(py_error) + EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) => { + return Err(py_error( + "Codex requested an unsupported kind of interactive approval.", + )); + } + EventMsg::RawResponseItem(raw) => lease.operations.observe_item(&raw.item), + _ => {} + } + serde_json::to_string(&event).map_err(|_| py_error("Unable to decode the Codex event.")) + }) + } + + fn resolve_approval<'py>( + &self, + py: Python<'py>, + call_json: String, + request_json: String, + preconditions_json: String, + ) -> PyResult> { + let call = + serde_json::from_str(&call_json).map_err(|_| py_error("Invalid saved Codex call."))?; + let request = serde_json::from_str(&request_json) + .map_err(|_| py_error("Invalid saved Codex request."))?; + let preconditions = serde_json::from_str(&preconditions_json) + .map_err(|_| py_error("Invalid saved Codex preconditions."))?; + let lease = self.lease.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + lease.ensure_open().map_err(py_error)?; + native_approval::resolve( + &lease.thread, + call, + request, + preconditions, + &lease.operations, + ) + .await + .map_err(py_error) }) } diff --git a/native/codex/src/native_approval.rs b/native/codex/src/native_approval.rs new file mode 100644 index 0000000000..7349af3e2e --- /dev/null +++ b/native/codex/src/native_approval.rs @@ -0,0 +1,134 @@ +//! Resume one native call after the SDK has atomically claimed its saved approval. + +use codex_core_api::{CodexThread, StartIfIdleSubmission}; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::{EventMsg, Op, ReviewDecision}; + +use crate::NativeResult; + +pub(super) async fn resolve( + thread: &CodexThread, + call: ResponseItem, + expected: EventMsg, + preconditions: serde_json::Value, + operations: &crate::operation_tracing::Root, +) -> NativeResult { + let (call_id, patch) = match &expected { + EventMsg::ExecApprovalRequest(request) => (request.call_id.clone(), false), + EventMsg::ApplyPatchApprovalRequest(request) => (request.call_id.clone(), true), + _ => return Err("Unsupported Codex approval request.".into()), + }; + // Restoration dispatches this call through Core. Core checks the saved request + // against its freshly prepared action before it can ask us for the decision. + let StartIfIdleSubmission::Started { turn_id, .. } = + Box::pin(thread.restore_tool_call_if_idle(call, expected, preconditions)) + .await + .map_err(|_| "Could not restore the pending Codex approval.")? + else { + return Err("The Codex thread is busy.".into()); + }; + + let mut answered = false; + let mut finished = false; + let mut result = None; + loop { + let event = thread + .next_event() + .await + .map_err(|_| "The restored Codex turn ended unexpectedly.")?; + if event.id != turn_id { + continue; + } + let decision = match event.msg { + EventMsg::ExecApprovalRequest(request) + if !patch + && !answered + && request.call_id == call_id + && request.approval_id.is_none() + && request.turn_id == turn_id + && request + .effective_available_decisions() + .contains(&ReviewDecision::Approved) => + { + Op::ExecApproval { + id: request.call_id, + turn_id: Some(turn_id.clone()), + decision: ReviewDecision::Approved, + } + } + EventMsg::ApplyPatchApprovalRequest(request) + if patch + && !answered + && request.call_id == call_id + && request.turn_id == turn_id => + { + Op::PatchApproval { + id: request.call_id, + decision: ReviewDecision::Approved, + } + } + EventMsg::ExecApprovalRequest(_) + | EventMsg::ApplyPatchApprovalRequest(_) + | EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::ElicitationRequest(_) + | EventMsg::DynamicToolCallRequest(_) => { + return Err("Codex requested another decision while restoring the call.".into()); + } + EventMsg::PatchApplyEnd(end) + if answered && patch && end.call_id == call_id && end.turn_id == turn_id => + { + finished = true; + continue; + } + EventMsg::RawResponseItem(item) => { + operations.observe_item(&item.item); + let output = match item.item { + ResponseItem::FunctionCallOutput { + call_id: Some(id), + output, + .. + } if !patch && id == call_id => output, + ResponseItem::CustomToolCallOutput { + call_id: id, + output, + .. + } if patch && id == call_id => output, + _ => continue, + }; + if !answered || result.is_some() { + return Err( + "Codex could not confirm the saved approval before dispatch.".into(), + ); + } + if !patch { + // Core only returns a successful restored command result after exit. + // Its asynchronous ExecCommandEnd event may follow TurnComplete. + finished = output.success == Some(true); + } + result = output.body.to_text(); + continue; + } + EventMsg::TurnComplete(_) => { + if !finished { + return Err("Codex could not confirm that the approved call finished.".into()); + } + return result.ok_or_else(|| "Codex did not complete the approved call.".into()); + } + EventMsg::TurnAborted(_) | EventMsg::Error(_) | EventMsg::ShutdownComplete => { + if answered { + return Err("Codex could not confirm that the approved call finished.".into()); + } + return Err( + "Codex could not complete the restored approval under current policy.".into(), + ); + } + _ => continue, + }; + thread + .submit(decision) + .await + .map_err(|_| "Could not submit the saved approval to Codex.")?; + answered = true; + } +} diff --git a/native/codex/src/operation_tracing.rs b/native/codex/src/operation_tracing.rs new file mode 100644 index 0000000000..93bbe1b54f --- /dev/null +++ b/native/codex/src/operation_tracing.rs @@ -0,0 +1,673 @@ +//! Model and tool spans for the root thread. Content capture requires explicit opt-in. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[path = "operation_tracing_content.rs"] +mod content; + +use codex_extension_api::ToolPayload; +use codex_extension_api::{ + CommandStartInput, ExtensionData, ModelRequestContent, ModelRequestContributor, + ModelRequestInput, ModelRequestKind, ModelResponseError, ModelResponseInterceptor, + ModelResponseStream, ResponseEvent, ToolCallOutcome, ToolFinishInput, ToolLifecycleContributor, + ToolLifecycleFuture, ToolStartInput, +}; +use codex_protocol::ToolName; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::TokenUsage; +use futures::StreamExt; +use serde_json::{Value, json}; + +const MAX_EVENTS: usize = 512; +const MAX_STRING_BYTES: usize = 256; + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or_default() +} + +#[derive(Debug, Default)] +pub(crate) struct Buffer { + events: Mutex, + sequence: AtomicU64, + content: AtomicBool, + pending: Mutex, bool)>>, + pub(crate) updates: tokio::sync::Notify, +} + +#[derive(Debug, Default)] +struct Events { + items: Vec, + dropped: bool, + bytes: usize, +} + +impl Buffer { + fn push(&self, event: Value) { + let mut events = self + .events + .lock() + .unwrap_or_else(|error| error.into_inner()); + let size = serde_json::to_vec(&event) + .map(|v| v.len()) + .unwrap_or(usize::MAX); + if events.items.len() < MAX_EVENTS && events.bytes.saturating_add(size) <= 1024 * 1024 { + events.bytes += size; + events.items.push(event); + } else { + events.dropped = true; + } + self.updates.notify_one(); + } + + fn bounded(&self, value: &str) -> bool { + let valid = !value.is_empty() && value.len() <= MAX_STRING_BYTES; + if !valid { + self.events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .dropped = true; + } + valid + } + + pub(crate) fn drain(&self) -> Option { + let events = { + let mut buffered = self + .events + .lock() + .unwrap_or_else(|error| error.into_inner()); + buffered.bytes = 0; + let mut events = std::mem::take(&mut buffered.items); + if std::mem::take(&mut buffered.dropped) { + events.push(json!({"kind": "dropped"})); + } + events + }; + if events.is_empty() { + return None; + } + Some(serde_json::to_string(&events).unwrap_or_else(|_| "[{\"kind\":\"dropped\"}]".into())) + } + + fn bounded_tool(&self, call_id: &str, name: &ToolName) -> bool { + self.bounded(call_id) + && self.bounded(&name.name) + && name + .namespace + .as_deref() + .is_none_or(|value| self.bounded(value)) + } + + fn tool(&self, phase: &str, call_id: &str, name: &ToolName, outcome: Option<&str>) { + if !self.bounded_tool(call_id, name) { + return; + } + let mut event = json!({ + "kind": "tool", "phase": phase, "id": call_id, "at": now(), + "name": name.name, "namespace": name.namespace, + }); + if let Some(outcome) = outcome { + event["outcome"] = json!(outcome); + } + self.push(event); + } +} + +#[derive(Debug, Default)] +pub(crate) struct Observers { + roots: Mutex>>, +} + +impl Observers { + pub(crate) fn root(self: &Arc, id: String) -> Arc { + Arc::new(Root { + id, + observers: self.clone(), + buffer: Arc::new(Buffer::default()), + enabled: AtomicBool::new(false), + }) + } + + fn buffer(&self, id: &str) -> Option> { + self.roots + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(id)? + .upgrade() + } +} + +pub(crate) struct Root { + id: String, + observers: Arc, + pub(crate) buffer: Arc, + enabled: AtomicBool, +} + +impl Root { + pub(crate) fn enable( + &self, + store: &ExtensionData, + mode: Option<&str>, + ) -> Result<(), &'static str> { + let capture = match mode { + None | Some("metadata") => false, + Some("content") => true, + _ => return Err("Invalid Codex operation capture mode."), + }; + self.buffer.content.store(capture, Ordering::Release); + if !self.enabled.swap(true, Ordering::AcqRel) { + store.insert_arc(self.buffer.clone()); + self.observers + .roots + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(self.id.clone(), Arc::downgrade(&self.buffer)); + } + Ok(()) + } + + pub(crate) fn observe_item(&self, item: &ResponseItem) { + if !self.enabled.load(Ordering::Acquire) || !self.buffer.content.load(Ordering::Acquire) { + return; + } + match item { + ResponseItem::FunctionCallOutput { call_id: Some(id), output, .. } if self.buffer.bounded(id) => + self.buffer.push(json!({"kind":"tool", "phase":"output", "id":id, "output":content::capture(output)})), + ResponseItem::CustomToolCallOutput { call_id: id, output, .. } if self.buffer.bounded(id) => + self.buffer.push(json!({"kind":"tool", "phase":"output", "id":id, "output":content::capture(output)})), + _ => {} + } + } + + pub(crate) fn close(&self) { + let mut roots = self + .observers + .roots + .lock() + .unwrap_or_else(|error| error.into_inner()); + if roots + .get(&self.id) + .is_some_and(|buffer| buffer.ptr_eq(&Arc::downgrade(&self.buffer))) + { + roots.remove(&self.id); + } + } +} + +impl Drop for Root { + fn drop(&mut self) { + self.close(); + } +} + +struct ModelObservation { + buffer: Arc, + id: String, + finished: bool, + output: Vec, + output_bytes: usize, + response_id: Option, +} + +impl ModelObservation { + fn finish(&mut self, outcome: &str, usage: Option<&TokenUsage>) { + if self.finished { + return; + } + self.finished = true; + let mut event = json!({ + "kind": "model", "phase": "end", "id": self.id, + "at": now(), "outcome": outcome, + }); + if self.buffer.content.load(Ordering::Acquire) { + event["response_id"] = json!(self.response_id); + event["output"] = content::capture(&self.output); + } + if let Some(usage) = usage { + event["usage"] = json!(usage); + } + self.buffer.push(event); + } +} + +impl Drop for ModelObservation { + fn drop(&mut self) { + self.finish("interrupted", None); + } +} + +impl ModelResponseInterceptor for ModelObservation { + fn request_failed(&mut self) { + self.finish("failed", None); + } + + fn observe_request(&mut self, request: ModelRequestContent<'_>) { + if !self.buffer.content.load(Ordering::Acquire) { + return; + } + let value = match request { + ModelRequestContent::Responses(value) => content::request( + &value.input, + value.tools.as_ref(), + None, + &value.tool_choice, + value.parallel_tool_calls, + ), + ModelRequestContent::WebSocket(value) => content::request( + value.input, + value.tools, + value.previous_response_id.as_deref(), + value.tool_choice, + value.parallel_tool_calls, + ), + }; + self.buffer + .push(json!({"kind":"model", "phase":"input", "id":self.id, "input":value})); + } + + fn intercept(self: Box, stream: ModelResponseStream) -> ModelResponseStream { + let mut observation = *self; + // Inspect borrows each event and yields the original event without buffering or rewriting. + Box::pin(stream.inspect(move |event| match event { + Ok(ResponseEvent::OutputItemDone(item)) + if observation.buffer.content.load(Ordering::Acquire) => + { + let value = content::capture(item); + let size = serde_json::to_vec(&value) + .map(|v| v.len()) + .unwrap_or(content::LIMIT); + if observation.output_bytes.saturating_add(size) <= content::LIMIT + && observation.output.len() < 128 + { + observation.output_bytes += size; + observation.output.push(value); + } else if observation.output.last() != Some(&content::omitted()) { + observation.output.push(content::omitted()); + observation.output_bytes = content::LIMIT; + } + } + Ok(ResponseEvent::Completed { + token_usage, + response_id, + .. + }) => { + if observation.buffer.content.load(Ordering::Acquire) + && response_id.len() <= MAX_STRING_BYTES + { + observation.response_id = Some(response_id.clone()); + } + observation.finish("completed", token_usage.as_ref()); + } + Err(ModelResponseError::ResponseIncomplete(error)) => { + observation.finish("failed", error.token_usage.as_ref()); + } + Err(_) => observation.finish("failed", None), + _ => {} + })) + } +} + +impl ModelRequestContributor for Observers { + fn request(&self, input: ModelRequestInput<'_>) -> Option> { + if input.kind != ModelRequestKind::Generation { + return None; + } + let buffer = self.buffer(input.thread_id)?; + if !buffer.bounded(input.model) { + return None; + } + let id = format!("model-{}", buffer.sequence.fetch_add(1, Ordering::Relaxed)); + buffer.push(json!({ + "kind": "model", "phase": "start", "id": id, + "at": now(), "model": input.model, + })); + Some(Box::new(ModelObservation { + buffer, + id, + finished: false, + output: Vec::new(), + output_bytes: 0, + response_id: None, + })) + } +} + +impl ToolLifecycleContributor for Observers { + fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() + && buffer.bounded_tool(input.call_id, input.tool_name) + { + let capture = buffer + .content + .load(Ordering::Acquire) + .then(|| match input.payload { + ToolPayload::Function { arguments } => content::capture(arguments), + ToolPayload::Custom { input } => content::capture(input), + _ => content::omitted(), + }); + let delayed = matches!( + input.tool_name.name.as_str(), + "exec_command" | "apply_patch" | "shell" | "shell_command" + ); + let mut pending = buffer.pending.lock().unwrap_or_else(|e| e.into_inner()); + if pending.len() < 128 { + pending.insert( + input.call_id.to_owned(), + (input.tool_name.clone(), capture, false), + ); + } else { + buffer.push(json!({"kind": "dropped"})); + } + drop(pending); + if !delayed { + start_tool(&buffer, input.call_id); + } + } + Box::pin(std::future::ready(())) + } + + fn on_command_start<'a>(&'a self, input: CommandStartInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() { + if buffer.content.load(Ordering::Acquire) { + if let Some((_, payload, _)) = buffer + .pending + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get_mut(input.call_id) + { + *payload = Some(content::capture( + &json!({"command":content::capture(input.command), "cwd":content::capture(input.cwd)}), + )); + } + } + } + Box::pin(std::future::ready(())) + } + + fn on_tool_execution_start(&self, thread_id: &str, _turn_id: &str, call_id: &str) { + if let Some(buffer) = self.buffer(thread_id) { + start_tool(&buffer, call_id); + } + } + + fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> { + if let Some(buffer) = input.thread_store.get::() { + let outcome = match input.outcome { + ToolCallOutcome::Completed { success: true } => "completed", + ToolCallOutcome::Aborted => "interrupted", + _ => "failed", + }; + buffer + .pending + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(input.call_id); + buffer.tool("end", input.call_id, input.tool_name, Some(outcome)); + } + Box::pin(std::future::ready(())) + } +} + +fn start_tool(buffer: &Buffer, call_id: &str) { + let mut pending = buffer.pending.lock().unwrap_or_else(|e| e.into_inner()); + let Some((name, payload, started)) = pending.get_mut(call_id) else { + return; + }; + if *started { + return; + } + *started = true; + buffer.tool("start", call_id, name, None); + if let Some(payload) = payload { + buffer.push(json!({"kind":"tool", "phase":"input", "id":call_id, "input":payload})); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::stream; + + fn request( + observers: &Observers, + root: &str, + kind: ModelRequestKind, + ) -> Option> { + observers.request(ModelRequestInput { + kind, + thread_id: root, + client_metadata: &mut None, + model: "fixture-model", + }) + } + + fn drain(buffer: &Buffer) -> Vec { + buffer + .drain() + .map(|value| serde_json::from_str(&value).unwrap()) + .unwrap_or_default() + } + + #[tokio::test] + async fn model_observation_forwards_original_events_and_keeps_only_metadata() { + let observers = Arc::new(Observers::default()); + let root = observers.root("root".into()); + assert!(request(&observers, "root", ModelRequestKind::Generation).is_none()); + root.enable(&ExtensionData::new("root"), None).unwrap(); + assert!(request(&observers, "child", ModelRequestKind::Generation).is_none()); + assert!(request(&observers, "root", ModelRequestKind::Warmup).is_none()); + let interceptor = request(&observers, "root", ModelRequestKind::Generation).unwrap(); + let usage = TokenUsage { + input_tokens: 11, + output_tokens: 3, + total_tokens: 14, + ..Default::default() + }; + let mut observed = interceptor.intercept(Box::pin(stream::iter([ + Ok(ResponseEvent::Created { + response_id: Some("private-response-id".into()), + }), + Ok(ResponseEvent::Completed { + response_id: "private-response-id".into(), + token_usage: Some(usage), + usage_metadata: None, + end_turn: Some(true), + }), + ]))); + assert!( + matches!(observed.next().await, Some(Ok(ResponseEvent::Created { response_id: Some(id) })) if id == "private-response-id") + ); + assert!( + matches!(observed.next().await, Some(Ok(ResponseEvent::Completed { response_id, .. })) if response_id == "private-response-id") + ); + assert!(observed.next().await.is_none()); + drop(observed); + let events = drain(&root.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[0]["id"], events[1]["id"]); + assert_eq!(events[0]["model"], "fixture-model"); + assert_eq!(events[1]["usage"]["total_tokens"], 14); + assert_eq!(events[1]["outcome"], "completed"); + assert!(events[0]["at"].as_u64().unwrap() > 1_000_000_000_000); + assert!(!format!("{events:?}").contains("private-response-id")); + assert!(root.buffer.drain().is_none()); + } + + #[tokio::test] + async fn failure_drop_and_close_remain_isolated_and_drainable() { + let observers = Arc::new(Observers::default()); + let first = observers.root("first".into()); + let second = observers.root("second".into()); + first.enable(&ExtensionData::new("first"), None).unwrap(); + second.enable(&ExtensionData::new("second"), None).unwrap(); + let pending = request(&observers, "second", ModelRequestKind::Generation).unwrap(); + let mut failed = request(&observers, "first", ModelRequestKind::Generation).unwrap(); + failed.request_failed(); + drop(failed); + let events = drain(&first.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[1]["outcome"], "failed"); + let interceptor = request(&observers, "first", ModelRequestKind::Generation).unwrap(); + let mut observed = interceptor.intercept(Box::pin(stream::iter([Err( + ModelResponseError::Stream("private failure detail".into()), + )]))); + assert!( + matches!(observed.next().await, Some(Err(ModelResponseError::Stream(message))) if message == "private failure detail") + ); + drop(observed); + first.close(); + assert!(request(&observers, "first", ModelRequestKind::Generation).is_none()); + let events = drain(&first.buffer); + assert_eq!(events.len(), 2); + assert_eq!(events[1]["outcome"], "failed"); + assert!(!format!("{events:?}").contains("private failure detail")); + assert_eq!(drain(&second.buffer).len(), 1); + second.close(); + drop(pending); + assert_eq!(drain(&second.buffer)[0]["outcome"], "interrupted"); + } + + #[tokio::test] + async fn tool_outcomes_preserve_namespace_without_payloads_or_false_success() { + let observers = Observers::default(); + let store = ExtensionData::new("root"); + let buffer = Arc::new(Buffer::default()); + store.insert_arc(buffer.clone()); + let name = ToolName::namespaced("functions", "lookup"); + let outcomes = [ + ToolCallOutcome::Completed { success: true }, + ToolCallOutcome::Completed { success: false }, + ToolCallOutcome::Blocked, + ToolCallOutcome::Failed { + handler_executed: false, + }, + ToolCallOutcome::Aborted, + ]; + for (index, outcome) in outcomes.into_iter().enumerate() { + let id = format!("call-{index}"); + buffer.tool("start", &id, &name, None); + observers + .on_tool_finish(ToolFinishInput { + session_store: &store, + thread_store: &store, + turn_store: &store, + turn_id: "turn", + call_id: &id, + tool_name: &name, + source: codex_extension_api::ToolCallSource::Direct, + outcome, + }) + .await; + } + let events = drain(&buffer); + let outcomes = ["completed", "failed", "failed", "failed", "interrupted"]; + for (pair, outcome) in events.chunks_exact(2).zip(outcomes) { + assert_eq!(pair[0]["id"], pair[1]["id"]); + assert_eq!(pair[0]["namespace"], "functions"); + assert_eq!(pair[1]["namespace"], "functions"); + assert_eq!(pair[1]["name"], "lookup"); + assert_eq!(pair[1]["outcome"], outcome); + assert!( + pair.iter() + .all(|event| event.get("input").is_none() && event.get("output").is_none()) + ); + } + } + + #[test] + fn bounds_events_and_omits_oversized_identifiers_without_truncating() { + let buffer = Buffer::default(); + buffer.tool( + "start", + &"x".repeat(MAX_STRING_BYTES + 1), + &ToolName::plain("tool"), + None, + ); + assert_eq!(drain(&buffer), vec![json!({"kind": "dropped"})]); + for index in 0..MAX_EVENTS + 2 { + buffer.tool("start", &index.to_string(), &ToolName::plain("tool"), None); + } + let events = drain(&buffer); + assert_eq!(events.len(), MAX_EVENTS + 1); + assert_eq!(events.last().unwrap(), &json!({"kind": "dropped"})); + assert!(buffer.drain().is_none()); + } + + #[test] + fn concurrent_handlers_and_drains_do_not_drop_observations() { + let buffer = Arc::new(Buffer::default()); + let completed = Arc::new(AtomicU64::new(0)); + let received = std::thread::scope(|scope| { + for worker in 0..4 { + let buffer = buffer.clone(); + let completed = completed.clone(); + scope.spawn(move || { + for item in 0..32 { + buffer.tool( + "start", + &format!("{worker}-{item}"), + &ToolName::plain("tool"), + None, + ); + } + completed.fetch_add(1, Ordering::Release); + }); + } + let mut events = Vec::new(); + while completed.load(Ordering::Acquire) != 4 { + events.extend(drain(&buffer)); + std::thread::yield_now(); + } + events.extend(drain(&buffer)); + events + }); + assert_eq!(received.len(), 128); + assert!(received.iter().all(|event| event["kind"] == "tool")); + } +} + +#[cfg(test)] +mod content_tests { + use super::*; + + #[test] + fn native_result_capture_is_explicit_and_runtime_start_is_not_repeated() { + let observers = Arc::new(Observers::default()); + let root = observers.root("root".into()); + let store = ExtensionData::new("root"); + assert!(root.enable(&store, Some("unknown")).is_err()); + let item: ResponseItem = serde_json::from_value(json!({ + "type":"function_call_output", "call_id":"call", "output":"accepted" })) + .unwrap(); + root.enable(&store, None).unwrap(); + root.observe_item(&item); + assert!(root.buffer.drain().is_none()); + root.enable(&store, Some("content")).unwrap(); + root.observe_item(&item); + let output = root.buffer.drain().unwrap(); + assert!(output.contains("accepted")); + root.buffer.pending.lock().unwrap().insert( + "call".into(), + ( + ToolName { + name: "exec_command".into(), + namespace: None, + }, + None, + false, + ), + ); + assert!(root.buffer.drain().is_none()); + observers.on_tool_execution_start("root", "turn", "call"); + observers.on_tool_execution_start("root", "turn", "call"); + let events: Vec = serde_json::from_str(&root.buffer.drain().unwrap()).unwrap(); + assert_eq!(events.iter().filter(|e| e["phase"] == "start").count(), 1); + } +} diff --git a/native/codex/src/operation_tracing_content.rs b/native/codex/src/operation_tracing_content.rs new file mode 100644 index 0000000000..fdceedef0f --- /dev/null +++ b/native/codex/src/operation_tracing_content.rs @@ -0,0 +1,203 @@ +//! Bounded, opt-in projections; no transport headers or binary/reasoning payloads. +use serde::Serialize; +use serde_json::{Value, json}; +use std::io::{self, Write}; + +pub(super) const LIMIT: usize = 32 * 1024; + +struct Bounded(Vec); +impl Write for Bounded { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.0.len().saturating_add(bytes.len()) > LIMIT { + return Err(io::Error::other("capture budget")); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(super) fn omitted() -> Value { + json!({"omitted": true, "reason": "unsupported_or_oversized_content"}) +} + +pub(super) fn capture(value: &T) -> Value { + let mut bytes = Bounded(Vec::new()); + if serde_json::to_writer(&mut bytes, value).is_err() { + return omitted(); + } + let projected = serde_json::from_slice(&bytes.0) + .map(|value| project(value, 0)) + .unwrap_or_else(|_| omitted()); + let mut check = Bounded(Vec::new()); + if serde_json::to_writer(&mut check, &projected).is_err() { + omitted() + } else { + projected + } +} + +/// Prioritize actual messages; optional tool schemas share the retained budget. +pub(super) fn request( + input: &[codex_protocol::models::ResponseItem], + tools: Option<&T>, + previous: Option<&str>, + tool_choice: &str, + parallel_tool_calls: bool, +) -> Value { + // Newest messages and accepted results take priority over old instructions. + let mut items = Vec::new(); + let mut remaining = LIMIT - 2048; + let mut omitted_input = input.len() > 128; + for item in input.iter().rev().take(128) { + let item = capture(item); + let size = serde_json::to_vec(&item) + .map(|v| v.len() + 1) + .unwrap_or(LIMIT); + if item.get("omitted") == Some(&Value::Bool(true)) || size > remaining { + omitted_input = true; + continue; + } + remaining -= size; + items.push(item); + } + if omitted_input { + items.push(omitted()); + } + items.reverse(); + let mut value = json!({"input":items, "tool_choice":capture(tool_choice), "parallel_tool_calls":parallel_tool_calls}); + if let Some(previous) = previous.filter(|id| id.len() <= 256) { + value["previous_response_id"] = json!(previous); + } + if let Some(tools) = tools { + value["tools"] = capture(tools); + let mut check = Bounded(Vec::new()); + if serde_json::to_writer(&mut check, &value).is_err() { + value["tools"] = json!([omitted()]); + } + } + capture(&value) +} + +fn project(value: Value, depth: usize) -> Value { + if depth > 24 { + return omitted(); + } + match value { + Value::Object(mut fields) => { + if fields + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| { + matches!( + kind, + "reasoning" + | "reasoning_text" + | "thinking" + | "compaction" + | "input_image" + | "output_image" + | "image" + | "input_audio" + | "output_audio" + | "input_file" + | "file" + | "audio" + | "video" + | "image_generation_call" + ) + }) + { + return omitted(); + } + fields.retain(|key, _| { + !matches!( + key.to_ascii_lowercase().replace('_', "").as_str(), + "apikey" + | "authorization" + | "accesstoken" + | "refreshtoken" + | "password" + | "headers" + | "clientmetadata" + | "encryptedcontent" + | "reasoning" + | "reasoningcontent" + | "reasoningdetails" + | "thinking" + | "imageurl" + | "filedata" + | "imagedata" + | "audiodata" + | "b64json" + | "xapikey" + ) + }); + Value::Object( + fields + .into_iter() + .map(|(k, v)| (k, project(v, depth + 1))) + .collect(), + ) + } + Value::Array(items) => { + Value::Array(items.into_iter().map(|v| project(v, depth + 1)).collect()) + } + Value::String(text) => { + if text.to_ascii_lowercase().contains("data:") { + return omitted(); + } + if matches!(text.trim_start().as_bytes().first(), Some(b'{' | b'[')) { + return match serde_json::from_str::(&text) { + Ok(value) => Value::String(project(value, depth + 1).to_string()), + Err(_) => omitted(), + }; + } + Value::String(text) + } + value => value, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_projection_omits_nested_credentials_media_and_oversized_values() { + let value = capture(&json!({ + "input": [{"type":"message", "content":[{"type":"input_text", "text":"keep"}, + {"type":"input_image", "image_url":"hidden-image"}]}], + "arguments": "{\"password\":\"hidden-password\",\"query\":\"keep\"}", + "client_metadata": {"secret":"hidden-metadata"}, + "encrypted_content":"hidden-reasoning" + })); + let encoded = value.to_string(); + assert!(encoded.contains("keep")); + assert!(!encoded.contains("hidden")); + assert_eq!(capture(&"x".repeat(LIMIT + 1)), omitted()); + } + #[test] + fn request_prioritizes_recent_prompt_and_accepted_result_over_large_instructions() { + let input: Vec = serde_json::from_value(json!([ + {"type":"message", "role":"developer", "content":[{"type":"input_text", "text":"x".repeat(LIMIT + 1)}]}, + {"type":"message", "role":"user", "content":[{"type":"input_text", "text":"recent user prompt"}]}, + {"type":"function_call_output", "call_id":"call", "output":"recent accepted result"} + ])).unwrap(); + let tools = serde_json::value::RawValue::from_string( + json!([{"name":"large", "description":"x".repeat(LIMIT)}]).to_string(), + ) + .unwrap(); + let value = request(&input, Some(&tools), Some("previous"), "auto", true); + let encoded = value.to_string(); + assert!(encoded.len() <= LIMIT); + assert!( + encoded.contains("recent user prompt") && encoded.contains("recent accepted result") + ); + assert_eq!(value["input"][0]["omitted"], true); + assert!(value["tools"].to_string().contains("omitted")); + assert_eq!(value["previous_response_id"], "previous"); + } +} diff --git a/native/codex/verify_installed.py b/native/codex/verify_installed.py index bb46961639..4064568129 100644 --- a/native/codex/verify_installed.py +++ b/native/codex/verify_installed.py @@ -6,6 +6,7 @@ import argparse import asyncio +import base64 import importlib import importlib.abc import importlib.machinery @@ -135,16 +136,22 @@ def respond(request): async def native_route(folder): + from unittest.mock import Mock + from pydantic import BaseModel from agents import Agent, RunConfig, Runner, RunState from agents.decorators import tool + from agents.memory.resumable_session import ResumableSession from agents.models.codex import CodexProvider + from agents.tracing import TracingProcessor, set_trace_processors, set_tracing_disabled class Answer(BaseModel): label: str + image_path = folder / "workspace" / "trace-image.png" requests = [] + approval_item = None errors = [] tool_calls = [] live_delta = threading.Event() @@ -168,7 +175,7 @@ def do_POST(self): ) body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) requests.append(body) - assert len(requests) <= 7, "Unexpected extra inference request." + assert len(requests) <= 13, "Unexpected extra inference request." number = len(requests) history = json.dumps(body["input"]) response = {"id": f"response-{number}"} @@ -176,7 +183,10 @@ def do_POST(self): self.send_header("Content-Type", "text/event-stream") self.end_headers() self.send_event({"type": "response.created", "response": response}) - if number == 1: + if number > 9 and number % 2 == 0: + assert approval_item is not None + item = approval_item + elif number == 1: item = { "type": "function_call", "call_id": "route-call", @@ -190,8 +200,19 @@ def do_POST(self): "name": "lookup", "arguments": '{"value":"record"}', } + elif number == 8: + item = { + "type": "function_call", + "call_id": "native-image-call", + "name": "view_image", + "arguments": json.dumps({"path": str(image_path)}), + } else: - if "blocked-input" in history: + if "approval-input" in history: + text = "approval resolved" + elif "content-input" in history: + text = "native-content" + elif "blocked-input" in history: assert ( "survivor-input" not in history and "after-cancel-input" not in history ) @@ -287,7 +308,15 @@ def lookup(value: str) -> str: encoding="utf-8", ) provider = CodexProvider(cwd=folder / "workspace", api_key="synthetic-installed-key") - config = RunConfig(model_provider=provider, tracing_disabled=True) + # The isolated worker collects spans locally, without a network exporter. + processor = Mock(spec=TracingProcessor) + set_trace_processors([processor]) + set_tracing_disabled(False) + config = RunConfig( + model_provider=provider, + trace_include_sensitive_data=False, + tracing={"operation_detail": "metadata"}, + ) try: paused = Runner.run_streamed(agent, "private-input", run_config=config) async for event in paused.stream_events(): @@ -317,7 +346,13 @@ def lookup(value: str) -> str: ) assert usage.input_tokens_details.cached_tokens == 6 assert usage.output_tokens_details.reasoning_tokens == 3 + spans = [call.args[0].span_data for call in processor.on_span_end.call_args_list] + samples = [span for span in spans if span.type == "generation"] + assert len(samples) == 3 + assert all(span.input is None and span.output is None for span in samples) + assert all(span.usage["input_tokens"] == 12 for span in samples) + config.tracing_disabled = True streamed = Runner.run_streamed(plain, "fresh-input", run_config=config) deltas = [] async for event in streamed.stream_events(): @@ -357,16 +392,119 @@ async def consume_blocked(): await asyncio.gather(consumer, return_exceptions=True) reused = await Runner.run(plain, "after-cancel-input", run_config=config) assert reused.final_output == "reused" and len(requests) == 7 + image_path.write_bytes( + base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNoaGgAAAMEAYFL09IQAAAAAElFTkSuQmCC" + ) + ) + before = len(processor.on_span_end.call_args_list) + content_config = RunConfig( + model_provider=provider, + trace_include_sensitive_data=True, + tracing={"operation_detail": "content"}, + ) + content_result = await Runner.run(plain, "content-input", run_config=content_config) + assert content_result.final_output == "native-content" and len(requests) == 9 + captured = [ + call.args[0].span_data for call in processor.on_span_end.call_args_list[before:] + ] + native_models = [span for span in captured if span.type == "generation"] + native_tools = [ + span + for span in captured + if span.type == "function" and span.name in {"view_image", "functions.view_image"} + ] + request_sizes = [len(json.dumps(body).encode()) for body in requests[-2:]] + assert len(native_models) == 2 and len(native_tools) == 1 + assert "content-input" in json.dumps(native_models[0].input), ( + f"Native content omitted; request sizes: {request_sizes}" + ) + assert "native-content" in json.dumps(native_models[-1].output) + assert json.loads(native_tools[0].input)["path"] == str(image_path) + assert "omitted" in native_tools[0].output + rendered = json.dumps([span.export() for span in captured]) + assert ( + "synthetic-installed-key" not in rendered + and "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" not in rendered + ) + assert "opaque-late-reasoning" not in rendered + + class ApprovalStore: + value = None + + async def load(self, _session_id): + return self.value + + async def save(self, _session_id, value, *, expected): + if self.value != expected: + return False + self.value = value + return True + + def fresh_provider(): + return CodexProvider( + cwd=folder / "workspace", + api_key="synthetic-installed-key", + config_overrides=[ + 'approval_policy="on-request"', + 'approvals_reviewer="user"', + ], + ) + + await provider.aclose() + approval_checks = [] + for approved in (False, True): + target = folder / "workspace" / f"builtin-command-{approved}.txt" + call_id = f"builtin-command-{approved}" + approval_item = { + "type": "function_call", + "name": "exec_command", + "call_id": call_id, + "arguments": json.dumps( + { + # Keep the process alive long enough for Codex's async exit watcher. + "cmd": f'sleep 1; echo builtin-approved > "{target}"', + "login": False, + "sandbox_permissions": "require_escalated", + } + ), + } + provider = fresh_provider() + options = RunConfig(model_provider=provider, tracing_disabled=True) + store = ApprovalStore() + session = await ResumableSession.create(store) + paused = await Runner.run(plain, "approval-input", session=session, run_config=options) + assert len(paused.interruptions) == 1 and not target.exists() + stored = store.value + await provider.aclose() + provider = fresh_provider() + options.model_provider = provider + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(plain) + assert store.value == stored and not target.exists() + (pending,) = state.get_interruptions() + assert pending.raw_item.call_id == call_id + state.approve(pending) if approved else state.reject(pending) + result = await Runner.run(plain, state, run_config=options) + assert result.final_output == "approval resolved" + assert target.exists() == approved + if approved: + assert b"builtin-approved" in target.read_bytes().replace(b"\x00", b"") + approval_checks.append("approve" if approved else "deny") + await provider.aclose() assert "agents.models._codex_native" in sys.modules assert "agents.models._codex_app_server" not in sys.modules assert not config_path.with_name("auth.json").exists(), "The API key persisted to disk." assert not errors, errors return { "provider_requests": len(requests), + "native_content_request_bytes": request_sizes, + "native_content_roundtrip": True, "tool_effects": len(tool_calls), "resumed_tokens": usage.total_tokens, "live_streaming": True, "concurrent_survivor_and_reuse_after_cancel": True, + "native_approval_checks": approval_checks, } finally: dynamic_delta.set() diff --git a/scripts/build_codex_wheel.py b/scripts/build_codex_wheel.py index a476e83952..2d82651933 100644 --- a/scripts/build_codex_wheel.py +++ b/scripts/build_codex_wheel.py @@ -53,11 +53,41 @@ def stage_asset(asset: dict[str, str], destination: Path, cache: Path) -> None: destination.chmod(0o755) +def patch_codex_source(source: Path, upstream: Path, metadata: dict) -> None: + """Stage the pinned Codex workspace and apply its documented changes in order.""" + patches = sorted((source / "native/codex/patches").glob("*.patch")) + if not patches: + raise ValueError("The Codex wheel is missing its patches.") + for patch in patches: + guide = patch.with_suffix(".md") + if not guide.is_file() or not guide.read_text(encoding="utf-8").strip(): + raise ValueError(f"Document {patch.name} in {guide.name} before building.") + + staged = source / "native/codex-upstream" + shutil.copytree(upstream, staged, ignore=shutil.ignore_patterns(".git")) + subprocess.run( + ["git", "apply", "--whitespace=error", *(str(p.resolve()) for p in patches)], + cwd=staged, + env=os.environ | {"GIT_CEILING_DIRECTORIES": str(staged.parent.resolve())}, + check=True, + ) + # All Codex crates must share the same types. Redirect only the resolved Codex workspace. + lines = ['[patch."https://github.com/openai/codex"]'] + for package in metadata["packages"]: + if (package["source"] or "").startswith("git+https://github.com/openai/codex?"): + relative = Path(package["manifest_path"]).parent.relative_to(upstream) + lines.append(f"{package['name']} = {{ path = {json.dumps(str(staged / relative))} }}") + config = source / ".cargo/config.toml" + config.parent.mkdir(exist_ok=True) + config.write_text("\n".join(lines) + "\n", encoding="utf-8") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, default=ROOT / "dist") parser.add_argument("--target", choices=TARGETS) parser.add_argument("--compatibility") + parser.add_argument("--test-native", action="store_true", help="Test the staged SDK binding.") parser.add_argument("--profile", choices=("dev", "release"), default="release") args = parser.parse_args() environment = os.environ.copy() @@ -148,18 +178,20 @@ def build_native( "--target", target, ] + patch_codex_source(source, upstream, metadata) + # Resolve local paths in the temporary lockfile without fetching different dependencies. subprocess.run( - [ - "cargo", - "build", - "--locked", - "--manifest-path", - str(manifest), - *target_flags, - "--bin", - "codex-sdk-runtime", - *profile, - ], + ["cargo", "metadata", "--offline", "--format-version=1", "--manifest-path", manifest], + cwd=source, + env=environment, + stdout=subprocess.DEVNULL, + check=True, + ) + flags = ["--locked", "--manifest-path", str(manifest), *target_flags, *profile] + if args.test_native: + subprocess.run(["cargo", "test", "--lib", *flags], cwd=source, env=environment, check=True) + subprocess.run( + ["cargo", "build", "--bin", "codex-sdk-runtime", *flags], cwd=source, env=environment, check=True, @@ -177,6 +209,7 @@ def build_native( stage_asset(asset, path, cache) licenses = staged / "licenses" licenses.mkdir() + shutil.copytree(native / "patches", licenses / "codex-patches") for name, path in { "CODEX_LICENSE": upstream / "LICENSE", "CODEX_NOTICE": upstream / "NOTICE", diff --git a/src/agents/extensions/tracing/opentelemetry.py b/src/agents/extensions/tracing/opentelemetry.py index 2883788459..7824698337 100644 --- a/src/agents/extensions/tracing/opentelemetry.py +++ b/src/agents/extensions/tracing/opentelemetry.py @@ -82,8 +82,18 @@ def _messages(value: Any, *, output: bool = False) -> list[dict[str, Any]]: if isinstance(item.get(flag), bool): part[flag] = item[flag] parts.append(part) - elif item.get("truncated") is True and "content" not in item and "tool_calls" not in item: - parts.append({"type": "openai.omitted", "reason": "truncated"}) + elif ( + (item.get("truncated") is True or item.get("omitted") is True) + and "content" not in item + and "tool_calls" not in item + ): + reason = item.get("reason") + parts.append( + { + "type": "openai.omitted", + "reason": reason if isinstance(reason, str) else "truncated", + } + ) elif kind in ("function_call", "custom_tool_call"): role = "assistant" parts.append( @@ -328,6 +338,27 @@ def _attributes(self, span: Span[Any]) -> dict[str, Any]: ): if isinstance(usage.get(source), int): attributes[destination] = usage[source] + for group, source, destination in ( + ( + "input_tokens_details", + "cached_tokens", + "gen_ai.usage.cache_read.input_tokens", + ), + ( + "input_tokens_details", + "cache_write_tokens", + "gen_ai.usage.cache_write.input_tokens", + ), + ( + "output_tokens_details", + "reasoning_tokens", + "gen_ai.usage.reasoning.output_tokens", + ), + ): + details = usage.get(group) + value = details.get(source) if isinstance(details, Mapping) else None + if isinstance(value, int) and value != 0: + attributes[destination] = value if data.operation.response_model: attributes["gen_ai.response.model"] = data.operation.response_model if data.operation.response_id: diff --git a/src/agents/memory/resumable_session.py b/src/agents/memory/resumable_session.py index a9f1314edd..abb58b9b85 100644 --- a/src/agents/memory/resumable_session.py +++ b/src/agents/memory/resumable_session.py @@ -283,6 +283,7 @@ async def load_state( return state def _validate_run(self, input: Any) -> None: + from ..run_internal.native_approvals import native_receipts_from_snapshot from ..run_state import RunState if self._checkpoint.status == "active": @@ -300,6 +301,22 @@ def _validate_run(self, input: Any) -> None: or binding.revision != self._checkpoint.revision ): raise ResumableSessionError("invalid_state") + invalid_native_state = False + try: + assert self._checkpoint.run_state is not None + original = json.loads(self._checkpoint.run_state) + original_receipts = native_receipts_from_snapshot(original) + processed = input._last_processed_response + if original_receipts is not None or ( + processed is not None and processed.native_approvals + ): + invalid_native_state = ( + native_receipts_from_snapshot(input.to_json()) != original_receipts + ) + except Exception: + invalid_native_state = True + if invalid_native_state: + raise ResumableSessionError("invalid_state") def _get_history(self) -> list[TResponseInputItem]: return copy.deepcopy(self._checkpoint.history) diff --git a/src/agents/models/_codex_items.py b/src/agents/models/_codex_items.py index 66ee061da9..1fbac165a3 100644 --- a/src/agents/models/_codex_items.py +++ b/src/agents/models/_codex_items.py @@ -1,9 +1,18 @@ -"""Internal output types attesting to work already completed by the Codex adapter.""" +"""Internal output types attesting to work observed directly by the Codex adapter.""" + +import json +from pathlib import Path +from typing import Any, cast from openai.types.responses import ( + ResponseCustomToolCall, ResponseCustomToolCallOutputItem, + ResponseFunctionToolCall, ResponseFunctionToolCallOutputItem, ) +from pydantic import Field + +from ..exceptions import UserError class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): @@ -12,3 +21,66 @@ class CodexFunctionToolCallOutput(ResponseFunctionToolCallOutputItem): class CodexCustomToolCallOutput(ResponseCustomToolCallOutputItem): """A matching native custom call has already completed in Codex.""" + + +class CodexPendingFunctionCall(ResponseFunctionToolCall): + """A native function call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +class CodexPendingCustomCall(ResponseCustomToolCall): + """A native custom call has paused at Codex's own approval boundary.""" + + checkpoint: dict[str, Any] = Field(exclude=True, repr=False) + + +def native_approval_checkpoint(raw_item: Any) -> dict[str, Any] | None: + """Validate an adapter-owned receipt against the call shown for approval.""" + # An unrelated approval may define __getattribute__("__class__"). Don't run it + # while taking a detached copy of a pending call or a session checkpoint. + raw_type = type(raw_item) + if raw_type is not CodexPendingFunctionCall and raw_type is not CodexPendingCustomCall: + return None + checkpoint = raw_item.checkpoint + if ( + not isinstance(checkpoint, dict) + or set(checkpoint) != {"version", "cwd", "call", "request", "model", "preconditions"} + or type(checkpoint["version"]) is not int + or checkpoint["version"] != 1 + or not isinstance(checkpoint["cwd"], str) + or not Path(checkpoint["cwd"]).is_absolute() + or not isinstance(checkpoint["model"], str) + or not checkpoint["model"] + or not isinstance(checkpoint["request"], dict) + or not isinstance(checkpoint["call"], dict) + or not isinstance(checkpoint["preconditions"], dict) + ): + raise UserError("Invalid Codex native approval checkpoint.") + model = ( + ResponseFunctionToolCall + if isinstance(raw_item, CodexPendingFunctionCall) + else ResponseCustomToolCall + ) + display_metadata = {"codex_reasoning_summary"} + displayed = model.model_validate(raw_item.model_dump()).model_dump(exclude=display_metadata) + stored = model.model_validate(checkpoint["call"]).model_dump(exclude=display_metadata) + if displayed != stored or raw_item.name not in {"exec_command", "apply_patch"}: + raise UserError("Codex native approval no longer matches its original call.") + return cast(dict[str, Any], json.loads(json.dumps(checkpoint, allow_nan=False))) + + +def restore_native_approval_call(raw_item: Any, checkpoint: Any) -> Any: + """Restore only explicit SDK checkpoint metadata, never provider output fields.""" + if not isinstance(raw_item, dict) or not isinstance(checkpoint, dict): + raise UserError("Invalid Codex native approval checkpoint.") + item: CodexPendingFunctionCall | CodexPendingCustomCall + item_type = raw_item.get("type") + if item_type == "function_call": + item = CodexPendingFunctionCall(**raw_item, checkpoint=checkpoint) + elif item_type == "custom_tool_call": + item = CodexPendingCustomCall(**raw_item, checkpoint=checkpoint) + else: + raise UserError("Invalid Codex native approval call type.") + native_approval_checkpoint(item) + return item diff --git a/src/agents/models/_codex_tracing.py b/src/agents/models/_codex_tracing.py new file mode 100644 index 0000000000..59557de152 --- /dev/null +++ b/src/agents/models/_codex_tracing.py @@ -0,0 +1,218 @@ +"""Translate bounded native observations without affecting model or tool execution.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any + +from openai.types.responses import Response + +from ..tracing import ( + Span, + custom_span, + function_span, + get_current_span, + get_current_trace, + response_span, +) +from ..tracing._content import _json, bound_content +from ..tracing._model import acquire_model_span, finish_model_span, record_model_input +from ..tracing.config import get_operation_detail +from ..tracing.provider import _is_noop_span +from ..tracing.span_data import FunctionSpanData, GenerationSpanData, ResponseSpanData +from ..usage import _response_usage_to_usage, model_usage_to_span_usage +from ._codex_usage import _usage +from .interface import ModelTracing + + +class CodexOperationTracing: + """Own observations for one ephemeral native thread, including tool restoration.""" + + def __init__( + self, sdk_tools: set[tuple[str | None, str]], *, expect_model: bool = True + ) -> None: + self._sdk_tools = sdk_tools + self._expect_model = expect_model + self._parent = get_current_span() or get_current_trace() + self._thread: Any = None + self._active: dict[tuple[str, str], Span[Any]] = {} + self._ends: dict[tuple[str, str], tuple[str, str]] = {} + self._ignored: set[str] = set() + self._reason: str | None = None + self._saw_model = False + self._tracing = ModelTracing.ENABLED_WITHOUT_DATA + self._capture = False + + def bind(self, thread: Any, tracing: ModelTracing) -> None: + if tracing.is_disabled() or self._parent is None: + return + if _is_noop_span(response_span(parent=self._parent)): + return + self._capture = ( + tracing.include_data() and get_operation_detail(include_data=True) == "content" + ) + self._tracing = tracing if self._capture else ModelTracing.ENABLED_WITHOUT_DATA + try: + if self._capture: + thread.enable_tracing("content") + else: + thread.enable_tracing() + self._thread = thread + except Exception: + self._reason = "unavailable" + + async def watch(self) -> None: + if self._thread is None: + return + try: + while True: + await self._thread.wait_for_operations() + self.drain() + except Exception: + self._reason = "telemetry_error" + + def drain(self) -> None: + if self._thread is None: + return + try: + encoded = self._thread.drain_operations() + if not encoded or encoded == "[]": + return + for event in json.loads(encoded): + self._observe(event) + except Exception: + self._reason = "telemetry_error" + + def _observe(self, event: dict[str, Any]) -> None: + kind = event["kind"] + if kind == "dropped": + self._reason = "dropped_operations" + return + phase = event["phase"] + if kind == "tool": + if phase == "start" and (event.get("namespace"), event["name"]) in self._sdk_tools: + self._ignored.add(event["id"]) + if event["id"] in self._ignored: + return + key = kind, event["id"] + if phase in ("input", "output"): + span = self._active.get(key) + if span is not None: + self._content(span, kind, event) + if phase == "output" and key in self._ends: + outcome, at = self._ends.pop(key) + self._finish(key, outcome, at) + return + at = datetime.fromtimestamp(event["at"] / 1000, tz=timezone.utc).isoformat() + if phase == "start": + if key in self._active or len(self._active) >= 128: + self._reason = "dropped_operations" + return + if kind == "model": + span = acquire_model_span( + self._tracing, event["model"], parent=self._parent, started_at=at + ) + self._saw_model = True + elif kind == "tool": + name = event["name"] + if event.get("namespace"): + name = f"{event['namespace']}.{name}" + span = function_span(name, parent=self._parent) + span.span_data.operation.detail = get_operation_detail(include_data=self._capture) + span.span_data.operation.tool_call_id = event["id"] + span.start_at(at) + else: + raise ValueError("Unknown native operation.") + self._active[key] = span + return + if key not in self._active: + # Rejected calls never reached the post-authorization execution boundary. + return + span = self._active[key] + if kind == "model": + if event.get("usage") is not None: + span.span_data.usage = model_usage_to_span_usage( + _response_usage_to_usage(_usage(event["usage"])) + ) + self._content(span, kind, event) + if kind == "tool" and self._capture: + # Core publishes the accepted model-visible output after the lifecycle callback. + self._ends[key] = event["outcome"], at + else: + self._finish(key, event["outcome"], at) + + def _content(self, span: Span[Any], kind: str, event: dict[str, Any]) -> None: + if not self._capture: + return + data = span.span_data + if kind == "tool" and isinstance(data, FunctionSpanData): + for direction in ("input", "output"): + if direction in event: + value = bound_content(event[direction]) + setattr(data, direction, value if isinstance(value, str) else _json(value)) + return + if "input" in event: + request = bound_content(event["input"]) + if isinstance(data, GenerationSpanData): + record_model_input(data, request) + if isinstance(request, dict): + data.tools = bound_content(request.get("tools")) + config = { + key: request[key] + for key in ("tool_choice", "parallel_tool_calls", "text", "service_tier") + if key in request + } + data.model_config = {**(data.model_config or {}), **config} or None + elif isinstance(data, ResponseSpanData): + data.input = ( + request.get("input", [request]) if isinstance(request, dict) else request + ) + if "output" in event: + output = bound_content(event["output"]) + if isinstance(data, GenerationSpanData): + data.output = output + elif isinstance(data, ResponseSpanData) and isinstance(event.get("response_id"), str): + data.response = Response.model_construct(id=event["response_id"], output=output) + if isinstance(event.get("response_id"), str): + data.operation.response_id = event["response_id"] + + def _finish(self, key: tuple[str, str], outcome: str, at: str | None = None) -> None: + span = self._active.pop(key) + span.span_data.operation.outcome = outcome + span.span_data.operation.estimated_end = at is None + if outcome != "completed": + span.set_error( + {"message": "Native operation did not complete successfully.", "data": None} + ) + if key[0] == "model": + finish_model_span(span, ended_at=at) + elif at is None: + span.finish() + else: + span.finish_at(at) + + def close(self) -> None: + self.drain() + if self._thread is not None and self._expect_model and not self._saw_model: + self._reason = self._reason or "missing_operations" + for key in list(self._active): + if key in self._ends: + outcome, at = self._ends.pop(key) + if self._capture: + self._content( + self._active[key], + key[0], + {"output": {"omitted": True, "reason": "output_unavailable"}}, + ) + self._finish(key, outcome, at) + else: + self._reason = self._reason or "interrupted" + self._finish(key, "interrupted") + if self._reason is not None: + with custom_span( + "codex.tracing.coverage", + {"coverage": "partial", "provider": "codex", "reason": self._reason}, + parent=self._parent, + ): + pass diff --git a/src/agents/models/_codex_usage.py b/src/agents/models/_codex_usage.py new file mode 100644 index 0000000000..11f2608efb --- /dev/null +++ b/src/agents/models/_codex_usage.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import Any + +from openai.types.responses import ResponseUsage +from openai.types.responses.response_usage import OutputTokensDetails + +from ..usage import _make_input_tokens_details, _response_usage_to_usage + + +def _usage(total: dict[str, Any]) -> ResponseUsage: + return ResponseUsage( + input_tokens=total.get("input_tokens", 0), + output_tokens=total.get("output_tokens", 0), + total_tokens=total.get( + "total_tokens", total.get("input_tokens", 0) + total.get("output_tokens", 0) + ), + input_tokens_details=_make_input_tokens_details( + cached_tokens=total.get("cached_input_tokens", 0), + cache_write_tokens=total.get("cache_write_input_tokens"), + ), + output_tokens_details=OutputTokensDetails( + reasoning_tokens=total.get("reasoning_output_tokens", 0) + ), + ) + + +def _add_sample_usage(previous: ResponseUsage | None, sample: dict[str, Any]) -> ResponseUsage: + current = _usage(sample) + if previous is None: + return current + total = _response_usage_to_usage(previous) + total.add(_response_usage_to_usage(current)) + return ResponseUsage( + input_tokens=total.input_tokens, + output_tokens=total.output_tokens, + total_tokens=total.total_tokens, + input_tokens_details=total.input_tokens_details, + output_tokens_details=total.output_tokens_details, + ) diff --git a/src/agents/models/_run_context.py b/src/agents/models/_run_context.py index ba57e3f19d..9a9fccbf57 100644 --- a/src/agents/models/_run_context.py +++ b/src/agents/models/_run_context.py @@ -19,6 +19,7 @@ class ModelToolHooks: context: RunContextWrapper[Any] lifecycle: RunHooks[Any] settings: ToolExecutionConfig | None + resumable: bool = False _MODEL_RUN_CONTEXT: ContextVar[tuple[object | None, ModelToolHooks | None]] = ContextVar( diff --git a/src/agents/models/codex.py b/src/agents/models/codex.py index ca13c586db..4f6d838fbc 100644 --- a/src/agents/models/codex.py +++ b/src/agents/models/codex.py @@ -4,7 +4,7 @@ import json import os import time -from collections.abc import AsyncIterator, Awaitable, Sequence +from collections.abc import AsyncIterator, Sequence from contextlib import AsyncExitStack from dataclasses import fields from importlib import import_module @@ -31,16 +31,15 @@ ResponseTextDeltaEvent, ResponseToolSearchCall, ResponseToolSearchOutputItem, - ResponseUsage, ) from openai.types.responses.response_prompt_param import ResponsePromptParam from openai.types.responses.response_reasoning_summary_part_added_event import Part as SummaryPart -from openai.types.responses.response_usage import OutputTokensDetails from ..agent_output import AgentOutputSchemaBase from ..exceptions import UserError from ..handoffs import Handoff from ..items import ModelResponse, TResponseInputItem, TResponseStreamEvent +from ..memory.session import await_session_mutation as _join_close from ..model_settings import ModelSettings from ..retry import ModelRetryAdvice, ModelRetryAdviceRequest from ..tool import ApplyPatchTool, ShellTool, Tool, WebSearchTool @@ -48,21 +47,28 @@ Usage, _attach_raw_usage_snapshot, _extract_raw_usage_snapshot, - _make_input_tokens_details, _response_usage_to_usage, ) from ._codex_inputs import prepare_codex_history, prepare_dynamic_tools, sdk_tool_call -from ._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, +) from ._codex_tool_hooks import CodexToolHooks -from ._run_context import get_model_tool_hooks +from ._codex_tracing import CodexOperationTracing +from ._codex_usage import _add_sample_usage, _usage +from ._run_context import ModelToolHooks, get_model_tool_hooks from .interface import Model, ModelProvider, ModelTracing class CodexProvider(ModelProvider): """Run opt-in Agents SDK requests through embedded Codex Core. - Codex owns workspace tools and authentication; the Runner owns SDK function tools, - handoffs, approvals, and conversation history. Each request uses an ephemeral thread. + Codex handles workspace tools and authentication. Each request uses a fresh + Codex thread. Pass a ResumableSession to the Runner to save a pending local + command or edit for approval. The SDK session ID stays the same across workers. """ _supports_stored_response_id = False @@ -111,6 +117,100 @@ async def aclose(self) -> None: def get_model(self, model_name: str | None) -> Model: return _CodexModel(model_name, self) + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], tracing: ModelTracing + ) -> str: + """Run only the saved invocation, without a model turn or its old Codex home.""" + if checkpoint.get("version") != 1 or checkpoint.get("cwd") != self._cwd: + raise UserError("This Codex approval belongs to a different workspace or SDK version.") + + settings = {"model": checkpoint["model"]} + observations = CodexOperationTracing(set(), expect_model=False) + thread = await self._native_manager().start_thread("[]", None, json.dumps(settings)) + watcher: asyncio.Task[None] | None = None + failure: BaseException | None = None + try: + observations.bind(thread, tracing) + watcher = asyncio.create_task(observations.watch()) + return str( + await thread.resolve_approval( + json.dumps(checkpoint["call"]), + json.dumps(checkpoint["request"]), + json.dumps(checkpoint["preconditions"]), + ) + ) + except BaseException as error: + failure = error + raise + finally: + + async def close() -> None: + try: + await thread.close() + finally: + if watcher is not None: + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + observations.close() + + try: + await _join_close(close()) + except asyncio.CancelledError: + if isinstance(failure, asyncio.CancelledError): + raise failure from None + raise + except Exception: + if failure is None: + raise + + +def _pending_native_call( + call: Any, event: dict[str, Any], cwd: str, model: str +) -> CodexPendingFunctionCall | CodexPendingCustomCall: + request = event["msg"] + wire = call.model_dump(exclude_none=True) + if event.get("preconditions") is None: + raise UserError("Codex could not confirm that this tool is still waiting to run.") + checkpoint = { + "version": 1, + "cwd": cwd, + "model": model, + "call": wire, + "request": request, + "preconditions": event["preconditions"], + } + if getattr(call, "namespace", None) not in (None, "functions"): + raise UserError("Only built-in Codex tools support native approval recovery.") + if ( + isinstance(call, ResponseFunctionToolCall) + and call.name == "exec_command" + and request["type"] == "exec_approval_request" + and request.get("kind", "command") == "command" + and request.get("approval_id") is None + and request.get("network_approval_context") is None + and request.get("environmentId") in (None, "local") + ): + return CodexPendingFunctionCall(**wire, checkpoint=checkpoint) + if ( + isinstance(call, ResponseCustomToolCall) + and call.name == "apply_patch" + and request["type"] == "apply_patch_approval_request" + ): + return CodexPendingCustomCall(**wire, checkpoint=checkpoint) + raise UserError("This Codex approval cannot safely be resumed on another worker.") + + +def _uses_sdk_workspace(tools: list[Tool], bindings: ModelToolHooks | None) -> bool: + settings = bindings.settings if bindings else None + return bool( + settings + and settings.sdk_only_tool_hooks + and (settings.prepare_tool_input or settings.prepare_tool_output) + ) or any( + getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) + for tool in tools + ) + def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: supported = { @@ -146,20 +246,6 @@ def _settings(settings: ModelSettings) -> tuple[str | None, str | None]: return (reasoning.effort if reasoning else None, reasoning.summary if reasoning else None) -async def _join_close(operation: Awaitable[None]) -> None: - """Keep cleanup owned through repeated cancellation of the awaiting task.""" - task = asyncio.ensure_future(operation) - cancelled = None - while not task.done(): - try: - await asyncio.shield(task) - except asyncio.CancelledError as error: - cancelled = error - task.result() - if cancelled is not None: - raise cancelled - - class _CodexModel(Model): _supports_stored_response_id = False _sandbox_model_capabilities: ClassVar[dict[str, Any]] = { @@ -176,6 +262,16 @@ def __init__(self, name: str | None, provider: CodexProvider) -> None: def get_default_model_settings(self) -> ModelSettings: return ModelSettings() + async def _run_approved_native_call( + self, checkpoint: dict[str, Any], *, tools: list[Tool], tracing: ModelTracing + ) -> str: + bindings = get_model_tool_hooks() + if _uses_sdk_workspace(tools, bindings) or CodexToolHooks.for_run(bindings, set()): + raise UserError( + "The current workspace or native tool hooks do not allow restoring this call." + ) + return await self._provider._run_approved_native_call(checkpoint, tracing) + def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice: # Native Codex tools may have changed the workspace before a failed response. return ModelRetryAdvice(replay_safety="unsafe") @@ -274,19 +370,7 @@ async def stream_response( ) provider = self._provider bindings = get_model_tool_hooks() - tool_execution = bindings.settings if bindings is not None else None - classic_preparation = ( - tool_execution is not None - and tool_execution.sdk_only_tool_hooks - and ( - tool_execution.prepare_tool_input is not None - or tool_execution.prepare_tool_output is not None - ) - ) - sdk_sandbox = classic_preparation or any( - getattr(tool, "_sdk_sandbox", False) or isinstance(tool, (ShellTool, ApplyPatchTool)) - for tool in tools - ) + sdk_sandbox = _uses_sdk_workspace(tools, bindings) native_hooks = None if sdk_sandbox else CodexToolHooks.for_run(bindings, identities) if sdk_sandbox: system_instructions = "\n\n".join( @@ -355,6 +439,7 @@ async def stream_response( and output_schema.is_strict_json_schema() else None ) + observations = CodexOperationTracing(identities) resources = AsyncExitStack() output: dict[str, Any] = {} streamed: set[str] = set() @@ -364,6 +449,7 @@ async def stream_response( sdk_calls: dict[str, Any] = {} native_pending: dict[str, tuple[str, str]] = {} native_completed: set[str] = set() + pending_approval: dict[str, Any] | None = None sampling_pending = False interrupt_requested = False sequence = 0 @@ -389,7 +475,15 @@ async def interrupt_when_ready() -> None: native_hooks.verify(thread) if history: await thread.inject_items(json.dumps(history)) - turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema)) + observations.bind(thread, tracing) + recover = bool( + bindings + and bindings.resumable + and not native_hooks + and not sdk_sandbox + and not require_sdk_call + ) + turn_id = await thread.start_turn(json.dumps(turn_input), json.dumps(schema), recover) response = Response( id=turn_id, created_at=time.time(), @@ -406,7 +500,20 @@ async def interrupt_when_ready() -> None: ) sequence += 1 while True: + if pending_approval is not None and not sampling_pending: + # All calls from this sampling response are now known. A sibling + # still in flight has an unknown outcome and cannot be dropped. + call_id = pending_approval["msg"]["call_id"] + if set(native_pending) != {call_id} or sdk_calls: + raise UserError("Codex can only pause one outstanding native call.") + item_id, _ = native_pending.pop(call_id) + output[item_id] = _pending_native_call( + output[item_id], pending_approval, provider._cwd, thread.model + ) + # Close the waiting thread before handing the pause to the Runner. + break encoded = await thread.next_event() + observations.drain() if encoded is None: raise RuntimeError("Codex finished without a completed response.") event = json.loads(encoded) @@ -425,6 +532,20 @@ async def interrupt_when_ready() -> None: ) if event.get("id") not in ("", turn_id): continue + if kind in ("exec_approval_request", "apply_patch_approval_request"): + if ( + message.get("turn_id") != turn_id + or pending_approval is not None + or native_hooks is not None + or sdk_sandbox + or require_sdk_call + ): + raise UserError( + "Codex can only pause a single built-in tool call before execution, " + "with native workspace tools enabled and without custom tool hooks." + ) + pending_approval = event + continue if kind == "dynamic_tool_call_request": if message.get("turnId") != turn_id: continue @@ -494,8 +615,9 @@ async def interrupt_when_ready() -> None: raw_usage["token_count"] = info continue if kind == "raw_response_completed": - if usage is None and message.get("token_usage") is not None: - usage = _usage(message["token_usage"]) + # An SDK tool can interrupt before the next cumulative token count. + if message.get("token_usage") is not None: + usage = _add_sample_usage(usage, message["token_usage"]) if ( model_settings.preserve_raw_usage and message.get("usage_metadata") is not None @@ -707,6 +829,8 @@ async def interrupt_when_ready() -> None: # Keep the attempt failure primary when joined cleanup also fails. if failure is None: raise + finally: + observations.close() if native_pending: if not interrupt_requested: raise RuntimeError("Codex finished with an incomplete native tool call.") @@ -757,20 +881,3 @@ async def interrupt_when_ready() -> None: yield ResponseCompletedEvent( response=response, sequence_number=sequence, type="response.completed" ) - - -def _usage(total: dict[str, Any]) -> ResponseUsage: - return ResponseUsage( - input_tokens=total.get("input_tokens", 0), - output_tokens=total.get("output_tokens", 0), - total_tokens=total.get( - "total_tokens", total.get("input_tokens", 0) + total.get("output_tokens", 0) - ), - input_tokens_details=_make_input_tokens_details( - cached_tokens=total.get("cached_input_tokens", 0), - cache_write_tokens=total.get("cache_write_input_tokens"), - ), - output_tokens_details=OutputTokensDetails( - reasoning_tokens=total.get("reasoning_output_tokens", 0) - ), - ) diff --git a/src/agents/run.py b/src/agents/run.py index 57ca88bbe2..715b5b4f24 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -871,6 +871,9 @@ async def _run_impl( run_state._reasoning_item_id_policy = resolved_reasoning_item_id_policy run_state.set_trace(get_current_trace()) + if isinstance(session, ResumableRun): + session.bind_state(run_state) + use_task_and_turn_spans = include_task_and_turn_spans(run_config.tracing) current_task_span: Span[TaskSpanData] | None = ( task_span(name=trace_workflow_name) if use_task_and_turn_spans else None @@ -2912,6 +2915,7 @@ async def run_loop() -> None: if isinstance(session, ResumableRun): try: await session.begin(input) + session.bind_state(run_state) except BaseException: streamed_result.is_complete = True streamed_result._event_queue.put_nowait(QueueCompleteSentinel()) diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 77ae747a8e..6ddaa62170 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -1189,6 +1189,10 @@ def _apply_approval_decision( rejection_message: str | None = None, ) -> None: """Record an approval or rejection decision.""" + from .models._codex_items import native_approval_checkpoint + + if native_approval_checkpoint(approval_item.raw_item) is not None and always: + raise UserError("Native approvals require a decision for the original call only.") hosted_request = get_hosted_mcp_approval_request_identity(approval_item) if hosted_request is not None: call_id = hosted_request.request_id diff --git a/src/agents/run_internal/native_approvals.py b/src/agents/run_internal/native_approvals.py new file mode 100644 index 0000000000..2e9b379d19 --- /dev/null +++ b/src/agents/run_internal/native_approvals.py @@ -0,0 +1,220 @@ +"""Keep native approval state and execution separate from SDK tool dispatch.""" + +from __future__ import annotations + +from typing import Any, cast + +from ..exceptions import UserError +from ..items import ToolCallOutputItem, ToolCallOutputTypes +from ..memory.resumable_session import _ResumableSessionBinding +from ..models._codex_items import native_approval_checkpoint +from ..models._run_context import ModelToolHooks, model_run_context +from .run_steps import NextStepInterruption, NextStepRunAgain, SingleStepResult +from .tool_execution import pending_tool_span +from .turn_preparation import get_all_tools, get_model + + +def native_receipts_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any] | None: + """Select the immutable native proposals while leaving decisions mutable.""" + processed = snapshot.get("last_processed_response") or {} + approvals = processed.get("native_approvals", []) + if not approvals: + return None + call_ids = {item["raw_item"]["call_id"] for item in approvals} + step = snapshot.get("current_step") or {} + pending = (step.get("data") or {}).get("interruptions", []) + return { + "plans": approvals, + "pending": [ + item for item in pending if item.get("raw_item", {}).get("call_id") in call_ids + ], + "generated": [ + item + for item in snapshot.get("generated_items", []) + if item.get("raw_item", {}).get("call_id") in call_ids + ], + } + + +def _validate_native_plan(processed: Any) -> None: + if len(processed.native_approvals) != 1 or any( + ( + processed.functions, + processed.custom_tool_calls, + processed.handoffs, + processed.computer_actions, + processed.local_shell_calls, + processed.shell_calls, + processed.apply_patch_calls, + processed.function_tools_not_found, + processed.mcp_approval_requests, + ) + ): + raise UserError( + "Native approvals require a single pending native call without SDK actions." + ) + item = processed.native_approvals[0] + if native_approval_checkpoint(item.raw_item) is None or item.tool_name != item.raw_item.name: + raise UserError("Invalid native approval plan.") + + +def native_pending_step( + *, + bindings: Any, + original_input: Any, + pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + run_state: Any, + run_config: Any, +) -> SingleStepResult: + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError( + "Start the run with session=ResumableSession to recover native tool approvals." + ) + pending = list(processed_response.native_approvals) + processed_response.interruptions = pending + with pending_tool_span(run_config, pending[0].tool_name, pending[0].raw_item.call_id) as span: + span.span_data.operation.outcome = "paused" + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=pre_step_items, + new_step_items=[*processed_response.new_items, *pending], + next_step=NextStepInterruption(interruptions=pending), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + + +async def resolve_native_approval_step( + *, + bindings: Any, + original_input: Any, + original_pre_step_items: list[Any], + new_response: Any, + processed_response: Any, + context_wrapper: Any, + hooks: Any, + run_config: Any, + run_state: Any, +) -> SingleStepResult: + from ..models.codex import _CodexModel + + _validate_native_plan(processed_response) + binding = run_state._resumable_binding if run_state is not None else None + if ( + not isinstance(binding, _ResumableSessionBinding) + or binding.session._checkpoint.status != "active" + ): + raise UserError("Native approval recovery requires a claimed ResumableSession.") + pending = processed_response.native_approvals[0] + checkpoint = native_approval_checkpoint(pending.raw_item) + assert checkpoint is not None + call = pending.raw_item + key = context_wrapper._resolve_approval_key(pending) + key = context_wrapper._function_approval_key(key, pending) or key + decision = context_wrapper._approvals.get(key) + if decision is not None and (decision.approved is True or decision.rejected is True): + raise UserError("Native approvals require a decision for the original call only.") + approved = context_wrapper.get_approval_status( + pending.tool_name, + call.call_id, + existing_pending=pending, + current_invocation=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + if approved is None: + with pending_tool_span(run_config, pending.tool_name, call.call_id) as span: + span.span_data.operation.outcome = "paused" + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[], + next_step=NextStepInterruption(interruptions=[pending]), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) + # A new checkpoint can carry the same model call ID. Claiming that checkpoint + # doesn't authorize repeating an invocation that already used its decision. + status = context_wrapper._tool_invocation_status( + call, tool_lookup_key=pending.tool_lookup_key, tool_name=pending.tool_name + ) + if status is None: + raise UserError("Cannot match the native approval to a pending invocation.") + _, completed, executed = status + if executed or completed: + raise UserError( + "Cannot reuse a native call ID after the call has executed or completed. " + "Start a new invocation with a new call ID." + ) + if approved: + from ..tracing.model_tracing import get_model_tracing_impl + + model = bindings.resolved_model or get_model(bindings.execution_agent, run_config) + if not isinstance(model, _CodexModel): + raise UserError("A pending native approval requires its current Codex model.") + tools = await get_all_tools(bindings.execution_agent, context_wrapper) + with model_run_context( + run_state, + ModelToolHooks( + bindings.public_agent, context_wrapper, hooks, run_config.tool_execution + ), + ): + output = await model._run_approved_native_call( + checkpoint, + tools=tools, + tracing=get_model_tracing_impl( + run_config.tracing_disabled, run_config.trace_include_sensitive_data + ), + ) + else: + from .tool_execution import resolve_approval_rejection_message + + with pending_tool_span(run_config, pending.tool_name, call.call_id): + output = await resolve_approval_rejection_message( + context_wrapper=context_wrapper, + run_config=run_config, + tool_type="shell" if call.name == "exec_command" else "apply_patch", + tool_name=call.name, + call_id=call.call_id, + tool_call=call, + existing_pending=pending, + tool_lookup_key=pending.tool_lookup_key, + tool_namespace=pending.tool_namespace, + ) + result_item = ToolCallOutputItem( + agent=bindings.public_agent, + raw_item=cast( + ToolCallOutputTypes, + { + "type": "function_call_output" + if call.type == "function_call" + else "custom_tool_call_output", + "call_id": call.call_id, + "output": output, + }, + ), + output=output, + ) + context_wrapper._mark_tool_call_completed(result_item.raw_item) + processed_response.interruptions = [] + return SingleStepResult( + original_input=original_input, + model_response=new_response, + pre_step_items=original_pre_step_items, + new_step_items=[result_item], + next_step=NextStepRunAgain(), + tool_input_guardrail_results=[], + tool_output_guardrail_results=[], + processed_response=processed_response, + ) diff --git a/src/agents/run_internal/resumable_session.py b/src/agents/run_internal/resumable_session.py index a55fba5c31..9754d31c09 100644 --- a/src/agents/run_internal/resumable_session.py +++ b/src/agents/run_internal/resumable_session.py @@ -37,6 +37,13 @@ async def begin(self, input: Any) -> None: await self._source._begin_run(input) self._items = self._source._get_history() + def bind_state(self, state: Any) -> None: + """Attach the current store claim to the internal execution state.""" + assert self._source is not None + state._resumable_binding = _ResumableSessionBinding( + self._source, self._source._checkpoint.revision + ) + async def finish(self, result: RunResult | RunResultStreaming) -> None: assert self._source is not None state = result.to_state() if result.interruptions else None diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 659642845a..7646456f04 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -164,7 +164,7 @@ ) from .oai_conversation import OpenAIServerConversationTracker from .prompt_cache_key import PromptCacheKeyResolver, model_settings_with_prompt_cache_key -from .resumable_session import checkpoint_streamed_result +from .resumable_session import ResumableRun, checkpoint_streamed_result from .run_steps import ( NextStepFinalOutput, NextStepHandoff, @@ -2692,7 +2692,13 @@ def get_model_stream() -> AsyncIterator[TResponseStreamEvent]: model_run_context_stream( retry_stream, tool_use_tracker, - ModelToolHooks(public_agent, context_wrapper, hooks, run_config.tool_execution), + ModelToolHooks( + public_agent, + context_wrapper, + hooks, + run_config.tool_execution, + resumable=isinstance(session, ResumableRun), + ), ) ) as model_events: async for event in model_events: @@ -3117,7 +3123,13 @@ async def rewind_model_request() -> None: with ( model_run_context( tool_use_tracker, - ModelToolHooks(public_agent, context_wrapper, hooks, run_config.tool_execution), + ModelToolHooks( + public_agent, + context_wrapper, + hooks, + run_config.tool_execution, + resumable=isinstance(session, ResumableRun), + ), ), record_failed_attempt_usage_on_error(context_wrapper.usage) as reported_usage, ): diff --git a/src/agents/run_internal/run_steps.py b/src/agents/run_internal/run_steps.py index e98177a1d4..4e786e46e4 100644 --- a/src/agents/run_internal/run_steps.py +++ b/src/agents/run_internal/run_steps.py @@ -137,6 +137,7 @@ class ProcessedResponse: # Original invocation recipients, independent of the currently available callables. tool_output_guardrail_result_start: int | None = None # First checkpointed tool output guardrail result owned by this response. + native_approvals: list[ToolApprovalItem] = dataclasses.field(default_factory=list) def has_tools_or_approvals_to_run(self) -> bool: # Handoffs, functions and computer actions need local processing @@ -152,6 +153,7 @@ def has_tools_or_approvals_to_run(self) -> bool: self.apply_patch_calls, self.mcp_approval_requests, self.function_tools_not_found, + self.native_approvals, ] ) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 04fad384e2..e847c9bddd 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -101,7 +101,13 @@ ) from ..lifecycle import RunHooks from ..logger import log_tool_action_error, logger -from ..models._codex_items import CodexCustomToolCallOutput, CodexFunctionToolCallOutput +from ..models._codex_items import ( + CodexCustomToolCallOutput, + CodexFunctionToolCallOutput, + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -151,6 +157,7 @@ run_item_to_input_item, shell_rejection_item, ) +from .native_approvals import native_pending_step, resolve_native_approval_step from .run_steps import ( NOT_FINAL_OUTPUT, NextStepFinalOutput, @@ -987,6 +994,18 @@ async def execute_tools_and_side_effects( """Run one turn of the loop, coordinating tools, approvals, guardrails, and handoffs.""" public_agent = bindings.public_agent + if processed_response.native_approvals: + _register_tool_call_items(context_wrapper, processed_response.new_items) + return native_pending_step( + bindings=bindings, + original_input=original_input, + pre_step_items=pre_step_items, + new_response=new_response, + processed_response=processed_response, + run_state=run_state, + run_config=run_config, + ) + execute_final_output_call = execute_final_output execute_handoffs_call = execute_handoffs @@ -1406,6 +1425,18 @@ async def resolve_interrupted_turn( """Continue a turn that was previously interrupted waiting for tool approval.""" public_agent = bindings.public_agent execution_agent = bindings.execution_agent + if processed_response.native_approvals: + return await resolve_native_approval_step( + bindings=bindings, + original_input=original_input, + original_pre_step_items=original_pre_step_items, + new_response=new_response, + processed_response=processed_response, + context_wrapper=context_wrapper, + hooks=hooks, + run_config=run_config, + run_state=run_state, + ) output_index = _build_tool_output_index(original_pre_step_items) response_boundary = _current_response_boundary((), processed_response, run_state) generated_output_start = response_boundary.generated_start @@ -3172,6 +3203,7 @@ def process_model_response( allow_apply_patch_function_fallback: bool = True, ) -> ProcessedResponse: items: list[RunItem] = [] + native_approvals: list[ToolApprovalItem] = [] # Only the Codex adapter creates these types, after validating native call/output pairing. provider_completed = { @@ -3276,6 +3308,12 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: output.__class__.__name__ if hasattr(output, "__class__") else type(output), ) caller = get_mapping_or_attr(output, "caller") + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + items.append(ToolCallItem(raw_item=cast(Any, output), agent=agent)) + native_approvals.append(ToolApprovalItem(agent=agent, raw_item=output)) + tools_used.append(output.name) + continue is_program_owned = get_mapping_or_attr(caller, "type") == "program" if ( not is_program_owned @@ -3964,6 +4002,7 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tools_used=tools_used, mcp_approval_requests=mcp_approval_requests, interruptions=[], + native_approvals=native_approvals, function_tools_not_found=function_tools_not_found, mcp_tool_bindings={ run.tool_call.call_id: binding @@ -3999,6 +4038,9 @@ def _preflight_response_invocations( if isinstance(item, (CodexFunctionToolCallOutput, CodexCustomToolCallOutput)) } for index, output in enumerate(response.output): + if isinstance(output, (CodexPendingFunctionCall, CodexPendingCustomCall)): + native_approval_checkpoint(output) + continue if isinstance(output, (ResponseFunctionToolCall, ResponseCustomToolCall)) and ( output.call_id in provider_completed ): diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 20c8ce2755..29d5d6458e 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -110,6 +110,10 @@ log_model_and_tool_data_warning, logger, ) +from .models._codex_items import ( + native_approval_checkpoint, + restore_native_approval_call, +) from .run_context import RunContextWrapper, _ApprovalRecord, _FunctionToolApprovalKey from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -312,7 +316,7 @@ def _default_run_state_validation_error( ), "1.23": ( "Binds prepared tool inputs to their original proposals across retries and resume, " - "and records required ResumableSession ownership for paused runs." + "records required ResumableSession ownership, and preserves native approval receipts." ), } SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES) @@ -785,6 +789,11 @@ def _restore_pydantic_fields_set(value: Any, source: Any) -> None: def _copy_tool_approval_raw_item(raw_item: Any) -> Any: """Copy a supported approval raw item through public Pydantic APIs.""" + native_checkpoint = native_approval_checkpoint(raw_item) + if native_checkpoint is not None: + return restore_native_approval_call( + raw_item.model_dump(exclude_unset=True), native_checkpoint + ) active_container_ids: set[int] = set() raw_item_mro = _static_type_mro(raw_item) for model_type, model_adapter in _TOOL_APPROVAL_MODEL_ADAPTERS: @@ -1237,6 +1246,13 @@ def _find_current_approval_item( ) return canonical_matches[0] if canonical_matches else None + def _decision_item(self, proposal: ToolApprovalItem) -> ToolApprovalItem: + current = self._find_current_approval_item(proposal) + expected = native_approval_checkpoint(current.raw_item) if current else None + if native_approval_checkpoint(proposal.raw_item) != expected: + raise UserError("Native approval does not match the current pending call.") + return current or proposal + def _find_nested_approval_state( self, approval_item: ToolApprovalItem, @@ -1409,9 +1425,8 @@ def approve(self, approval_item: ToolApprovalItem, always_approve: bool = False) nested_state, nested_item = nested_approval nested_state.approve(nested_item, always_approve=always_approve) return - current_approval_item = self._find_current_approval_item(approval_item) self._context.approve_tool( - current_approval_item or approval_item, + self._decision_item(approval_item), always_approve=always_approve, ) @@ -1441,7 +1456,7 @@ def reject( ) return self._context.reject_tool( - self._find_current_approval_item(approval_item) or approval_item, + self._decision_item(approval_item), always_reject=always_reject, rejection_message=rejection_message, ) @@ -2167,6 +2182,15 @@ def _serialize_processed_response( for call_id, binding in processed_response.mcp_tool_bindings.items() }, } + if processed_response.native_approvals: + result["native_approvals"] = [ + _serialize_tool_approval_interruption( + item, + include_tool_name=True, + agent_identity_keys_by_id=agent_identity_keys_by_id, + ) + for item in processed_response.native_approvals + ] if processed_response.tool_output_guardrail_result_start is not None: result["tool_output_guardrail_result_start"] = ( processed_response.tool_output_guardrail_result_start @@ -2226,6 +2250,9 @@ def _serialize_item( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(item.raw_item) + if native_checkpoint is not None: + result["codex_native_checkpoint"] = native_checkpoint if isinstance(item, InputItem): result["input_id"] = item.input_id @@ -2540,6 +2567,9 @@ def validation_error_factory( validation_error_factory=validation_error_factory, ) + if _unsupported_native_approvals(state_json): + raise validation_error_factory(_UNSUPPORTED_NATIVE_APPROVALS, UserError) + from .sandbox._mount_security import sanitize_run_state_sandbox_mount_authority if "sandbox" in state_json: @@ -2590,6 +2620,35 @@ def validation_error_factory( # -------------------------- +_UNSUPPORTED_NATIVE_APPROVALS = ( + "Run state contains unsupported native tool approvals. " + "Check for prior effects before starting a new run with SDK tools." +) + + +def _unsupported_native_approvals(state: dict[str, Any], *, nested: bool = False) -> bool: + """Check nested agent runs before best-effort context restoration can skip a child.""" + response = state.get("last_processed_response") + if not isinstance(response, dict): + return False + approvals = response.get("native_approvals") + if approvals: + # Only the root session has a claimed checkpoint. A nested state cannot + # authorize an action, even when it contains an otherwise valid receipt. + if nested or not isinstance(approvals, list) or len(approvals) != 1: + return True + item = approvals[0] + if not isinstance(item, dict) or not isinstance(item.get("codex_native_checkpoint"), dict): + return True + functions = response.get("functions") + if isinstance(functions, list): + for entry in functions: + child = entry.get("agent_run_state") if isinstance(entry, dict) else None + if isinstance(child, dict) and _unsupported_native_approvals(child, nested=True): + return True + return False + + def _validate_run_state_json_value(value: object) -> None: """Validate the exact built-in JSON tree without invoking caller-defined protocols.""" if type(value) is dict: @@ -2896,6 +2955,9 @@ def _serialize_tool_approval_interruption( agent_identity_keys_by_id=agent_identity_keys_by_id, ), } + native_checkpoint = native_approval_checkpoint(interruption.raw_item) + if native_checkpoint is not None: + interruption_dict["codex_native_checkpoint"] = native_checkpoint if include_tool_name and interruption.tool_name is not None: interruption_dict["tool_name"] = interruption.tool_name if interruption.tool_namespace is not None: @@ -3643,6 +3705,19 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: if approval_item is not None: interruptions.append(approval_item) + native_approvals: list[ToolApprovalItem] = [] + for approval_data in processed_response_data.get("native_approvals", []): + approval = _deserialize_tool_approval_item( + approval_data, + agent_map=agent_map, + agent_identity_map=agent_identity_map, + fallback_agent=current_agent, + validation_error_factory=validation_error_factory, + ) + if approval is None or native_approval_checkpoint(approval.raw_item) is None: + raise validation_error_factory("Invalid native approval plan", UserError) + native_approvals.append(approval) + saved_bindings = processed_response_data.get("mcp_tool_bindings", {}) mcp_tool_bindings = { call_id: cast(tuple[str, str, int | None], tuple(binding)) @@ -3656,6 +3731,7 @@ def _deserialize_function_actions() -> list[_DeserializedFunctionAction]: return ProcessedResponse( new_items=new_items, + native_approvals=native_approvals, handoffs=handoffs, functions=functions, computer_actions=computer_actions, @@ -3835,6 +3911,8 @@ def _deserialize_tool_approval_item( tool_lookup_key = deserialize_function_tool_lookup_key(item_data.get("tool_lookup_key")) allow_bare_name_alias = item_data.get("allow_bare_name_alias") is True raw_item = _deserialize_tool_approval_raw_item(raw_item_data) + if "codex_native_checkpoint" in item_data: + raw_item = restore_native_approval_call(raw_item_data, item_data["codex_native_checkpoint"]) return ToolApprovalItem( agent=agent, raw_item=raw_item, @@ -4236,6 +4314,10 @@ async def _build_run_state_from_json( validation_error_factory=validation_error_factory, ) schema_major, schema_minor = (int(part) for part in schema_version.split(".", maxsplit=1)) + if (schema_major, schema_minor) < (1, 23) and ( + (state_json.get("last_processed_response") or {}).get("native_approvals") + ): + raise validation_error_factory("Native approvals require RunState schema 1.23.", UserError) programmatic_major, programmatic_minor = ( int(part) for part in _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION.split(".", maxsplit=1) ) @@ -5343,6 +5425,10 @@ def _resolve_agent_info( # Tool call items can be function calls, shell calls, apply_patch calls, # MCP calls, etc. Check the type field to determine which type to deserialize as raw_item_tool = _deserialize_tool_call_raw_item(normalized_raw_item) + if "codex_native_checkpoint" in item_data: + raw_item_tool = restore_native_approval_call( + normalized_raw_item, item_data["codex_native_checkpoint"] + ) # Preserve display metadata if it was stored with the item. description = item_data.get("description") title = item_data.get("title") @@ -5705,6 +5791,7 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]: "RunState tool invocation must be a mapping.", "RunState tool invocation contains invalid lifecycle data.", "Tool input rewrites require RunState schema 1.23.", + _UNSUPPORTED_NATIVE_APPROVALS, "RunState tool input rewrite is invalid.", "Hosted MCP approval decisions require a non-empty request id.", ( diff --git a/tests/memory/test_native_approval_recovery.py b/tests/memory/test_native_approval_recovery.py new file mode 100644 index 0000000000..5d9ef00a95 --- /dev/null +++ b/tests/memory/test_native_approval_recovery.py @@ -0,0 +1,475 @@ +from __future__ import annotations + +import asyncio +import copy +import json + +import pytest + +from agents import Agent, ResumableSession, RunConfig, Runner, RunState +from agents.exceptions import AgentsException, UserError +from agents.items import ModelResponse +from agents.memory.resumable_session import ResumableSessionError +from agents.models._codex_items import CodexPendingCustomCall, CodexPendingFunctionCall +from agents.models.codex import CodexProvider +from agents.testing import ScriptedModel +from agents.usage import Usage +from tests.memory.test_resumable_session import MemoryStore, run_agent +from tests.models.test_codex_provider import FakeThread, threads as threads +from tests.test_responses import get_text_message +from tests.testing_processor import fetch_ordered_spans + + +def pending_call(tmp_path, kind): + call = { + "type": "function_call" if kind == "exec_command" else "custom_tool_call", + "id": "native-item", + "call_id": "native-call", + "name": kind, + "status": "in_progress", + } + if kind == "exec_command": + call["arguments"] = '{"cmd":"echo harmless"}' + item_type = CodexPendingFunctionCall + else: + call["input"] = "*** Begin Patch\n*** Add File: harmless.txt\n+safe\n*** End Patch\n" + item_type = CodexPendingCustomCall + checkpoint = { + "version": 1, + "cwd": str(tmp_path.resolve()), + "call": copy.deepcopy(call), + "request": {"type": "native_approval", "call_id": "native-call"}, + "model": "fixture-model", + "preconditions": {"version": 1, "first_approval": True}, + } + item = item_type(**call, checkpoint=checkpoint) + return item.model_copy(update={"codex_reasoning_summary": "Review this effect."}) + + +async def setup_native(tmp_path, monkeypatch, kind="exec_command"): + provider = CodexProvider(cwd=tmp_path) + model = provider.get_model("fixture-model") + pending = pending_call(tmp_path, kind) + calls = [] + requests = [] + + async def respond(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = [pending] if len(requests) == 1 else [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint) + return "native accepted" + + monkeypatch.setattr(model, "get_response", respond) + monkeypatch.setattr(model, "_run_approved_native_call", execute, raising=False) + agent = Agent(name="worker", model=model) + store = MemoryStore() + session = await ResumableSession.create(store) + config = RunConfig(tracing_disabled=True) + return agent, store, session, config, pending, calls, requests + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"]) +@pytest.mark.parametrize("approval_after_count", [False, True], ids=["before-count", "after-count"]) +async def test_native_pause_preserves_all_sampling_usage_across_session_resume( + tmp_path, monkeypatch, threads, streamed, approval_after_count +): + from tests.models.test_codex_pending_approval import ApprovalThread, approval, raw + from tests.models.test_codex_provider import native_pair + + earlier = { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + "cached_input_tokens": 1, + "cache_write_input_tokens": 1, + "reasoning_output_tokens": 1, + } + current = { + "input_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + "cached_input_tokens": 2, + "cache_write_input_tokens": 3, + "reasoning_output_tokens": 2, + } + cumulative = {key: earlier[key] + current[key] for key in earlier} + + # The wire double preserves the completion/count ordering around a real Runner pause. + class SamplingApprovalThread(ApprovalThread): + async def events(self): + yield { + "type": "raw_response_completed", + "response_id": "earlier", + "token_usage": earlier, + } + yield {"type": "token_count", "info": {"total_token_usage": earlier}} + call, _ = native_pair(call_id="pending") + yield raw(call) + if not approval_after_count: + yield approval() + yield { + "type": "raw_response_completed", + "response_id": "current", + "token_usage": current, + } + yield {"type": "token_count", "info": {"total_token_usage": cumulative}} + if approval_after_count: + yield approval() + + def tokens(usage): + return ( + usage.input_tokens, + usage.output_tokens, + usage.total_tokens, + usage.input_tokens_details.cached_tokens, + usage.input_tokens_details.cache_write_tokens, + usage.output_tokens_details.reasoning_tokens, + ) + + model = CodexProvider(cwd=tmp_path).get_model("fixture-model") + agent = Agent(name="worker", model=model) + first = SamplingApprovalThread() + threads[0].extend([first, FakeThread(text="finished")]) + calls = [] + + async def execute(checkpoint, *, tools, tracing): + calls.append(checkpoint["call"]["call_id"]) + return "native accepted" + + monkeypatch.setattr(model, "_run_approved_native_call", execute) + store = MemoryStore() + session = await ResumableSession.create(store) + paused = await run_agent(agent, "start", session, streamed) + assert first.recover_approvals + assert first.closed and len(paused.interruptions) == 1 and calls == [] + assert tokens(paused.context_wrapper.usage) == (15, 5, 20, 3, 4, 3) + assert store.checkpoint(session.session_id)["status"] == "paused" + + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + completed = await run_agent(agent, state, None, streamed) + assert completed.final_output == "finished" and calls == ["pending"] + # The fresh native thread adds its own 10 input and 7 output tokens exactly once. + assert tokens(completed.context_wrapper.usage) == (25, 12, 37, 6, 4, 5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("approved", [True, False]) +async def test_native_approval_round_trip_uses_original_runtime_only( + tmp_path, monkeypatch, kind, approved +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + first = await Runner.run(agent, "do the native operation", session=session, run_config=config) + assert len(first.interruptions) == 1 + assert calls == [] + assert store.checkpoint(session.session_id)["status"] == "paused" + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + interruption = state.get_interruptions()[0] + assert interruption.arguments == ( + pending.arguments if kind == "exec_command" else pending.input + ) + if approved: + state.approve(interruption) + else: + state.reject(interruption, rejection_message="Leave the release branch alone.") + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert result.final_output == "finished" + assert calls == ([pending.checkpoint] if approved else []) + assert "checkpoint" not in repr(requests[-1]) + expected_message = "native accepted" if approved else "Leave the release branch alone." + assert any( + item.get("call_id") == interruption.call_id and item.get("output") == expected_message + for item in requests[-1] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_plain_runner_rejects_unrecoverable_native_pause(tmp_path, monkeypatch, streamed): + agent, _, _, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + with pytest.raises(UserError, match="session=ResumableSession"): + if streamed: + result = Runner.run_streamed(agent, "start", run_config=config) + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "start", run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +@pytest.mark.parametrize("field", ["name", "call_id", "id", "payload"]) +async def test_display_mutation_cannot_be_approved(tmp_path, monkeypatch, kind, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch, kind) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + item = state.get_interruptions()[0] + field = ("arguments" if kind == "exec_command" else "input") if field == "payload" else field + setattr(item.raw_item, field, "changed") + with pytest.raises(UserError): + state.approve(item) + assert not calls + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["payload", "cwd", "request", "model"]) +async def test_coordinated_receipt_mutation_fails_before_claim(tmp_path, monkeypatch, field): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + # Rewrite every serialized copy consistently; the store's original still binds authority. + snapshot = state.to_json() + + def rewrite(value): + if isinstance(value, dict): + if field == "payload" and value.get("type") == "function_call": + value["arguments"] = '{"cmd":"changed"}' + if value.get("version") == 1 and "cwd" in value and "request" in value: + if field == "cwd": + value["cwd"] = str(tmp_path / "other") + elif field == "request": + value["request"] = {"type": "changed"} + elif field == "model": + value["model"] = "changed" + for child in value.values(): + rewrite(child) + elif isinstance(value, list): + for child in value: + rewrite(child) + + rewrite(snapshot) + changed = await RunState.from_string(agent, json.dumps(snapshot)) + changed._resumable_binding = state._resumable_binding + original = store.records[session.session_id] + with pytest.raises(ResumableSessionError): + await Runner.run(agent, changed, session=recovered, run_config=config) + assert store.records[session.session_id] == original + assert not calls + + +@pytest.mark.asyncio +async def test_no_decision_repauses_and_unbound_copy_never_dispatches(tmp_path, monkeypatch): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + first = await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(result.interruptions) == 1 + assert not calls + copied = await RunState.from_string(agent, first.to_state().to_string()) + copied.approve(copied.get_interruptions()[0]) + with pytest.raises(UserError): + await Runner.run(agent, copied, run_config=config) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["exec_command", "apply_patch"]) +async def test_completed_native_call_id_cannot_reuse_a_previous_decision( + tmp_path, monkeypatch, kind +): + agent, store, session, config, pending, calls, requests = await setup_native( + tmp_path, monkeypatch, kind + ) + + async def repeat_native_call(*args, **kwargs): + requests.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + if len(requests) <= 2: + # The adapter can attest a new native request with a repeated model call ID. + repeated = pending.model_copy(deep=True) + repeated.checkpoint["request"]["turn_id"] = f"native-turn-{len(requests)}" + output = [repeated] + else: + output = [get_text_message("finished")] + return ModelResponse(output=output, usage=Usage(), response_id=None) + + monkeypatch.setattr(agent.model, "get_response", repeat_native_call) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + + # Native execution stays at the adapter seam while public resume owns approval state. + with pytest.raises(UserError, match="completed"): + repeated_result = await Runner.run(agent, state, session=recovered, run_config=config) + assert len(repeated_result.interruptions) == 1 + assert len(calls) == 1 + replacement = await ResumableSession.resume(store, session.session_id) + repeated_state = await replacement.load_state(agent) + await Runner.run(agent, repeated_state, session=replacement, run_config=config) + assert len(calls) == 1, "The previous decision authorized another native dispatch." + + assert len(calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("approved", [True, False]) +async def test_sticky_native_decisions_are_rejected(tmp_path, monkeypatch, approved): + agent, store, session, config, _, calls, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + state = await session.load_state(agent) + item = state.get_interruptions()[0] + with pytest.raises(UserError, match="original call only"): + if approved: + state.approve(item, always_approve=True) + else: + state.reject(item, always_reject=True) + assert not calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_waits_for_checkpoint_ack(tmp_path, monkeypatch, streamed): + agent, store, session, config, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + store.block_checkpoint = True + events = [] + if streamed: + scripted = ScriptedModel([[pending]]) + monkeypatch.setattr(agent.model, "stream_response", scripted.stream_response) + result = Runner.run_streamed(agent, "start", session=session, run_config=config) + + async def consume(): + async for event in result.stream_events(): + events.append(event) + return result + + task = asyncio.create_task(consume()) + else: + task = asyncio.create_task(Runner.run(agent, "start", session=session, run_config=config)) + try: + await asyncio.wait_for(store.saving.wait(), 5) + assert not task.done() + assert store.checkpoint(session.session_id)["status"] == "active" + assert not calls + if streamed: + assert not result.interruptions + assert not any( + getattr(event, "name", None) == "tool_approval_requested" for event in events + ) + with pytest.raises(AgentsException): + result.to_state() + finally: + store.release.set() + finished = await asyncio.wait_for(task, 5) + assert len(finished.interruptions) == 1 + assert store.checkpoint(session.session_id)["status"] == "paused" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_data,cancel", [(False, False), (True, False), (False, True)]) +async def test_restored_native_execution_observes_live_start_and_interruption( + tmp_path, monkeypatch, threads, include_data, cancel +): + from agents.tracing import get_current_span + from tests.testing_processor import SPAN_PROCESSOR_TESTING + + started = asyncio.Event() + release = asyncio.Event() + closed = asyncio.Event() + updates = asyncio.Event() + events = [] + calls = [] + original_start = SPAN_PROCESSOR_TESTING.on_span_start + caller_spans = [] + + def on_start(span): + original_start(span) + if span.span_data.type == "function": + caller_spans.append(get_current_span()) + started.set() + + monkeypatch.setattr(SPAN_PROCESSOR_TESTING, "on_span_start", on_start) + + # The FFI double preserves suspension and cancellation at the native execution boundary. + class RestoredThread(FakeThread): + def enable_tracing(self, mode="metadata"): + calls.append(mode) + + async def wait_for_operations(self): + await updates.wait() + + def drain_operations(self): + encoded = json.dumps(events) + events.clear() + updates.clear() + return encoded + + async def resolve_approval(self, call_json, request, preconditions): + key = {"kind": "tool", "name": "exec_command", "id": json.loads(call_json)["call_id"]} + events.append(dict(key, phase="start", at=1_795_154_400_000)) + if calls == ["content"]: + events.append(dict(key, phase="input", input="private command")) + updates.set() + await release.wait() + events.append(dict(key, phase="end", outcome="completed", at=1_795_154_402_000)) + if calls == ["content"]: + events.append(dict(key, phase="output", output="native accepted")) + updates.set() + return "native accepted" + + async def close(self): + await super().close() + closed.set() + + agent, store, session, config, _, _, _ = await setup_native(tmp_path, monkeypatch) + await Runner.run(agent, "start", session=session, run_config=config) + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.approve(state.get_interruptions()[0]) + monkeypatch.delattr(agent.model, "_run_approved_native_call") + threads[0].append(RestoredThread()) + task = asyncio.create_task( + Runner.run( + agent, + state, + run_config=RunConfig( + tracing={"operation_detail": "content"}, + trace_include_sensitive_data=include_data, + ), + ) + ) + try: + await asyncio.wait_for(started.wait(), 2) + assert not task.done() + assert calls == ["content" if include_data else "metadata"] + assert caller_spans[0] is not None + assert caller_spans[0].span_data.type == "agent" + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + result = await asyncio.wait_for(task, 2) + assert result.final_output == "finished" + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert closed.is_set() + (span,) = [ + span + for span in fetch_ordered_spans() + if span.span_data.type == "function" + and span.span_data.operation.tool_call_id == "native-call" + ] + assert span.span_data.operation.outcome == ("interrupted" if cancel else "completed") + assert span.started_at == "2026-11-20T06:00:00+00:00" + assert ("private command" in repr(span.span_data.input)) == include_data + assert span.parent_id == caller_spans[0].span_id diff --git a/tests/memory/test_native_approval_tracing.py b/tests/memory/test_native_approval_tracing.py new file mode 100644 index 0000000000..2a563b0ba9 --- /dev/null +++ b/tests/memory/test_native_approval_tracing.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agents import ResumableSession, RunConfig, Runner +from agents.extensions.tracing import OpenTelemetryTracingProcessor +from agents.testing import ScriptedModel +from agents.tracing import set_trace_processors +from tests.memory.test_native_approval_recovery import setup_native +from tests.test_responses import get_text_message +from tests.testing_processor import SPAN_PROCESSOR_TESTING, fetch_ordered_spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["default", "metadata", "content", "veto", "disabled"]) +@pytest.mark.parametrize("streamed", [False, True]) +async def test_native_pause_and_denial_emit_only_approval_metadata( + tmp_path, monkeypatch, mode, streamed +): + agent, store, session, _, pending, calls, _ = await setup_native(tmp_path, monkeypatch) + if streamed: + script = ScriptedModel([[pending], [get_text_message("finished")]]) + monkeypatch.setattr(agent.model, "stream_response", script.stream_response) + config = RunConfig( + tracing={} + if mode == "default" + else {"operation_detail": "metadata" if mode == "metadata" else "content"}, + tracing_disabled=mode == "disabled", + trace_include_sensitive_data=mode != "veto", + ) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor = OpenTelemetryTracingProcessor(tracer_provider=provider) + set_trace_processors([SPAN_PROCESSOR_TESTING, processor]) + + async def run(input, current): + if streamed: + result = Runner.run_streamed(agent, input, session=current, run_config=config) + async for _ in result.stream_events(): + pass + return result + return await Runner.run(agent, input, session=current, run_config=config) + + try: + first = await run("start", session) + assert len(first.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + second = await run(await recovered.load_state(agent), recovered) + assert len(second.interruptions) == 1 + recovered = await ResumableSession.resume(store, session.session_id) + state = await recovered.load_state(agent) + state.reject(state.get_interruptions()[0], rejection_message="DENIAL_CANARY") + result = await run(state, recovered) + assert result.final_output == "finished" + assert not calls + spans = fetch_ordered_spans() + if mode == "default": + decisions = [s.span_data for s in spans if s.span_data.type == "function"] + assert [s.operation.outcome for s in decisions] == ["paused", "paused", "rejected"] + assert all(s.operation.tool_call_id == pending.call_id for s in decisions) + assert all(s.input is None and s.output is None for s in decisions) + else: + assert not [s for s in spans if s.span_data.type == "function"] + decisions = [ + s.span_data.data + for s in spans + if getattr(s.span_data, "name", None) == "tool_approval" + ] + assert decisions == ( + [] + if mode == "disabled" + else [ + {"tool_name": pending.name, "tool_call_id": pending.call_id, "outcome": outcome} + for outcome in ["paused", "paused", "rejected"] + ] + ) + otel = exporter.get_finished_spans() + approvals = [s for s in otel if s.name == "openai.agents.tool_approval exec_command"] + assert len(approvals) == (0 if mode == "disabled" else 3) + assert not [s for s in otel if s.name.startswith("execute_tool")] + assert "DENIAL_CANARY" not in json.dumps([dict(s.attributes or {}) for s in approvals]) + finally: + set_trace_processors([SPAN_PROCESSOR_TESTING]) + provider.shutdown() diff --git a/tests/memory/test_resumable_session.py b/tests/memory/test_resumable_session.py index 0b8a7b60d2..69987751ca 100644 --- a/tests/memory/test_resumable_session.py +++ b/tests/memory/test_resumable_session.py @@ -5,13 +5,14 @@ import asyncio import copy import json +from pathlib import Path from typing import Any import pytest from agents import Agent, RunConfig, Runner, RunState, Session, SQLiteSession, ToolExecutionConfig from agents.decorators import tool -from agents.exceptions import AgentsException +from agents.exceptions import AgentsException, UserError from agents.memory.resumable_session import ResumableSession, ResumableSessionError from agents.result import RunResult, RunResultStreaming from agents.run import AgentRunner @@ -52,6 +53,85 @@ def checkpoint(self, session_id: str) -> dict[str, Any]: CONFIG = RunConfig(tracing_disabled=True) +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", [False, True], ids=["direct", "agent-as-tool"]) +@pytest.mark.parametrize( + "with_context_callback", [False, True], ids=["default", "context-callback"] +) +@pytest.mark.parametrize("loader", ["run-state", "session"]) +async def test_saved_native_approval_cannot_become_an_sdk_tool_approval( + nested: bool, with_context_callback: bool, loader: str, tmp_path: Path +) -> None: + store = MemoryStore() + session = await ResumableSession.create(store) + workspace = tmp_path / "unchanged.txt" + workspace.write_text("pre-existing work") + + @tool(needs_approval=True) + def exec_command(command: str) -> str: + workspace.write_text(command) + pytest.fail("A native approval cannot authorize a matching SDK tool.") + + agent = Agent( + name="assistant", + model=ScriptedModel([[get_function_tool_call("exec_command", '{"command":"original"}')]]), + tools=[exec_command], + ) + if nested: + agent = Agent( + name="coordinator", + model=ScriptedModel([[get_function_tool_call("delegate", '{"input":"Run command"}')]]), + tools=[agent.as_tool("delegate", "Delegate the command")], + ) + await Runner.run( + agent, "Run command", session=session, context={"test": True}, run_config=CONFIG + ) + checkpoint = store.checkpoint(session.session_id) + snapshot = json.loads(checkpoint["run_state"]) + pending_state = ( + snapshot["last_processed_response"]["functions"][0]["agent_run_state"] + if nested + else snapshot + ) + processed = pending_state["last_processed_response"] + # An earlier preview stored native actions outside the SDK function queue. + processed["native_approvals"] = copy.deepcopy(processed["interruptions"]) + if nested: + item = processed["native_approvals"][0] + item["codex_native_checkpoint"] = { + "version": 1, + "cwd": str(tmp_path.resolve()), + "model": "fixture-model", + "call": item["raw_item"], + "request": {"type": "exec_approval_request", "call_id": item["raw_item"]["call_id"]}, + "preconditions": {"version": 1, "first_approval": True}, + } + processed["functions"] = [] + checkpoint["run_state"] = json.dumps(snapshot) + stored = json.dumps(checkpoint) + store.records[session.session_id] = stored + + context_callbacks: list[str] = [] + + def restore_context(value: dict[str, Any]) -> dict[str, Any]: + context_callbacks.append("restored") + return value + + context_deserializer = restore_context if with_context_callback else None + if loader == "run-state": + with pytest.raises(UserError, match="unsupported native tool approvals"): + await RunState.from_json(agent, snapshot, context_deserializer=context_deserializer) + else: + resumed = await ResumableSession.resume(store, session.session_id) + with pytest.raises(ResumableSessionError) as failure: + await resumed.load_state(agent, context_deserializer=context_deserializer) + assert failure.value.reason == "invalid_state" + assert store.records[session.session_id] == stored + assert store.checkpoint(session.session_id)["status"] == "paused" + assert workspace.read_text() == "pre-existing work" + assert not context_callbacks + + async def run_agent( agent: Agent[Any], input: Any, session: Session | ResumableSession | None, streamed: bool ) -> RunResult | RunResultStreaming: diff --git a/tests/models/test_codex_pending_approval.py b/tests/models/test_codex_pending_approval.py new file mode 100644 index 0000000000..396404ce47 --- /dev/null +++ b/tests/models/test_codex_pending_approval.py @@ -0,0 +1,166 @@ +import json + +import pytest + +from agents import ModelSettings, UserError +from agents.models._codex_items import ( + CodexPendingCustomCall, + CodexPendingFunctionCall, + native_approval_checkpoint, +) +from agents.models.codex import CodexProvider +from agents.models.interface import ModelTracing + +from .test_codex_provider import FakeThread, native_pair, threads as threads + + +class ApprovalThread(FakeThread): + async def next_event(self): + encoded = await super().next_event() + if encoded is None: + return None + event = json.loads(encoded) + if event["msg"]["type"] in ( + "exec_approval_request", + "apply_patch_approval_request", + ): + event["preconditions"] = {"version": 1, "first_approval": True} + return json.dumps(event) + + +def approval(call_id="pending"): + return { + "type": "exec_approval_request", + "call_id": call_id, + "turn_id": "turn_1", + "kind": "command", + "command": ["sh", "-c", "printf native-result"], + "cwd": "/workspace", + } + + +def raw(item): + return {"type": "raw_response_item", "item": item} + + +async def response(tmp_path, settings=None): + return ( + await CodexProvider(cwd=tmp_path) + .get_model(None) + .get_response( + None, + "Continue", + settings or ModelSettings(), + [], + None, + [], + ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ) + + +@pytest.mark.asyncio +async def test_approval_waits_for_sampling_completion_and_rejects_late_sibling(tmp_path, threads): + call, _ = native_pair(call_id="pending") + sibling, _ = native_pair(call_id="later") + thread = ApprovalThread( + events=[ + raw(call), + approval(), + raw(sibling), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + with pytest.raises(UserError, match="one outstanding native call"): + await response(tmp_path) + assert thread.closed + + +@pytest.mark.asyncio +async def test_completed_read_stays_completed_while_next_native_call_pauses(tmp_path, threads): + read, read_output = native_pair(call_id="read") + read["arguments"] = json.dumps({"cmd": "cat README.md"}) + pending, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + raw(read), + raw(read_output), + raw(pending), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert [item.call_id for item in result.output] == ["read", "read", "pending"] + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert result.output[1].output == read_output["output"] + assert isinstance(result.output[2], CodexPendingFunctionCall) + assert result.output[2].checkpoint["call"]["call_id"] == "pending" + assert thread.closed + + +@pytest.mark.asyncio +async def test_raw_model_metadata_cannot_forge_a_pending_native_approval(tmp_path, threads): + call, output = native_pair() + call["checkpoint"] = {"request": approval("native_1"), "version": 1} + call["native_approval"] = True + thread = FakeThread(raw_items=[call, output]) + threads[0].append(thread) + result = await response(tmp_path) + assert not isinstance(result.output[0], CodexPendingFunctionCall) + assert not hasattr(result.output[0], "checkpoint") + assert "native_approval" not in result.output[0].model_dump() + assert thread.closed + + +@pytest.mark.asyncio +async def test_patch_begin_does_not_mean_an_awaiting_patch_has_executed(tmp_path, threads): + call, _ = native_pair("custom_tool_call", "pending", "apply_patch") + + thread = ApprovalThread( + events=[ + raw(call), + {"type": "patch_apply_begin", "call_id": "pending"}, + { + "type": "apply_patch_approval_request", + "call_id": "pending", + "turn_id": "turn_1", + "changes": {}, + }, + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path) + assert isinstance(result.output[0], CodexPendingCustomCall) + assert result.output[0].checkpoint["preconditions"]["first_approval"] is True + assert thread.closed + + +@pytest.mark.asyncio +async def test_reasoning_summary_does_not_change_the_approved_action(tmp_path, threads): + call, _ = native_pair(call_id="pending") + thread = ApprovalThread( + events=[ + { + "type": "item_completed", + "item": {"type": "Reasoning", "summary_text": ["Reviewing the command."]}, + }, + raw(call), + approval(), + {"type": "raw_response_completed", "response_id": "response_1"}, + ] + ) + threads[0].append(thread) + result = await response(tmp_path, ModelSettings(reasoning={"summary": "auto"})) + pending = result.output[-1] + assert pending.codex_reasoning_summary == ["Reviewing the command."] + checkpoint = native_approval_checkpoint(pending) + assert checkpoint is not None + assert checkpoint["call"]["call_id"] == "pending" + assert thread.closed diff --git a/tests/models/test_codex_provider.py b/tests/models/test_codex_provider.py index 43e2f6ee89..f593214edf 100644 --- a/tests/models/test_codex_provider.py +++ b/tests/models/test_codex_provider.py @@ -39,7 +39,8 @@ def __init__(self, text="A Codex response", call=None, gate=None, raw_items=(), async def inject_items(self, items): self.history = json.loads(items) - async def start_turn(self, inputs, schema): + async def start_turn(self, inputs, schema, recover_approvals=False): + self.recover_approvals = recover_approvals self.inputs = json.loads(inputs) self.schema = json.loads(schema) self.started.set() @@ -1204,6 +1205,66 @@ async def close(self): assert thread.closed +@pytest.mark.asyncio +@pytest.mark.parametrize("primary", ["success", "failure", "cancel"]) +@pytest.mark.parametrize("late_cancel", [False, True]) +async def test_native_cleanup_preserves_cancellation_and_primary_failure( + tmp_path, threads, primary, late_cancel +): + close_started, release_close = asyncio.Event(), asyncio.Event() + closed = [] + + class HeldCloseThread(FakeThread): + async def close(self): + close_started.set() + try: + await release_close.wait() + raise RuntimeError("cleanup failure") + finally: + closed.append(True) + await super().close() + + thread = HeldCloseThread( + events=[{"type": "error"}] if primary == "failure" else [], + gate=asyncio.Event() if primary == "cancel" else None, + ) + threads[0].append(thread) + provider = CodexProvider(cwd=tmp_path) + operation = Runner.run( + Agent(name="test"), "Answer", run_config=RunConfig(model_provider=provider) + ) + task = asyncio.create_task(operation) + try: + await asyncio.wait_for(thread.started.wait(), 2) + if primary == "cancel": + task.cancel("primary cancellation") + await asyncio.wait_for(close_started.wait(), 2) + if late_cancel: + task.cancel("cleanup cancellation") + await asyncio.sleep(0) + task.cancel("repeated cancellation") + await asyncio.sleep(0) + assert not task.done() + release_close.set() + error, reason = RuntimeError, "cleanup failure" + if primary == "failure": + reason = "Codex model turn failed" + if primary == "cancel" or late_cancel: + error = asyncio.CancelledError + first_cancel = primary == "cancel" and not late_cancel + reason = "primary cancellation" if first_cancel else "cleanup cancellation" + # Python 3.10 has not been forwarding cancellation reasons across task boundaries. + match = None if error is asyncio.CancelledError and sys.version_info < (3, 11) else reason + with pytest.raises(error, match=match): + await task + assert closed == [True] + finally: + release_close.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await provider.aclose() + + @pytest.mark.asyncio async def test_closing_stream_during_tool_publication_retires_pending_native_call( tmp_path, threads @@ -1346,7 +1407,8 @@ async def events(self): "output": "Invalid arguments", }, } - yield {"type": "token_count", "info": {"total_token_usage": {"total_tokens": 20}}} + if first_callback_after_completion: + yield {"type": "token_count", "info": {"total_token_usage": {"total_tokens": 20}}} yield {"type": "turn_aborted"} queue, _ = threads @@ -1916,3 +1978,174 @@ async def events(self): "value": "prepared", "environment_id": "workspace", } + + +class TracedThread(FakeThread): + # The native binding delivers passive, source-timed observations outside the response stream. + def __init__(self, *observations, **kwargs): + super().__init__(**kwargs) + self.observations = list(observations) + self.tracing = False + + def enable_tracing(self, mode=None): + self.tracing = True + self.capture_mode = mode + + def drain_operations(self): + assert self.tracing + observations, self.observations = self.observations, [] + return json.dumps(observations) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_metadata_operations_keep_source_times(tmp_path, threads, streamed): + from tests.testing_processor import fetch_ordered_spans + + model = {"kind": "model", "id": "1", "model": "native-default"} + tool = {"kind": "tool", "id": "c1", "name": "view_image"} + thread = TracedThread( + {**model, "phase": "start", "at": 1000}, + {**tool, "phase": "start", "at": 1010}, + { + **model, + "phase": "end", + "at": 1015, + "outcome": "completed", + "usage": {"input_tokens": 11, "output_tokens": 5}, + }, + {**tool, "phase": "end", "at": 1020, "outcome": "failed"}, + ) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": "metadata"}, + trace_include_sensitive_data=True, + ) + if streamed: + stream = Runner.run_streamed(Agent(name="test"), "Private input", run_config=cfg) + async for _ in stream.stream_events(): + pass + assert stream.final_output == thread.text + else: + result = await Runner.run(Agent(name="test"), "Private input", run_config=cfg) + assert result.final_output == thread.text + spans = fetch_ordered_spans() + (turn,) = [s for s in spans if s.span_data.type == "turn"] + (request,) = [s for s in spans if s.span_data.type == "generation"] + (execution,) = [s for s in spans if s.span_data.type == "function"] + assert request.parent_id == execution.parent_id == turn.span_id + assert request.started_at and request.ended_at + assert execution.started_at and execution.ended_at + assert request.started_at < execution.started_at < request.ended_at < execution.ended_at + assert request.span_data.usage["input_tokens"] == 11 + assert request.span_data.operation.model == "native-default" + assert request.span_data.operation.outcome == "completed" + assert execution.span_data.operation.tool_call_id == "c1" + assert execution.span_data.operation.outcome == "failed" + for span in (request, execution): + assert span.span_data.operation.detail == "metadata" + assert span.span_data.input is span.span_data.output is None + assert thread.capture_mode is None + assert not [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] + + +@pytest.mark.asyncio +async def test_native_operation_loss_does_not_block_runner_or_invent_success(tmp_path, threads): + from tests.testing_processor import fetch_ordered_spans + + thread = TracedThread( + {"kind": "model", "phase": "start", "id": "1", "model": "native-default", "at": 1000}, + {"kind": "dropped"}, + ) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), tracing={"operation_detail": "metadata"} + ) + result = await Runner.run(Agent(name="test"), "Hello", run_config=cfg) + assert result.final_output == thread.text + spans = fetch_ordered_spans() + (request,) = [s for s in spans if s.span_data.type == "generation"] + assert request.span_data.operation.outcome == "interrupted" + assert request.span_data.operation.estimated_end + (notice,) = [s for s in spans if getattr(s.span_data, "name", "") == "codex.tracing.coverage"] + assert notice.span_data.data["reason"] == "dropped_operations" + + +@pytest.mark.asyncio +async def test_disabled_tracing_never_subscribes_to_native_operations(tmp_path, threads): + thread = TracedThread() + threads[0].append(thread) + await Runner.run(Agent(name="test"), "Hello", run_config=config(tmp_path)) + assert not thread.tracing + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "detail,sensitive,captures", + [(None, True, False), ("content", False, False), ("content", True, True)], +) +async def test_native_content_requires_explicit_policy_and_correlates_final_output( + tmp_path, threads, detail, sensitive, captures +): + from tests.testing_processor import fetch_ordered_spans + + model = {"kind": "model", "id": "request", "model": "fixture"} + tool = {"kind": "tool", "id": "command", "name": "exec_command"} + thread = TracedThread( + {**model, "phase": "start", "at": 1000}, + { + **model, + "phase": "input", + "input": { + "input": [{"role": "user", "content": "request text"}], + "tools": [{"type": "function", "name": "lookup"}], + "client_metadata": {"api_key": "never-export"}, + }, + }, + {**tool, "phase": "start", "at": 1100}, + { + **tool, + "phase": "input", + "input": {"command": ["echo", "prepared"], "password": "never-export"}, + }, + {**tool, "phase": "end", "at": 1200, "outcome": "completed"}, + {"kind": "tool", "phase": "output", "id": "unrelated", "output": "wrong output"}, + {"kind": "tool", "phase": "output", "id": "command", "output": "accepted result"}, + { + **model, + "phase": "end", + "at": 1300, + "outcome": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "response text"}], + } + ], + }, + ) + threads[0].append(thread) + cfg = RunConfig( + model_provider=CodexProvider(cwd=tmp_path), + tracing={"operation_detail": detail} if detail else None, + trace_include_sensitive_data=sensitive, + ) + await Runner.run(Agent(name="test"), "sdk boundary", run_config=cfg) + spans = fetch_ordered_spans() + (request,) = [s for s in spans if s.span_data.type in ("generation", "response")] + (execution,) = [s for s in spans if s.span_data.type == "function"] + assert thread.capture_mode == ("content" if captures else None) + if captures: + assert "request text" in json.dumps(request.span_data.input) + assert "response text" in json.dumps(request.span_data.output) + assert execution.span_data.output == "accepted result" + assert "prepared" in execution.span_data.input + assert "never-export" not in json.dumps([s.export() for s in (request, execution)]) + assert "wrong output" not in execution.span_data.output + else: + assert request.span_data.input is None + assert execution.span_data.input is execution.span_data.output is None + assert execution.ended_at is not None and request.ended_at is not None + assert execution.ended_at < request.ended_at diff --git a/tests/test_build_codex_wheel.py b/tests/test_build_codex_wheel.py new file mode 100644 index 0000000000..65190714f3 --- /dev/null +++ b/tests/test_build_codex_wheel.py @@ -0,0 +1,57 @@ +"""Regression coverage for the native wheel's local Core patch boundary.""" + +import importlib.util +import json +import subprocess +from pathlib import Path + +import pytest + + +def test_core_patch_applies_inside_an_ancestor_git_checkout(tmp_path: Path) -> None: + script = Path(__file__).resolve().parents[1] / "scripts/build_codex_wheel.py" + spec = importlib.util.spec_from_file_location("build_codex_wheel", script) + assert spec is not None and spec.loader is not None + build = importlib.util.module_from_spec(spec) + spec.loader.exec_module(build) + subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True) + upstream = tmp_path / "upstream" + crate = upstream / "codex-rs/core" + crate.mkdir(parents=True) + (crate / "Cargo.toml").write_text('[package]\nname = "codex-core"\n') + (crate / "value.txt").write_bytes(b"stock\n") + source = tmp_path / "target/staged" + patches = source / "native/codex/patches" + patches.mkdir(parents=True) + (patches / "0001.patch").write_text( + "diff --git a/codex-rs/core/value.txt b/codex-rs/core/value.txt\n" + "--- a/codex-rs/core/value.txt\n" + "+++ b/codex-rs/core/value.txt\n" + "@@ -1 +1 @@\n-stock\n+patched\n", + newline="\n", + ) + metadata = { + "packages": [ + { + "name": "codex-core", + "source": "git+https://github.com/openai/codex?rev=pinned#pinned", + "manifest_path": str(crate / "Cargo.toml"), + }, + {"name": "sdk", "source": None}, + ] + } + + guide = patches / "0001.md" + for text in (None, " \n"): + if text is not None: + guide.write_text(text) + with pytest.raises(ValueError, match=r"Document 0001\.patch in 0001\.md"): + build.patch_codex_source(source, upstream, metadata) + + guide.write_text("Replace the stock value for the build test.") + build.patch_codex_source(source, upstream, metadata) + patched = source / "native/codex-upstream" + + assert (patched / "codex-rs/core/value.txt").read_text() == "patched\n" + assert (crate / "value.txt").read_text() == "stock\n" + assert json.dumps(str(patched / "codex-rs/core")) in (source / ".cargo/config.toml").read_text() diff --git a/tests/tracing/test_opentelemetry.py b/tests/tracing/test_opentelemetry.py index 1c65745494..42f2508da7 100644 --- a/tests/tracing/test_opentelemetry.py +++ b/tests/tracing/test_opentelemetry.py @@ -46,7 +46,12 @@ def test_mapping_and_legacy_content_privacy(otel) -> None: with trace("group"), task_span("assistant"), agent_span("Worker"), turn_span(1, "Worker"): with generation_span(model="test-model", input=[{"content": "private-input"}]) as model: model.span_data.output = [{"content": "private-output"}] - model.span_data.usage = {"input_tokens": 13, "output_tokens": 5} + model.span_data.usage = { + "input_tokens": 13, + "output_tokens": 5, + "input_tokens_details": {"cached_tokens": 3, "cache_write_tokens": 4}, + "output_tokens_details": {"reasoning_tokens": 2}, + } with function_span("lookup", input="private-arguments") as tool: tool.span_data.output = "private-result" spans = exporter.get_finished_spans() @@ -63,6 +68,9 @@ def test_mapping_and_legacy_content_privacy(otel) -> None: assert inference.name == "chat test-model" assert inference.attributes["gen_ai.usage.input_tokens"] == 13 assert inference.attributes["gen_ai.usage.output_tokens"] == 5 + assert inference.attributes["gen_ai.usage.cache_read.input_tokens"] == 3 + assert inference.attributes["gen_ai.usage.cache_write.input_tokens"] == 4 + assert inference.attributes["gen_ai.usage.reasoning.output_tokens"] == 2 assert inference.parent.span_id == by_operation["openai.agents.turn"].context.span_id assert by_operation["execute_tool"].parent.span_id == inference.parent.span_id assert "private-" not in str([span.attributes for span in spans]) @@ -342,16 +350,26 @@ def test_tool_approval_mapping_and_partial_coverage(otel) -> None: ) -def test_marker_only_content_preserves_omission(otel) -> None: +@pytest.mark.parametrize( + "marker,reason", + [ + ({"truncated": True}, "truncated"), + ( + {"omitted": True, "reason": "unsupported_or_oversized_content"}, + "unsupported_or_oversized_content", + ), + ], +) +def test_marker_only_content_preserves_omission(otel, marker, reason) -> None: _, exporter = otel with trace("group"), generation_span(model="test-model") as model: model.span_data.operation.detail = "content" - model.span_data.input = [{"truncated": True}] - model.span_data.output = [{"truncated": True}] + model.span_data.input = [marker] + model.span_data.output = [marker] span = exporter.get_finished_spans()[0] for attribute in ("gen_ai.input.messages", "gen_ai.output.messages"): assert list(span.attributes[attribute][0]["parts"]) == [ - {"type": "openai.omitted", "reason": "truncated"} + {"type": "openai.omitted", "reason": reason} ] @@ -500,3 +518,29 @@ async def chunks(): {"type": "text", "content": "answer"}, {"type": "tool_call", "id": "call-lookup", "name": "lookup", "arguments": {"key": "value"}}, ] + + +@pytest.mark.parametrize("details", [None, 0]) +def test_missing_or_zero_usage_details_do_not_add_optional_counts(otel, details) -> None: + _, exporter = otel + usage: dict[str, object] = {"input_tokens": 13, "output_tokens": 5} + if details is not None: + usage.update( + { + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + ) + with trace("usage"), generation_span(model="test-model") as model: + model.span_data.usage = usage + (span,) = exporter.get_finished_spans() + assert span.attributes["gen_ai.usage.input_tokens"] == 13 + assert span.attributes["gen_ai.usage.output_tokens"] == 5 + assert not any( + key in span.attributes + for key in ( + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cache_write.input_tokens", + "gen_ai.usage.reasoning.output_tokens", + ) + )