From a1c94d7a3aafb353036619db712ddb5e108d3321 Mon Sep 17 00:00:00 2001 From: Awesome Rustacean Date: Sat, 7 Nov 2026 01:08:12 +0300 Subject: [PATCH] test(eval): give srt a short runtime directory Bind each sandbox's own tmp/ at /tmp/symposium-srt in the outer bwrap and make that srt's TMPDIR, so its proxy sockets fit Linux's 107-byte limit however deep the evaluation root. Each namespace maps the alias to its own sandbox, so concurrent sandboxes never share it. Run the command through env with its sandbox's tmp/ as TMPDIR, over the /tmp/claude srt would set, and drop the evaluation root length check. --- symposium-eval/src/run/adversarial.rs | 18 +--- symposium-eval/src/run/error.rs | 5 - symposium-eval/src/run/mod.rs | 2 + symposium-eval/src/run/runtime_tmp.rs | 136 ++++++++++++++++++++++++ symposium-eval/src/run/srt.rs | 137 ++++++++++++++----------- symposium-eval/src/run/test_support.rs | 30 +++++- 6 files changed, 244 insertions(+), 84 deletions(-) create mode 100644 symposium-eval/src/run/runtime_tmp.rs diff --git a/symposium-eval/src/run/adversarial.rs b/symposium-eval/src/run/adversarial.rs index 3f051354a..c7fc7bedb 100644 --- a/symposium-eval/src/run/adversarial.rs +++ b/symposium-eval/src/run/adversarial.rs @@ -1,30 +1,18 @@ //! Real sandbox regressions with fake credentials and no model requests. use std::fs; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use tempfile::TempDir; -use super::test_support::{RealSrt, api_key, on_path}; +use super::test_support::{RealSrt, api_key, on_path, wait_for}; use super::{ApiKey, Configuration}; const DEADLINE: Duration = Duration::from_secs(60); -fn wait_for(path: &Path) { - let deadline = Instant::now() + DEADLINE; - while !path.is_file() { - assert!( - Instant::now() < deadline, - "never created {}", - path.display() - ); - thread::sleep(Duration::from_millis(20)); - } -} - #[test] #[ignore = "needs node, srt, and bubblewrap installed"] fn concurrent_trials_keep_temporary_files_and_processes_separate() { diff --git a/symposium-eval/src/run/error.rs b/symposium-eval/src/run/error.rs index d9e8549a9..6f608bb3f 100644 --- a/symposium-eval/src/run/error.rs +++ b/symposium-eval/src/run/error.rs @@ -69,11 +69,6 @@ pub(crate) enum RunError { PathList(#[source] JoinPathsError), #[error("`{}` has not been written, so the trial cannot run", .0.display())] NoPolicy(PathBuf), - #[error( - "`{}` is too long for `srt`'s sockets; choose an evaluation root with a shorter path", - .0.display() - )] - TmpTooLong(PathBuf), #[error( "Cargo or rustfmt would read `{}` while grading, which could change the result; choose an evaluation root beneath no readable Cargo manifests, Cargo configuration, or rustfmt configuration", .0.display() diff --git a/symposium-eval/src/run/mod.rs b/symposium-eval/src/run/mod.rs index 421b61345..acd26fe68 100644 --- a/symposium-eval/src/run/mod.rs +++ b/symposium-eval/src/run/mod.rs @@ -31,6 +31,8 @@ mod grading; mod inputs; mod policy; mod prelaunch; +#[cfg(all(test, target_os = "linux"))] +mod runtime_tmp; pub(crate) mod setup; #[cfg(all(test, target_os = "linux"))] mod smoke; diff --git a/symposium-eval/src/run/runtime_tmp.rs b/symposium-eval/src/run/runtime_tmp.rs new file mode 100644 index 000000000..f1c776a21 --- /dev/null +++ b/symposium-eval/src/run/runtime_tmp.rs @@ -0,0 +1,136 @@ +//! `srt`'s short `TMPDIR` alias under real `srt`: run on Linux with +//! `cargo test -p symposium-eval -- --ignored`. +//! +//! Each command's sandbox has its own `tmp/` at `/tmp/symposium-srt`, where `srt` makes its +//! proxy's sockets, so a deep evaluation root still runs, the command keeps its own +//! `TMPDIR`, and no two sandboxes share the alias. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use super::test_support::{RealSrt, api_key, wait_for}; +use super::{Configuration, Trial}; +use crate::grade::process::{Backend, Finished, Invocation}; +use crate::grade::{Grading, Outcome}; + +const DEADLINE: Duration = Duration::from_secs(120); + +/// An evaluation root 150 bytes deep, whose trial and grading `tmp/` paths are far too +/// long for a socket. +fn deep() -> PathBuf { + std::iter::repeat_n("a-directory-of-thirty-bytes-xx", 5).collect() +} + +/// The lines `script` prints, run with `sh -c` in `trial`'s sandbox. +fn lines(finished: &Finished) -> Vec { + let stdout = std::fs::read_to_string(finished.stdout.log()).unwrap(); + assert!(finished.succeeded(), "{}", finished.stderr.tail()); + stdout.lines().map(str::to_owned).collect() +} + +/// Prints the command's `TMPDIR`, then what is at the alias. +const SHOW_TMP: &str = r#"printf '%s\n' "$TMPDIR"; ls -a /tmp/symposium-srt"#; + +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn a_deep_trial_runs_with_its_own_tmpdir_and_srts_sockets_at_the_alias() { + let real = RealSrt::nested(api_key(), &deep()); + let trial = real.trial_of( + &crate::task::TaskName::new("formatting").unwrap(), + 1, + Configuration::Baseline, + ); + + let lines = lines(&real.run(&trial, SHOW_TMP, &[], DEADLINE)); + + assert_eq!(Path::new(&lines[0]), trial.tmp(), "{lines:?}"); + // The trial's network goes through a proxy, whose sockets `srt` made at the alias. + assert!( + lines.iter().any(|line| line.starts_with("claude-http-")), + "{lines:?}" + ); +} + +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn a_deep_trial_grades_with_its_own_tmpdir() { + let real = RealSrt::nested(api_key(), &deep()); + let trial = real.trial_of( + &crate::task::TaskName::new("formatting").unwrap(), + 1, + Configuration::Baseline, + ); + let solution = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../evals/tasks/formatting/solution/src/lib.rs"); + std::fs::copy(solution, trial.project().join("src/lib.rs")).unwrap(); + + let grading = real.grade(&trial, "formatting", &trial.project()); + let shown = lines(&grading_run(&real, &trial, SHOW_TMP)); + + let Grading::Graded(report) = &grading else { + panic!("expected a report, got {grading:?}"); + }; + assert!( + report + .steps + .iter() + .all(|step| step.outcome == Outcome::Passed), + "{report:?}" + ); + assert_eq!(Path::new(&shown[0]), trial.grading_tmp(), "{shown:?}"); +} + +/// Two sandboxes running at once each see only their own directory at the alias. +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn concurrent_sandboxes_never_share_the_alias() { + let real = RealSrt::new(api_key()); + let first = real.trial(1, Configuration::Baseline); + let second = real.trial(2, Configuration::Baseline); + // Each leaves a marker in its own `tmp/` and says it is ready, then lists the alias + // once released, which happens only when both are ready: both are running then. + let script = r#"touch "$TMPDIR/marker-$1" +touch "$TMPDIR/ready" +while [ ! -f "$TMPDIR/release" ]; do sleep 0.02; done +ls /tmp/symposium-srt"#; + + let (first_lines, second_lines) = thread::scope(|scope| { + let first_run = + scope.spawn(|| lines(&real.run(&first, script, &["first".into()], DEADLINE))); + let second_run = + scope.spawn(|| lines(&real.run(&second, script, &["second".into()], DEADLINE))); + for trial in [&first, &second] { + wait_for(&trial.tmp().join("ready")); + } + for trial in [&first, &second] { + std::fs::write(trial.tmp().join("release"), "").unwrap(); + } + (first_run.join().unwrap(), second_run.join().unwrap()) + }); + + for (own, other, lines) in [ + ("marker-first", "marker-second", &first_lines), + ("marker-second", "marker-first", &second_lines), + ] { + assert!(lines.iter().any(|line| line == own), "{lines:?}"); + assert!(!lines.iter().any(|line| line == other), "{lines:?}"); + } +} + +/// Runs `script` with `sh -c` under `trial`'s grading policy. +fn grading_run(real: &RealSrt, trial: &Trial, script: &str) -> Finished { + let backend = real.srt().for_grading(trial).unwrap(); + let invocation = Invocation { + program: "sh".into(), + args: vec!["-c".into(), script.into()], + dir: trial.grading_work(), + env: vec![("PATH".into(), OsString::from("/usr/bin:/bin"))], + log_name: "show-tmp".to_owned(), + }; + std::fs::create_dir_all(trial.grading_tmp()).unwrap(); + backend + .run(&invocation, DEADLINE, &trial.grading_logs()) + .unwrap() +} diff --git a/symposium-eval/src/run/srt.rs b/symposium-eval/src/run/srt.rs index d67718d60..dfe7a270b 100644 --- a/symposium-eval/src/run/srt.rs +++ b/symposium-eval/src/run/srt.rs @@ -15,6 +15,13 @@ //! sandbox inside, with its own PID namespace (`--unshare-pid --die-with-parent`), so //! killing the tree reaches every process, including one that called `setsid`. //! +//! The same outer namespace binds that `tmp/` at a short alias too, [`SRT_RUNTIME_TMP`], +//! which is `srt`'s own `TMPDIR`. `srt` makes its proxy's Unix sockets there, and a +//! socket's path is limited to 107 bytes, so a long evaluation root would otherwise +//! leave no room. Each outer namespace maps the alias to its own sandbox's directory, so +//! concurrent sandboxes never share one. The command itself still gets its own `tmp/` as +//! `TMPDIR`: it runs through `env`, since `srt` would otherwise set `/tmp/claude`. +//! //! Enforced mode is Linux only. On macOS trials would share `/tmp/claude`, and a process //! that leaves the process group outlives the deadline, since there is no PID namespace. @@ -39,6 +46,22 @@ const PROBE_DEADLINE: Duration = Duration::from_secs(30); /// Where `srt` lets every sandbox write. const SRT_SHARED_TMP: &str = "/tmp/claude"; +/// `srt`'s own `TMPDIR` inside each outer namespace: the sandbox's `tmp/`, by a path +/// short enough for the proxy's sockets whatever the evaluation root. +const SRT_RUNTIME_TMP: &str = "/tmp/symposium-srt"; + +/// The longest Unix socket path Linux allows, in bytes, its terminating NUL left out. +const MAX_SOCKET_PATH: usize = 107; + +/// What `srt` 0.0.81 adds to its `TMPDIR` for its proxy's sockets: +/// `/claude-socks-<16 hex>.sock` (`linux-sandbox-utils.js`, `initializeLinuxNetworkBridge`). +const SRT_SOCKET_NAME: usize = "/claude-socks-0123456789abcdef.sock".len(); + +const _: () = assert!(SRT_RUNTIME_TMP.len() + SRT_SOCKET_NAME <= MAX_SOCKET_PATH); + +/// Runs a command with the environment variables before it set. +const ENV: &str = "/usr/bin/env"; + /// The `srt` installation trials run under, on Linux. #[derive(Debug, Clone)] pub(crate) struct Srt { @@ -165,10 +188,6 @@ impl Srt { if !policy.is_file() { return Err(RunError::NoPolicy(policy)); } - // Only on Linux does `srt` bridge its proxy through Unix sockets. - if cfg!(target_os = "linux") && tmp.as_os_str().len() + SRT_SOCKET_NAME > MAX_SOCKET_PATH { - return Err(RunError::TmpTooLong(tmp)); - } Ok(Sandboxed { srt: self, policy, @@ -177,13 +196,6 @@ impl Srt { } } -/// The longest Unix socket path Linux allows, in bytes, its terminating NUL left out. -const MAX_SOCKET_PATH: usize = 107; - -/// What `srt` 0.0.81 adds to its `TMPDIR` for its proxy's sockets: -/// `/claude-socks-<16 hex>.sock` (`linux-sandbox-utils.js`, `initializeLinuxNetworkBridge`). -const SRT_SOCKET_NAME: usize = "/claude-socks-0123456789abcdef.sock".len(); - /// Whether `node --version` output, such as `v22.12.0`, is at least [`NODE_MINIMUM`]. fn node_is_supported(version: &str) -> bool { let mut parts = version.trim_start_matches('v').split('.'); @@ -196,18 +208,18 @@ fn node_is_supported(version: &str) -> bool { impl Srt { /// `invocation`, run by `srt` under `policy` inside the outer `bwrap`, which binds - /// `tmp` over `/tmp/claude`. + /// `tmp` over `/tmp/claude` and at [`SRT_RUNTIME_TMP`]. The command gets `tmp` as its + /// `TMPDIR` through `env`, over the one `srt` sets. fn wrap(&self, policy: &Path, tmp: &Path, invocation: &Invocation) -> Invocation { - let mut args: Vec = vec![ - "--dev-bind".into(), - "/".into(), - "/".into(), - // A bind needs its destination, and `srt` may not have created it yet. - "--dir".into(), - SRT_SHARED_TMP.into(), - "--bind".into(), - tmp.into(), - SRT_SHARED_TMP.into(), + let mut command_tmpdir = OsString::from("TMPDIR="); + command_tmpdir.push(tmp); + let mut args: Vec = vec!["--dev-bind".into(), "/".into(), "/".into()]; + for alias in [SRT_SHARED_TMP, SRT_RUNTIME_TMP] { + // A bind needs its destination, which may not exist yet. + args.extend(["--dir".into(), alias.into()]); + args.extend(["--bind".into(), tmp.into(), alias.into()]); + } + args.extend([ "--die-with-parent".into(), "--".into(), self.node.clone().into(), @@ -215,30 +227,28 @@ impl Srt { "--settings".into(), policy.into(), "--".into(), + ENV.into(), + command_tmpdir, invocation.program.clone(), - ]; + ]); args.extend(invocation.args.iter().cloned()); + let mut env = invocation.env.clone(); + env.retain(|(name, _)| name != "TMPDIR"); + env.push(("TMPDIR".into(), SRT_RUNTIME_TMP.into())); Invocation { program: self.bwrap.clone().into(), args, dir: invocation.dir.clone(), - env: invocation.env.clone(), + env, log_name: invocation.log_name.clone(), } } } impl Sandboxed<'_> { - /// `invocation`, run by `srt` under the policy, with its own `/tmp/claude`, and the - /// sandbox's directory as `TMPDIR`: `srt` makes its proxy's sockets there, so its - /// path must stay short. + /// `invocation`, run by `srt` under the policy, with this sandbox's own `tmp/`. fn wrap(&self, invocation: &Invocation) -> Invocation { - let mut invocation = invocation.clone(); - invocation.env.retain(|(name, _)| name != "TMPDIR"); - invocation - .env - .push(("TMPDIR".into(), self.tmp.clone().into())); - self.srt.wrap(&self.policy, &self.tmp, &invocation) + self.srt.wrap(&self.policy, &self.tmp, invocation) } } @@ -287,37 +297,31 @@ mod tests { .unwrap() } - #[test] - fn a_sandboxed_command_has_the_sandboxs_tmpdir() { - let host = Host::new(); - let trial = written_trial(&host); - let srt = srt(&host); - let mut invocation = probe(&trial); - invocation.env.push(("TMPDIR".into(), "/elsewhere".into())); - - let wrapped = srt.for_trial(&trial).unwrap().wrap(&invocation); - - let tmpdirs: Vec<_> = wrapped - .env - .iter() - .filter(|(name, _)| name == "TMPDIR") - .map(|(_, value)| PathBuf::from(value)) - .collect(); - assert_eq!(tmpdirs, [trial.tmp()]); + /// The `--bind` pairs the outer `bwrap` gets, as `(source, destination)`. + fn binds(wrapped: &Invocation) -> Vec<(OsString, OsString)> { + wrapped + .args + .windows(3) + .filter(|window| window[0] == "--bind") + .map(|window| (window[1].clone(), window[2].clone())) + .collect() } - #[cfg(target_os = "linux")] #[test] - fn a_tmp_too_long_for_srts_sockets_is_refused() { + fn grading_gets_its_own_tmp() { let host = Host::new(); let trial = written_trial(&host); - let srt = srt(&host); - let fits = PathBuf::from("/").join("t".repeat(MAX_SOCKET_PATH - SRT_SOCKET_NAME - 1)); - let long = fits.join("x"); + fs::create_dir_all(trial.grading()).unwrap(); + fs::write(trial.grading_policy_file(), "{}").unwrap(); + + let wrapped = srt(&host).for_grading(&trial).unwrap().wrap(&probe(&trial)); - assert!(srt.sandboxed(trial.policy_file(), fits).is_ok()); - let result = srt.sandboxed(trial.policy_file(), long); - assert!(matches!(result, Err(RunError::TmpTooLong(_))), "{result:?}"); + let sources: Vec<_> = binds(&wrapped) + .into_iter() + .map(|(source, _)| source) + .collect(); + let grading_tmp = trial.grading_tmp().into_os_string(); + assert_eq!(sources, [grading_tmp.clone(), grading_tmp]); } #[test] @@ -337,8 +341,13 @@ mod tests { let trial = written_trial(&host); let srt = srt(&host); - let wrapped = srt.for_trial(&trial).unwrap().wrap(&probe(&trial)); + let mut invocation = probe(&trial); + invocation.env.push(("TMPDIR".into(), "/elsewhere".into())); + + let wrapped = srt.for_trial(&trial).unwrap().wrap(&invocation); + let mut command_tmpdir = OsString::from("TMPDIR="); + command_tmpdir.push(trial.tmp()); let expected: Vec = vec![ "--dev-bind".into(), "/".into(), @@ -348,6 +357,11 @@ mod tests { "--bind".into(), trial.tmp().into(), "/tmp/claude".into(), + "--dir".into(), + "/tmp/symposium-srt".into(), + "--bind".into(), + trial.tmp().into(), + "/tmp/symposium-srt".into(), "--die-with-parent".into(), "--".into(), host.join("bin/node").into(), @@ -355,6 +369,8 @@ mod tests { "--settings".into(), trial.policy_file().into(), "--".into(), + "/usr/bin/env".into(), + command_tmpdir, "sh".into(), "-c".into(), "echo hello".into(), @@ -362,8 +378,9 @@ mod tests { assert_eq!(wrapped.program, host.join("bin/bwrap").as_os_str()); assert_eq!(wrapped.args, expected); assert_eq!(wrapped.dir, trial.project()); + // `srt` itself gets the short alias, never the command's own `TMPDIR`. let mut env = probe(&trial).env; - env.push(("TMPDIR".into(), trial.tmp().into())); + env.push(("TMPDIR".into(), "/tmp/symposium-srt".into())); assert_eq!(wrapped.env, env); } diff --git a/symposium-eval/src/run/test_support.rs b/symposium-eval/src/run/test_support.rs index 7fd6bb59f..11c132268 100644 --- a/symposium-eval/src/run/test_support.rs +++ b/symposium-eval/src/run/test_support.rs @@ -101,19 +101,27 @@ pub(crate) struct RealSrt { #[cfg(target_os = "linux")] impl RealSrt { pub(crate) fn new(api_key: ApiKey) -> Self { + Self::nested(api_key, std::path::Path::new("")) + } + + /// As [`Self::new`], with the evaluation root at `under` inside its temporary + /// directory, so a test can make the root's path as long as it needs. + pub(crate) fn nested(api_key: ApiKey, under: &std::path::Path) -> Self { let srt = installed_srt(); let decoy_home = decoy_home(); let machine = Machine::new(vec![decoy_home.path().to_owned()], &checkout(), &sysroot()).unwrap(); - let eval_root = TempDir::new().unwrap(); - srt.preflight(eval_root.path()).unwrap(); - let batch = Batch::create(eval_root.path(), &BatchName::new("batch").unwrap()).unwrap(); + let temp = TempDir::new().unwrap(); + let eval_root = temp.path().join(under); + fs::create_dir_all(&eval_root).unwrap(); + srt.preflight(&eval_root).unwrap(); + let batch = Batch::create(&eval_root, &BatchName::new("batch").unwrap()).unwrap(); Self { srt, machine, batch, api_key, - _eval_root: eval_root, + _eval_root: temp, decoy_home, } } @@ -386,6 +394,20 @@ fn decoy_home() -> TempDir { home } +/// Waits up to a minute for a sandboxed command to create the file at `path`. +#[cfg(target_os = "linux")] +pub(crate) fn wait_for(path: &std::path::Path) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while !path.is_file() { + assert!( + std::time::Instant::now() < deadline, + "never created {}", + path.display() + ); + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + /// Writes an executable shell script at `path`. #[cfg(unix)] pub(crate) fn executable(path: &std::path::Path, script: &str) {