diff --git a/symposium-eval/src/run/adversarial.rs b/symposium-eval/src/run/adversarial.rs index 57f493c3..784adaae 100644 --- a/symposium-eval/src/run/adversarial.rs +++ b/symposium-eval/src/run/adversarial.rs @@ -1,30 +1,18 @@ //! Real sandbox regressions with fake credentials and no model requests. use std::fs; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use tempfile::TempDir; -use super::test_support::{RealSrt, api_key, on_path}; +use super::test_support::{RealSrt, api_key, on_path, wait_for}; use super::{ApiKey, Configuration}; const DEADLINE: Duration = Duration::from_secs(60); -fn wait_for(path: &Path) { - let deadline = Instant::now() + DEADLINE; - while !path.is_file() { - assert!( - Instant::now() < deadline, - "never created {}", - path.display() - ); - thread::sleep(Duration::from_millis(20)); - } -} - #[test] #[ignore = "needs node, srt, and bubblewrap installed"] fn concurrent_trials_keep_temporary_files_and_processes_separate() { diff --git a/symposium-eval/src/run/error.rs b/symposium-eval/src/run/error.rs index d14e036c..6d7ce5b4 100644 --- a/symposium-eval/src/run/error.rs +++ b/symposium-eval/src/run/error.rs @@ -69,11 +69,6 @@ pub(crate) enum RunError { PathList(#[source] JoinPathsError), #[error("`{}` has not been written, so the trial cannot run", .0.display())] NoPolicy(PathBuf), - #[error( - "`{}` is too long for `srt`'s sockets; choose an evaluation root with a shorter path", - .0.display() - )] - TmpTooLong(PathBuf), #[error( "Cargo or rustfmt would read `{}` while grading, which could change the result; choose an evaluation root beneath no readable Cargo manifests, Cargo configuration, or rustfmt configuration", .0.display() diff --git a/symposium-eval/src/run/mod.rs b/symposium-eval/src/run/mod.rs index a3ccd9b4..8f488c8b 100644 --- a/symposium-eval/src/run/mod.rs +++ b/symposium-eval/src/run/mod.rs @@ -31,6 +31,8 @@ mod grading; mod inputs; mod policy; mod prelaunch; +#[cfg(all(test, target_os = "linux"))] +mod runtime_tmp; pub(crate) mod setup; #[cfg(all(test, target_os = "linux"))] mod smoke; diff --git a/symposium-eval/src/run/runtime_tmp.rs b/symposium-eval/src/run/runtime_tmp.rs new file mode 100644 index 00000000..f1c776a2 --- /dev/null +++ b/symposium-eval/src/run/runtime_tmp.rs @@ -0,0 +1,136 @@ +//! `srt`'s short `TMPDIR` alias under real `srt`: run on Linux with +//! `cargo test -p symposium-eval -- --ignored`. +//! +//! Each command's sandbox has its own `tmp/` at `/tmp/symposium-srt`, where `srt` makes its +//! proxy's sockets, so a deep evaluation root still runs, the command keeps its own +//! `TMPDIR`, and no two sandboxes share the alias. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use super::test_support::{RealSrt, api_key, wait_for}; +use super::{Configuration, Trial}; +use crate::grade::process::{Backend, Finished, Invocation}; +use crate::grade::{Grading, Outcome}; + +const DEADLINE: Duration = Duration::from_secs(120); + +/// An evaluation root 150 bytes deep, whose trial and grading `tmp/` paths are far too +/// long for a socket. +fn deep() -> PathBuf { + std::iter::repeat_n("a-directory-of-thirty-bytes-xx", 5).collect() +} + +/// The lines `script` prints, run with `sh -c` in `trial`'s sandbox. +fn lines(finished: &Finished) -> Vec { + let stdout = std::fs::read_to_string(finished.stdout.log()).unwrap(); + assert!(finished.succeeded(), "{}", finished.stderr.tail()); + stdout.lines().map(str::to_owned).collect() +} + +/// Prints the command's `TMPDIR`, then what is at the alias. +const SHOW_TMP: &str = r#"printf '%s\n' "$TMPDIR"; ls -a /tmp/symposium-srt"#; + +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn a_deep_trial_runs_with_its_own_tmpdir_and_srts_sockets_at_the_alias() { + let real = RealSrt::nested(api_key(), &deep()); + let trial = real.trial_of( + &crate::task::TaskName::new("formatting").unwrap(), + 1, + Configuration::Baseline, + ); + + let lines = lines(&real.run(&trial, SHOW_TMP, &[], DEADLINE)); + + assert_eq!(Path::new(&lines[0]), trial.tmp(), "{lines:?}"); + // The trial's network goes through a proxy, whose sockets `srt` made at the alias. + assert!( + lines.iter().any(|line| line.starts_with("claude-http-")), + "{lines:?}" + ); +} + +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn a_deep_trial_grades_with_its_own_tmpdir() { + let real = RealSrt::nested(api_key(), &deep()); + let trial = real.trial_of( + &crate::task::TaskName::new("formatting").unwrap(), + 1, + Configuration::Baseline, + ); + let solution = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../evals/tasks/formatting/solution/src/lib.rs"); + std::fs::copy(solution, trial.project().join("src/lib.rs")).unwrap(); + + let grading = real.grade(&trial, "formatting", &trial.project()); + let shown = lines(&grading_run(&real, &trial, SHOW_TMP)); + + let Grading::Graded(report) = &grading else { + panic!("expected a report, got {grading:?}"); + }; + assert!( + report + .steps + .iter() + .all(|step| step.outcome == Outcome::Passed), + "{report:?}" + ); + assert_eq!(Path::new(&shown[0]), trial.grading_tmp(), "{shown:?}"); +} + +/// Two sandboxes running at once each see only their own directory at the alias. +#[test] +#[ignore = "needs node, srt, and bubblewrap installed"] +fn concurrent_sandboxes_never_share_the_alias() { + let real = RealSrt::new(api_key()); + let first = real.trial(1, Configuration::Baseline); + let second = real.trial(2, Configuration::Baseline); + // Each leaves a marker in its own `tmp/` and says it is ready, then lists the alias + // once released, which happens only when both are ready: both are running then. + let script = r#"touch "$TMPDIR/marker-$1" +touch "$TMPDIR/ready" +while [ ! -f "$TMPDIR/release" ]; do sleep 0.02; done +ls /tmp/symposium-srt"#; + + let (first_lines, second_lines) = thread::scope(|scope| { + let first_run = + scope.spawn(|| lines(&real.run(&first, script, &["first".into()], DEADLINE))); + let second_run = + scope.spawn(|| lines(&real.run(&second, script, &["second".into()], DEADLINE))); + for trial in [&first, &second] { + wait_for(&trial.tmp().join("ready")); + } + for trial in [&first, &second] { + std::fs::write(trial.tmp().join("release"), "").unwrap(); + } + (first_run.join().unwrap(), second_run.join().unwrap()) + }); + + for (own, other, lines) in [ + ("marker-first", "marker-second", &first_lines), + ("marker-second", "marker-first", &second_lines), + ] { + assert!(lines.iter().any(|line| line == own), "{lines:?}"); + assert!(!lines.iter().any(|line| line == other), "{lines:?}"); + } +} + +/// Runs `script` with `sh -c` under `trial`'s grading policy. +fn grading_run(real: &RealSrt, trial: &Trial, script: &str) -> Finished { + let backend = real.srt().for_grading(trial).unwrap(); + let invocation = Invocation { + program: "sh".into(), + args: vec!["-c".into(), script.into()], + dir: trial.grading_work(), + env: vec![("PATH".into(), OsString::from("/usr/bin:/bin"))], + log_name: "show-tmp".to_owned(), + }; + std::fs::create_dir_all(trial.grading_tmp()).unwrap(); + backend + .run(&invocation, DEADLINE, &trial.grading_logs()) + .unwrap() +} diff --git a/symposium-eval/src/run/srt.rs b/symposium-eval/src/run/srt.rs index d67718d6..dfe7a270 100644 --- a/symposium-eval/src/run/srt.rs +++ b/symposium-eval/src/run/srt.rs @@ -15,6 +15,13 @@ //! sandbox inside, with its own PID namespace (`--unshare-pid --die-with-parent`), so //! killing the tree reaches every process, including one that called `setsid`. //! +//! The same outer namespace binds that `tmp/` at a short alias too, [`SRT_RUNTIME_TMP`], +//! which is `srt`'s own `TMPDIR`. `srt` makes its proxy's Unix sockets there, and a +//! socket's path is limited to 107 bytes, so a long evaluation root would otherwise +//! leave no room. Each outer namespace maps the alias to its own sandbox's directory, so +//! concurrent sandboxes never share one. The command itself still gets its own `tmp/` as +//! `TMPDIR`: it runs through `env`, since `srt` would otherwise set `/tmp/claude`. +//! //! Enforced mode is Linux only. On macOS trials would share `/tmp/claude`, and a process //! that leaves the process group outlives the deadline, since there is no PID namespace. @@ -39,6 +46,22 @@ const PROBE_DEADLINE: Duration = Duration::from_secs(30); /// Where `srt` lets every sandbox write. const SRT_SHARED_TMP: &str = "/tmp/claude"; +/// `srt`'s own `TMPDIR` inside each outer namespace: the sandbox's `tmp/`, by a path +/// short enough for the proxy's sockets whatever the evaluation root. +const SRT_RUNTIME_TMP: &str = "/tmp/symposium-srt"; + +/// The longest Unix socket path Linux allows, in bytes, its terminating NUL left out. +const MAX_SOCKET_PATH: usize = 107; + +/// What `srt` 0.0.81 adds to its `TMPDIR` for its proxy's sockets: +/// `/claude-socks-<16 hex>.sock` (`linux-sandbox-utils.js`, `initializeLinuxNetworkBridge`). +const SRT_SOCKET_NAME: usize = "/claude-socks-0123456789abcdef.sock".len(); + +const _: () = assert!(SRT_RUNTIME_TMP.len() + SRT_SOCKET_NAME <= MAX_SOCKET_PATH); + +/// Runs a command with the environment variables before it set. +const ENV: &str = "/usr/bin/env"; + /// The `srt` installation trials run under, on Linux. #[derive(Debug, Clone)] pub(crate) struct Srt { @@ -165,10 +188,6 @@ impl Srt { if !policy.is_file() { return Err(RunError::NoPolicy(policy)); } - // Only on Linux does `srt` bridge its proxy through Unix sockets. - if cfg!(target_os = "linux") && tmp.as_os_str().len() + SRT_SOCKET_NAME > MAX_SOCKET_PATH { - return Err(RunError::TmpTooLong(tmp)); - } Ok(Sandboxed { srt: self, policy, @@ -177,13 +196,6 @@ impl Srt { } } -/// The longest Unix socket path Linux allows, in bytes, its terminating NUL left out. -const MAX_SOCKET_PATH: usize = 107; - -/// What `srt` 0.0.81 adds to its `TMPDIR` for its proxy's sockets: -/// `/claude-socks-<16 hex>.sock` (`linux-sandbox-utils.js`, `initializeLinuxNetworkBridge`). -const SRT_SOCKET_NAME: usize = "/claude-socks-0123456789abcdef.sock".len(); - /// Whether `node --version` output, such as `v22.12.0`, is at least [`NODE_MINIMUM`]. fn node_is_supported(version: &str) -> bool { let mut parts = version.trim_start_matches('v').split('.'); @@ -196,18 +208,18 @@ fn node_is_supported(version: &str) -> bool { impl Srt { /// `invocation`, run by `srt` under `policy` inside the outer `bwrap`, which binds - /// `tmp` over `/tmp/claude`. + /// `tmp` over `/tmp/claude` and at [`SRT_RUNTIME_TMP`]. The command gets `tmp` as its + /// `TMPDIR` through `env`, over the one `srt` sets. fn wrap(&self, policy: &Path, tmp: &Path, invocation: &Invocation) -> Invocation { - let mut args: Vec = vec![ - "--dev-bind".into(), - "/".into(), - "/".into(), - // A bind needs its destination, and `srt` may not have created it yet. - "--dir".into(), - SRT_SHARED_TMP.into(), - "--bind".into(), - tmp.into(), - SRT_SHARED_TMP.into(), + let mut command_tmpdir = OsString::from("TMPDIR="); + command_tmpdir.push(tmp); + let mut args: Vec = vec!["--dev-bind".into(), "/".into(), "/".into()]; + for alias in [SRT_SHARED_TMP, SRT_RUNTIME_TMP] { + // A bind needs its destination, which may not exist yet. + args.extend(["--dir".into(), alias.into()]); + args.extend(["--bind".into(), tmp.into(), alias.into()]); + } + args.extend([ "--die-with-parent".into(), "--".into(), self.node.clone().into(), @@ -215,30 +227,28 @@ impl Srt { "--settings".into(), policy.into(), "--".into(), + ENV.into(), + command_tmpdir, invocation.program.clone(), - ]; + ]); args.extend(invocation.args.iter().cloned()); + let mut env = invocation.env.clone(); + env.retain(|(name, _)| name != "TMPDIR"); + env.push(("TMPDIR".into(), SRT_RUNTIME_TMP.into())); Invocation { program: self.bwrap.clone().into(), args, dir: invocation.dir.clone(), - env: invocation.env.clone(), + env, log_name: invocation.log_name.clone(), } } } impl Sandboxed<'_> { - /// `invocation`, run by `srt` under the policy, with its own `/tmp/claude`, and the - /// sandbox's directory as `TMPDIR`: `srt` makes its proxy's sockets there, so its - /// path must stay short. + /// `invocation`, run by `srt` under the policy, with this sandbox's own `tmp/`. fn wrap(&self, invocation: &Invocation) -> Invocation { - let mut invocation = invocation.clone(); - invocation.env.retain(|(name, _)| name != "TMPDIR"); - invocation - .env - .push(("TMPDIR".into(), self.tmp.clone().into())); - self.srt.wrap(&self.policy, &self.tmp, &invocation) + self.srt.wrap(&self.policy, &self.tmp, invocation) } } @@ -287,37 +297,31 @@ mod tests { .unwrap() } - #[test] - fn a_sandboxed_command_has_the_sandboxs_tmpdir() { - let host = Host::new(); - let trial = written_trial(&host); - let srt = srt(&host); - let mut invocation = probe(&trial); - invocation.env.push(("TMPDIR".into(), "/elsewhere".into())); - - let wrapped = srt.for_trial(&trial).unwrap().wrap(&invocation); - - let tmpdirs: Vec<_> = wrapped - .env - .iter() - .filter(|(name, _)| name == "TMPDIR") - .map(|(_, value)| PathBuf::from(value)) - .collect(); - assert_eq!(tmpdirs, [trial.tmp()]); + /// The `--bind` pairs the outer `bwrap` gets, as `(source, destination)`. + fn binds(wrapped: &Invocation) -> Vec<(OsString, OsString)> { + wrapped + .args + .windows(3) + .filter(|window| window[0] == "--bind") + .map(|window| (window[1].clone(), window[2].clone())) + .collect() } - #[cfg(target_os = "linux")] #[test] - fn a_tmp_too_long_for_srts_sockets_is_refused() { + fn grading_gets_its_own_tmp() { let host = Host::new(); let trial = written_trial(&host); - let srt = srt(&host); - let fits = PathBuf::from("/").join("t".repeat(MAX_SOCKET_PATH - SRT_SOCKET_NAME - 1)); - let long = fits.join("x"); + fs::create_dir_all(trial.grading()).unwrap(); + fs::write(trial.grading_policy_file(), "{}").unwrap(); + + let wrapped = srt(&host).for_grading(&trial).unwrap().wrap(&probe(&trial)); - assert!(srt.sandboxed(trial.policy_file(), fits).is_ok()); - let result = srt.sandboxed(trial.policy_file(), long); - assert!(matches!(result, Err(RunError::TmpTooLong(_))), "{result:?}"); + let sources: Vec<_> = binds(&wrapped) + .into_iter() + .map(|(source, _)| source) + .collect(); + let grading_tmp = trial.grading_tmp().into_os_string(); + assert_eq!(sources, [grading_tmp.clone(), grading_tmp]); } #[test] @@ -337,8 +341,13 @@ mod tests { let trial = written_trial(&host); let srt = srt(&host); - let wrapped = srt.for_trial(&trial).unwrap().wrap(&probe(&trial)); + let mut invocation = probe(&trial); + invocation.env.push(("TMPDIR".into(), "/elsewhere".into())); + + let wrapped = srt.for_trial(&trial).unwrap().wrap(&invocation); + let mut command_tmpdir = OsString::from("TMPDIR="); + command_tmpdir.push(trial.tmp()); let expected: Vec = vec![ "--dev-bind".into(), "/".into(), @@ -348,6 +357,11 @@ mod tests { "--bind".into(), trial.tmp().into(), "/tmp/claude".into(), + "--dir".into(), + "/tmp/symposium-srt".into(), + "--bind".into(), + trial.tmp().into(), + "/tmp/symposium-srt".into(), "--die-with-parent".into(), "--".into(), host.join("bin/node").into(), @@ -355,6 +369,8 @@ mod tests { "--settings".into(), trial.policy_file().into(), "--".into(), + "/usr/bin/env".into(), + command_tmpdir, "sh".into(), "-c".into(), "echo hello".into(), @@ -362,8 +378,9 @@ mod tests { assert_eq!(wrapped.program, host.join("bin/bwrap").as_os_str()); assert_eq!(wrapped.args, expected); assert_eq!(wrapped.dir, trial.project()); + // `srt` itself gets the short alias, never the command's own `TMPDIR`. let mut env = probe(&trial).env; - env.push(("TMPDIR".into(), trial.tmp().into())); + env.push(("TMPDIR".into(), "/tmp/symposium-srt".into())); assert_eq!(wrapped.env, env); } diff --git a/symposium-eval/src/run/test_support.rs b/symposium-eval/src/run/test_support.rs index 7ab1a6ed..8fb0c039 100644 --- a/symposium-eval/src/run/test_support.rs +++ b/symposium-eval/src/run/test_support.rs @@ -101,19 +101,27 @@ pub(crate) struct RealSrt { #[cfg(target_os = "linux")] impl RealSrt { pub(crate) fn new(api_key: ApiKey) -> Self { + Self::nested(api_key, std::path::Path::new("")) + } + + /// As [`Self::new`], with the evaluation root at `under` inside its temporary + /// directory, so a test can make the root's path as long as it needs. + pub(crate) fn nested(api_key: ApiKey, under: &std::path::Path) -> Self { let srt = installed_srt(); let decoy_home = decoy_home(); let machine = Machine::new(vec![decoy_home.path().to_owned()], &checkout(), &sysroot()).unwrap(); - let eval_root = TempDir::new().unwrap(); - srt.preflight(eval_root.path()).unwrap(); - let batch = Batch::create(eval_root.path(), &BatchName::new("batch").unwrap()).unwrap(); + let temp = TempDir::new().unwrap(); + let eval_root = temp.path().join(under); + fs::create_dir_all(&eval_root).unwrap(); + srt.preflight(&eval_root).unwrap(); + let batch = Batch::create(&eval_root, &BatchName::new("batch").unwrap()).unwrap(); Self { srt, machine, batch, api_key, - _eval_root: eval_root, + _eval_root: temp, decoy_home, } } @@ -386,6 +394,20 @@ fn decoy_home() -> TempDir { home } +/// Waits up to a minute for a sandboxed command to create the file at `path`. +#[cfg(target_os = "linux")] +pub(crate) fn wait_for(path: &std::path::Path) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while !path.is_file() { + assert!( + std::time::Instant::now() < deadline, + "never created {}", + path.display() + ); + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + /// Writes an executable shell script at `path`. #[cfg(unix)] pub(crate) fn executable(path: &std::path::Path, script: &str) {