Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions md/design/agent-eval.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion md/rfds/skill-evaluation/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ A hook record contradicted by the transcript, including more records than the tr

## Outcomes and counting

A trial can succeed only if its client session completed. A timeout or client error is a failure even when the code it left would pass, and so is a session the runner lost hold of or whose ending its transcript cannot show, since it did not complete either; that code is still graded and reported, and a grading infrastructure error during it does not move the trial out of the count. For a completed trial:
A trial can succeed only if its client session completed. A timeout or client error is a failure even when the code it left would pass, and so is a session the runner lost hold of or whose ending its transcript cannot show, since it did not complete either. Its code is still graded and reported when process containment is certain, and a grading infrastructure error during it does not move the trial out of the count. When the runner cannot prove every writer stopped, it records an `uncontained_session` failure without inspecting configuration, collecting filesystem evidence, or grading the submission; `code` and `expectations` are then `null`. For a completed trial:

| Kind | Succeeds when | Inconclusive when |
| ---------- | ---------------------------------------------- | --------------------------------------------------------------- |
Expand Down
24 changes: 24 additions & 0 deletions symposium-eval/src/client/claude.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,14 @@ impl Claude {
}

impl ClientRunner for Claude {
fn client(&self) -> Client {
Client::ClaudeCode
}

fn model(&self) -> &str {
&self.model
}

fn invocation(
&self,
trial: &Trial,
Expand Down Expand Up @@ -182,6 +190,22 @@ mod tests {
);
}

#[test]
fn the_model_launched_is_the_model_reported() {
let host = Host::new();
let trial = trial(&host.batch(), 1, Configuration::Symposium);
let claude = Claude::enforced(model(), version::MINIMUM);

let invocation = claude.invocation(&trial, "do the task", Vec::new());

let flag = invocation.args.iter().position(|arg| arg == "--model");
let launched = flag.and_then(|flag| invocation.args.get(flag + 1));
assert_eq!(
launched.map(|model| model.as_os_str()),
Some(claude.model().as_ref())
);
}

#[test]
fn an_exploratory_session_uses_the_trials_own_sandbox() {
let host = Host::new();
Expand Down
123 changes: 111 additions & 12 deletions symposium-eval/src/client/claude/transcript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ use super::hooks::{HookFields, Hooks};
use super::loads::Skills;
use super::version::Version;
use crate::client::{
Completion, InstalledSkill, Load, Malformation, ProvenanceError, Reading, Summary, Tokens,
TranscriptError, Unusable, Usage,
Completion, InstalledSkill, Load, Malformation, ProvenanceError, Reading, SessionIntroduction,
Summary, Tokens, TranscriptError, Unusable, Usage,
};
use crate::jsonl::{self, Line, MAX_LINE};

Expand Down Expand Up @@ -102,6 +102,10 @@ struct Reader<'a> {
/// Whether an `init` has been seen. One that cannot be read counts: the transcript
/// is then malformed, not unattributed.
introduced: bool,
/// A matching result appeared before an introduction. This is not itself a
/// provenance error: if no introduction follows, it proves only that the client
/// never reached the session boundary.
ended_before_introduction: bool,
/// Why the transcript cannot say how the session ended, if it cannot.
unusable: Option<Unusable>,
calls: Calls,
Expand Down Expand Up @@ -158,6 +162,7 @@ impl<'a> Reader<'a> {
usage: Usage::Unknown,
response: None,
introduced: false,
ended_before_introduction: false,
unusable: None,
calls: Calls::new(project),
hooks: Hooks::new(),
Expand Down Expand Up @@ -241,12 +246,19 @@ impl<'a> Reader<'a> {
Ok(()) => self.model = init.model,
Err(error) => self.mismatched(error),
}
if self.ended_before_introduction {
self.mismatched(ProvenanceError::Unstated);
}
}

fn ending(&mut self, ending: Ending<'_>) {
if !self.is_ours(&ending.session_id) {
return;
}
if !self.introduced {
self.mismatched(ProvenanceError::Unstated);
} else if self.is_ours(&ending.session_id) {
self.ended_before_introduction = true;
self.damage();
} else {
self.usage = ending.usage();
(self.completion, self.response) = ending.completion();
self.calls.finish();
Expand Down Expand Up @@ -334,6 +346,11 @@ impl<'a> Reader<'a> {
},
};
Reading {
introduction: if self.introduced {
SessionIntroduction::Observed
} else {
SessionIntroduction::NotObserved
},
summary,
completion: self.unusable.map_or(Ok(self.completion), Err),
loads: (self.calls.outcomes(&Skills::new(self.skills)))
Expand Down Expand Up @@ -427,14 +444,17 @@ mod tests {
session: Uuid,
version: Version,
) -> Result<Completion, Unusable> {
reading_as(transcript, session, version).completion
}

fn reading_as(transcript: &str, session: Uuid, version: Version) -> Reading {
read_text(
transcript,
Path::new("/trial/project"),
session,
version,
&[],
)
.completion
}

/// As the recorded session, run by the version it was recorded with.
Expand Down Expand Up @@ -872,23 +892,84 @@ mod tests {
}

#[test]
fn a_result_without_an_init_is_unstated() {
fn a_matching_result_without_an_init_only_shows_no_introduction() {
let transcript = without(FINISHED, "system/init");
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
completion(&transcript),
Err(Unusable::Provenance(ProvenanceError::Unstated))
));
assert_eq!(reading.completion.unwrap(), Completion::Missing);
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}

#[test]
fn a_result_before_the_init_is_unstated() {
let transcript = format!("{}\n{FINISHED}", recorded_result());
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
completion(&transcript),
reading.completion,
Err(Unusable::Provenance(ProvenanceError::Unstated))
));
assert_eq!(reading.introduction, SessionIntroduction::Observed);
}

#[test]
fn a_foreign_result_without_an_init_is_mismatched() {
let other = Uuid::new_v4();
let transcript = format!("{}\n", as_session(recorded_result(), other));
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
reading.completion,
Err(Unusable::Provenance(ProvenanceError::Session { .. }))
));
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}

#[test]
fn a_foreign_message_without_an_init_is_mismatched() {
let other = Uuid::new_v4();
let assistant = first(FINISHED, "assistant");
let transcript = format!("{}\n", as_session(assistant, other));
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
reading.completion,
Err(Unusable::Provenance(ProvenanceError::Session { .. }))
));
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}

#[test]
fn transcript_damage_outranks_a_matching_result_without_an_init_in_either_order() {
for transcript in [
format!("not json\n{}\n", recorded_result()),
format!("{}\nnot json\n", recorded_result()),
] {
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
reading.completion,
Err(Unusable::Transcript(TranscriptError::Malformed { .. }))
));
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}
}

#[test]
fn foreign_evidence_outranks_a_matching_result_without_an_init_in_either_order() {
let foreign = as_session(first(FINISHED, "assistant"), Uuid::new_v4());
for transcript in [
format!("{}\n{foreign}\n", recorded_result()),
format!("{foreign}\n{}\n", recorded_result()),
] {
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
reading.completion,
Err(Unusable::Provenance(ProvenanceError::Session { .. }))
));
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}
}

#[test]
Expand Down Expand Up @@ -963,8 +1044,26 @@ mod tests {
fn an_init_without_its_version_is_malformed() {
let init = without_field(recorded_init(), "claude_code_version");
let transcript = format!("{init}\n{}\n", recorded_result());
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert_eq!(malformed_line(completion(&transcript)), 1);
assert_eq!(malformed_line(reading.completion), 1);
assert_eq!(reading.introduction, SessionIntroduction::Observed);
}

#[test]
fn damaged_output_without_an_init_does_not_prove_absence() {
for transcript in [
"not json\n".to_owned(),
format!("{}\n", "x".repeat(MAX_LINE + 1)),
] {
let reading = reading_as(&transcript, recorded_session(), version::MINIMUM);

assert!(matches!(
reading.completion,
Err(Unusable::Transcript(TranscriptError::Malformed { .. }))
));
assert_eq!(reading.introduction, SessionIntroduction::NotObserved);
}
}

#[test]
Expand Down
13 changes: 13 additions & 0 deletions symposium-eval/src/client/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,19 @@ use std::process::ExitStatus;

use thiserror::Error;

use crate::grade::ProcessError;

/// Why a client never reached the beginning of a session.
#[derive(Debug, Error)]
pub(crate) enum LaunchError {
/// The runner could not start the client process.
#[error(transparent)]
Process(#[from] ProcessError),
/// A wrapper process started, but the client exited before introducing a session.
#[error("the client exited with {status} before starting a session: {stderr}")]
Exited { status: ExitStatus, stderr: String },
}

/// Why a session failed, by the client's own doing.
#[derive(Debug, Error)]
pub(crate) enum ClientError {
Expand Down
Loading
Loading