Refactor: Centralize benchmark model configuration

Move benchmark model definitions to a new config file and update scripts to use it.

Co-authored-by: frank <frank@anomalyinnovations.com>

Author: cursoragent

scripts/benchmark-config.ts

@@ -0,0 +1,23 @@
+export const BENCHMARK_MODELS: Record<string, string[]> = {
+  "claude-code": [
+    "claude-sonnet-4-5",
+    "claude-opus-4-5",
+  ],
+  codex: [
+    "gpt-5-codex",
+    "gpt-5.1-codex",
+  ],
+  opencode: [
+    "opencode/gpt-5-codex",
+    "opencode/gpt-5.1-codex",
+    "opencode/claude-sonnet-4-5",
+    "opencode/claude-opus-4-5",
+    "opencode/glm-4.6",
+    "opencode/glm-4.7-free",
+    "opencode/gemini-3-pro",
+    "opencode/qwen3-coder",
+    "opencode/kimi-k2",
+    "opencode/grok-code",
+    "opencode/alpha-gd4",
+  ],
+};

scripts/generate-benchmark-matrix.ts

@@ -1,17 +1,19 @@
 #!/usr/bin/env bun
 import { Agent } from "~/agents/index.js";
 import { Task } from "~/src/tasks/index.js";
+import { BENCHMARK_MODELS } from "./benchmark-config.js";
 
 const agents = Agent.list();
 const tasks = await Task.listNames();
 const include = tasks.flatMap((task) =>
-  agents.flatMap((agent) =>
-    agent.models.map((model) => ({
+  agents.flatMap((agent) => {
+    const models = BENCHMARK_MODELS[agent.name] || [];
+    return models.map((model) => ({
       eval: task,
       model,
       agent: agent.name,
-    })),
-  ),
+    }));
+  }),
 );
 
 const matrix = JSON.stringify({ include });

scripts/sync-workflow-inputs.ts

@@ -9,6 +9,7 @@
 
 import { readFileSync, writeFileSync } from "node:fs";
 import { Agent } from "~/agents/index.js";
+import { BENCHMARK_MODELS } from "./benchmark-config.js";
 import YAML from "yaml";
 
 interface WorkflowInput {
@@ -43,7 +44,8 @@ async function main(): Promise<void> {
   const combinations: Array<{ agent: string; model: string }> = [];
 
   for (const agent of agents) {
-    for (const model of agent.models) {
+    const models = BENCHMARK_MODELS[agent.name] || [];
+    for (const model of models) {
       combinations.push({ agent: agent.name, model });
     }
   }

src/agents/claude-code.ts

@@ -8,14 +8,6 @@ import { Logger } from "../util/logger.js";
 
 const sessionCache = new Map<string, string>();
 
-export const models: string[] = [
-  "claude-sonnet-4-5",
-  "claude-opus-4-5",
-  // "claude-sonnet-4",
-  // "claude-opus-4-1",
-  // "claude-3-5-haiku",
-];
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }

src/agents/codex.ts

@@ -16,14 +16,6 @@ const DEFAULT_SANDBOX: SandboxMode = "workspace-write";
 const codexClient = new Codex();
 const threadCache = new Map<string, Thread>();
 
-export const models = [
-  "gpt-5-codex",
-  "gpt-5.1-codex",
-  // "gpt-5",
-  // "o3",
-  // "o4-mini"
-] as const;
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }
@@ -67,7 +59,7 @@ function getOrCreateThread(model: string, cwd: string): Thread {
   return thread;
 }
 
-const codexAgent: Agent.Definition<(typeof models)[number]> = {
+const codexAgent: Agent.Definition = {
   async run(model, prompt, options) {
     options.logger.log(
       `codex-sdk --model ${model} --sandbox ${DEFAULT_SANDBOX} ${prompt}`,

src/agents/index.ts

@@ -20,9 +20,9 @@ export namespace Agent {
     prompt: Prompt,
   ) => CommandSpec | Promise<CommandSpec>;
 
-  export interface Definition<TModel extends string = string> {
+  export interface Definition {
     run: (
-      model: TModel,
+      model: string,
       prompt: Prompt,
       options: RunOptions,
     ) => Promise<RunResult>;
@@ -43,33 +43,29 @@ export namespace Agent {
     logger: Logger.Instance;
   }
 
-  export interface Registration<TModel extends string = string> {
+  export interface Registration {
     name: string;
-    definition: Definition<TModel>;
-    models: ReadonlyArray<TModel>;
+    definition: Definition;
   }
 
-  const agents: Record<string, Registration<any>> = {
+  const agents: Record<string, Registration> = {
     // Only keep opencode active while debugging timeouts for specific models.
     opencode: createRegistration("opencode", opencodeAgent),
     //codex: createRegistration("codex", codexAgent),
     //"claude-code": createRegistration("claude-code", claudeCodeAgent),
   };
 
-  function createRegistration<TModel extends string>(
+  function createRegistration(
     name: string,
     module: {
-      default?: Definition<TModel>;
-      models?: ReadonlyArray<TModel>;
+      default?: Definition;
     },
-  ): Registration<TModel> {
+  ): Registration {
     const definition = module.default;
-    const models = module.models;
 
     assert(definition, `Agent module ${name} is missing a default export.`);
-    assert(models, `Agent module ${name} is missing the exported models list.`);
 
-    return { name, definition, models };
+    return { name, definition };
   }
 
   export function get(name: string): Registration {
@@ -78,13 +74,6 @@ export namespace Agent {
     return agent;
   }
 
-  export function validateModel(agent: Registration, model: string) {
-    if (!agent.models.find((entry) => entry === model))
-      throw new Error(
-        `Model ${model} is not registered for agent ${agent.name}.`,
-      );
-  }
-
   export function list() {
     return Object.values(agents);
   }

src/agents/opencode.ts

@@ -36,20 +36,6 @@ const opencode = await createOpencode({
 
 const sessionCache = new Map<string, string>();
 
-export const models: string[] = [
-  "opencode/gpt-5-codex",
-  "opencode/gpt-5.1-codex",
-  "opencode/claude-sonnet-4-5",
-  "opencode/claude-opus-4-5",
-  "opencode/glm-4.6",
-  "opencode/glm-4.7-free",
-  "opencode/gemini-3-pro",
-  "opencode/qwen3-coder",
-  "opencode/kimi-k2",
-  "opencode/grok-code",
-  "opencode/alpha-gd4",
-];
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }

src/eval.ts

@@ -47,7 +47,6 @@ export namespace Eval {
     },
   ) {
     const agent = Agent.get(agentName);
-    Agent.validateModel(agent, modelId);
     const task = await Task.get(taskId);
     const cwd = await mkdtemp(join(tmpdir(), "openreval-"));
     $.cwd(cwd);