Refactor: Use env vars for agent/model selection

Remove hardcoded agent/model lists and use environment variables for flexibility.

Co-authored-by: frank <frank@anomalyinnovations.com>

Author: cursoragent

package-lock.json

@@ -1,17 +1,32 @@
 #!/usr/bin/env bun
-import { Agent } from "~/agents/index.js";
 import { Task } from "~/src/tasks/index.js";
 
-const agents = Agent.list();
+// Models are passed via BENCHMARK_MODELS environment variable (comma-separated agent:model pairs)
+// Example: BENCHMARK_MODELS="opencode:opencode/gpt-5-codex,opencode:opencode/claude-sonnet-4-5"
+const modelsEnv = process.env.BENCHMARK_MODELS;
+if (!modelsEnv) {
+  console.error(
+    "Error: BENCHMARK_MODELS environment variable is required (comma-separated agent:model pairs)",
+  );
+  process.exit(1);
+}
+
+const agentModelPairs = modelsEnv.split(",").map((pair) => {
+  const [agent, model] = pair.split(":");
+  if (!agent || !model) {
+    console.error(`Invalid agent:model pair: ${pair}`);
+    process.exit(1);
+  }
+  return { agent, model };
+});
+
 const tasks = await Task.listNames();
 const include = tasks.flatMap((task) =>
-  agents.flatMap((agent) =>
-    agent.models.map((model) => ({
-      eval: task,
-      model,
-      agent: agent.name,
-    })),
-  ),
+  agentModelPairs.map(({ agent, model }) => ({
+    eval: task,
+    model,
+    agent,
+  })),
 );
 
 const matrix = JSON.stringify({ include });

scripts/generate-benchmark-matrix.ts

@@ -1,14 +1,12 @@
 #!/usr/bin/env bun
 /**
- * Syncs workflow_dispatch inputs in compare-models.yml with available agent:model combinations.
- * Run this after modifying agent model lists to keep the workflow in sync.
+ * Syncs workflow_dispatch inputs in compare-models.yml with provided agent:model combinations.
  *
  * Usage:
- *   bun run scripts/sync-workflow-inputs.ts
+ *   WORKFLOW_MODELS="opencode:opencode/gpt-5-codex,opencode:opencode/claude-sonnet-4-5" bun run scripts/sync-workflow-inputs.ts
  */
 
 import { readFileSync, writeFileSync } from "node:fs";
-import { Agent } from "~/agents/index.js";
 import YAML from "yaml";
 
 interface WorkflowInput {
@@ -34,24 +32,27 @@ function toDescription(agent: string, model: string): string {
 async function main(): Promise<void> {
   const workflowPath = ".github/workflows/compare-models.yml";
 
-  // Load the workflow file
-  const workflowContent = readFileSync(workflowPath, "utf8");
-  const workflow = YAML.parse(workflowContent);
-
-  // Get all available agent:model combinations
-  const agents = Agent.list();
-  const combinations: Array<{ agent: string; model: string }> = [];
+  // Models are passed via WORKFLOW_MODELS environment variable (comma-separated agent:model pairs)
+  const modelsEnv = process.env.WORKFLOW_MODELS;
+  if (!modelsEnv) {
+    console.error(
+      "Error: WORKFLOW_MODELS environment variable is required (comma-separated agent:model pairs)",
+    );
+    process.exit(1);
+  }
 
-  for (const agent of agents) {
-    for (const model of agent.models) {
-      combinations.push({ agent: agent.name, model });
+  const combinations = modelsEnv.split(",").map((pair) => {
+    const [agent, model] = pair.split(":");
+    if (!agent || !model) {
+      console.error(`Invalid agent:model pair: ${pair}`);
+      process.exit(1);
     }
-  }
+    return { agent, model };
+  });
 
-  if (combinations.length === 0) {
-    console.error("No agent:model combinations found");
-    process.exit(1);
-  }
+  // Load the workflow file
+  const workflowContent = readFileSync(workflowPath, "utf8");
+  const workflow = YAML.parse(workflowContent);
 
   // Build new inputs
   const inputs: WorkflowInputs = {};

scripts/sync-workflow-inputs.ts

@@ -8,14 +8,6 @@ import { Logger } from "../util/logger.js";
 
 const sessionCache = new Map<string, string>();
 
-export const models: string[] = [
-  "claude-sonnet-4-5",
-  "claude-opus-4-5",
-  // "claude-sonnet-4",
-  // "claude-opus-4-1",
-  // "claude-3-5-haiku",
-];
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }

src/agents/claude-code.ts

@@ -16,14 +16,6 @@ const DEFAULT_SANDBOX: SandboxMode = "workspace-write";
 const codexClient = new Codex();
 const threadCache = new Map<string, Thread>();
 
-export const models = [
-  "gpt-5-codex",
-  "gpt-5.1-codex",
-  // "gpt-5",
-  // "o3",
-  // "o4-mini"
-] as const;
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }
@@ -67,7 +59,7 @@ function getOrCreateThread(model: string, cwd: string): Thread {
   return thread;
 }
 
-const codexAgent: Agent.Definition<(typeof models)[number]> = {
+const codexAgent: Agent.Definition = {
   async run(model, prompt, options) {
     options.logger.log(
       `codex-sdk --model ${model} --sandbox ${DEFAULT_SANDBOX} ${prompt}`,

src/agents/codex.ts

@@ -1,8 +1,8 @@
-import { strict as assert } from "node:assert";
 import * as opencodeAgent from "./opencode.js";
 import * as codexAgent from "./codex.js";
 import * as claudeCodeAgent from "./claude-code.js";
 import { Logger } from "../util/logger.js";
+import { strict as assert } from "node:assert";
 
 export namespace Agent {
   export type Prompt = string;
@@ -43,33 +43,29 @@ export namespace Agent {
     logger: Logger.Instance;
   }
 
-  export interface Registration<TModel extends string = string> {
+  export interface Registration {
     name: string;
-    definition: Definition<TModel>;
-    models: ReadonlyArray<TModel>;
+    definition: Definition;
   }
 
-  const agents: Record<string, Registration<any>> = {
+  const agents: Record<string, Registration> = {
     // Only keep opencode active while debugging timeouts for specific models.
     opencode: createRegistration("opencode", opencodeAgent),
     //codex: createRegistration("codex", codexAgent),
     //"claude-code": createRegistration("claude-code", claudeCodeAgent),
   };
 
-  function createRegistration<TModel extends string>(
+  function createRegistration(
     name: string,
     module: {
-      default?: Definition<TModel>;
-      models?: ReadonlyArray<TModel>;
+      default?: Definition;
     },
-  ): Registration<TModel> {
+  ): Registration {
     const definition = module.default;
-    const models = module.models;
 
     assert(definition, `Agent module ${name} is missing a default export.`);
-    assert(models, `Agent module ${name} is missing the exported models list.`);
 
-    return { name, definition, models };
+    return { name, definition };
   }
 
   export function get(name: string): Registration {
@@ -78,13 +74,6 @@ export namespace Agent {
     return agent;
   }
 
-  export function validateModel(agent: Registration, model: string) {
-    if (!agent.models.find((entry) => entry === model))
-      throw new Error(
-        `Model ${model} is not registered for agent ${agent.name}.`,
-      );
-  }
-
   export function list() {
     return Object.values(agents);
   }

src/agents/index.ts

@@ -36,20 +36,6 @@ const opencode = await createOpencode({
 
 const sessionCache = new Map<string, string>();
 
-export const models: string[] = [
-  "opencode/gpt-5-codex",
-  "opencode/gpt-5.1-codex",
-  "opencode/claude-sonnet-4-5",
-  "opencode/claude-opus-4-5",
-  "opencode/glm-4.6",
-  "opencode/glm-4.7-free",
-  "opencode/gemini-3-pro",
-  "opencode/qwen3-coder",
-  "opencode/kimi-k2",
-  "opencode/grok-code",
-  "opencode/alpha-gd4",
-];
-
 function sessionKey(model: string, cwd: string): string {
   return `${cwd}::${model}`;
 }

src/agents/opencode.ts

@@ -47,7 +47,6 @@ export namespace Eval {
     },
   ) {
     const agent = Agent.get(agentName);
-    Agent.validateModel(agent, modelId);
     const task = await Task.get(taskId);
     const cwd = await mkdtemp(join(tmpdir(), "openreval-"));
     $.cwd(cwd);