wip

Author: Aslemammad

agents/codex.ts

@@ -70,10 +70,14 @@ function logTurnItems(
     try {
       writeLog(process.stdout, JSON.stringify(item), options?.logPrefix);
     } catch (error) {
-      const fallback = isCommandExecutionItem(item)
+      const sanitizedItem = isCommandExecutionItem(item)
         ? { ...item, aggregated_output: "<omitted>" }
         : item;
-      writeLog(process.stdout, JSON.stringify(fallback), options?.logPrefix);
+      writeLog(
+        process.stdout,
+        JSON.stringify(sanitizedItem),
+        options?.logPrefix,
+      );
       if (error instanceof Error) {
         writeLog(
           process.stderr,

agents/opencode.ts

@@ -71,9 +71,9 @@ const sessionCache = new Map<string, string>();
 
 export const models: string[] = [
   // "opencode/gpt-5",
-  "opencode/gpt-5-codex",
-  // "opencode/claude-sonnet-4-5",
-  // "opencode/big-pickle",
+  // "opencode/gpt-5-codex",
+  "opencode/claude-sonnet-4-5",
+  "opencode/big-pickle",
   // "opencode/claude-sonnet-4",
   // "opencode/claude-3-5-haiku",
   // "opencode/claude-opus-4-1",

judges.ts

@@ -1,17 +1,14 @@
 import type { Judge } from "~/lib/judgeTypes.js";
 import { getZenLanguageModel } from "~/lib/zenModels.js";
 
-const fallback = (envName: string, defaultValue: string): string =>
-  process.env[envName]?.trim() || defaultValue;
-
 function resolveJudgeModelId(judgeName: Judge["name"]): string {
   switch (judgeName) {
     case "claude-4.5":
-      return fallback("CLAUDE_MODEL", "opencode/claude-sonnet-4-5");
+      return "opencode/claude-sonnet-4-5";
     case "gpt-5-codex":
-      return fallback("GPT5_CODEX_MODEL", "opencode/gpt-5-codex");
+      return "opencode/gpt-5-codex";
     case "kimi":
-      return fallback("KIMI_MODEL", "opencode/kimi-k2");
+      return "opencode/kimi-k2";
     default:
       return judgeName;
   }

lib/planner.ts

@@ -5,9 +5,6 @@ import type { DatasetEval } from "~/lib/dataset.js";
 import { plannerExamples } from "~/lib/plannerExamples.js";
 import { getZenLanguageModel } from "~/lib/zenModels.js";
 
-const fallback = (envName: string, defaultValue: string): string =>
-  process.env[envName]?.trim() || defaultValue;
-
 export interface PlannerCommitDiff {
   sha: string;
   title: string;
@@ -61,7 +58,7 @@ What NOT to include:
 
 Always respond strictly as JSON conforming to the schema. Do not add commentary.`;
 
-const plannerModelId = fallback("PLANNER_MODEL", "opencode/claude-sonnet-4-5");
+const plannerModelId = "opencode/claude-sonnet-4-5";
 
 function buildSystemPrompt(): string {
   if (plannerExamples.length === 0) {

lib/summarizer.ts

@@ -4,9 +4,6 @@ import { z } from "zod";
 import type { DatasetEval } from "~/lib/dataset.js";
 import { getZenLanguageModel } from "~/lib/zenModels.js";
 
-const fallback = (envName: string, defaultValue: string): string =>
-  process.env[envName]?.trim() || defaultValue;
-
 export interface EpisodeActions {
   episodeIndex: number;
   actions: string[];
@@ -43,10 +40,7 @@ Guidelines:
 - Note any errors or issues encountered
 - Be objective and descriptive, not evaluative`;
 
-const summarizerModelId = fallback(
-  "SUMMARIZER_MODEL",
-  "opencode/claude-sonnet-4-5",
-);
+const summarizerModelId = "opencode/claude-sonnet-4-5";
 
 export async function generateActionsSummary(
   evaluation: DatasetEval,

scripts/analysis.ts

@@ -33,13 +33,7 @@ Focus on concrete observations from the data provided. Look for patterns such as
 
 Provide a concise, insightful analysis that helps developers understand agent behavior and improve the evaluation system.`;
 
-const fallback = (envName: string, defaultValue: string): string =>
-  process.env[envName]?.trim() || defaultValue;
-
-const analyzerModelId = fallback(
-  "ANALYZER_MODEL",
-  "opencode/claude-sonnet-4-5",
-);
+const analyzerModelId = "opencode/claude-sonnet-4-5";
 
 function buildDynamicContext(runs: EvaluationRunExport[]): string {
   const repo = runs[0].evaluation.repo;
@@ -99,25 +93,16 @@ function formatFallbackSummary(runs: EvaluationRunExport[]): string {
   return lines.join("\n").trimEnd();
 }
 
-async function generateAnalysis(
-  runs: EvaluationRunExport[],
-): Promise<string> {
+async function generateAnalysis(runs: EvaluationRunExport[]): Promise<string> {
   const context = buildDynamicContext(runs);
 
-  try {
-    const { text } = await generateText({
-      model: getZenLanguageModel(analyzerModelId),
-      system: AGENT_ANALYSIS_PROMPT,
-      prompt: context,
-      temperature: 0.3,
-    });
-    return text.trim();
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return `Failed to generate AI analysis (${message}).\n\n${formatFallbackSummary(
-      runs,
-    )}`;
-  }
+  const { text } = await generateText({
+    model: getZenLanguageModel(analyzerModelId),
+    system: AGENT_ANALYSIS_PROMPT,
+    prompt: context,
+    temperature: 0.3,
+  });
+  return text.trim();
 }
 
 function usage(): void {
@@ -137,33 +122,24 @@ async function main(): Promise<void> {
   }
 
   const filePath = args[0];
-  let parsed: unknown;
+  let runs: EvaluationRunExport[];
 
   try {
     const fileContent = readFileSync(filePath, "utf-8");
-    parsed = JSON.parse(fileContent) as unknown;
+    runs = JSON.parse(fileContent) as EvaluationRunExport[];
   } catch (error) {
     console.error(`Error reading file ${filePath}:`, error);
     process.exit(1);
   }
 
-  const runs = (Array.isArray(parsed)
-    ? (parsed as EvaluationRunExport[])
-    : [parsed as EvaluationRunExport]).filter(
-    (run): run is EvaluationRunExport =>
-      run != null && typeof run === "object" && "finalScore" in run,
-  );
-
   if (runs.length === 0) {
     console.error("No evaluation runs found in the provided file.");
     process.exit(1);
   }
 
-  const orderedRuns = [...runs].sort(
-    (a, b) => b.finalScore - a.finalScore,
-  );
+  runs.sort((a, b) => b.finalScore - a.finalScore);
 
-  const output = await generateAnalysis(orderedRuns);
+  const output = await generateAnalysis(runs);
   process.stdout.write(`${output.trimEnd()}\n`);
 }