Merge branch 'main' into tmd/add-judges-summary

Author: Aslemammad

.github/workflows/benchmark-reusable.yml

@@ -134,11 +134,19 @@ jobs:
             COMMAND="bunx \"${PACKAGE_SPEC}\" \"${BENCHMARK_AGENT}\" --eval \"${BENCHMARK_EVAL}\" --model \"${BENCHMARK_MODEL}\" --output \"${OUTPUT_FILE}\""
 
             echo "Executing: ${COMMAND}"
-            if ! bunx "${PACKAGE_SPEC}" "${BENCHMARK_AGENT}" --eval "${BENCHMARK_EVAL}" --model "${BENCHMARK_MODEL}" --output "${OUTPUT_FILE}"; then
-              echo "openreval benchmark failed, dumping OpenCode logs..." >&2
-              find /home/runner/.local/share/opencode/log -type f -print -exec cat {} + || true
-              exit 1
-            fi
+            bunx "${PACKAGE_SPEC}" "${BENCHMARK_AGENT}" --eval "${BENCHMARK_EVAL}" --model "${BENCHMARK_MODEL}" --output "${OUTPUT_FILE}"
+
+      - name: Log benchmark summary
+        if: always()
+        run: |
+          set -euo pipefail
+          if [ -f benchmark.json ]; then
+            echo "=== Benchmark Summary ==="
+            jq -r '.summary // "No summary available"' benchmark.json
+            echo "========================"
+          else
+            echo "benchmark.json not found, skipping summary log"
+          fi
 
       - name: Generate Judges Summary for this Evaluation
         env:

.github/workflows/compare-models.yml

@@ -2,26 +2,10 @@ name: Compare Models
 on:
   workflow_dispatch:
     inputs:
-      codex_gpt_5_codex:
-        description: codex:gpt-5-codex
-        type: boolean
-        default: false
       opencode_opencode_gpt_5_codex:
         description: opencode:opencode/gpt-5-codex
         type: boolean
         default: false
-      opencode_opencode_claude_sonnet_4_5:
-        description: opencode:opencode/claude-sonnet-4-5
-        type: boolean
-        default: false
-      opencode_opencode_big_pickle:
-        description: opencode:opencode/big-pickle
-        type: boolean
-        default: false
-      claude_code_claude_sonnet_4_5:
-        description: claude-code:claude-sonnet-4-5
-        type: boolean
-        default: false
 permissions:
   contents: read
   actions: read

agents/claude-code.ts

@@ -12,7 +12,7 @@ import type {
 const sessionCache = new Map<string, string>();
 
 export const models: string[] = [
-  "claude-sonnet-4-5",
+  // "claude-sonnet-4-5",
   // "claude-sonnet-4",
   // "claude-opus-4-1",
   // "claude-3-5-haiku",
@@ -106,6 +106,12 @@ const claudeCodeAgent: AgentDefinition = {
     const cacheKey = sessionKey(cwd, model);
     const existingSessionID = sessionCache.get(cacheKey);
 
+    const actions: string[] = [];
+    const usage = {
+      input: 0,
+      output: 0,
+    };
+
     try {
       const result = query({
         prompt,
@@ -122,6 +128,14 @@ const claudeCodeAgent: AgentDefinition = {
       for await (const message of result) {
         // Extract and cache session ID from messages
         sessionCache.set(cacheKey, message.session_id);
+
+        // Accumulate token usage if available (only SDKResultMessage has usage)
+        if (message.type === "result" && "usage" in message) {
+          usage.input += message.usage.input_tokens || 0;
+          usage.output += message.usage.output_tokens || 0;
+        }
+
+        actions.push(JSON.stringify(message));
         logJson(message, options);
       }
     } catch (error) {
@@ -137,7 +151,7 @@ const claudeCodeAgent: AgentDefinition = {
       throw error;
     }
 
-    return { command: displayCommand };
+    return { command: displayCommand, actions, usage };
   },
 };
 

agents/codex.ts

@@ -3,6 +3,7 @@ import process from "node:process";
 
 import {
   Codex,
+  Usage,
   type CommandExecutionItem,
   type SandboxMode,
   type Thread,
@@ -21,7 +22,7 @@ const codexClient = new Codex();
 const threadCache = new Map<string, Thread>();
 
 export const models: string[] = [
-  "gpt-5-codex",
+  // "gpt-5-codex",
   // "gpt-5",
   // "o3",
   // "o4-mini"
@@ -122,15 +123,27 @@ const codexAgent: AgentDefinition = {
     const key = sessionKey(model, cwd);
     const thread = getOrCreateThread(model, cwd);
 
+    const actions: string[] = [];
+    let usage: Usage;
     try {
       const turn = await thread.run(prompt);
+      assert(turn.usage, "The agent did not emit the usage information.");
+      usage = turn.usage;
+      actions.push(...turn.items.map((item) => JSON.stringify(item)));
       logTurnItems(turn.items, options);
     } catch (error) {
       threadCache.delete(key);
       throw error;
     }
 
-    return { command: displayCommand };
+    return {
+      command: displayCommand,
+      actions,
+      usage: {
+        input: usage.input_tokens,
+        output: usage.output_tokens,
+      },
+    };
   },
 };
 

agents/opencode.ts

@@ -22,23 +22,58 @@ const DEFAULT_PERMISSION_CONFIG: NonNullable<OpencodeConfig["permission"]> = {
   webfetch: "allow",
 };
 
+// Custom fetch with 25-minute timeout
+const customFetch = async (request: Request): Promise<Response> => {
+  const startTime = Date.now();
+
+  try {
+    const response = await fetch(request, {
+      signal: AbortSignal.timeout(1_500_000),
+    });
+    const duration = Date.now() - startTime;
+    console.error(`[opencode] Request completed - Duration: ${duration}ms`);
+    return response;
+  } catch (error) {
+    const duration = Date.now() - startTime;
+    console.error(`[opencode] Request failed - Duration: ${duration}ms`);
+    throw error;
+  }
+};
+
 const opencodePort = await detectPort(4096);
 
+// Set OpenCode config before server starts to ensure timeout is applied
+const opencodeConfig = {
+  permission: DEFAULT_PERMISSION_CONFIG,
+  provider: {
+    opencode: {
+      options: {
+        timeout: false as false, // Disable timeout for OpenCode provider requests
+      },
+    },
+  },
+} satisfies OpencodeConfig;
+
+// CRITICAL: Set via environment variable BEFORE importing/creating anything
+// The SDK reads this when spawning the server process
+const configJson = JSON.stringify(opencodeConfig);
+process.env.OPENCODE_CONFIG_CONTENT = configJson;
+
+console.error(`[opencode] Setting config: ${configJson}`);
+
 const opencode = await createOpencode({
   port: opencodePort,
-  config: {
-    permission: DEFAULT_PERMISSION_CONFIG,
-  },
+  timeout: 1_500_000, // 25 minutes timeout for server startup
+  config: opencodeConfig,
 });
-process.once("beforeExit", () => opencode.server.close());
 
 const sessionCache = new Map<string, string>();
 
 export const models: string[] = [
   // "opencode/gpt-5",
   "opencode/gpt-5-codex",
-  "opencode/claude-sonnet-4-5",
-  "opencode/big-pickle",
+  // "opencode/claude-sonnet-4-5",
+  // "opencode/big-pickle",
   // "opencode/claude-sonnet-4",
   // "opencode/claude-3-5-haiku",
   // "opencode/claude-opus-4-1",
@@ -73,33 +108,36 @@ function writeLog(
 
 function logJson(value: unknown, options: AgentRunOptions | undefined): void {
   try {
-    writeLog(process.stdout, JSON.stringify(value), options?.logPrefix);
+    const message = JSON.stringify(value);
+    writeLog(process.stdout, message, options?.logPrefix);
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    writeLog(
-      process.stdout,
-      JSON.stringify({ error: "serialization_failed", reason }),
-      options?.logPrefix,
-    );
+    const errorMessage = JSON.stringify({
+      error: "serialization_failed",
+      reason,
+    });
+    writeLog(process.stdout, errorMessage, options?.logPrefix);
   }
 }
 
 function logError(value: unknown, options: AgentRunOptions | undefined): void {
   try {
-    writeLog(process.stderr, JSON.stringify(value), options?.logPrefix);
+    const message = JSON.stringify(value);
+    writeLog(process.stderr, message, options?.logPrefix);
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    writeLog(
-      process.stderr,
-      JSON.stringify({ error: "serialization_failed", reason }),
-      options?.logPrefix,
-    );
+    const errorMessage = JSON.stringify({
+      error: "serialization_failed",
+      reason,
+    });
+    writeLog(process.stderr, errorMessage, options?.logPrefix);
   }
 }
 
 function logPromptResult(
   result: { info: AssistantMessage; parts: Part[] },
   options: AgentRunOptions | undefined,
+  logs?: string[],
 ): void {
   logJson({ info: result.info }, options);
   if (Array.isArray(result.parts)) {
@@ -167,8 +205,14 @@ const opencodeAgent: AgentDefinition = {
       sessionCache.set(cacheKey, sessionID);
     }
 
+    const actions: string[] = [];
+    const usage = {
+      input: 0,
+      output: 0,
+    };
     try {
       const [providerID, modelID] = model.split("/");
+
       const { data } = await opencode.client.session.prompt({
         path: { id: sessionID! },
         query: { directory: cwd },
@@ -180,10 +224,29 @@ const opencodeAgent: AgentDefinition = {
           parts: [{ type: "text", text: prompt }],
         },
         throwOnError: true,
+        fetch: customFetch,
       });
 
+      if (data.info?.tokens) {
+        usage.input = data.info.tokens.input || 0;
+        usage.output = data.info.tokens.output || 0;
+      } else {
+        console.error(
+          `[opencode] WARNING: No token usage in response. Available fields: ${Object.keys(data.info || {}).join(", ")}`,
+        );
+      }
+
+      actions.push(JSON.stringify(data.info));
+      if (Array.isArray(data.parts)) {
+        data.parts.forEach((part) => actions.push(JSON.stringify(part)));
+      }
+
       logPromptResult(data, options);
     } catch (error) {
+      console.error(
+        `[opencode] Error in ${model}:`,
+        error instanceof Error ? error.message : String(error),
+      );
       sessionCache.delete(cacheKey);
       logError(
         {
@@ -195,7 +258,10 @@ const opencodeAgent: AgentDefinition = {
       throw error;
     }
 
-    return { command: displayCommand };
+    return { command: displayCommand, actions, usage };
+  },
+  cleanup() {
+    opencode.server.close();
   },
 };