feat: add cost and overall summary to evals (#8)

* feat: add cost and overall summary to evals

* feat: add cost and overall summary to evals

* wip

* things in a better shape

* minor changes

---------

Co-authored-by: Mohammad Bagher Abiyat <37929992+Aslemammad@users.noreply.github.com>

Author: tmickleydoyle

.github/workflows/benchmark-reusable.yml

@@ -140,6 +140,18 @@ jobs:
               exit 1
             fi
 
+      - name: Log benchmark summary
+        if: always()
+        run: |
+          set -euo pipefail
+          if [ -f benchmark.json ]; then
+            echo "=== Benchmark Summary ==="
+            jq -r '.summary // "No summary available"' benchmark.json
+            echo "========================"
+          else
+            echo "benchmark.json not found, skipping summary log"
+          fi
+
       - name: Prepare artifact name
         id: artifact
         env:

agents/claude-code.ts

@@ -106,6 +106,12 @@ const claudeCodeAgent: AgentDefinition = {
     const cacheKey = sessionKey(cwd, model);
     const existingSessionID = sessionCache.get(cacheKey);
 
+    const actions: string[] = [];
+    const usage = {
+      input: 0,
+      output: 0,
+    };
+
     try {
       const result = query({
         prompt,
@@ -122,6 +128,14 @@ const claudeCodeAgent: AgentDefinition = {
       for await (const message of result) {
         // Extract and cache session ID from messages
         sessionCache.set(cacheKey, message.session_id);
+
+        // Accumulate token usage if available (only SDKResultMessage has usage)
+        if (message.type === "result" && "usage" in message) {
+          usage.input += message.usage.input_tokens || 0;
+          usage.output += message.usage.output_tokens || 0;
+        }
+
+        actions.push(JSON.stringify(message));
         logJson(message, options);
       }
     } catch (error) {
@@ -137,7 +151,7 @@ const claudeCodeAgent: AgentDefinition = {
       throw error;
     }
 
-    return { command: displayCommand };
+    return { command: displayCommand, actions, usage };
   },
 };
 

agents/codex.ts

@@ -3,6 +3,7 @@ import process from "node:process";
 
 import {
   Codex,
+  Usage,
   type CommandExecutionItem,
   type SandboxMode,
   type Thread,
@@ -122,15 +123,27 @@ const codexAgent: AgentDefinition = {
     const key = sessionKey(model, cwd);
     const thread = getOrCreateThread(model, cwd);
 
+    const actions: string[] = [];
+    let usage: Usage;
     try {
       const turn = await thread.run(prompt);
+      assert(turn.usage, "The agent did not emit the usage information.");
+      usage = turn.usage;
+      actions.push(...turn.items.map((item) => JSON.stringify(item)));
       logTurnItems(turn.items, options);
     } catch (error) {
       threadCache.delete(key);
       throw error;
     }
 
-    return { command: displayCommand };
+    return {
+      command: displayCommand,
+      actions,
+      usage: {
+        input: usage.input_tokens,
+        output: usage.output_tokens,
+      },
+    };
   },
 };
 

agents/opencode.ts

@@ -73,33 +73,36 @@ function writeLog(
 
 function logJson(value: unknown, options: AgentRunOptions | undefined): void {
   try {
-    writeLog(process.stdout, JSON.stringify(value), options?.logPrefix);
+    const message = JSON.stringify(value);
+    writeLog(process.stdout, message, options?.logPrefix);
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    writeLog(
-      process.stdout,
-      JSON.stringify({ error: "serialization_failed", reason }),
-      options?.logPrefix,
-    );
+    const errorMessage = JSON.stringify({
+      error: "serialization_failed",
+      reason,
+    });
+    writeLog(process.stdout, errorMessage, options?.logPrefix);
   }
 }
 
 function logError(value: unknown, options: AgentRunOptions | undefined): void {
   try {
-    writeLog(process.stderr, JSON.stringify(value), options?.logPrefix);
+    const message = JSON.stringify(value);
+    writeLog(process.stderr, message, options?.logPrefix);
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    writeLog(
-      process.stderr,
-      JSON.stringify({ error: "serialization_failed", reason }),
-      options?.logPrefix,
-    );
+    const errorMessage = JSON.stringify({
+      error: "serialization_failed",
+      reason,
+    });
+    writeLog(process.stderr, errorMessage, options?.logPrefix);
   }
 }
 
 function logPromptResult(
   result: { info: AssistantMessage; parts: Part[] },
   options: AgentRunOptions | undefined,
+  logs?: string[],
 ): void {
   logJson({ info: result.info }, options);
   if (Array.isArray(result.parts)) {
@@ -167,6 +170,11 @@ const opencodeAgent: AgentDefinition = {
       sessionCache.set(cacheKey, sessionID);
     }
 
+    const actions: string[] = [];
+    const usage = {
+      input: 0,
+      output: 0,
+    };
     try {
       const [providerID, modelID] = model.split("/");
       const { data } = await opencode.client.session.prompt({
@@ -182,6 +190,14 @@ const opencodeAgent: AgentDefinition = {
         throwOnError: true,
       });
 
+      usage.input = data.info.tokens.input;
+      usage.output = data.info.tokens.output;
+
+      actions.push(JSON.stringify(data.info));
+      if (Array.isArray(data.parts)) {
+        data.parts.forEach((part) => actions.push(JSON.stringify(part)));
+      }
+
       logPromptResult(data, options);
     } catch (error) {
       sessionCache.delete(cacheKey);
@@ -195,7 +211,7 @@ const opencodeAgent: AgentDefinition = {
       throw error;
     }
 
-    return { command: displayCommand };
+    return { command: displayCommand, actions, usage };
   },
 };