feat: add tokens per second metric

Author: tmickleydoyle

cli.ts

@@ -295,11 +295,13 @@ async function main(): Promise<void> {
         logs: string[];
         actions: string[];
         usage: Usage;
+        durationMs: number;
       }
 
       const runEpisode = async (
         episodeIndex: number,
       ): Promise<EpisodeResult> => {
+        const episodeStartTime = Date.now();
         const episodeTag = `[episode ${episodeIndex}/${EPISODES}]`;
         const baselineCommit = evalDefinition.from;
         const prefix = `${episodeTag} [${combinationLabel}]`;
@@ -456,6 +458,7 @@ async function main(): Promise<void> {
             logs: [],
             actions: episodeActions,
             usage,
+            durationMs: Date.now() - episodeStartTime,
           };
         } finally {
           if (cwd) {
@@ -513,6 +516,16 @@ async function main(): Promise<void> {
         { input: 0, output: 0 },
       );
 
+      // Calculate total duration and tokens per second
+      const totalDurationMs = episodeResults.reduce(
+        (sum, result) => sum + result.durationMs,
+        0,
+      );
+      const totalTokens = averageUsage.input + averageUsage.output;
+      const durationSeconds = totalDurationMs / 1000;
+      const tokensPerSecond =
+        durationSeconds > 0 ? totalTokens / durationSeconds : 0;
+
       for (const result of episodeResults) {
         mergeAggregationInputs(aggregatedInputs, result.aggregation);
         episodeExports.push({
@@ -560,6 +573,8 @@ async function main(): Promise<void> {
         episodeExports,
         averageUsage,
         summary,
+        totalDurationMs,
+        tokensPerSecond,
       );
     };
 
@@ -578,8 +593,14 @@ async function main(): Promise<void> {
     });
 
     if (evaluationResult.exportData) {
-      const { episodes, finalScore, baseScore, variancePenalty } =
-        evaluationResult.exportData;
+      const {
+        episodes,
+        finalScore,
+        baseScore,
+        variancePenalty,
+        durationMs,
+        tokensPerSecond,
+      } = evaluationResult.exportData;
       if (episodes.length > 0) {
         console.log("[debug] Episode recap:");
         episodes.forEach((episode, index) => {
@@ -599,6 +620,9 @@ async function main(): Promise<void> {
           3,
         )})`,
       );
+      console.log(
+        `[debug] Performance: ${(durationMs / 1000).toFixed(1)}s, ${tokensPerSecond.toFixed(1)} tokens/sec`,
+      );
 
       // Generate and log radar chart URL
       const chartUrl = buildRadarChartUrl({
@@ -780,6 +804,8 @@ function summarizeAggregation(
   episodes: Episode[],
   usage: Usage,
   summary: string,
+  durationMs: number,
+  tokensPerSecond: number,
 ): { lines: string[]; exportData: EvaluationRunExport } {
   const evalId = datasetEval.repo;
   const runContext = contextLabel ? `${evalId} [${contextLabel}]` : evalId;
@@ -842,6 +868,8 @@ function summarizeAggregation(
     episodes,
     usage,
     summary,
+    durationMs,
+    tokensPerSecond,
   };
 
   return { lines, exportData };

scripts/discord-sample.ts

@@ -231,6 +231,8 @@ const sampleExport: EvaluationRunExport[] = [
     episodes: claudeEpisodes,
     usage: { input: 50000, output: 10100 },
     summary: "",
+    durationMs: 180000,
+    tokensPerSecond: 334.0,
   },
   {
     agent: "opencode",
@@ -250,6 +252,8 @@ const sampleExport: EvaluationRunExport[] = [
     episodes: gptEpisodes,
     usage: { input: 48167, output: 9633 },
     summary: "",
+    durationMs: 165000,
+    tokensPerSecond: 350.9,
   },
 ];
 

types/export.ts

@@ -54,4 +54,6 @@ export interface EvaluationRunExport {
   episodes: Episode[];
   usage: Usage;
   summary: string;
+  durationMs: number;
+  tokensPerSecond: number;
 }