chore: focus opencode agent and quiet rustfmt diffs

Author: Aslemammad

agents/index.ts

@@ -27,15 +27,16 @@ function createAgentRegistration<TModel extends string>(
 }
 
 const agents: Record<string, AgentRegistration<any>> = {
-  codex: createAgentRegistration("codex", await import("~/agents/codex.js")),
+  // Only keep opencode active while debugging timeouts for specific models.
   opencode: createAgentRegistration(
     "opencode",
     await import("~/agents/opencode.js"),
   ),
-  "claude-code": createAgentRegistration(
-    "claude-code",
-    await import("~/agents/claude-code.js"),
-  ),
+  // codex: createAgentRegistration("codex", await import("~/agents/codex.js")),
+  // "claude-code": createAgentRegistration(
+  //   "claude-code",
+  //   await import("~/agents/claude-code.js"),
+  // ),
 };
 
 export async function getAgent(

dataset.yaml

@@ -41,7 +41,7 @@
         setup:
           - cargo fetch
         commands:
-          - cargo fmt --all -- --check
+          - cargo fmt --all -- --check --quiet
 - repo: HelixDB/helix-db
   from: 35a9e733c659762487ba03ee68184f91639b912a
   to: 12ebac2e330aa4dfd5509852653508de51be4186
@@ -68,7 +68,7 @@
         setup:
           - cargo fetch
         commands:
-          - cargo fmt --all -- --check
+          - cargo fmt --all -- --check --quiet
 - repo: getsentry/sentry
   from: 62c4c6573f53f39b2f5b73356e26f330a501e7ea
   to: b950f7636b79d9431b800d6fbe7c166f3ff00d29

scores/checks.ts

@@ -1,4 +1,4 @@
-import { spawnSync } from "node:child_process";
+import { spawn } from "node:child_process";
 import { strict as assert } from "node:assert";
 
 import { generateObject } from "ai";
@@ -125,22 +125,21 @@ const COMMAND_TIMEOUT_MS = 5 * 60 * 1000;
 type ChecksConfig = z.infer<typeof commandConfigSchema>;
 
 export default createScore<PreparedCheck[], ChecksConfig>({
-  prepare: ({ cwd, evaluation, config }) => {
+  prepare: async ({ cwd, evaluation, config }) => {
     const parsedConfig = commandConfigSchema.parse(config ?? {});
 
-    parsedConfig.setup.forEach((command) => {
-      const result = runCommand(command, cwd);
+    for (const command of parsedConfig.setup) {
+      const result = await runCommand(command, cwd);
       logSetupExecution(command, result);
-    });
+    }
 
-    const results: PreparedCheck[] = parsedConfig.commands.map((command) => {
-      const baseline = runCommand(command, cwd);
+    const results: PreparedCheck[] = [];
+
+    for (const command of parsedConfig.commands) {
+      const baseline = await runCommand(command, cwd);
       logExecution("baseline", command, baseline);
-      return {
-        command,
-        baseline,
-      };
-    });
+      results.push({ command, baseline });
+    }
 
     assert(
       results.length > 0,
@@ -152,12 +151,12 @@ export default createScore<PreparedCheck[], ChecksConfig>({
   evaluate: async ({ evaluation, cwd, judge, reference, config: _config }) => {
     finalizeAgentChanges(evaluation, cwd, evaluation.from);
 
-    reference.forEach((entry) => {
+    for (const entry of reference) {
       if (!entry.after) {
-        entry.after = runCommand(entry.command, cwd);
+        entry.after = await runCommand(entry.command, cwd);
         logExecution("after", entry.command, entry.after);
       }
-    });
+    }
 
     const prompt = buildJudgePrompt(reference);
 
@@ -181,46 +180,67 @@ export default createScore<PreparedCheck[], ChecksConfig>({
   },
 });
 
-function runCommand(command: string, cwd: string): CommandExecution {
+async function runCommand(command: string, cwd: string): Promise<CommandExecution> {
   const start = Date.now();
-  const result = spawnSync(command, {
-    cwd,
-    shell: true,
-    encoding: "utf8",
-    timeout: COMMAND_TIMEOUT_MS,
-    env: {
-      ...process.env,
-      CI: process.env.CI ?? "1",
-    },
-  });
 
-  const runtimeMs = Date.now() - start;
-  const stdout = (result.stdout ?? "").toString();
-  const stderr = (result.stderr ?? "").toString();
+  return await new Promise<CommandExecution>((resolve) => {
+    const child = spawn(command, {
+      cwd,
+      shell: true,
+      env: {
+        ...process.env,
+        CI: process.env.CI ?? "1",
+      },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
 
-  let exitCode: number | null = null;
-  let success = false;
-  let errorMessage: string | undefined;
+    let stdout = "";
+    let stderr = "";
+    let errorMessage: string | undefined;
+    let timeout: NodeJS.Timeout | undefined;
+    let settled = false;
 
-  if (typeof result.status === "number") {
-    exitCode = result.status;
-    success = exitCode === 0;
-  }
+    const finalize = (exitCode: number | null) => {
+      if (settled) return;
+      settled = true;
+      if (timeout) clearTimeout(timeout);
 
-  if (result.error) {
-    success = false;
-    errorMessage = result.error.message;
-  }
+      const runtimeMs = Date.now() - start;
+      const success = exitCode === 0 && !errorMessage;
+
+      resolve({
+        command,
+        success,
+        exitCode,
+        stdout,
+        stderr,
+        runtimeMs,
+        errorMessage,
+      });
+    };
+
+    timeout = setTimeout(() => {
+      errorMessage = `Timed out after ${COMMAND_TIMEOUT_MS}ms`;
+      child.kill("SIGKILL");
+    }, COMMAND_TIMEOUT_MS);
+
+    child.stdout?.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
 
-  return {
-    command,
-    success,
-    exitCode,
-    stdout,
-    stderr,
-    runtimeMs,
-    errorMessage,
-  };
+    child.stderr?.on("data", (chunk) => {
+      stderr += chunk.toString();
+    });
+
+    child.on("error", (error) => {
+      errorMessage = error.message;
+    });
+
+    child.on("close", (code) => {
+      const exitCode = typeof code === "number" ? code : null;
+      finalize(exitCode);
+    });
+  });
 }
 
 function buildJudgePrompt(entries: PreparedCheck[]): string {