benchmark-reusable.yml

name: Reusable Benchmark Workflow

on:
  workflow_call:
    inputs:
      matrix:
        description: 'JSON matrix with agent/model/eval combinations'
        required: true
        type: string
      package_urls:
        description: 'Package URLs from publish step'
        required: true
        type: string

permissions:
  contents: read
  actions: read

jobs:
  prepare-analysis:
    name: Prepare Judge Analysis Matrix
    runs-on: blacksmith-4vcpu-ubuntu-2404
    outputs:
      evals: ${{ steps.compute.outputs.evals }}
    steps:
      - name: Extract unique evaluations
        id: compute
        env:
          MATRIX_JSON: ${{ inputs.matrix }}
        run: |
          set -euo pipefail
          evals=$(jq -c '.include | unique_by(.eval) | map({ eval: .eval, safe: (.eval | gsub("/"; "-")) })' <<<"$MATRIX_JSON")

          if [ -z "${evals}" ] || [ "${evals}" = "null" ]; then
            echo "No evaluations found in matrix definition." >&2
            evals="[]"
          fi

          echo "Analysis eval matrix: ${evals}"
          echo "evals=${evals}" >> "$GITHUB_OUTPUT"

  benchmark:
    name: Benchmark ${{ matrix.agent }} / ${{ matrix.model }} / ${{ matrix.eval }}
    runs-on: blacksmith-4vcpu-ubuntu-2404
    permissions:
      contents: read
      actions: read
    environment: production
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(inputs.matrix) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Bun
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: 1.2.21

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Install OpenCode CLI
        run: bun add -g opencode-ai@dev @openai/codex-sdk

      - name: Determine benchmark job URL
        id: job_url
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BENCHMARK_AGENT: ${{ matrix.agent }}
          BENCHMARK_MODEL: ${{ matrix.model }}
          BENCHMARK_EVAL: ${{ matrix.eval }}
        run: |
          set -euo pipefail
          job_pattern="Benchmark ${BENCHMARK_AGENT} / ${BENCHMARK_MODEL} / ${BENCHMARK_EVAL}"
          job_url="$(bun run scripts/determine-job-url.ts --pattern "${job_pattern}")"

          echo "Job URL: ${job_url}"
          echo "GITHUB_BENCHMARK_JOB_URL=${job_url}" >> "$GITHUB_ENV"
          echo "url=${job_url}" >> "$GITHUB_OUTPUT"

      - name: Run openreval benchmark
        uses: nick-fields/retry@v2
        env:
          OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
          CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_BENCHMARK_JOB_URL: ${{ steps.job_url.outputs.url }}
          URLS: ${{ inputs.package_urls }}
          BENCHMARK_EVAL: ${{ matrix.eval }}
          BENCHMARK_MODEL: ${{ matrix.model }}
          BENCHMARK_AGENT: ${{ matrix.agent }}
        with:
          max_attempts: 3
          timeout_minutes: 90
          retry_on: error
          command: |
            set -euo pipefail
            PACKAGE_URL="$(printf '%s\n' "${URLS}" | awk 'NF {print $1; exit}')"

            if [ -z "${BENCHMARK_EVAL}" ]; then
              echo "Matrix entry missing evaluation identifier." >&2
              exit 1
            fi

            if [ -z "${BENCHMARK_MODEL}" ]; then
              echo "Matrix entry missing model identifier." >&2
              exit 1
            fi

            if [ -z "${BENCHMARK_AGENT}" ]; then
              echo "Matrix entry missing agent identifier." >&2
              exit 1
            fi

            if [ -z "${PACKAGE_URL}" ]; then
              echo "No package URL found in publish outputs." >&2
              exit 1
            fi

            PACKAGE_SPEC="orvl@${PACKAGE_URL}"
            OUTPUT_FILE="benchmark.json"
            COMMAND="bunx \"${PACKAGE_SPEC}\" \"${BENCHMARK_AGENT}\" --eval \"${BENCHMARK_EVAL}\" --model \"${BENCHMARK_MODEL}\" --output \"${OUTPUT_FILE}\""

            echo "Executing: ${COMMAND}"
            bunx "${PACKAGE_SPEC}" "${BENCHMARK_AGENT}" --eval "${BENCHMARK_EVAL}" --model "${BENCHMARK_MODEL}" --output "${OUTPUT_FILE}"

      - name: Log benchmark summary
        if: always()
        run: |
          set -euo pipefail
          if [ -f benchmark.json ]; then
            echo "=== Benchmark Summary ==="
            jq -r '.summary // "No summary available"' benchmark.json
            echo "========================"
          else
            echo "benchmark.json not found, skipping summary log"
          fi

      - name: Prepare artifact name
        id: artifact
        env:
          BENCHMARK_AGENT: ${{ matrix.agent }}
          BENCHMARK_MODEL: ${{ matrix.model }}
          BENCHMARK_EVAL: ${{ matrix.eval }}
        run: |
          set -euo pipefail
          agent="${BENCHMARK_AGENT//\//-}"
          model="${BENCHMARK_MODEL//\//-}"
          eval="${BENCHMARK_EVAL//\//-}"
          echo "name=benchmark-${agent}-${model}-${eval}" >> "$GITHUB_OUTPUT"

      - name: Upload benchmark artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.artifact.outputs.name }}
          path: benchmark.json

  eval-analysis:
    name: Judge Analysis - ${{ matrix.eval }}
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs:
      - benchmark
      - prepare-analysis
    if: needs.prepare-analysis.outputs.evals != '[]'
    environment: production
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.prepare-analysis.outputs.evals) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Bun
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: 1.2.21

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Download benchmark artifacts for eval
        uses: actions/download-artifact@v4
        with:
          path: eval-benchmarks
          pattern: benchmark-*-*-${{ matrix.safe }}

      - name: Merge benchmark exports
        id: merge
        run: |
          set -euo pipefail

          if [ ! -d eval-benchmarks ] || ! find eval-benchmarks -type f -name '*.json' -print -quit | grep -q .; then
            echo "No benchmark artifacts found for eval ${{ matrix.eval }}; skipping analysis."
            echo "has_data=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          bun run scripts/merge-benchmark-exports.ts eval-benchmarks merged-benchmark.json
          echo "Merged benchmark export ready for analysis."
          echo "has_data=true" >> "$GITHUB_OUTPUT"

      - name: Run judges analysis
        if: steps.merge.outputs.has_data == 'true'
        id: analysis
        env:
          OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
        run: |
          set -euo pipefail
          bun run scripts/analysis.ts merged-benchmark.json > analysis.txt
          cat analysis.txt

      - name: Determine analysis job URL
        if: steps.merge.outputs.has_data == 'true'
        id: analysis_url
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          MATRIX_EVAL: ${{ matrix.eval }}
          MATRIX_SAFE: ${{ matrix.safe }}
        run: |
          set -euo pipefail
          job_pattern="Judge Analysis - ${MATRIX_EVAL}"
          job_url="$(bun run scripts/determine-job-url.ts --pattern "${job_pattern}")"

          step_url="${job_url}#step:7:0"

          jq -n \
            --arg eval "${MATRIX_EVAL}" \
            --arg safe "${MATRIX_SAFE}" \
            --arg url "${step_url}" \
            '{eval: $eval, safe: $safe, url: $url}' > analysis-info.json

      - name: Upload analysis artifact
        if: steps.merge.outputs.has_data == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: analysis-${{ matrix.safe }}
          path: |
            analysis.txt
            analysis-info.json

  notify:
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs:
      - benchmark
      - eval-analysis
    if: needs.benchmark.result == 'success'
    environment: production
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Bun
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: 1.2.21

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Download benchmark artifacts
        uses: actions/download-artifact@v4
        with:
          path: benchmarks

      - name: Download analysis artifacts
        uses: actions/download-artifact@v4
        with:
          path: analysis
          pattern: analysis-*

      - name: Merge benchmark exports
        run: bun run scripts/merge-benchmark-exports.ts benchmarks merged-benchmark.json

      - name: Build analysis link map
        run: |
          set -euo pipefail
          mkdir -p analysis
          mapfile -d '' info_files < <(find analysis -type f -name 'analysis-info.json' -print0 2>/dev/null || true)
          if [ "${#info_files[@]}" -eq 0 ]; then
            echo "[]" > analysis/analysis-links.json
          else
            jq -s 'map({eval: .eval, url: .url, safe: .safe})' "${info_files[@]}" > analysis/analysis-links.json
          fi

      - name: Send Discord notification
        env:
          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
          ANALYSIS_LINKS_FILE: ${{ github.workspace }}/analysis/analysis-links.json
        run: |
          set -euo pipefail
          if [ ! -f merged-benchmark.json ]; then
            echo "merged-benchmark.json not found; skipping Discord notification." >&2
            exit 0
          fi

          bun run scripts/discord-sample.ts merged-benchmark.json