diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5cdff5f6..2d70927d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -484,6 +484,257 @@ jobs: AGENTEYE_TESTS_REQUIRE_FRAMEWORKS: "1" run: uv run pytest tests/integrations -q + # The TypeScript telemetry SDK (`@failproofai/sdk`). Separate from `quality` + # and `test` because it is its own npm package with its own lockfile, its own + # tsconfig and its own vitest config — running it inside the root project's + # jobs would mean the root's dependency tree decided whether this package's + # zero-dependency claim holds. + # + # The matrix is Node's supported majors, not a single version. This package's + # floor is 20.9 and its `using` support, `AsyncLocalStorage.enterWith`, + # `worker_threads` resource limits and `Symbol.dispose` shim all behave + # differently across that range — which is precisely the range a customer's + # agent runs on. + failproofai-ts-sdk: + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: sdk/typescript + strategy: + fail-fast: false + matrix: + include: + # The `engines` FLOOR. This leg proves the published artifact runs on + # it, and deliberately does not run the suite: vitest 5 pulls vite 8, + # which pulls rolldown, which imports `styleText` from `node:util` — + # added in Node 20.12. The TEST RUNNER's floor is not the PACKAGE's + # floor, and the honest way to say so is to keep the floor at 20.9 and + # prove it with the thing a consumer actually gets. + # + # Pinning the runner back to something 20.9 can load is the tail + # wagging the dog: it means carrying the CVEs vitest 5 fixed (one + # Critical, one High) so that a test runner can start on a Node + # release nobody runs the tests on. + - node-version: "20.9" + suite: false + - node-version: "20.x" + suite: true + - node-version: "22.x" + suite: true + - node-version: "24.x" + suite: true + steps: + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + + - name: Install dependencies + uses: nick-fields/retry@v4 + with: + max_attempts: 3 + timeout_minutes: 5 + command: cd sdk/typescript && npm ci --no-audit --no-fund + + - name: Typecheck + if: matrix.suite + run: npm run typecheck + + - name: Lint + if: matrix.suite + run: npm run lint + + # Runs on EVERY leg, including the floor: `tsc` is the thing that produces + # what ships, so "it builds on 20.9" is a claim worth checking there. + - name: Build + run: npm run build + + # The sandbox suite needs `dist/` present — the evaluator sandbox is a + # real `worker_threads` entry and cannot load a `.ts` file — and it + # asserts rather than skips when it is missing, so the build above is a + # prerequisite rather than a duplicate. + - name: Test + if: matrix.suite + run: npx vitest run + + # Everything above ran against the source tree. These two steps run + # against the ARTIFACT, because the failures they catch — a missing export + # condition, a CommonJS build Node reads as ESM, a `dist/` path the files + # list does not ship — are invisible from inside the package and total + # from outside it. + - name: Pack + run: npm pack --pack-destination /tmp + + - name: Smoke-test the packed tarball with no dependencies + run: | + mkdir -p /tmp/ts-sdk-smoke && cd /tmp/ts-sdk-smoke + npm init -y >/dev/null + # `--omit=optional --omit=peer` is the assertion, not an optimisation. + # "Zero dependencies" is the reason this package is safe to drop into + # someone else's agent, so prove it against the built artifact: install + # it with nothing else present, emit real events, and read them back + # off disk. + npm install --no-audit --no-fund --omit=optional --omit=peer /tmp/failproofai-sdk-*.tgz + test ! -d node_modules/@failproofai/sdk/node_modules \ + || { echo "the published package brought transitive dependencies"; exit 1; } + + cat > esm.mjs <<'EOF' + import * as fp from "@failproofai/sdk"; + fp.configure({ baseDir: process.env.SPOOL }); + await fp.agent("smoke", { goal: "ci" }, async () => { + await fp.toolCall("t", { input: { q: 1 } }, () => "ok"); + }); + await fp.flush(); + console.log(fp.version); + EOF + SPOOL=/tmp/ts-sdk-spool node esm.mjs + + cat > cjs.cjs <<'EOF' + const fp = require("@failproofai/sdk"); + fp.configure({ baseDir: process.env.SPOOL }); + fp.event.agentStart({ sessionId: "cjs", goal: "ci" }); + fp.flushSync(); + console.log(fp.version); + EOF + SPOOL=/tmp/ts-sdk-spool node cjs.cjs + + # The evaluator loads from its own subpath, and its CLI has to be + # executable — a `bin` that is not fails on every platform where the + # installer links rather than copies. + node -e "const e = require('@failproofai/sdk/evaluator'); if (typeof e.Evaluator !== 'function') throw new Error('evaluator subpath is broken')" + npx --no-install failproofai-evaluator --help > /dev/null + + node - <<'EOF' + const { readdirSync, readFileSync } = require("node:fs"); + const { join } = require("node:path"); + const dir = "/tmp/ts-sdk-spool/events"; + const events = readdirSync(dir) + .filter((f) => f.endsWith(".jsonl")) + .flatMap((f) => readFileSync(join(dir, f), "utf8").split("\n").filter(Boolean)) + .map((line) => JSON.parse(line)); + const types = new Set(events.map((e) => e.type)); + const want = ["agent_start", "agent_end", "tool_use", "tool_result"]; + for (const type of want) { + if (!types.has(type)) throw new Error(`the installed artifact never wrote ${type}`); + } + // Emitted by the artifact, so this also proves the wire format + // survived packaging rather than only surviving an in-tree import. + if (!events.every((e) => "environment" in e && "session_id" in e)) { + throw new Error("an event reached disk missing a required field"); + } + console.log(`${events.length} events written by the installed artifact`); + EOF + + # The evaluator sandbox in an INSTALLED package resolves its worker + # through the package's own `./sandbox-worker` export, with no env + # override in sight. That resolution is the one part of the sandbox that + # cannot be exercised from inside this repository, and a failure in it + # means managed evaluations refuse to run for every customer. + - name: Verify the evaluator sandbox resolves from an installed package + run: | + cd /tmp/ts-sdk-smoke + node - <<'EOF' + const { compileEvaluator, sessionTranscriptFromWire } = require("@failproofai/sdk/evaluator"); + const session = sessionTranscriptFromWire({ + schema_version: "2", + assignment_id: "a", session_id: "s", session_revision_id: "r", + agent_id: "main", environment: "dev", + started_at: "2026-01-01T00:00:00.000000Z", + ended_at: "2026-01-01T00:01:00.000000Z", + event_count: 1, + events: [{ id: "1", ts: "2026-01-01T00:00:01.000000Z", event_type: "tool_use", payload: {} }], + }); + compileEvaluator("EvalResult({ score: Score(session.count('tool_use') > 0 ? 1 : 0) })", { evalKey: "k" })(session) + .then((result) => { + if (result.score.value !== 1) throw new Error(`unexpected score ${result.score.value}`); + console.log("sandbox resolved and evaluated from the installed package"); + }) + .catch((error) => { console.error(error); process.exit(1); }); + EOF + + # The TypeScript SDK's framework adapters, against REAL framework releases — + # the counterpart of `failproofai-sdk-integrations` above. `failproofai-ts-sdk` + # runs the adapters with no framework installed, which proves their logic and + # nothing about whether it ever reaches a framework: the first release passed + # 242 unit tests with every adapter recording nothing in an ES-module app. + # + # Each fixture under `sdk/typescript/integration/fixtures/` is a consumer + # project with its own lockfile pinning one framework release. The packed + # tarball is extracted into each, and one agent is run as BOTH an ES module and + # CommonJS, because the two module systems load different copies of a + # dual-published framework. A fixture that fails to install fails the job — + # there is no skip path to read as green. + failproofai-ts-sdk-integrations: + name: failproofai-ts-sdk-integrations (${{ matrix.shard }}, node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: sdk/typescript + strategy: + fail-fast: false + matrix: + # Sharded by what a shard needs installed, because the whole suite — + # four frameworks at both ends of their ranges, Bun and Deno parity over + # every fixture, and five real `next build`s — is ~23 CPU-minutes, too + # much for one runner inside a timeout. Each shard installs only its own + # fixtures (FAILPROOFAI_IT_FIXTURES) and runs only its own files. + # + # frameworks: Node's oldest and newest supported majors — the ESM/CJS + # split this job exists for behaves differently once `require(esm)` is + # unflagged. runtimes and nextjs: one Node, because what they vary is + # the runtime or the bundler, not Node. + include: + - shard: frameworks + node-version: "20.x" + fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,langchain-dup-core,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,types,vanilla + files: integration/ai.test.ts integration/langchain.test.ts integration/mastra.test.ts integration/mastra-coverage.test.ts integration/llamaindex.test.ts integration/types.test.ts integration/vanilla.test.ts + - shard: frameworks + node-version: "24.x" + fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,langchain-dup-core,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,types,vanilla + files: integration/ai.test.ts integration/langchain.test.ts integration/mastra.test.ts integration/mastra-coverage.test.ts integration/llamaindex.test.ts integration/types.test.ts integration/vanilla.test.ts + - shard: runtimes + node-version: "24.x" + fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,runtimes + files: integration/runtimes.core.test.ts integration/runtimes.bun.test.ts integration/runtimes.deno.test.ts + - shard: nextjs + node-version: "24.x" + fixtures: nextjs,langchain-1,ai-7,mastra-1,llamaindex-0.12 + files: integration/nextjs.test.ts + steps: + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: | + sdk/typescript/package-lock.json + sdk/typescript/integration/fixtures/*/package-lock.json + + - name: Install dependencies + uses: nick-fields/retry@v4 + with: + max_attempts: 3 + timeout_minutes: 5 + command: cd sdk/typescript && npm ci --no-audit --no-fund + + # `test:integration` builds, packs, `npm ci`s this shard's fixtures and + # runs its files; the fixture installs are the network-heavy part, so + # retry them as a whole rather than failing the job on a registry blip. + - name: Test against real releases (${{ matrix.shard }}) + uses: nick-fields/retry@v4 + env: + FAILPROOFAI_IT_FIXTURES: ${{ matrix.fixtures }} + with: + max_attempts: 2 + timeout_minutes: 25 + command: cd sdk/typescript && npm run test:integration -- ${{ matrix.files }} + test: runs-on: ubuntu-latest # The retry above nominally allows 3 attempts x 10 minutes. Capping the job diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index d6a5c889c..0c9c44f3f 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -62,7 +62,7 @@ jobs: # nothing at all — not this job, which was only ever given `bun.lock`, and # not Dependabot, which had no `cargo` ecosystem — for a TLS stack that # compiles into a root-installed system service. - - name: Scan bun.lock and Cargo.lock for known-vulnerable / malicious dependencies + - name: Scan every lockfile for known-vulnerable / malicious dependencies id: scan uses: google/osv-scanner-action/osv-scanner-action@f4cfcc01edc9c8b756a9b873b7a623ca674da51e # v2.3.8 with: @@ -79,6 +79,7 @@ jobs: --lockfile=Cargo.lock --lockfile=fp-cloud-cli/uv.lock --lockfile=sdk/python/uv.lock + --lockfile=sdk/typescript/package-lock.json # Only the schedule run notifies — nothing on main touched the lockfile, # so nobody is watching it the way a PR author watches their own checks # or a push failure shows up against the commit they just merged. Same diff --git a/.github/workflows/publish-failproofai-ts-sdk.yml b/.github/workflows/publish-failproofai-ts-sdk.yml new file mode 100644 index 000000000..c0c9d0088 --- /dev/null +++ b/.github/workflows/publish-failproofai-ts-sdk.yml @@ -0,0 +1,421 @@ +name: Publish @failproofai/sdk to npm + +# The TypeScript telemetry SDK ships on its own cadence, separate from the +# `failproofai` npm package in publish.yml, from the daemon binaries, and from +# its Python sibling in publish-failproofai-sdk.yml. It is a pure-JavaScript +# package with no platform matrix and no release assets, so it shares nothing +# with those pipelines beyond credentials it does not use. +# +# AUTHENTICATION is the `NPM_TOKEN` secret publish.yml already uses, plus +# `--provenance`, which needs `id-token: write` on the publish job and nothing +# configured out of band. Provenance is what lets a consumer verify this tarball +# was built by this workflow from this commit, and it costs one flag. +# +# TWO SETTINGS LIVE OUTSIDE THIS FILE, and they are the ones a branch edit +# cannot reach. Every check below lives on the ref being dispatched, so a writer +# can delete them on a branch and click Run: +# +# 1. Repo -> Settings -> Environments -> `npm-failproofai-ts-sdk`: +# Deployment branches: `main` only (add required reviewers if you want a +# second pair of eyes on every release). +# A job referencing this environment from any other ref is refused before +# it starts. NOTE: GitHub creates a missing environment implicitly, WITHOUT +# protection rules — so this is not self-configuring; set it up by hand. +# 2. The `NPM_TOKEN` secret scoped to that environment rather than to the +# repository, so a workflow on another ref cannot read it. +# +# WHO MAY PUBLISH. Every publish is restricted to the logins in +# `RELEASE_ACTORS`, checked against BOTH `github.actor` and +# `github.triggering_actor`. Both, because on a RE-RUN `github.actor` stays the +# user who started the ORIGINAL run while `triggering_actor` is whoever pressed +# re-run: a guard reading only the former lets anyone with write access re-run a +# maintainer's failed publish and ship from it. +# +# VERSION MANAGEMENT lives in `sdk/typescript/scripts/release.mjs`, which this +# workflow calls rather than restating: +# +# beta X.Y.Z-beta.N -> X.Y.Z-beta.(N+1) +# stable X.Y.Z -> X.Y.(Z+1)-beta.0 +# +# `preflight` resolves it, refuses one npm has already taken, and computes what +# `main` moves to; `bump` pushes that to `main` after a successful upload — the +# version line AND a stub CHANGELOG section for it, in one commit, because a +# version with no section is a release `preflight` refuses. Cutting a release is +# therefore: edit ONE line only when you want to LEAVE the current beta line (a +# stable cut, or a minor/major bump), then dispatch. A plain beta needs no edit +# at all. +# +# The bump pushes to `main`, which the org ruleset protects, so it authenticates +# as the version-bot GitHub App — the SAME app and the SAME two secrets +# (`VERSION_BOT_APP_ID` / `VERSION_BOT_PRIVATE_KEY`) publish.yml and its +# siblings already use. No new out-of-band setup. + +on: + workflow_dispatch: + inputs: + dry_run: + description: Build and verify, but do not upload to npm + type: boolean + default: false + +concurrency: + group: publish-failproofai-ts-sdk + cancel-in-progress: false + +env: + RELEASE_ACTORS: NiveditJain + +jobs: + # Cheap, fail-fast, and holding NO identity of any kind: it resolves the + # version, refuses one npm has already taken, checks the CHANGELOG section + # exists, and computes what `main` moves to afterwards. Everything here is + # stdlib Node, `git` and one `curl` — no dependency is installed, so there is + # no third-party code in the job that decides whether a release may proceed. + # + # Separate from `build` because failing here is nearly free, and because npm — + # like PyPI — never releases a version for reuse, so discovering a burned one + # after a full build costs the whole run and fixes nothing. + preflight: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + version: ${{ steps.resolve.outputs.version }} + next_version: ${{ steps.resolve.outputs.next_version }} + is_prerelease: ${{ steps.resolve.outputs.is_prerelease }} + dist_tag: ${{ steps.resolve.outputs.dist_tag }} + tag: ${{ steps.resolve.outputs.tag }} + steps: + - name: Authorize actor and branch + env: + ACTOR: ${{ github.actor }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + run: | + set -euo pipefail + if [ "${{ github.ref }}" != "refs/heads/main" ]; then + echo "::error::releases are cut from main only (got ${{ github.ref }})" + exit 1 + fi + for who in "$ACTOR" "$TRIGGERING_ACTOR"; do + allowed=0 + for candidate in $RELEASE_ACTORS; do + if [ "$(echo "$who" | tr 'A-Z' 'a-z')" = "$(echo "$candidate" | tr 'A-Z' 'a-z')" ]; then + allowed=1 + fi + done + if [ "$allowed" -ne 1 ]; then + echo "::error::$who is not authorized to publish @failproofai/sdk" + exit 1 + fi + done + + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-node@v5 + with: + node-version: 22.x + + - name: Resolve the version + id: resolve + working-directory: sdk/typescript + run: node scripts/release.mjs resolve + + # Refused here, before anything is built. npm returns the version's + # metadata for a version that exists and a 404 for one that does not, so + # this is one request and no ambiguity. + - name: Refuse a version npm has already taken + env: + VERSION: ${{ steps.resolve.outputs.version }} + run: | + set -euo pipefail + status=$(curl -sS -o /dev/null -w '%{http_code}' \ + "https://registry.npmjs.org/@failproofai%2Fsdk/$VERSION") + case "$status" in + 404) echo "$VERSION is available" ;; + 200) + echo "::error::@failproofai/sdk@$VERSION is already published. npm never releases a version for reuse — move src/version.ts and package.json to the next one." + exit 1 + ;; + *) + echo "::error::could not query the registry (HTTP $status); refusing to guess" + exit 1 + ;; + esac + + # The GitHub Release body. A version whose changes nobody can read is a + # version that may as well not have shipped, so its absence stops the run + # here rather than producing a release with an empty description. + - name: Require a CHANGELOG section for this version + working-directory: sdk/typescript + run: node scripts/release.mjs changelog "${{ steps.resolve.outputs.version }}" > /dev/null + + # Everything the CI job does, plus the checks that only make sense against a + # release artifact. Holds no identity either: it uploads the tarball for the + # publish job rather than publishing it, so the job that CAN publish does no + # building and runs no third-party build script. + build: + needs: preflight + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + defaults: + run: + working-directory: sdk/typescript + steps: + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-node@v5 + with: + node-version: 22.x + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + + - run: npm ci --no-audit --no-fund + + - run: npm run typecheck + - run: npm run lint + - run: npm run build + - run: npx vitest run + + - name: Pack + run: npm pack --pack-destination "$RUNNER_TEMP" + + # The tarball is the thing being released, so assert its contents rather + # than the source tree's. A `files` list that stopped shipping `dist/` + # would publish a SUCCESSFUL but empty package, and npm would install it. + - name: Verify the tarball actually contains the package + run: | + set -euo pipefail + tarball=$(ls "$RUNNER_TEMP"/failproofai-sdk-*.tgz) + names=$(tar -tzf "$tarball") + echo "$names" | grep -q 'package/dist/esm/index.js' || { echo "::error::dist/esm is missing"; exit 1; } + echo "$names" | grep -q 'package/dist/cjs/index.js' || { echo "::error::dist/cjs is missing"; exit 1; } + echo "$names" | grep -q 'package/dist/esm/index.d.ts' || { echo "::error::type declarations are missing"; exit 1; } + echo "$names" | grep -q 'package/dist/cjs/package.json' || { echo "::error::the CommonJS type marker is missing — Node would read that build as ESM"; exit 1; } + echo "$names" | grep -q 'package/dist/esm/evaluator/cli.js' || { echo "::error::the evaluator CLI is missing"; exit 1; } + echo "$names" | grep -q 'package/dist/cjs/evaluator/sandbox-worker.js' || { echo "::error::the sandbox worker is missing — managed evaluations would be refused"; exit 1; } + echo "$names" | grep -q 'package/LICENSE' || { echo "::error::LICENSE is missing"; exit 1; } + echo "$names" | grep -q 'package/README.md' || { echo "::error::README.md is missing"; exit 1; } + echo "$names" | grep -qv 'package/src/' || true + modules=$(echo "$names" | grep -c 'package/dist/esm/.*\.js$' || true) + if [ "$modules" -lt 15 ]; then + echo "::error::the tarball looks empty — only $modules modules under dist/esm" + exit 1 + fi + echo "$modules modules, both builds, types, CLI and sandbox present" + + - uses: actions/upload-artifact@v4 + with: + name: ts-sdk-tarball + path: ${{ runner.temp }}/failproofai-sdk-*.tgz + if-no-files-found: error + retention-days: 7 + + publish: + needs: [preflight, build] + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: npm-failproofai-ts-sdk + permissions: + contents: write + # `--provenance` mints an OIDC token to attest that this tarball came from + # this workflow at this commit. + id-token: write + steps: + - name: Authorize actor + env: + ACTOR: ${{ github.actor }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + run: | + set -euo pipefail + for who in "$ACTOR" "$TRIGGERING_ACTOR"; do + allowed=0 + for candidate in $RELEASE_ACTORS; do + if [ "$(echo "$who" | tr 'A-Z' 'a-z')" = "$(echo "$candidate" | tr 'A-Z' 'a-z')" ]; then + allowed=1 + fi + done + if [ "$allowed" -ne 1 ]; then + echo "::error::$who is not authorized to publish @failproofai/sdk" + exit 1 + fi + done + + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-node@v5 + with: + node-version: 22.x + registry-url: "https://registry.npmjs.org" + + # Into RUNNER_TEMP, never the checkout: a tarball inside the working tree + # is a tarball the next `npm pack` would include. + - uses: actions/download-artifact@v4 + with: + name: ts-sdk-tarball + path: ${{ runner.temp }}/tarball + + - name: Verify the token can authenticate before doing anything else + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + if ! who=$(npm whoami 2>&1); then + echo "::error::NPM_TOKEN cannot authenticate with the registry: $who" + exit 1 + fi + echo "authenticated as $who" + + # `--ignore-scripts`, because a tarball's lifecycle scripts must not run + # on a machine holding a publish token. The package was already built and + # verified in the `build` job; there is nothing left for a script to do. + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + DIST_TAG: ${{ needs.preflight.outputs.dist_tag }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + tarball=$(ls "$RUNNER_TEMP"/tarball/failproofai-sdk-*.tgz) + if [ "$DRY_RUN" = "true" ]; then + # --provenance is dropped here on purpose: attestation needs a real + # publish to attach to, and a dry run has none. + npm publish "$tarball" --dry-run --ignore-scripts --tag "$DIST_TAG" --access public + else + npm publish "$tarball" --provenance --ignore-scripts --tag "$DIST_TAG" --access public + fi + + - name: Extract the release notes + if: inputs.dry_run != true + id: notes + working-directory: sdk/typescript + run: node scripts/release.mjs changelog "${{ needs.preflight.outputs.version }}" + + - name: Create the GitHub Release + if: inputs.dry_run != true + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.preflight.outputs.tag }} + VERSION: ${{ needs.preflight.outputs.version }} + PRERELEASE: ${{ needs.preflight.outputs.is_prerelease }} + NOTES: ${{ steps.notes.outputs.body }} + run: | + set -euo pipefail + flags=(--title "@failproofai/sdk $VERSION" --notes "$NOTES" --target "$GITHUB_SHA") + if [ "$PRERELEASE" = "true" ]; then flags+=(--prerelease); fi + gh release create "$TAG" "${flags[@]}" + + # The registry is eventually consistent, so this runs after the publish rather + # than as part of it: a package that published and cannot be installed is the + # failure worth catching, and only a real install from the real registry can. + verify-install: + needs: [preflight, publish] + if: inputs.dry_run != true + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/setup-node@v5 + with: + node-version: 22.x + + - name: Install the published version and emit real events + env: + VERSION: ${{ needs.preflight.outputs.version }} + run: | + set -euo pipefail + mkdir -p /tmp/verify && cd /tmp/verify + npm init -y > /dev/null + for attempt in 1 2 3 4 5; do + if npm install --no-audit --no-fund --omit=optional --omit=peer "@failproofai/sdk@$VERSION"; then + break + fi + echo "the registry has not caught up yet (attempt $attempt); waiting" + sleep 20 + done + test ! -d node_modules/@failproofai/sdk/node_modules \ + || { echo "::error::the published package brought transitive dependencies"; exit 1; } + cat > verify.mjs <<'EOF' + import * as fp from "@failproofai/sdk"; + fp.configure({ baseDir: "/tmp/verify-spool" }); + await fp.agent("verify", { goal: "post-publish" }, () => undefined); + await fp.flush(); + console.log(fp.version); + EOF + published=$(node verify.mjs) + test "$published" = "$VERSION" || { echo "::error::installed $published, expected $VERSION"; exit 1; } + test -n "$(ls /tmp/verify-spool/events/*.jsonl)" \ + || { echo "::error::the published package wrote no events"; exit 1; } + echo "installed $VERSION and it works" + + # Opens the next line on main: the version AND a stub CHANGELOG section for + # it, in one commit. Both halves together on purpose — a version with no + # section is what `preflight` refuses at release time, and a bump commit + # carries a skip-ci marker, so that state would go red on the next unrelated + # PR rather than on itself. + bump: + needs: [preflight, publish] + if: inputs.dry_run != true + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.VERSION_BOT_APP_ID }} + private-key: ${{ secrets.VERSION_BOT_PRIVATE_KEY }} + + - uses: actions/checkout@v7.0.1 + with: + ref: main + token: ${{ steps.app-token.outputs.token }} + + - uses: actions/setup-node@v5 + with: + node-version: 22.x + + - name: Move to the next version and open its CHANGELOG section + env: + NEXT: ${{ needs.preflight.outputs.next_version }} + RELEASED: ${{ needs.preflight.outputs.version }} + working-directory: sdk/typescript + run: | + set -euo pipefail + current=$(node -e "process.stdout.write(require('./package.json').version)") + if [ "$current" != "$RELEASED" ]; then + echo "main has moved to $current since $RELEASED was released; leaving it alone" + exit 0 + fi + node scripts/release.mjs write "$NEXT" + today=$(date -u +%Y-%m-%d) + node - "$NEXT" "$today" <<'EOF' + const { readFileSync, writeFileSync } = require("node:fs"); + const [version, today] = process.argv.slice(2); + const path = "CHANGELOG.md"; + const text = readFileSync(path, "utf8"); + const marker = "\n## "; + const at = text.indexOf(marker); + if (at === -1) throw new Error("CHANGELOG.md has no version sections"); + const stub = `\n## ${version} — ${today}\n\n### Fixes\n\n- _Nothing yet._\n`; + writeFileSync(path, text.slice(0, at) + stub + text.slice(at)); + EOF + + - name: Commit + env: + NEXT: ${{ needs.preflight.outputs.next_version }} + run: | + set -euo pipefail + if git diff --quiet; then + echo "nothing to commit" + exit 0 + fi + git config user.name "failproofai-version-bot[bot]" + git config user.email "failproofai-version-bot[bot]@users.noreply.github.com" + git add sdk/typescript/src/version.ts sdk/typescript/package.json sdk/typescript/CHANGELOG.md + git commit -m "chore(@failproofai/sdk): open $NEXT after publishing ${{ needs.preflight.outputs.version }} [skip ci]" + git push origin main diff --git a/.gitignore b/.gitignore index d40eec4cd..7fdf07091 100644 --- a/.gitignore +++ b/.gitignore @@ -61,8 +61,14 @@ next-env.d.ts # custom hooks loader temp files *.__failproofai_tmp__.* -# package manager lockfiles (bun.lock is tracked; bun.lockb is binary) +# package manager lockfiles (bun.lock is tracked; bun.lockb is binary). +# This rule exists to stop an `npm install` in the ROOT project producing a +# second, divergent lockfile beside bun.lock. `sdk/typescript` is a separate npm +# package that is installed with npm and nothing else, so its lockfile is real: +# its CI job runs `npm ci`, which refuses to run without one, and the release +# workflow's whole point is installing exactly what CI installed. package-lock.json +!sdk/typescript/package-lock.json # translation cache .translation-cache.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f730a6e..4b89ebd9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,25 @@ ### Added +- **`@failproofai/sdk`, the TypeScript telemetry SDK, lands at `sdk/typescript`.** The counterpart to the Python `failproofai-sdk`: the same 15 events, the same wire format, the same spool directory, the same Evaluator v2 protocol — so a fleet running Node agents and Python agents writes into one pipe, and the dashboard cannot tell which wrote what. Scopes (`session`/`agent`/`toolCall`) carry identity on `AsyncLocalStorage` and come in a callback form and a `using` form; `instrument()` wires LangChain.js/LangGraph.js, the Vercel AI SDK, Mastra and LlamaIndex.TS. Zero runtime dependencies, dual ESM+CommonJS, Node >= 20.9, with its own CI job and its own release workflow. Versioned independently of the npm `failproofai` package and of both Python packages. (#830) +- **The TypeScript SDK's framework adapters are held to the Python SDK's bar.** A new `failproofai-ts-sdk-integrations` CI job runs every adapter against real framework releases at both ends of its declared range, as an ES module and as CommonJS, from the packed tarball — the counterpart of `failproofai-sdk-integrations`. It exists because testing the adapters with no framework installed let every one of them ship recording nothing in an ES-module app: they patched the CommonJS copy of the framework while the app ran the ESM one. The adapters now patch the copy the app loads; LangChain/LangGraph traces match the Python adapter's; the Vercel AI SDK is supported through `ai` 7; Mastra's range starts at 0.20 and LlamaIndex.TS's at 0.11.4, the oldest releases the suite proves; CommonJS and `moduleResolution: node` consumers get working type declarations (TypeScript ≥ 5.4); and an adversarial review's findings for long-running servers are fixed — live runs no longer lose events once 10k others have finished, `instrument("ai")` no longer takes the application's OpenTelemetry slot, and concurrent requests on one shared LlamaIndex engine stay apart. A coverage sweep then extended the suite to every commonly used surface of each framework and to Bun, Deno and Next.js; Next.js apps get `withFailproofai()` from `@failproofai/sdk/next`, and `instrument()` now warns instead of recording nothing when Next bundles a framework. (#830) +- **Server-authored evaluations run through a parser and an interpreter, not `eval` and not `node:vm`.** The Python SDK validates an AST and then calls `eval` with empty builtins; the same shape in JavaScript is not safe, because `x["constructor"]["constructor"]("…")()` reaches arbitrary code through a key computed at RUNTIME, which no source-level allowlist can see, and a `vm` context has its own `Function` to reach. So the TypeScript sandbox interprets: every property read goes through one function that checks the actual key at the moment of the read, and the interpreter never constructs a function. A `worker_threads` sandbox with V8 heap limits, a wall-clock `terminate()`, a bounded result and a concurrency cap sits around that as the RESOURCE bound. An evaluation that cannot be sandboxed is refused, never run unbounded. (#830) - `fp issues` gains `close`, `archive`, `unarchive` and `clear`. `close` is the second terminal state — "we're done with it", not "we fixed it" — and the difference is what happens next: a **resolved** issue reopens when its audit finding recurs, a **closed** one does not. `archive`/`unarchive` toggle a flag that is orthogonal to state, so taking an issue off the board never overwrites how it ended. `clear` is the bulk operation behind "we changed our agents, give us a fresh board": it resolves every open issue in a scope (`--audit `, `--all-audits`, or `--everything` — exactly one required, no default) plus the audit findings behind them, needs `issues:close` **and** `audits:write`, and previews with `--dry-run` from the same server-side scope predicate the write uses. It writes **no** suppression, so a pattern that survived the agent changes reopens its issue on the next run rather than staying hidden. `--state closed` is accepted by `issues list`, and `Incident` carries `closed_at` / `archived_at`. (#815) - **`fp-cloud-cli` cuts its first stable release, `0.0.1`.** The commands above drive server endpoints that are being deployed to FailproofAI Cloud, so the CLI half stops being a pre-release: the `0.0.1b3` line the `bump` job opened is cut stable rather than published, `Development Status` moves to `5 - Production/Stable` so the classifier agrees with the one string pip reads, and `0.0.2b0` opens the next beta line. Nothing about the command surface changes at the cut — `pipx install fp-cloud-cli` already resolved the betas, because they were the only releases; now it resolves a version that says so. This is the PyPI package's own line and does not move the npm version, which stays at `1.0.7-beta.0`. (#815) ### Fixes +- **The TypeScript SDK, after a user-style test on six real agent setups against FailproofAI Cloud.** (1) Events emitted in the same millisecond tied on their timestamp, and the dashboard showed a `tool_result` before its `tool_use`. The digits below the millisecond are now a per-process sequence, so emission order is kept, and it re-anchors when the wall clock steps back. (2) SIGTERM left every open run showing as running forever. The exit hook now closes what is still open: tools get a `ProcessExit` error, and agents (hand-written and adapter-opened) get `error` then `agent_end` with `outcome: "failed"`. (3) `configure()` only reached its own copy of the SDK, so a Next.js route bundled without `withFailproofai` reported `environment: dev`. `environment` and `baseDir` are now process-wide. (4) An `agent()` wrapper around a framework agent of the same name produced two `agent_start`s and an agent that was its own parent; it now joins it. (5) LlamaIndex 0.12 tool errors read `Error: Error(Error): …`. (6) openai's errors were recorded as `error_type: Error`; the class name is used now. (7) An unawaited `instrument()` is now warned about by the LangChain adapter, whether the graph's nodes arrive under an unknown parent or as roots, instead of producing a silently wrong trace. (8) A second round of the same tests found shutdown still left the open tool, node and model call of an adapter run — and a hand-written `event.modelRequest` — unclosed. Every tool, hook and model call is now tracked where all of them are emitted, the event namespace, and closed at exit, except in a session paused on a human. (9) A third round found the close order wrong for nested runs (a planner's delegate tool closed before the writer sub-agent it started) and a crash's cause missing from the trace. Everything open now closes in one most-recently-opened-first order across tools, hooks, model calls and agents; the closing messages name the uncaught exception a crash exited on; a model call closed at exit carries its duration; and ProcessExit errors no longer carry the SDK's own stack as a traceback. The vanilla example records the model's tool calls as `fw_tool_calls` (the adapters' field), keeps its error path to the provider call, records a call with malformed arguments instead of dropping it, and narrows openai's union tool types; the skill's copy of it is now type-checked under `--strict` against the real `openai` client in CI. (#830) +- **`npx failproofai-evaluator` refused every CommonJS evaluator file.** The bin is the ESM build, and a CommonJS file (plain `tsc` output, `require`) builds its `Evaluator` from `dist/cjs`, a second copy of the class. The loader's `instanceof` check was false across the two copies, so it failed with "resolved to Evaluator, not an Evaluator". It now checks for a `Symbol.for` marker that both builds share, and a packaging test runs a CommonJS file and an ESM file through the built bin. The TypeScript SDK's Vercel AI SDK warnings also told readers to call `failproofai.ai.telemetry()`, which the root package does not export. They now name `import { telemetry, wrapModel } from "@failproofai/sdk/ai"`. (#830) - **One event, two answers: "a recurring finding reopens the resolved issue" and "it opens a new one" were both written down, four lines apart.** `fp audits resolve --help`'s confirm line said `a genuine recurrence re-opens as new`, the skill and its command reference said the same, and the new `clear` text inherited it — while `issues close --help`, the audits guide's own table and the Cloud CLI reference all said the issue **reopens**. Only the second reading makes the rest coherent: a recurrence that opened a different issue would leave `close` with nothing to stay closed *through*, and could not un-archive "a live issue" the way archiving documents. The recurrence sentence now says `reopens` everywhere it appears, in the confirm text a user reads before resolving and in the skill a model reads before acting. (#815) - **`clear`'s help promised the preview count could not disagree with the write, which nothing enforces.** The dry run and the write share a scope predicate, not a row set: the write re-runs it, so an issue that enters the scope in between is cleared without being in the number the user confirmed. That is the right behaviour for a command whose argument is a *scope* — but the guarantee as written was a stronger one than the two requests can make. The help now says what holds (the count is the real size of the scope, taken server-side) and what does not (it is a count, not a lease), and points at the closing line, which reports what actually changed. (#815) - **The skill mapped "clear all our issues" to `--all-audits`, which is not all issues.** `--all-audits` leaves alert-born and hand-opened issues on the board, so a model following that row would clear part of it and report a fresh start. Both the intent table and the workflow note now make settling the scope the first step, with what each flag does and does not cover. (#815) ### Docs +- A TypeScript reference page sits beside the Python custom-agents one, registered in the English navigation; the translate pipeline picks up the other fourteen locales on its next run. The cross-link runs both ways and both sides say the same thing in the same place — the two SDKs write the same events into the same spool, so a fleet with Node agents and Python agents produces one set of sessions, not two. That is the fact a reader needs before they start choosing, and it appeared nowhere the choice is actually made. (#830) +- The `failproofai-sdk` skill's TypeScript page now covers what six user-style tests found missing: how each framework names its agent and how to own the session id (`session({ sessionId })`), adapter options, where streamed-usage flags go (the Vercel AI SDK needs one too, and LlamaIndex needs it even without streaming), verifying on a machine whose daemon empties the spool, Next.js `configure()` placement, and the new shutdown behaviour. The id-uniqueness rule no longer contradicts itself between SKILL.md and events.md. (#830) +- The `failproofai-sdk` skill now covers TypeScript/JavaScript agents and the customer evaluator worker. New `references/typescript.md` covers install, names, the four adapters, Next.js and bundlers, the three-edit-site wiring for a hand-built loop, shutdown and verification. New `references/evaluator.md` covers writing, running and deploying an evaluator worker (the eval pod) in Python and TypeScript. The description no longer routes to the retired `agenteye-evaluator`, and the Codex `openai.yaml` picks up the `policy:` nesting the skills repo already carries, so the next sync does not revert it. A new `sdk/typescript/test/skill-snippets.test.ts` parses every TypeScript block in the skill and fails when one calls an SDK name that no longer exists. (#830) - Document ending an issue three ways (resolve / close / archive) and clearing a board after an agent change, in the audits guide and the Cloud CLI reference. (#815) ## 1.0.6 — 2026-09-16 diff --git a/CLAUDE.md b/CLAUDE.md index a60d2de75..685f01b26 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -898,6 +898,7 @@ finish. If any job fails, **stop and fix it before continuing**. Never leave a r | build | `bun run build` (Next.js + `dist/index.js` + `dist/cli.mjs` + `dist/worker.mjs`) | | test-e2e | `bun run test:e2e` | | docs | docs build/validation | +| failproofai-ts-sdk | `sdk/typescript`, across four Node majors: `npm run typecheck` + `npm run lint` + `npm run build` + `vitest run`, then `npm pack` and a smoke test of the TARBALL — installed with `--omit=peer`, loaded from both ESM and CommonJS, events read back off disk, and the evaluator sandbox resolved through the package's own `./sandbox-worker` export. That last one cannot be exercised from inside the repo and is the difference between managed evaluations working and being refused. | A separate `.github/workflows/build-daemon.yml` ("Build failproofaid") cross-compiles the 4 real `failproofaid` release binaries (linux-x64/arm64, darwin-x64/arm64), gzips each one @@ -1273,9 +1274,47 @@ sdk/python/ The telemetry SDK (Python, uv, pytest). PyPI dist or the SDK writes where no daemon reads, with NO error on either side. tests/test_spool_contract.py checks the Rust and the TypeScript directly and never skips +sdk/typescript/ The telemetry SDK (TypeScript, npm, vitest). npm package + `@failproofai/sdk`. The same 15 events, wire format, + spool directory and Evaluator v2 protocol as + sdk/python — a fleet running both writes into one + pipe. Zero runtime dependencies for the same reason, + enforced by a test AND by scripts/finalize-build.mjs. + Its OWN npm package: its own package-lock.json, + tsconfig, eslint config and vitest config, excluded + from the root tsconfig and the root eslint config so + this project's dependency tree cannot decide whether + that package's zero-dependency claim holds. + src/writer.ts Interval flush, `unref`'d so importing the SDK never + stops a script exiting; sync flush on `process.exit`. + `renameSync` inside an otherwise-async write is + load-bearing: it is what lets the exit path tell a + durable chunk from an un-written one with no window + src/context.ts AsyncLocalStorage, the analogue of Python's + contextvars. The agent stack is a FROZEN array, + replaced never mutated — a store object is shared by + reference with every async branch below it + src/evaluator/expression.ts A restricted expression language, PARSED AND + INTERPRETED. Not `eval`, not `node:vm`: JavaScript + reaches arbitrary code through `x["constructor"]`, + whose key is computed at runtime, so no source-level + allowlist closes it and a vm context has its own + `Function`. `readProperty` checks the ACTUAL key at + the moment of the read; that is the boundary + src/evaluator/source.ts The worker_threads sandbox around it — V8 heap + limits, wall-clock terminate(), bounded result, + concurrency cap. The RESOURCE bound, and it fails + CLOSED: no sandbox means managed source is refused + scripts/release.mjs The npm release scheme (`X.Y.Z-beta.N`), and the only + place it is written down; the ts-sdk release workflow + calls it. Node stdlib only, because it runs in the + preflight job, which installs nothing precisely so no + third-party code decides whether a release proceeds __tests__/ Unit + e2e tests (vitest) — TypeScript only; the Python components' tests live in fp-cloud-cli/tests/ and - sdk/python/tests/ and run under pytest + sdk/python/tests/ and run under pytest, and the + TypeScript SDK's live in sdk/typescript/test/ under + its own vitest config examples/ Sample custom policy files ``` @@ -1318,7 +1357,10 @@ any `packages/*/package.json` files against root; that directory does not curren That is the **npm** version, and it governs the CLI, the daemon and the Cargo workspace. The two Python packages version **independently of it and of each other** — `fp-cloud-cli` and `failproofai-sdk` share no version line with the npm package and never have. Do not move -them to match it. +them to match it. Neither does `@failproofai/sdk`: it is a second npm package with its own +line in `sdk/typescript/src/version.ts` and `sdk/typescript/package.json`, which +`sdk/typescript/scripts/release.mjs` keeps in step and `__tests__/ci/ts-sdk-pipeline.test.ts` +asserts agree. ### The Python packages' scheme diff --git a/__tests__/ci/ts-sdk-integration-shards.test.ts b/__tests__/ci/ts-sdk-integration-shards.test.ts new file mode 100644 index 000000000..77db7c078 --- /dev/null +++ b/__tests__/ci/ts-sdk-integration-shards.test.ts @@ -0,0 +1,54 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +/** + * The TypeScript SDK's real-framework suite runs in CI as SHARDS, each listing + * the integration files it runs and the fixtures it installs. Those lists are + * hand-maintained, so a new `integration/*.test.ts` that nobody adds to a shard + * would simply never run in CI — green forever, testing nothing. This is the + * tripwire for that, and for a shard naming a fixture that does not exist + * (which `npm ci` would never reach, because the fixture installs by name). + */ + +const root = join(__dirname, "..", ".."); +const sdk = join(root, "sdk", "typescript"); + +interface Shard { + shard: string; + fixtures: string; + files: string; +} + +const workflow = parse(readFileSync(join(root, ".github", "workflows", "ci.yml"), "utf8")) as { + jobs: Record; +}; +const shards = workflow.jobs["failproofai-ts-sdk-integrations"]?.strategy?.matrix?.include ?? []; + +describe("failproofai-ts-sdk-integrations shards", () => { + it("has shards", () => { + expect(shards.length).toBeGreaterThan(0); + }); + + it("runs every integration test file in at least one shard", () => { + const files = readdirSync(join(sdk, "integration")) + .filter((name) => name.endsWith(".test.ts")) + .map((name) => `integration/${name}`); + const covered = new Set(shards.flatMap((s) => s.files.split(/\s+/).filter(Boolean))); + expect(files.filter((file) => !covered.has(file))).toEqual([]); + }); + + it("names only files and fixtures that exist", () => { + const fixtures = new Set(readdirSync(join(sdk, "integration", "fixtures"))); + for (const s of shards) { + for (const file of s.files.split(/\s+/).filter(Boolean)) { + expect(() => readFileSync(join(sdk, file)), `${s.shard}: ${file}`).not.toThrow(); + } + for (const fixture of s.fixtures.split(",").filter(Boolean)) { + expect(fixtures.has(fixture), `${s.shard}: fixture ${fixture}`).toBe(true); + } + } + }); +}); diff --git a/__tests__/ci/ts-sdk-pipeline.test.ts b/__tests__/ci/ts-sdk-pipeline.test.ts new file mode 100644 index 000000000..d0bb44e70 --- /dev/null +++ b/__tests__/ci/ts-sdk-pipeline.test.ts @@ -0,0 +1,262 @@ +// @vitest-environment node +/** + * The release scheme for `@failproofai/sdk`, and the two pipelines that run it. + * + * npm never releases a version for reuse, so the arithmetic in + * `sdk/typescript/scripts/release.mjs` is the whole mechanism: get it wrong and + * the wrong thing ships under a number nobody can take back. So this file tests + * the script by RUNNING it, and then asserts that the workflows actually wire it + * to the jobs that make it a pipeline rather than a convention a maintainer has + * to remember. + * + * `node` is not guarded behind a skip. The package under test is a Node package + * and every runner has one; a skip here would mean the release scheme's only + * test silently stops running — the failure mode this repo keeps closing + * elsewhere (FAILPROOFAI_SDK_REQUIRE_CONTRACT, AGENTEYE_TESTS_REQUIRE_FRAMEWORKS). + */ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parse } from "yaml"; + +const ROOT = process.cwd(); +const SDK = resolve(ROOT, "sdk/typescript"); +const SCRIPT = resolve(SDK, "scripts/release.mjs"); +const PUBLISH_WORKFLOW = resolve(ROOT, ".github/workflows/publish-failproofai-ts-sdk.yml"); +const CI_WORKFLOW = resolve(ROOT, ".github/workflows/ci.yml"); + +function run(...args: string[]): string { + return execFileSync(process.execPath, [SCRIPT, ...args], { + cwd: SDK, + encoding: "utf8", + env: { ...process.env, GITHUB_OUTPUT: "" }, + }); +} + +function workflow(path: string): Record { + return parse(readFileSync(path, "utf8")) as Record; +} + +describe("the version scheme", () => { + it("advances a beta and opens one after a stable", () => { + expect(run("next", "0.0.1-beta.4").trim()).toBe("0.0.1-beta.5"); + expect(run("next", "1.2.3").trim()).toBe("1.2.4-beta.0"); + expect(run("next", "0.0.1-beta.9").trim()).toBe("0.0.1-beta.10"); + }); + + it("refuses a spelling npm would store as something else", () => { + // Every consumer in the pipeline compares version STRINGS — the source + // file, the tarball name, the dist-tag and the "is this published" query — + // so a spelling that round-trips differently is a release that half + // succeeds. + for (const bad of ["v1.0.0", "1.2.3-beta", "1.2.3-beta.01", "01.2.3", "1.2.3-rc.1"]) { + expect(() => run("next", bad)).toThrow(); + } + }); + + it("resolves the checked-in version and keeps both files in step", () => { + const output = run("resolve"); + const version = /^version=(.+)$/m.exec(output)?.[1]; + expect(version).toBeTruthy(); + + const manifest = JSON.parse(readFileSync(resolve(SDK, "package.json"), "utf8")) as { + version: string; + }; + const source = readFileSync(resolve(SDK, "src/version.ts"), "utf8"); + // They are read by different consumers — npm reads one, the SDK reports the + // other — and a release where they disagree publishes a package that + // misreports its own version to the dashboard. + expect(manifest.version).toBe(version); + expect(source).toContain(`export const VERSION = "${version!}";`); + }); + + it("picks the dist-tag from the channel, so a beta never becomes `latest`", () => { + const output = run("resolve"); + const isPrerelease = /^is_prerelease=(.+)$/m.exec(output)?.[1]; + const distTag = /^dist_tag=(.+)$/m.exec(output)?.[1]; + expect(distTag).toBe(isPrerelease === "true" ? "beta" : "latest"); + }); + + it("refuses a release whose CHANGELOG section is missing", () => { + // A published version nobody can read the changes for is a version that may + // as well not have shipped. + expect(() => run("changelog", "9.9.9")).toThrow(); + expect(run("changelog").length).toBeGreaterThan(50); + }); +}); + +describe("the publish pipeline", () => { + const jobs = workflow(PUBLISH_WORKFLOW).jobs as Record; + + it("resolves the version and refuses a burned one BEFORE anything is built", () => { + const steps = jobs.preflight.steps as Array>; + const names = steps.map((step) => String(step.name ?? step.uses ?? "")); + expect(names.some((name) => /already taken/i.test(name))).toBe(true); + // npm never releases a version for reuse, so discovering a burned one after + // a full build costs the whole run and fixes nothing. + expect(jobs.preflight.needs).toBeUndefined(); + expect(jobs.build.needs).toBe("preflight"); + }); + + it("gives the preflight job no identity at all", () => { + // Nothing in the job that decides whether a release may proceed should be + // able to perform one. + expect(jobs.preflight.permissions).toEqual({ contents: "read" }); + expect(jobs.preflight.environment).toBeUndefined(); + }); + + it("separates the job that builds from the job that publishes", () => { + // The job holding the token runs no build script. + expect(jobs.build.permissions).toEqual({ contents: "read" }); + expect(jobs.publish.needs).toEqual(["preflight", "build"]); + expect(jobs.publish.environment).toBe("npm-failproofai-ts-sdk"); + expect(jobs.publish.permissions["id-token"]).toBe("write"); + }); + + it("checks BOTH actors, so a re-run cannot launder a publish", () => { + // On a re-run `github.actor` stays the user who started the ORIGINAL run + // while `triggering_actor` is whoever pressed re-run. + for (const job of [jobs.preflight, jobs.publish]) { + const guard = (job.steps as Array>).find((step) => + /Authorize actor/i.test(String(step.name ?? "")), + ); + expect(guard).toBeDefined(); + expect(JSON.stringify(guard!.env)).toContain("github.actor"); + expect(JSON.stringify(guard!.env)).toContain("github.triggering_actor"); + } + }); + + it("publishes with provenance and without running the tarball's scripts", () => { + const publish = (jobs.publish.steps as Array>).find( + (step) => String(step.name ?? "") === "Publish", + ); + expect(publish).toBeDefined(); + const script = String(publish!.run); + expect(script).toContain("--provenance"); + // A tarball's lifecycle scripts must not run on a machine holding a publish + // token. + expect(script).toContain("--ignore-scripts"); + expect(script).toContain("--tag \"$DIST_TAG\""); + // The dry run drops provenance on purpose: attestation needs a real publish + // to attach to. + expect(script).toContain("--dry-run"); + }); + + it("verifies the install from the real registry, after the publish", () => { + expect(jobs["verify-install"].needs).toEqual(["preflight", "publish"]); + const script = JSON.stringify(jobs["verify-install"].steps); + // The zero-dependency promise is asserted against what npm actually + // resolved, not against the source tree. + expect(script).toContain("--omit=peer"); + expect(script).toContain("node_modules/@failproofai/sdk/node_modules"); + }); + + it("opens the next version AND its CHANGELOG section in one commit", () => { + // A version with no section is what `preflight` refuses at release time, + // and a bump commit carries a skip-ci marker — so that state would go red + // on the next unrelated PR rather than on itself. + const script = JSON.stringify(jobs.bump.steps); + expect(script).toContain("release.mjs write"); + expect(script).toContain("CHANGELOG.md"); + expect(script).toContain("[skip ci]"); + expect(script).toContain("VERSION_BOT_APP_ID"); + }); + + it("does not publish, release, verify or bump on a dry run", () => { + expect(jobs["verify-install"].if).toContain("dry_run"); + expect(jobs.bump.if).toContain("dry_run"); + const publishSteps = jobs.publish.steps as Array>; + const release = publishSteps.find((step) => + /Create the GitHub Release/i.test(String(step.name ?? "")), + ); + expect(String(release!.if)).toContain("dry_run"); + }); +}); + +describe("the CI job", () => { + const jobs = workflow(CI_WORKFLOW).jobs as Record; + const job = jobs["failproofai-ts-sdk"] as Record; + + it("exists and runs from the package directory", () => { + expect(job).toBeDefined(); + expect(job.defaults.run["working-directory"]).toBe("sdk/typescript"); + }); + + it("covers the Node range the package advertises, floor included", () => { + const manifest = JSON.parse(readFileSync(resolve(SDK, "package.json"), "utf8")) as { + engines: { node: string }; + }; + const floor = /(\d+)\.(\d+)/.exec(manifest.engines.node)!; + const legs = job.strategy.matrix.include as Array<{ "node-version": string; suite: boolean }>; + const versions = legs.map((leg) => leg["node-version"]); + // The floor itself has to be in the matrix, or "we support 20.9" is a claim + // nothing checks. + expect(versions).toContain(`${floor[1]!}.${floor[2]!}`); + expect(versions.length).toBeGreaterThanOrEqual(3); + // And at least one leg has to run the suite, or the matrix proves only that + // the package installs. + expect(legs.some((leg) => leg.suite)).toBe(true); + }); + + it("proves the floor with the ARTIFACT, since the test runner cannot start there", () => { + // vitest 5 pulls vite 8 pulls rolldown, which needs `styleText` from + // `node:util` — Node 20.12. The test runner's floor is not the package's + // floor, so the floor leg skips the suite and must still do the thing that + // actually demonstrates 20.9 support: build, pack, install, run. + const legs = job.strategy.matrix.include as Array<{ "node-version": string; suite: boolean }>; + const floorLeg = legs.find((leg) => !leg.suite); + expect(floorLeg).toBeDefined(); + + const steps = job.steps as Array>; + const unconditional = steps + .filter((step) => step.if === undefined) + .map((step) => String(step.name ?? step.uses ?? "")); + for (const required of ["Build", "Pack", "Smoke-test the packed tarball with no dependencies"]) { + expect(unconditional).toContain(required); + } + // Conversely, the steps that cannot run there must be gated rather than + // failing the leg. + for (const gated of ["Typecheck", "Lint", "Test"]) { + const step = steps.find((item) => String(item.name ?? "") === gated)!; + expect(step.if).toBe("matrix.suite"); + } + }); + + it("typechecks, lints, tests and proves the artifact installs", () => { + const script = JSON.stringify(job.steps); + for (const step of ["npm run typecheck", "npm run lint", "npm run build", "vitest run"]) { + expect(script).toContain(step); + } + // Everything above runs against the source tree; these run against the + // artifact, where a missing export condition is total and invisible. + expect(script).toContain("npm pack"); + expect(script).toContain("--omit=peer"); + // Both module systems, because half the ecosystem is still CommonJS. + expect(script).toContain("esm.mjs"); + expect(script).toContain("cjs.cjs"); + // The sandbox's self-resolution cannot be exercised from inside this repo. + expect(script).toContain("sandbox resolved and evaluated from the installed package"); + }); +}); + +describe("the package itself", () => { + it("ships no runtime dependencies", () => { + const manifest = JSON.parse(readFileSync(resolve(SDK, "package.json"), "utf8")) as { + dependencies?: Record; + optionalDependencies?: Record; + peerDependencies?: Record; + peerDependenciesMeta?: Record; + }; + // The reason this package is safe to drop into someone else's agent. + expect(manifest.dependencies ?? {}).toEqual({}); + expect(manifest.optionalDependencies ?? {}).toEqual({}); + for (const name of Object.keys(manifest.peerDependencies ?? {})) { + expect(manifest.peerDependenciesMeta?.[name]?.optional).toBe(true); + } + }); + + it("keeps its own lockfile, so CI installs what a release installs", () => { + expect(existsSync(resolve(SDK, "package-lock.json"))).toBe(true); + }); +}); diff --git a/docs/docs.json b/docs/docs.json index bbd01838b..9aee958a3 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -217,6 +217,7 @@ "reference/overview", "reference/harnesses", "reference/custom-agents", + "reference/custom-agents-typescript", "reference/evaluator-sdk", "reference/policy-sdk", "reference/self-hosting" diff --git a/docs/reference/custom-agents-typescript.mdx b/docs/reference/custom-agents-typescript.mdx new file mode 100644 index 000000000..16e66eb26 --- /dev/null +++ b/docs/reference/custom-agents-typescript.mdx @@ -0,0 +1,401 @@ +--- +title: "Custom agents (TypeScript)" +description: "Configuration, the event catalog, the scopes and the framework adapters for @failproofai/sdk." +icon: "square-js" +--- + +What every setting, method and field does for the TypeScript SDK. If you are instrumenting for the first time, start with the guide — this page is for looking things up. + + + + Install, instrument, the event methods, a worked example, and common problems. + + + The same events, the same wire format, the same spool — from Python. + + + +Node 20.9 or newer. ESM and CommonJS. No runtime dependencies. + + + This SDK and the Python one write **the same events into the same spool**. A fleet with Node agents and Python agents produces one set of sessions, not two, and nothing in the dashboard distinguishes them. Pick per service, not per company. + + +## Install + +```bash +npm install @failproofai/sdk +``` + +```ts +import * as failproofai from "@failproofai/sdk"; + +await failproofai.agent("planner", { goal: question }, async () => { + const hits = await failproofai.toolCall("web_search", { input: { q } }, () => search(q)); +}); +``` + +The framework adapters ship in the package itself. The frameworks are **optional peer dependencies** — declared so the supported ranges are visible, never installed on your behalf, and imported only when you call `instrument()`. + +## Connect the Failproof daemon + +Identical to the Python SDK: create an `events:add` key under **Admin → Keys**, then [connect the daemon](/start/setup#connect-a-machine-to-cloud) on the agent machine. The SDK writes to disk; the daemon ships. + +## Configuration + +```ts +failproofai.configure({ + environment: "production", + flushInterval: 0.5, + baseDir: undefined, +}); +``` + +| Option | What it does | +| --- | --- | +| `environment` | The label on every event — `production`, `staging`, `prod-eu`. Defaults to `dev`. | +| `flushInterval` | How often the timer writes to disk, in seconds. Defaults to `0.5`. | +| `baseDir` | Where to write. Defaults to the daemon's spool, which is what you want unless you know otherwise. | + +Nothing is applied unless all of it validates, so a rejected call leaves the SDK exactly as it was rather than with a new `baseDir` and the old interval. + +Set by environment variable instead: + +| Variable | What it does | +| --- | --- | +| `AGENTEYE_ENVIRONMENT` | Sets `environment` without a code change. A `configure()` option wins over it. | +| `FAILPROOFAI_HOME` | Moves the Failproof AI root that holds the spool. | +| `FAILPROOFAI_SDK_LOG_LEVEL` | `debug`, `info`, `warn` (default), `error`, `silent`. | +| `FAILPROOFAI_SDK_STRICT` | `1` makes instrumentation errors throw instead of being logged. | +| `FAILPROOFAI_SDK_STRICT_INTEGRATIONS` | `1` makes a framework-compatibility problem throw instead of warning and carrying on. | + + + **No commas in `environment`.** Ingest splits that field on commas to build its filters, and skips any event whose label contains one — so a whole run silently vanishes. Write `prod-eu`, not `prod,eu`. + + `configure({ environment: "prod,eu" })` throws so you find out immediately. `AGENTEYE_ENVIRONMENT` cannot throw — nothing is calling you — so it warns once and falls back to `dev`. + + +Route the SDK's own log lines into your logger with `failproofai.setLogger({ debug, info, warn, error })`. + +## Shutdown + +Buffered events are flushed on `process.on("exit")`. + +A process killed by a signal never reaches that, and Node's default for `SIGTERM` is to terminate without running exit handlers — so a containerised agent loses whatever the last interval had not written. + + + **This SDK will not install a signal handler for you.** Registering one changes your process's behaviour: a listener suppresses Node's default termination, so a library that added one would silently stop Ctrl-C from working. Add your own: + + ```ts + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + failproofai.flushSync(); + process.exit(0); + }); + } + ``` + + +A short-lived script or a serverless handler should `await failproofai.flush()` before returning — the interval alone does not guarantee delivery. + +## Identity + +Every event belongs to a session and an agent. **The scopes fill both in**, so you rarely pass them: + +```ts +await failproofai.session(async () => { + await failproofai.agent("planner", async () => { + failproofai.event.toolUse({ toolName: "search", toolCallId: "c1" }); + }); +}); +``` + +Passing `sessionId` or `agentId` explicitly still works and wins. With neither bound nor passed, the call throws rather than emitting an event Cloud would quietly discard. + + + Identity rides on `AsyncLocalStorage`. It follows `await`, `.then()`, timers and any callback created inside the scope. It does **not** follow a callback stored during one run and invoked during another, or work handed across a `worker_threads` boundary — wrap those in `failproofai.propagate()` or their events land unattached. + + +### Scopes + +| Scope | Emits | Returns | +| --- | --- | --- | +| `session(body)` | nothing — identity only | whatever `body` returns | +| `agent(id, options?, body)` | `agent_start`, then `agent_end` | whatever `body` returns | +| `toolCall(name, options?, body)` | `tool_use`, then `tool_result` | whatever `body` returns | + +A synchronous body stays synchronous: `agent("x", () => 1)` returns `1`, not a promise. + +`toolCall` records the body's resolved value as the tool's `output`, unless you assign `call.output` yourself. + + + +| What happened | Events | `outcome` | +| --- | --- | --- | +| the block returned | `agent_end` | `"success"`, or your `outcome` | +| the block threw | `error`, then `agent_end` | `"failed"` | +| an `AbortError` | `agent_end` only | `"cancelled"` | + +The error is always re-thrown. + +A tool failure is recorded on the leaf — `tool_result` with an `error` string — and emits **no** run-level `error` event. One the agent loop catches is not a run failure, and one that propagates is reported exactly once, by the enclosing `agent()`. + + + + + +When the work is not a single function — a scope opened in a constructor and closed in a teardown, or one that straddles existing control flow: + +```ts +{ + using span = failproofai.agent.open("planner", { goal }); + using call = failproofai.toolCall.open("search", { input: { q } }); + call.call.output = await search(q); +} // tool_result, then agent_end +``` + +Both forms emit byte-identical events. Prefer the callback form: it runs inside `AsyncLocalStorage.run()`, so there is nothing to unwind and the whole class of "opened here, closed over there" bugs is unreachable. + +A `using` block that catches its own failure reports it with `span.fail(error)` — the disposer has no exception channel of its own. + + + +## Event catalog + +The same fifteen methods as the Python SDK, in camelCase. Most come in **pairs** — you call the opener, then the closer, and the SDK times the gap. + +| | Opens | Closes | +| --- | --- | --- | +| **Agents** | `agentStart` | `agentEnd` | +| | `agentPause` | `agentResume` | +| **Models** | `modelRequest` | `modelResponse` | +| **Tools** | `toolUse` | `toolResult` | +| **Hooks** | `hookTriggered` | `hookCompleted` | +| **Humans** | `humanWait` | `humanInput` | + +Three stand alone: `error`, `humanPause`, `humanInterrupt`. + + + +Every method also takes `sessionId` and `agentId`, which the scopes fill in for you. Anything omitted is dropped rather than sent as JSON `null`. + +| Method | Required | Optional | +| --- | --- | --- | +| `agentStart` | — | `goal`, `parentId` | +| `agentEnd` | — | `outcome`, `summary` | +| `agentPause` | `pauseId` | `reason`, `userId` | +| `agentResume` | `pauseId` | `reason`, `userId` | +| `modelRequest` | — | `model`, `messages`, `system`, `tools`, `requestId` | +| `modelResponse` | — | `model`, `stopReason`, `inputTokens`, `outputTokens`, `content`, `role`, `requestId` | +| `toolUse` | `toolName`, `toolCallId` | `input` | +| `toolResult` | `toolName`, `toolCallId` | `output`, `error` | +| `hookTriggered` | `hookName`, `hookId` | `triggerEvent`, `input` | +| `hookCompleted` | `hookName`, `hookId` | `outcome`, `output`, `error` | +| `error` | `errorType`, `message` | `traceback` | +| `humanWait` | `inputId` | `prompt`, `options`, `reason` | +| `humanInput` | `inputId` | `response` | +| `humanPause` | — | `reason`, `userId` | +| `humanInterrupt` | — | `reason`, `userId`, `atStep` | + +Any other key you add becomes a custom payload field. Namespace anything framework-specific `fw_*`; a name that collides with a declared field is refused rather than silently overwriting a promoted column. + + + + + **`duration_ms` is computed, not accepted.** The four closing methods time the gap from their opener and refuse a caller-supplied `duration_ms` — a reported duration is unfalsifiable. + + Pairs are matched on the **session** and the id, never on the agent. A tool opened under `planner` and closed under `worker` still pairs, which is what nested multi-agent runs actually do. + + +## Framework adapters + +```ts +await failproofai.instrument(); // whatever it can find +await failproofai.instrument("langchain"); // exactly one +failproofai.uninstrument(); // put everything back +``` + +| Framework | Supported | How it attaches | +| --- | --- | --- | +| **LangChain.js / LangGraph.js** | `@langchain/core` 0.3 – 1.x, LangGraph.js 0.4 – 1.x | `CallbackManager.configure`, so every `invoke`/`stream`/`batch` is covered without passing `callbacks:` anywhere — or pass `langchainHandler()` yourself and patch nothing. | +| **Vercel AI SDK** | `ai` 4 – 7 | `telemetry()` at the call site, or `instrument("ai")` for the whole process on `ai` 7 (on 4–6 that is opt-in — see below). | +| **Mastra** | `@mastra/core` 0.20 – 1.x | `Agent.generate`/`.stream`, the agent's model and tool resolution, and the workflow run/step engine. | +| **LlamaIndex.TS** | `llamaindex` 0.11.4 – 0.x | `Settings.callbackManager` (subscribed) plus `AgentWorkflow.runStream`, for workflow runs and their steps. | + +Every range is tested against real framework releases, at both ends, as an ES module and as CommonJS, on every CI run. + +The mapping is the Python SDK's, so the same program draws the same tree in either language. A construct is an **agent** only if it owns an LLM decision loop — a graph or chain run, an AI SDK `generateText`/`streamText` call, a Mastra agent, a LlamaIndex agent run. A LangGraph node or a workflow step is a **hook** (`hook_triggered`/`hook_completed`), never a nested agent. Model calls are `model_request`/`model_response` pairs with token counts; tool calls carry the model's own tool call id. A failure is recorded once, on the event it happened in. + +An adapter that fails to install is logged and skipped; the others still install, because a broken LlamaIndex should not cost you LangGraph. + + + `instrument()` with no argument detects a framework by whether it **resolves**, not by whether it is already imported — Node exposes no equivalent of Python's `sys.modules` for ES modules. A framework you have installed but do not use will be imported and patched. Name the one you want if that matters. + + + + Most of these frameworks ship an ES-module build and a CommonJS build, which Node loads as two unrelated copies. The adapters patch the copy your application loads (and the CommonJS copy too if something already `require`d it), so both module systems work. A framework **bundled into your own output** by esbuild or webpack is out of reach — use the call-site helpers there: `langchainHandler()`, `telemetry()`, `wrapTool()`. + + +### LangChain without patching + +```ts +import { langchainHandler } from "@failproofai/sdk/langchain"; +await graph.invoke(input, { callbacks: [langchainHandler()] }); +``` + +The handler works with or without `instrument()` and never double-records. `instrument("langchain")` takes `sessionId`, `captureContent`, `includeChains`, `graphCallbacks` and `captureLimit`, as the Python adapter does; `metadata: { failproofai_sdk_session_id }` on a call picks the session for that invocation. + +### Vercel AI SDK + +The AI SDK exports plain functions from an ES module, and an ES module namespace is immutable by specification — there is nowhere to patch. It uses the extension points the SDK itself documents: + +```ts +import { telemetry } from "@failproofai/sdk/ai"; + +const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + // on ai 7, `telemetry: telemetry({ … })` — the same object, the new name +}); +``` + +That is the complete integration: an agent span, a model request/response pair per step with token counts, and every tool call. One call site works on every major — `ai` 4–6 read the tracer it carries, `ai` 7 the telemetry integration. + +`instrument("ai")` does the same process-wide **on `ai` 7**: every call, through the AI SDK's global telemetry-integration list, which is additive and takes nothing from anybody else's. + +**On `ai` 4–6, `instrument("ai")` records nothing by itself, and logs one warning saying so.** The only process-wide hook those majors have is the global OpenTelemetry tracer provider — a single slot OpenTelemetry refuses to hand over once taken. Registering ours would silently refuse your own `NodeSDK.start()` later in startup and send your http/database spans to a tracer that exports nothing. Use `telemetry()` at the call site or `wrapModel` there. If the process runs no OpenTelemetry of its own, opt in with `instrument("ai", { registerGlobalTracer: true })`: it then records every call that passes `experimental_telemetry: { isEnabled: true }`, and only takes the slot if it is still empty. `registerGlobalTracer: false` keeps the default and silences the warning. + +If you would rather wrap the model once, `wrapModel` sees model calls only, because tool calls happen above the model layer. A wrapped model called with nothing around it is recorded as its own run. A streamed call closes however the stream stops — `stop_reason: "cancelled"` when the consumer cancels it, `"error"` with the error when it fails part-way: + +```ts +import { wrapModel } from "@failproofai/sdk/ai"; +const model = await wrapModel(openai("gpt-4o")); +``` + +Using both is fine: the middleware notices the call is already being recorded and defers, so each call is recorded once. + +`functionId` names the agent span. Keep it low-cardinality — it lands in `agent_id`, the primary dashboard facet. + +### Next.js + +`next build` bundles your server's dependencies by default, and a framework bundled into the build is a copy `instrument()` cannot reach. Wrap the config once and call `instrument()` from Next's startup hook: + +```ts +// next.config.ts +import { withFailproofai } from "@failproofai/sdk/next"; +export default withFailproofai({ /* your config */ }); +``` + +```ts +// instrumentation.ts +export async function register() { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + const failproofai = await import("@failproofai/sdk"); + await failproofai.instrument(); +} +``` + +`withFailproofai` adds LangChain, Mastra, LlamaIndex and the SDK itself to `serverExternalPackages`, keeping your own list. Without it, `instrument()` warns once per framework it cannot reach rather than failing silently; if you list the packages yourself, set `FAILPROOFAI_NEXT_EXTERNALS=1`. The Vercel AI SDK and the call-site helpers work either way. An Edge route gets a no-op build: importing the SDK is safe and records nothing. + +### Token counts on streamed calls + +OpenAI-compatible APIs only report usage on a stream when the client asks. LangChain and the Vercel AI SDK ask; for LlamaIndex pass `additionalChatOptions: { stream_options: { include_usage: true } }` to its `OpenAI` LLM, and for Mastra build the model with usage enabled (for example `createOpenAICompatible({ includeUsage: true })`). Otherwise streamed model calls carry no token counts. + +### Runtimes + +Node ≥ 20.9, Bun and Deno — every framework, as an ES module and as CommonJS, is tested on each against Node's trace. The SDK runs beside the `failproofaid` daemon, which ships what it writes. + +## Your own agent — no framework + +For an agent loop you wrote yourself, or a framework without an adapter. You emit the events with the same API the adapters use underneath, so the trace has the same shape and quality. + +You don't need to know how the agent is organised. Every hand-built agent already has three places, whatever its functions are called, and those three are the whole integration: + +| Where | What to add | Emits | +| --- | --- | --- | +| Where **one run** starts and ends | `failproofai.agent("name", { goal }, async () => …)` | `agent_start` / `agent_end` | +| The **one function that calls the model** | `event.modelRequest` before, `event.modelResponse` after — both halves, even on failure | one pair per model turn | +| The **one function that runs tools** | `failproofai.toolCall(name, { toolCallId, input }, () => run())` | `tool_use` / `tool_result` | + +```ts +async function callModel(messages) { + const requestId = randomUUID(); + const started = Date.now(); + failproofai.event.modelRequest({ model: MODEL, requestId, messages }); + try { + const reply = await client.chat.completions.create({ model: MODEL, messages, tools }); + failproofai.event.modelResponse({ + model: reply.model, requestId, stopReason: reply.choices[0].finish_reason, + inputTokens: reply.usage?.prompt_tokens, outputTokens: reply.usage?.completion_tokens, + duration_ms: Date.now() - started, + }); + return reply.choices[0].message; + } catch (error) { + failproofai.event.modelResponse({ model: MODEL, requestId, stopReason: "error", + error: String(error), duration_ms: Date.now() - started }); + throw error; + } +} + +async function dispatch(call) { + const input = JSON.parse(call.function.arguments); + return failproofai.toolCall(call.function.name, { toolCallId: call.id, input }, + () => runTool(call.function.name, input)); +} + +await failproofai.agent("inventory", { goal: question }, async () => { + for (;;) { + const message = await callModel(messages); + if (!message.tool_calls?.length) return message.content; + for (const call of message.tool_calls) await dispatch(call); + } +}); +``` + +Identity is ambient: everything inside `agent()` lands on that run's session without taking an id, and nothing else in the program changes — including whatever the agent already writes to its own database. + +- **A service or a worker:** pass your own request or job id as `sessionId`, so a session on the dashboard and the record in your own logs or database are the same string. +- **Sub-agents:** nest `agent()` calls. The inner one joins the session with the outer as its `parent_id`. +- **Emit the pairs.** A `modelRequest` with no `modelResponse` is a span the dashboard shows as running forever — hence the `catch`. + +[`sdk/typescript/examples/research-agent.ts`](https://github.com/FailproofAI/failproofai/blob/main/sdk/typescript/examples/research-agent.ts) in the repository is the complete, runnable version: a real OpenAI tool loop instrumented exactly like this, run in CI on every change as an ES module and as CommonJS. + +## Evaluations + +```ts +import { Evaluator, EvalResult, Score } from "@failproofai/sdk/evaluator"; + +export const app = new Evaluator({ name: "my-evals", version: "1" }); + +app.eval("tool_success_rate", { version: "1" }, (session) => { + const results = session.eventsOfType("tool_result"); + const failures = results.filter((event) => event.payload.error != null).length; + return new EvalResult({ + score: new Score(results.length === 0 ? 1 : 1 - failures / results.length), + reasoning: `${failures} of ${results.length} tool calls failed`, + }); +}); +``` + +```bash +FAILPROOFAI_EVALUATOR_URL=… FAILPROOFAI_EVALUATOR_TOKEN=… \ + npx failproofai-evaluator ./my-evals.js +``` + +See the [Evaluator SDK reference](/reference/evaluator-sdk) for the protocol, the worker settings and the result types. + + + **An evaluation must yield.** A synchronous function that never returns blocks the one thread Node has, and no timeout can fire while it does. Write `async` evaluations. + + +## What it will not do to your process + +| | | +| --- | --- | +| **Block your agent loop** | Events go into an in-memory queue; a timer writes them. The timer is `unref`'d, so importing this package never stops a script exiting. | +| **Grow without bound** | The queue is capped by count *and* by measured bytes. Past either, the oldest events are discarded and a warning says so — a telemetry outage must not become an OOM kill. | +| **Take the process down** | One unencodable event is dropped alone, not the batch around it. A throwing getter, a circular reference, a `BigInt`, a lone surrogate: each is handled rather than propagated. | +| **Leave a half-written batch** | Content is `fsync`ed before an atomic rename, the directory is `fsync`ed after, and a failed write cleans up its temporary file. | +| **Leave transcripts readable** | Batches are `0600` inside a `0700` directory. They carry goals, prompts, tool arguments and tool output. | +| **Ship credentials** | API keys, tokens, JWTs, bearer headers and secret-shaped assignments are redacted before the bytes reach disk. The daemon redacts again before upload. | diff --git a/docs/reference/custom-agents.mdx b/docs/reference/custom-agents.mdx index ab9b7245d..cb891e148 100644 --- a/docs/reference/custom-agents.mdx +++ b/docs/reference/custom-agents.mdx @@ -10,12 +10,16 @@ What every setting, method and field does. If you are instrumenting for the firs Install, instrument, the event methods, a worked example, and common problems. - - LangChain, CrewAI, LlamaIndex and Pydantic AI instrument themselves with one call. + + The same events, the same wire format, the same spool — from Node. -Python 3.10 or newer. No runtime dependencies. +Python 3.10 or newer. No runtime dependencies. Using a framework? [LangChain, CrewAI, LlamaIndex and Pydantic AI](/start/integrations) instrument themselves with one call. + + + There is a **TypeScript SDK** too, and the two write the same events into the same spool. A fleet with Node agents and Python agents produces one set of sessions, not two. Pick per service, not per company. + ## Install diff --git a/eslint.config.mjs b/eslint.config.mjs index 35f081224..71e0f9789 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -6,7 +6,12 @@ const config = [ // is the brand team's reference HTML/JSX kit, not source code we ship. // integration-suite is a standalone Docker+shell test harness (Node CJS // scripts driven inside a container), not shipped source — like dist/assets. - { ignores: ["dist/", "assets/", "integration-suite/"] }, + // sdk/typescript is its OWN npm package with its own tsconfig, eslint config + // and vitest config, for the same reason sdk/python and fp-cloud-cli are + // their own projects: linting it from here would mean this project's rules + // and dependency tree decided whether that package's zero-dependency claim + // holds. Its CI job runs `npm run lint` inside it. + { ignores: ["dist/", "assets/", "integration-suite/", "sdk/typescript/"] }, ...nextConfig, { settings: { react: { version: "19" } } }, { diff --git a/osv-scanner.toml b/osv-scanner.toml index 16392c993..868ca6760 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -1,6 +1,6 @@ # OSV-Scanner allow-list — auto-loaded by the Supply Chain CI gate -# (.github/workflows/osv-scanner.yml) when it scans bun.lock, Cargo.lock and -# the two uv.lock files. +# (.github/workflows/osv-scanner.yml) when it scans bun.lock, Cargo.lock, the +# two uv.lock files and sdk/typescript/package-lock.json. # # The gate blocks on ANY known-vulnerable or malicious dependency. When a finding # has no available fix (e.g. an unpatched transitive advisory), add a justified, diff --git a/sdk/python/skill/SKILL.md b/sdk/python/skill/SKILL.md index bc1cd5275..e12ec1bd2 100644 --- a/sdk/python/skill/SKILL.md +++ b/sdk/python/skill/SKILL.md @@ -1,19 +1,28 @@ --- name: failproofai-sdk description: |- - The way to make an AI agent report what it did to Failproof AI — planning what to record, writing the instrumentation, and proving the events land. Reach for it on vague phrasing too: "add observability to my agent", "why isn't my agent showing up?" + Make a custom AI agent — Python or TypeScript/JavaScript, on a framework or hand-built — report what it did to Failproof AI, and run your own evaluator worker (the "eval pod") that scores those runs. Reach for it on vague phrasing too: "add observability to my agent", "why isn't my agent showing up?", "run an LLM judge on our own infra". Trigger when the user wants to: - • plan an integration — which points in their agent loop to record, and what the platform must see before sessions, errors, and evals work at all; - • write or fix instrumentation — add the `failproofai_sdk` Python SDK to an agent codebase, thread session/agent identity through it, emit tool, model, hook, or human events; - • verify it — confirm events are being written, or debug an integration that looks correct and produces nothing. + • plan an integration — which points in the agent loop to record; + • instrument — add `failproofai-sdk` (Python) or `@failproofai/sdk` (Node, Bun, Deno, Next.js): turn on an adapter (LangChain/LangGraph, CrewAI, LlamaIndex, Pydantic AI, Vercel AI SDK, Mastra) or wire a hand-built loop; + • verify — confirm events are written, or debug an integration that produces nothing; + • evaluate — write, deploy or debug an Evaluator worker in Python or TypeScript. - Served by the `failproofai_sdk` Python SDK, inside the user's own agent. - - NOT for reading telemetry that already landed or operating a deployment (that's `fp-cloud-cli`), or building the evaluator service that scores runs (that's `agenteye-evaluator`). + NOT for reading telemetry or scores that already landed (that's `fp-cloud-cli`), or deciding what is worth evaluating (that's `failproofai-eval-brainstorm`). --- -# Failproof AI Python SDK +# Failproof AI SDK — Python and TypeScript + +Two packages, one pipe: `failproofai-sdk` (Python, imported as `failproofai_sdk`) +and `@failproofai/sdk` (TypeScript/JavaScript). They write the same 15 events in +the same wire format into the same spool directory. Everything in this file is +language-neutral unless it says otherwise, and code blocks are Python. +**For a TypeScript or JavaScript agent, read `references/typescript.md` alongside +it**: it has the camelCase names, the adapters, bundler and Next.js setup, the +no-framework wiring, shutdown and verification. Both packages also ship the +**evaluator worker** that scores finished sessions on your own infrastructure (§7, +`references/evaluator.md`). The SDK records what your agent did, from inside your agent. You call it at points you choose; it appends structured events to local `.jsonl` files. A separate @@ -34,10 +43,14 @@ and it is verifiable on a laptop with no server, no API key, and no network. The API is small — 15 event methods, all keyword-only. The hard parts are **deciding where to call them** and **knowing which silences are bugs**, because this SDK does not raise when you get it wrong. Sections 1-3 are the plan, 4 is the -code, 5-6 are the proof. +code, 5-6 are the proof, 7 is scoring the runs. ## 1. Install it +TypeScript/JavaScript: `npm install @failproofai/sdk` — zero dependencies, Node ≥ +20.9, Bun or Deno, ESM and CommonJS. The npm name has no lookalike trap; the rest of +this section is Python. See `references/typescript.md`. + ```bash pip install failproofai-sdk # or: uv add failproofai-sdk ``` @@ -136,6 +149,10 @@ Full field-by-field catalog: `references/events.md`. Work with these; none of them raise, so none of them show up in testing. +(TypeScript: the same contract with camelCase spellings, the same `"dev"` default, +the same reserved names and the same `duration_ms` rule, but a different shutdown +recipe — `references/typescript.md` → *The contract, in TypeScript* and *Shutdown*.) + - **There IS an ambient session, and it is the ergonomic path.** `session()`, `agent()` and `tool_call()` bind identity on contextvars, so `session_id` and `agent_id` are optional on all 15 event methods — omitted, they resolve from @@ -277,7 +294,10 @@ Threading `session_id` and `agent_id` through every call site by hand is the thi that makes integrations ugly and abandoned. Don't. Bind identity once per run and let the call sites read it. -`references/frameworks.md` covers the four adapters. `references/integration.md` has the hand-written wrapper — one small +`references/frameworks.md` covers the four Python adapters; `references/typescript.md` +covers the four TypeScript ones (LangChain.js/LangGraph.js, Vercel AI SDK, Mastra, +LlamaIndex.TS), Next.js, bundlers, and the three-edit-site wiring for a hand-built +TypeScript loop. `references/integration.md` has the hand-written wrapper — one small module, correct under `asyncio` and threads, adaptable to any codebase — plus worked shapes for a tool dispatcher, an LLM client wrapper, and framework-specific callback layers. Read it before writing your own; the naive version (a module @@ -289,6 +309,9 @@ has a request context or a trace id, bind to that instead of inventing one. ## 5. Verify — watch the files +(TypeScript: same directory, same checklist; the commands are in +`references/typescript.md` → *Verify*.) + **This is the whole point of the file boundary: you can prove the integration without a server.** Run the agent and look. @@ -319,7 +342,7 @@ cat ~/.failproofai/custom-agents/events/*.jsonl | python -m json.tool --json-lin Then check, in this order — the first failure explains everything downstream: -1. **Any files at all — or do they stop mid-run?** Look at stderr for +1. **Any files at all — or do they stop mid-run?** (Python) Look at stderr for `Exception in thread failproofai-sdk-flush`. **This is the first thing to check and the worst thing to miss**: one non-JSON-serializable value killed the writer, and everything after it — including the at-exit flush — is gone (§3). The tell @@ -342,8 +365,8 @@ Then check, in this order — the first failure explains everything downstream: even when identity is a module global, and mixing only appears once two runs overlap — which is production, not your laptop (§4). 7. **Do `tool_use` and `tool_result` share a `tool_call_id`?** Unpaired means no - duration. Also confirm your ids are unique *process-wide* — a collision pairs - the wrong two events and reports a confident wrong duration (§3). + duration. Also confirm no id repeats *within a session* for the same kind — a + collision pairs the wrong two events and reports a confident wrong duration (§3). A test-mode loop that costs nothing: @@ -388,6 +411,10 @@ Compare the two paths first: print the directory your agent is actually writing (`python -c "import failproofai_sdk._resolver as r; print(r.get_base_dir())"` in the agent's own environment, with the agent's own env vars) and check the collector is running and pointed at the same one. A `.jsonl` count that only grows is the tell. +The reverse is healthy: with `failproofaid` running, the directory empties seconds +after each flush, because the daemon ships each file and deletes it — so an empty +real spool proves nothing either way. Verify content with the throwaway +`FAILPROOFAI_HOME` loop in §5, and arrival with `fp-cloud-cli`. Confirming events arrived on the *platform* is deliberately not this skill's job — that is the `fp-cloud-cli` skill, from a **separate environment** (§1). Collector @@ -396,4 +423,23 @@ setup and deployment are your platform's own documentation. If the files look right (§5) and the collector is running against the same directory, the integration is done. - +## 7. Score the runs — your own evaluator worker + +Sessions that land can be scored. Hosted evaluations are written in the dashboard +(**Analyze → eval authoring**) and run on Failproof AI's managed evaluator: that is +the default. When an evaluation needs your own model keys, packages, secrets, +private network or heavy compute, run it in **your own worker** — the eval pod. + +It ships in the same packages: `failproofai_sdk.evaluator` and +`@failproofai/sdk/evaluator`. You declare an `Evaluator`, register versioned +evaluations with an optional `when` condition, and start it with an +`evaluations:run` key in `FAILPROOFAI_EVALUATOR_TOKEN`. It only calls out over +HTTPS — claim finished sessions, score them, submit — so a pod needs egress and a +secret, and no ingress. Its results carry the **customer** tag. + +Two things to settle before writing one: the agents must already produce finished +sessions (§2 — nothing scores a run with no `agent_end`), and bump an evaluation's +`version` whenever its logic changes. Everything else — the API in both languages, +env vars, a Dockerfile, SIGTERM drain vs. the pod's grace period, scaling, and a +debugging order for a worker that scores nothing — is in `references/evaluator.md`. + diff --git a/sdk/python/skill/agents/openai.yaml b/sdk/python/skill/agents/openai.yaml index 42451fd83..18bc0c372 100644 --- a/sdk/python/skill/agents/openai.yaml +++ b/sdk/python/skill/agents/openai.yaml @@ -5,4 +5,5 @@ # Let Codex auto-select this skill when a task matches the description # (set to false to require explicit `$failproofai-sdk` invocation). -allow_implicit_invocation: true +policy: + allow_implicit_invocation: true diff --git a/sdk/python/skill/references/evaluator.md b/sdk/python/skill/references/evaluator.md new file mode 100644 index 000000000..ddac798c8 --- /dev/null +++ b/sdk/python/skill/references/evaluator.md @@ -0,0 +1,255 @@ +# Your own evaluator worker — the eval pod + +An evaluation scores a **finished** session. There are two places one can run: + +| | Hosted | Your own worker (this page) | +|---|---|---| +| Written | in the dashboard, **Analyze → eval authoring** | in Python or TypeScript, with the SDK you already install | +| Runs on | Failproof AI's managed evaluator, sandboxed | your infrastructure: a container, a pod, a VM | +| Use it for | deterministic checks and model-backed checks Failproof AI hosts | LLM judges on your own keys, packages, secrets, your private network, models you host, heavy processing | + +Default to hosted. Reach for a worker when the evaluation needs one of the +right-hand column's things. Deciding *what* is worth evaluating is the +`failproofai-eval-brainstorm` skill. This page is about building and running the +worker. + +**How it works.** The worker only makes outbound HTTPS calls, and nothing connects +in to it, so there is no port, no Service and no ingress: + +1. It registers its catalogue of evaluations. +2. It claims finished sessions and runs each applicable evaluation. +3. It submits the results under a heartbeat. + +Its results appear on the evaluations page tagged **customer**. Hosted results are +tagged **managed**. + +**Nothing scores a session that has no `agent_start`/`agent_end`.** The worker +evaluates sessions, and sessions exist only through those two events (`SKILL.md` +§2). Get instrumentation landing first. + +## Ships in the SDK + +| | Python | TypeScript | +|---|---|---| +| Install | `pip install failproofai-sdk` | `npm install @failproofai/sdk` | +| Import | `failproofai_sdk.evaluator` | `@failproofai/sdk/evaluator` | +| Run | `python evaluator.py` (with `app.run_from_env()`) or `python -m failproofai_sdk.evaluator evaluator:app` | `npx failproofai-evaluator ./evals.js` (export named `app`; `./evals.js#other` for another) or `app.runFromEnv()` | + +Importing the tracing SDK does not load the evaluator. In TypeScript the reverse +holds too; in Python, importing `failproofai_sdk.evaluator` also imports the tracing +package and starts its flush thread (harmless, but it is there). + +## Write the evaluations + +Python: + +```python +from failproofai_sdk.evaluator import ConditionResult, EvalResult, Evaluator, Metric, Score + +app = Evaluator(name="support-evals", version="2026.09.1") + + +@app.eval( + "tool_efficiency", + version="1.0.0", + labels=["tools", "deterministic"], + when=lambda s: ConditionResult(s.count("tool_use") > 0, "no_tool_calls"), +) +def tool_efficiency(session): + calls = session.events_of_type("tool_use") + distinct = {e.payload.get("tool_name") for e in calls if e.payload.get("tool_name")} + value = len(distinct) / len(calls) + return EvalResult( + score=Score(value, passed=value >= 0.7), + metrics={"tool_call_count": Metric(len(calls), unit="events")}, + reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls", + ) + + +@app.eval( + "answer_relevance", + version="judge-v1", + labels=["llm_judge"], + when=lambda s: ConditionResult(s.count("model_response") > 0, "no_model_response"), + timeout_seconds=30, +) +async def answer_relevance(session): + answer = session.events_of_type("model_response")[-1].payload.get("content") + value, why = await ask_judge(answer) # your model call: a 0-1 score and why + return EvalResult(score=Score(value, passed=value >= 0.7), reasoning=why) + + +if __name__ == "__main__": + app.run_from_env() +``` + +TypeScript (same API, camelCase, options objects): + +```ts +import { ConditionResult, EvalResult, Evaluator, Metric, Score } from "@failproofai/sdk/evaluator"; + +export const app = new Evaluator({ name: "support-evals", version: "2026.09.1" }); + +app.eval( + "tool_efficiency", + { + version: "1.0.0", + labels: ["tools", "deterministic"], + when: (s) => new ConditionResult(s.count("tool_use") > 0, "no_tool_calls"), + }, + (session) => { + const calls = session.eventsOfType("tool_use"); + const distinct = new Set(calls.map((e) => e.payload.tool_name).filter(Boolean)); + const value = distinct.size / calls.length; + return new EvalResult({ + score: new Score(value, { passed: value >= 0.7 }), + metrics: { tool_call_count: new Metric(calls.length, { unit: "events" }) }, + reasoning: `${distinct.size} distinct tools across ${calls.length} calls`, + }); + }, +); + +app.eval( + "answer_relevance", + { + version: "judge-v1", + labels: ["llm_judge"], + when: (s) => new ConditionResult(s.count("model_response") > 0, "no_model_response"), + timeoutSeconds: 30, + }, + async (session) => { + const answer = session.eventsOfType("model_response").at(-1)?.payload.content; + const { value, why } = await askJudge(answer); // your model call + return new EvalResult({ score: new Score(value, { passed: value >= 0.7 }), reasoning: why }); + }, +); +``` + +The evaluator module can be an ES module or CommonJS (`.js`, `.mjs`, `.cjs`, plain +`tsc` output). For a `.ts` file, either compile it and point `failproofai-evaluator` +at the `.js`, or call `await app.runFromEnv()` at the bottom and start it with +`npx tsx evals.ts`. + +### The rules that bite + +- **The key is what results chart under.** It must match `^[a-z][a-z0-9_]*$`. + **Bump `version` whenever the logic changes.** Each result keeps the version that produced it, so a chart shows + exactly when new logic took over. Keys must be unique, and one worker holds at + most 100 evaluations. +- **`when` is how you scope.** Return `ConditionResult(False, "")` to + skip a session, and the reason is recorded. A judge with no condition costs a + model call on every session. +- **Result shapes:** + - `result_kind` / `resultKind` is `"score"` by default. + - For a `"metric"` or `"assertion"` evaluation, name one `metrics` or + `assertions` entry after the key: that entry is the result. + - Every `EvalResult` carries at least one and at most 25 scores, metrics or + assertions. + - `Score` values are 0–1, and anything else throws. +- **Read payload keys off a real session.** Keys like `tool_name`, `content` and + `response` are whatever the agents emitted. The adapters add framework fields + under `fw_*`. +- **Evaluations must yield.** + - Every evaluation is bounded by `timeout_seconds` / `timeoutSeconds`, and by + 300 s when you set none. + - **Python:** a synchronous evaluation that overruns cannot be interrupted. Its + thread runs on, and a permanently blocked one leaks a thread per session. Write + judges and network calls as `async def`. + - **TypeScript:** a synchronous loop blocks the only thread, so no timeout can + fire. Write evaluations `async`. +- **The session object:** + - Python: `session_id`, `agent_id`, `environment`, `started_at`, `ended_at`, + `events`, `count(type)`, `events_of_type(type)`. + - TypeScript: `sessionId`, `agentId`, `environment`, `startedAt`, `endedAt`, + `events`, `count(type)`, `eventsOfType(type)`. + - Each event has `id`, `ts`, `event_type` (TS `eventType`) and `payload`. + +## Run it + +Create a key with the **`evaluations:run`** permission under **Administration → +Keys**. Inject it from your secret store; never paste it into a command line or an +image. + +| Variable | Default | | +|---|---|---| +| `FAILPROOFAI_EVALUATOR_URL` | required | `https://app.befailproof.ai` for Cloud, or your instance. HTTPS unless loopback | +| `FAILPROOFAI_EVALUATOR_TOKEN` | required | the `evaluations:run` key | +| `FAILPROOFAI_EVALUATOR_WORKER_ID` | `-` | names this worker | +| `FAILPROOFAI_EVALUATOR_CONCURRENCY` | `1` | sessions scored at once by this process | +| `FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS` | `30` | per request to Failproof AI | +| `FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS` | `60` | how long a stopping worker waits for runs in flight | +| `FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP` | `false` | plain HTTP to a non-loopback URL. **Cleartext token and transcripts.** Isolated dev networks only | +| `FAILPROOFAI_EVALUATOR_MODULE` | — | the module when the CLI is given none: `module:attr` (Python), `path#export` (TS) | + +Local smoke run against Cloud: + +```bash +export FAILPROOFAI_EVALUATOR_URL=https://app.befailproof.ai +export FAILPROOFAI_EVALUATOR_TOKEN="$(your-secret-store read failproofai/evaluator)" +python evaluator.py # or: npx failproofai-evaluator ./evals.js +``` + +Then finish a session from an instrumented agent and watch for a **customer** +result on it. Evaluation runs forward: a worker started now scores sessions that +finish while it runs. The dashboard's "score sessions you already have" backfill +covers hosted evaluations only; there is no backfill for a worker's own evaluations. + +## Deploy it as a pod + +Treat it as a long-running, outbound-only worker: + +```dockerfile +# Python +FROM python:3.12-slim +RUN pip install --no-cache-dir failproofai-sdk # plus whatever your judges need +COPY evaluator.py /app/evaluator.py +CMD ["python", "/app/evaluator.py"] +``` + +```dockerfile +# TypeScript (compiled to dist/evals.js) +FROM node:22-slim +WORKDIR /app +COPY package*.json ./ +RUN npm ci --omit=dev +COPY dist/ ./dist/ +CMD ["npx", "failproofai-evaluator", "./dist/evals.js"] +``` + +- **Secrets:** + - `FAILPROOFAI_EVALUATOR_TOKEN` comes from a Secret (Kubernetes) or your secret + manager. + - The judge's own model key belongs there too. + - Egress must reach the Failproof AI URL and your model endpoint. No ingress is + needed. +- **Shutdown:** both workers stop on `SIGTERM`/`SIGINT` and drain runs in flight + for up to `FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS`. Give the orchestrator + more grace than that. In Kubernetes, set `terminationGracePeriodSeconds` above + the drain timeout, e.g. 90 for the default 60. Otherwise a rolling deploy + hard-kills evaluations mid-run. +- **Scaling:** + - Raise `FAILPROOFAI_EVALUATOR_CONCURRENCY` for I/O-bound judges. + - Add replicas for more throughput. Each replica claims its own sessions. + - Leave `WORKER_ID` unset, or make it unique per replica: the default + `-` already is. +- **Releasing new logic.** Ship the image with a bumped eval `version`. Removing an + evaluation from the worker, or stopping the worker, stops it running. There is + nothing to disable in the dashboard, and worker-registered evaluations are not + listed on the eval authoring page. + +## Debugging a worker that scores nothing + +Work through these in order: + +1. **Does it start?** + - A missing URL or token fails at startup with the variable's name. + - An HTTP URL to a non-loopback host is refused unless you opt in. +2. **Is the key right?** It needs `evaluations:run` in the same organisation as the + agents. +3. **Are sessions finishing?** No `agent_end` means no finished session, and + nothing to claim. Check the instrumentation (`SKILL.md` §5). +4. **Does `when` skip everything?** The skip reason is recorded per session. A + condition reading a payload key the agents never emit skips every session. +5. **Are evaluations timing out?** Check the worker's own log. Make judges `async` + and give them a `timeout_seconds` / `timeoutSeconds` that covers one model + call. diff --git a/sdk/python/skill/references/events.md b/sdk/python/skill/references/events.md index 1fdf1dd16..ffcdb5159 100644 --- a/sdk/python/skill/references/events.md +++ b/sdk/python/skill/references/events.md @@ -1,5 +1,11 @@ # Event catalog +> TypeScript/JavaScript: the same 15 events on `failproofai.event`, camelCase +> (`toolUse({ toolName, toolCallId })`) taking one options object; the file written +> is identical. Name map in `typescript.md`. Code below is Python, and where it +> says `ValueError` or `TypeError`, TypeScript throws `TypeError` or a plain +> `Error` (listed in `typescript.md` → *The contract, in TypeScript*). + Every method lives on `failproofai_sdk.event`, is **keyword-only**, and returns `None`. Nothing here blocks or does I/O — the call queues the event and returns. @@ -93,16 +99,19 @@ its original) oldest-first, and will keep bracketing the wrong pairs. **Always pass `duration_ms` on the `model_response` too** — that is what keeps the reported duration right even when the bracketing is wrong. -**Correlation is a process-wide map keyed by your ids.** The SDK holds open -starts there until their matching end arrives. Consequences, in order of how much +**Correlation is a map keyed by session, kind and your id.** The SDK holds open +starts there until their matching end arrives. An id only has to be unique *within +one session, for one kind* (the second bullet). Consequences, in order of how much they hurt: -- **Per-run counters are unsafe.** `call_1`, `call_2` — common in home-grown loops - — collide across overlapping runs. The failure is not the missing duration the - docs might lead you to expect; it is a *plausible wrong number attributed to the - wrong run*, which is worse. Reuse your framework's id (Anthropic and OpenAI - tool-call ids are globally unique), or a `uuid4`. `failproofai_sdk.tool_call()` - generates a `uuid4` for you. +- **Per-run counters are unsafe inside a shared session.** `call_1`, `call_2` — + common in home-grown loops — are fine while every run has its own session. They + collide when two runs share one: a retried job that reuses its job id as the + session, or two agents in one session each counting from `call_1`. The failure is + not the missing duration the docs might lead you to expect; it is a *plausible + wrong number attributed to the wrong run*, which is worse. Reuse your framework's + id (Anthropic and OpenAI tool-call ids are globally unique), or a `uuid4`. + `failproofai_sdk.tool_call()` generates a `uuid4` for you. - **`tool_call_id` and `hook_id` no longer collide with each other.** They live in separate namespaces, so a `hook_completed(hook_id="x")` cannot pair with a pending `tool_use(tool_call_id="x")`. The key is `::`, so diff --git a/sdk/python/skill/references/frameworks.md b/sdk/python/skill/references/frameworks.md index 3e9cdfec9..69646ebb0 100644 --- a/sdk/python/skill/references/frameworks.md +++ b/sdk/python/skill/references/frameworks.md @@ -1,5 +1,8 @@ # Framework integrations +> TypeScript/JavaScript (LangChain.js/LangGraph.js, Vercel AI SDK, Mastra, +> LlamaIndex.TS, Next.js): see `typescript.md`. This page is the Python SDK. + If the agent runs on LangChain/LangGraph, CrewAI, LlamaIndex or Pydantic AI, you do not write the instrumentation — you turn it on. The adapters ship inside the SDK wheel and are imported only when you ask for them. diff --git a/sdk/python/skill/references/install.md b/sdk/python/skill/references/install.md index 50c03a636..c82e4f5cd 100644 --- a/sdk/python/skill/references/install.md +++ b/sdk/python/skill/references/install.md @@ -1,5 +1,8 @@ # Installing the SDK +> TypeScript/JavaScript: `npm install @failproofai/sdk` — see `typescript.md`. +> This page is the Python distribution. + ```bash pip install failproofai-sdk # or: uv add failproofai-sdk ``` diff --git a/sdk/python/skill/references/integration.md b/sdk/python/skill/references/integration.md index c3bff39da..9093e0803 100644 --- a/sdk/python/skill/references/integration.md +++ b/sdk/python/skill/references/integration.md @@ -1,5 +1,9 @@ # Writing the integration +> TypeScript/JavaScript: the same three scopes exist as `session()`, `agent()` and +> `toolCall()` on `AsyncLocalStorage`, and a hand-built loop is three edit sites — +> see `typescript.md` → *An agent with no framework*. This page is the Python SDK. + Identity is ambient. Bind it once per run with a context manager and every `event.*` call inside — including calls in functions that have never heard of Failproof AI — lands on the right session and agent. @@ -50,8 +54,8 @@ a session is *defined* as something that emitted `agent_start`. one run rather than splitting it in two. - `parent_id` defaults to the enclosing agent from the scope stack. Pass `parent_id=None` to force a root span, or a string to override. -- `tool_call_id` defaults to `uuid4().hex` — unique process-wide, which is what - the correlation map needs (see `events.md`). +- `tool_call_id` defaults to `uuid4().hex` — unique everywhere, so it can never + collide in the correlation map (see `events.md`). ## What `agent()` does on the way out diff --git a/sdk/python/skill/references/typescript.md b/sdk/python/skill/references/typescript.md new file mode 100644 index 000000000..0a6dc75bd --- /dev/null +++ b/sdk/python/skill/references/typescript.md @@ -0,0 +1,568 @@ +# TypeScript and JavaScript agents — `@failproofai/sdk` + +Everything in `SKILL.md` §2 (plan) and §6 (production) applies unchanged. The +TypeScript SDK writes the **same 15 events, the same wire format, into the same +spool directory** as the Python one, and `failproofaid` ships both. A fleet running +both languages writes into one pipe, and the dashboard cannot tell which language +wrote what. This page is what differs: install, names, adapters, bundlers, runtimes, +shutdown and verification. + +## Install + +```bash +npm install @failproofai/sdk # or: pnpm add / yarn add / bun add +``` + +Node ≥ 20.9, Bun, or Deno (`npm:@failproofai/sdk`). ES modules and CommonJS both +work. It has zero runtime dependencies. The framework packages are optional peer +dependencies, loaded only when you ask for them. + +```ts +import * as failproofai from "@failproofai/sdk"; // ESM +// const failproofai = require("@failproofai/sdk"); // CommonJS +``` + +Confirm what you have: + +```bash +node -e 'console.log(require("@failproofai/sdk").version)' +``` + +| Import | What it holds | +|---|---| +| `@failproofai/sdk` | scopes, `event.*`, `configure`, `flush`/`flushSync`, `instrument`/`uninstrument` | +| `@failproofai/sdk/langchain` | `langchainHandler()` — a callback handler, no patching | +| `@failproofai/sdk/ai` | `telemetry()`, `wrapModel()` for the Vercel AI SDK | +| `@failproofai/sdk/mastra` | `wrapTool()`, `workflow()` | +| `@failproofai/sdk/llamaindex` | the adapter's internals — nothing to import; use `instrument("llamaindex", { … })` | +| `@failproofai/sdk/next` | `withFailproofai()` for `next.config` | +| `@failproofai/sdk/evaluator` | the evaluator worker — see `evaluator.md` | + +The root import deliberately does not re-export the subpaths. + +## Names: camelCase in, snake_case on the wire + +Every Python name has a camelCase twin, and the file on disk is identical: + +| Python | TypeScript | +|---|---| +| `failproofai_sdk.agent("x", goal=g)` | `failproofai.agent("x", { goal: g }, fn)` | +| `failproofai_sdk.tool_call(...)` | `failproofai.toolCall(name, { toolCallId, input }, fn)` | +| `failproofai_sdk.session(session_id=...)` | `failproofai.session({ sessionId }, fn)` | +| `event.tool_use(tool_name=, tool_call_id=)` | `event.toolUse({ toolName, toolCallId })` | +| `event.model_response(input_tokens=, stop_reason=, request_id=)` | `event.modelResponse({ inputTokens, stopReason, requestId })` | +| `event.error(error_type=, message=)` | `event.error({ errorType, message })` | +| `event.human_wait(input_id=)` / `agent_pause(pause_id=)` / `hook_triggered(hook_id=, trigger_event=)` | `inputId` / `pauseId` / `hookId`, `triggerEvent` | +| `configure(base_dir=, flush_interval=, environment=)` | `configure({ baseDir, flushInterval, environment })` | +| `failproofai_sdk._writer.flush_now()` | `await failproofai.flush()` / `failproofai.flushSync()` | +| `propagate(fn)` | `propagate(fn)` | + +**Only declared option names are camelCase.** Any other key is a custom payload +field and is written **verbatim**, so `duration_ms` stays `duration_ms`. A camelCase +typo of a declared name (`toolCallID`) is not an error: it becomes a new field, and +nothing tells you. + +## The contract, in TypeScript + +`SKILL.md` §3 holds, with these spellings: + +- **Identity is ambient.** It rides on `AsyncLocalStorage`, so it follows `await`, + `.then()`, timers and callbacks created inside the scope. Two concurrent runs in + one process never mix. A callback **stored** in one run and invoked from another + does not inherit it: wrap it with `failproofai.propagate(fn)`. +- **No session bound and none passed throws `TypeError`** from `event.*` and + `toolCall()`. `agent()` and `session()` mint a fresh session instead. A non-string id also + throws `TypeError`. An empty or whitespace id throws `Error`. `agentId` falls back + to `"main"`. +- **A reserved extra throws.** `timestamp`, `session_id`, `agent_id`, `type` and + `environment` cannot be custom fields. +- **Token counts must be integers.** `inputTokens: 12.5` throws `TypeError`. + `null` is fine: the key is omitted. +- **`duration_ms` is refused on the four paired closers**: `toolResult`, + `hookCompleted`, `humanInput` and `agentResume`. They compute it. On every other + method, including `modelResponse`, it is accepted as a custom field, and that is + how you time a model call yourself. +- **`configure()`**: call it once, at startup, before the first event. + - Omitted options reset to their defaults. + - Nothing is applied unless every option validates. + - A comma in `environment` throws. + - A comma in `AGENTEYE_ENVIRONMENT` does not throw. It warns and falls back to + `"dev"`. + - `environment` and `baseDir` apply to the whole process — every copy of the + SDK loaded into it (the ESM and CommonJS builds, or a copy a bundler put in a + Next.js route) — so configure once, anywhere at startup. +- **`environment` defaults to `"dev"`.** Set it in `configure` or with + `AGENTEYE_ENVIRONMENT`. +- **Scope outcomes**: + - The body returned: `agent_end` with `outcome: "success"`. + - The body threw: `error`, then `agent_end` with `outcome: "failed"`. + - An `AbortError`: `agent_end` with `outcome: "cancelled"`. + + The error is always rethrown. A tool failure is recorded on its `tool_result` and + emits no run-level `error`. +- **`error_type` is the error's class.** When an error's `name` is just `"Error"` + (openai's `BadRequestError`, many SDK errors) the class name is recorded instead. +- **Timestamps order events within a millisecond.** The last three of the six + fractional digits are a per-process sequence, not a measurement, so events + emitted in the same millisecond still sort in the order they happened. +- **Unencodable values never take the process down.** Circular references, + `BigInt`, throwing getters and lone surrogates are handled. One bad event is + dropped alone, not the batch around it. +- **Credentials are redacted before the bytes reach disk.** That covers API keys, + tokens, JWTs and bearer headers. +- **The SDK's own warnings go to stderr, prefixed `[failproofai-sdk]`.** + - Route them with `failproofai.setLogger({ debug, info, warn, error })`. + - Set verbosity with `FAILPROOFAI_SDK_LOG_LEVEL`: `debug`, `info`, `warn` + (the default), `error` or `silent`. + - `FAILPROOFAI_SDK_STRICT=1` makes a swallowed adapter failure throw. + +## Shutdown — the part that loses data + +- **A normal exit is covered.** Buffered events flush on `process.on("exit")`, and + the flush timer is `unref`'d, so importing the SDK never keeps a script alive. +- **Short-lived work that returns rather than exits** — a serverless handler, a + queue job in a long-lived worker — should `await failproofai.flush()` before + returning: the process lives on, so no exit flush comes. +- **Signals.** `SIGTERM` (every deploy, `docker stop`, a Kubernetes eviction) kills + Node **without** running exit handlers. The SDK will not install a signal handler + in your process (that would change what Ctrl-C does), so add one at startup: + +```ts +for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]] as const) { + process.once(signal, () => { + failproofai.flushSync(); + process.exit(code); // 128 + signal number: the orchestrator sees a termination, not a success + }); +} +``` + + On the way out the SDK **closes whatever is still open**, most recently opened + first — including what an adapter opened: an interrupted tool gets its + `tool_result`, a hook its `hook_completed`, a model call its `model_response` + (`stop_reason: "error"`), each with a `ProcessExit: …` error, and each open agent + gets an `error` event and `agent_end` with `outcome: "failed"`. So a deploy never + leaves a run showing as running forever. (Python gets the same through + `SystemExit` unwinding.) A `flushSync()` while the process carries on closes + nothing, and a run paused on a human is left open for whoever resumes it. +- **Every exit path, not just signals.** The same closing happens on an uncaught + exception or unhandled rejection (the message names it — `… on an uncaught + QuotaError: …`), on `process.exit()` from anywhere, and when the event loop + drains with a run still pending. A run still open when the process ends is a + failed run, whatever the exit code. +- **Your own records.** `process.exit()` in that handler skips your code's + `catch`/`finally`, so a job that records its own result (a database row, a + status file) should record "interrupted" in the handler too, before + `flushSync()` — or the dashboard shows a failed run your own store never heard of. +- **Run `node` directly in services and containers.** `npx tsx` does not pass + SIGTERM on to the program, which then runs on, finishes, and records `success` + for a job that was killed. Use `node --import tsx app.ts`, or compile. +- **Next.js:** put that handler in its own module and import it from + `instrumentation.ts` behind the runtime check (see *Next.js* below) — `process.once` + in `instrumentation.ts` itself makes Turbopack warn about the Edge runtime. + +## A framework agent — turn it on + +```ts +import * as failproofai from "@failproofai/sdk"; + +failproofai.configure({ environment: "production" }); +await failproofai.instrument("langchain"); // name the framework you use +``` + +| Framework | Tested range | How it attaches | +|---|---|---| +| LangChain.js / LangGraph.js | `@langchain/core` 0.3 – 1.x, LangGraph 0.4 – 1.x | global callback configuration — no `callbacks:` needed. Or pass `langchainHandler()` yourself and patch nothing | +| Vercel AI SDK | `ai` 4 – 7 | `telemetry()` at the call site (every major); `instrument("ai")` process-wide on `ai` 7 only | +| Mastra | `@mastra/core` 0.20 – 1.x | `Agent.generate`/`.stream`, tool resolution, the workflow engine | +| LlamaIndex.TS | `llamaindex` 0.11.4 – 0.x | `Settings.callbackManager` plus `AgentWorkflow.runStream` | + +**`instrument()` — the rules:** + +- **Await it, before the first run.** An unawaited `instrument()` may happen to + work when the first run starts late — which is why it passes a local test and + fails under load. When it loses the race it does not record nothing; it records a + *wrong* trace: the root run starts before the adapter + exists, so a graph node, a tool or a bare model call becomes the session's agent, + or each becomes its own one-call session. The LangChain adapter warns when it sees + this (`… started under a parent run the langchain adapter never saw …`); the + others cannot tell it apart from a bare call. +- **Name the framework.** A bare `instrument()` patches *every* supported framework + that resolves from the project — including ones installed but unused, and ones + resolving from a parent directory's `node_modules`. It resolves to the names it + instrumented, and `[]` plus a warning when it found nothing. An unknown name throws. +- **Options** go in the second argument, `instrument("", { … })`: + + | Adapter | Options | + |---|---| + | `langchain` | `sessionId`, `captureContent` (default `true`), `includeChains`, `graphCallbacks`, `captureLimit` — also accepted by `langchainHandler({ … })` | + | `llamaindex` | `captureMessages` (default `true`), `steps` (default `true`; `false` drops the workflow-step hooks), `embeddings`, `staleAfter`, `reaperInterval`, `captureLimit` | + | `ai` | `registerGlobalTracer` (ai 4–6 only, see below), `captureLimit` | + | `mastra` | `captureLimit` | + + Content capture is **on** by default: a LangGraph node's `hook_triggered` carries + the message history as `input`, and a hand-written `model_request` carries what + you pass. `captureContent: false` / `captureMessages: false` keep structure, + durations and tokens and drop the text. + +**The mapping is the Python SDK's.** An **agent** is anything that owns an LLM +decision loop: a graph or chain run, `generateText`/`streamText`, a Mastra agent or +workflow, a LlamaIndex agent run. A LangGraph node or a workflow step is a **hook** +(`hook_triggered`/`hook_completed` with a `trigger_event`), never a nested agent. +Model pairs carry token counts; tool pairs carry the model's own tool-call id. A +failure is recorded once, where it happened — a tool that throws is an `error` on its +`tool_result`, and if the model then recovers the run still ends `success`. + +### Naming the agent, and owning the session id + +`agent_id` is the facet every dashboard view groups by, and each framework takes it +from a different place. Set it, or you get the framework's default: + +| Framework | `agent_id` comes from | Default when unset | +|---|---|---| +| LangGraph / LangChain | the root run's name: `createReactAgent({ name })`, or `.withConfig({ runName })` on a compiled graph | `"LangGraph"` | +| Vercel AI SDK | `functionId` in the call's telemetry settings — `telemetry({ functionId })`. An agent class's own `id` (`ToolLoopAgent({ id })`) is **not** passed through | `"ai.generateText"` / `"ai.streamText"` | +| Mastra | the agent's `name` (not its `id`, not its registration key). A workflow run is rooted under the **workflow's** id, with the agent nested under it | — | +| LlamaIndex.TS | `agent({ name })` | the class or operation name | + +Mastra's `tool_name` is the key in the agent's `tools` map, not the `createTool` id. + +An adapter mints a random session id when nothing is bound, and nothing reads it +back to you. To use your own — a request or job id, so a dashboard session and your +own logs or database share it — bind it around the framework call: + +```ts +await failproofai.session({ sessionId: requestId }, () => graph.invoke(input)); +``` + +`session()` emits nothing; each framework run inside it is its own agent — so +three AI SDK calls under one `session()` are three agents in one session. Wrapping in +`agent()` also works: under the **same** name as the framework agent it becomes that +agent (one `agent_start`, and every call inside joins it — how several AI SDK +calls become one agent); under a **different** name the framework agent nests +under yours (`parent_id` = your name), which is right when your wrapper is a real +outer agent. A joined agent's `agent_start` is yours, so put the `goal` on it. Inside either, `failproofai.current().sessionId` is the id. + +Two consequences of owning it: a **retried** job that reuses its job id lands in the +**same** session, as a second run in it — append the attempt (`${jobId}-2`) if you +want retries separate. And LangChain also accepts it per call, as +`metadata: { failproofai_sdk_session_id }`. + +### Per framework + +- **Vercel AI SDK.** `telemetry()` at each call site works on every major and is the + one way to name the agent. On `ai` 7, `instrument("ai")` covers every call in the + process too; using both does not double-record. + +```ts +import { telemetry } from "@failproofai/sdk/ai"; + +await generateText({ + model, + prompt, + telemetry: telemetry({ functionId: "answer-question" }), // ai 4–6: `experimental_telemetry:` +}); +``` + + On `ai` 4–6, `instrument("ai")` records nothing by itself and warns: the only + process-wide hook there is the global OpenTelemetry tracer, and taking it would + break the app's own tracing. `instrument("ai", { registerGlobalTracer: true })` + opts in when the process runs no OpenTelemetry of its own. An agent class takes + it the same way — `new ToolLoopAgent({ model, tools, telemetry: telemetry({ + functionId: "ai-research" }) })` — since its own `id` is not passed through. + `await wrapModel(model)` + (async — pass the resolved model) records model calls only; tools run above the + model layer. In a route handler on `ai` 7, pass `abortSignal: request.signal` so a + client that disconnects mid-stream closes the run instead of leaving it open. +- **Mastra.** `instrument("mastra")` is enough for agents on a `Mastra` instance, + their tools, and workflows. `wrapTool(tool)` is only for a tool called directly — + from your own code or a workflow step, not by an agent; `workflow(name, body)` only + groups work that is not a Mastra workflow under one named span. +- **LlamaIndex.TS.** A tool or LLM call made *outside* an agent workflow is recorded + as its own one-call run — that is how bare calls are shown, not a bug. Its + `openai()` client sends `temperature: 0.1` by default, which some models reject + with a 400 that crashes the workflow; set `temperature` on `openai({ … })` then. + +**Token counts — set usage on the model client, or they are silently missing.** +OpenAI-compatible APIs report usage on a stream only when asked, and some framework +clients stream even for a non-streaming call: + +| Framework | Where | Needed for | +|---|---|---| +| Vercel AI SDK + `@ai-sdk/openai-compatible` | `createOpenAICompatible({ …, includeUsage: true })` | `streamText` | +| Mastra (same provider package) | `createOpenAICompatible({ …, includeUsage: true })` | `.stream()` | +| LlamaIndex.TS | `openai({ …, additionalChatOptions: { stream_options: { include_usage: true } } })` | **every** agent run — its agent streams internally even for `run()` | + +LangChain's `ChatOpenAI` already asks. For any other client, check for +`input_tokens`/`output_tokens` on each `model_response` when you verify. + +### Bundlers — the silent one + +Most of these frameworks ship an ESM build and a CJS build. Node loads them as two +unrelated copies. The adapters patch the copy your app loads, plus a CJS copy that +something has already `require`d. + +No adapter can reach a framework **bundled into your own output** (esbuild, +webpack, `ncc`). The copy in `node_modules` is not the one running, and nothing is +recorded. Either keep the framework external in the bundler config, or use the +call-site helpers, which work bundled: `langchainHandler()`, `telemetry()`, +`wrapTool()`. + +### Next.js + +`next build` bundles server dependencies by default. Wrap the config, and configure +and instrument from Next's startup hook: + +```ts +// next.config.ts +import { withFailproofai } from "@failproofai/sdk/next"; +export default withFailproofai({ /* your config */ }); +``` + +```ts +// instrumentation.ts +export async function register() { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + const failproofai = await import("@failproofai/sdk"); + failproofai.configure({ environment: "production" }); + await failproofai.instrument("langchain"); + await import("./instrumentation-node"); // the signal handler from *Shutdown* +} +``` + +- `withFailproofai` adds LangChain, Mastra, LlamaIndex and the SDK to + `serverExternalPackages` and keeps your own list. Without it, `instrument()` warns + when `next start` boots — not at `next build` — for each framework it cannot + reach, and those frameworks record nothing. If you list the packages by hand, + `FAILPROOFAI_NEXT_EXTERNALS=1` silences the warning. +- The Vercel AI SDK is not externalized and does not need to be: `telemetry()` and + `instrument("ai")` on `ai` 7 both work in a bundled route. +- `configure()` in `register()` applies to the whole server process, including a copy + of the SDK bundled into a route. +- Route handlers have no request id of their own: use the `x-request-id` header your + platform or proxy sets, with a fallback — + `session({ sessionId: request.headers.get("x-request-id") ?? randomUUID() }, …)`. +- Pass the request's signal down, so a client that disconnects ends the run + (`cancelled`) instead of it running on: `graph.invoke(input, { signal: + request.signal })` for LangGraph, `abortSignal: request.signal` for the AI SDK. +- **Edge runtimes** (Next.js Edge routes, workers) get a no-op build. Importing is + safe, and nothing is recorded there. Instrument the Node side. + +## An agent with no framework — three edit sites + +This is the path for a hand-built loop: an OpenAI or Anthropic client, a `for` loop +and a tool table. It also covers any framework without an adapter, whatever else the +agent does, such as writing its own records to a database. You emit the events +yourself with the same API the adapters use, so the trace is the same shape — and, +if you record what the snippet below records, the same quality. + +Every hand-built agent already has these three places, whatever its functions are +called. Find them in the codebase first: + +| Where | Add | Emits | +|---|---|---| +| where **one run** starts and ends | `failproofai.agent("name", { goal }, async () => …)` | `agent_start` / `agent_end` | +| the **one function that calls the model** | `event.modelRequest` before, `event.modelResponse` after — **both halves, even on failure** | one pair per model turn | +| the **one function that runs tools** | `failproofai.toolCall(name, { toolCallId, input }, () => run())` | `tool_use` / `tool_result` | + +```ts +import { randomUUID } from "node:crypto"; +import * as failproofai from "@failproofai/sdk"; +import OpenAI from "openai"; +import type { ChatCompletionMessageParam } from "openai/resources/chat/completions"; + +// 1. the run — everything inside lands on this session, with no ids passed +const answer = await failproofai.agent("inventory", { goal: question }, async () => { + for (let turn = 0; turn < 6; turn++) { + const message = await callModel(messages); + const calls = (message.tool_calls ?? []).filter((c) => c.type === "function"); + if (calls.length === 0) return message.content ?? ""; + messages.push(message); + for (const call of calls) { + messages.push({ role: "tool", tool_call_id: call.id, content: await dispatch(call) }); + } + } + return "(gave up)"; +}); + +// 2. the model call — pair on requestId, time it yourself, close it on failure +async function callModel(messages: ChatCompletionMessageParam[]) { + const requestId = randomUUID(); + const started = Date.now(); + failproofai.event.modelRequest({ + model: MODEL, + requestId, + // Role and content, plus the ids linking a tool result to its call. (openai's + // own message types do not satisfy the SDK's JSON types under `tsc --strict`.) + messages: messages.map((m) => ({ + role: m.role, + content: typeof m.content === "string" ? m.content : m.content == null ? "" : JSON.stringify(m.content), + ...(m.role === "tool" ? { tool_call_id: m.tool_call_id } : {}), + ...(m.role === "assistant" && m.tool_calls + ? { tool_calls: m.tool_calls.map((c) => ({ id: c.id, name: c.type === "function" ? c.function.name : c.type })) } + : {}), + })), + // Tool and tool-call types are unions in openai ≥ 6 (custom tools): narrow them. + tools: TOOLS.flatMap((t) => (t.type === "function" ? [{ name: t.function.name, description: t.function.description ?? "" }] : [])), + }); + let reply: OpenAI.Chat.Completions.ChatCompletion; + try { + // Only the provider call in the `try`: nothing else can reach the error path. + reply = await client.chat.completions.create({ model: MODEL, messages, tools: TOOLS }); + } catch (error) { + failproofai.event.modelResponse({ + model: MODEL, + requestId, + stopReason: "error", + error: error instanceof Error ? `${error.constructor.name}: ${error.message}` : String(error), + duration_ms: Date.now() - started, + }); + throw error; // the enclosing agent() then ends "failed" + } + const choice = reply.choices[0]!; + const calls = (choice.message.tool_calls ?? []).filter((c) => c.type === "function"); + failproofai.event.modelResponse({ + model: reply.model, + requestId, + role: choice.message.role, + content: choice.message.content ?? "", + stopReason: choice.finish_reason, + inputTokens: reply.usage?.prompt_tokens ?? null, + outputTokens: reply.usage?.completion_tokens ?? null, + duration_ms: Date.now() - started, + // the field and shape the adapters write + fw_tool_calls: calls.map((c) => ({ toolCallId: c.id, toolName: c.function.name, input: c.function.arguments })), + }); + return choice.message; +} + +// 3. the tool dispatcher — reuse the model's own tool-call id +async function dispatch(call: { id: string; function: { name: string; arguments: string } }): Promise { + // Malformed arguments are still a tool call: recorded, and failed inside + // toolCall(), so the trace shows it and the model gets an error to recover from. + let input: Record = {}; + let malformed: unknown; + try { + const parsed: unknown = JSON.parse(call.function.arguments || "{}"); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new TypeError("tool arguments must be a JSON object"); + } + input = parsed as Record; + } catch (error) { + malformed = error; + input = { arguments: call.function.arguments }; + } + try { + return await failproofai.toolCall(call.function.name, { toolCallId: call.id, input }, async () => { + if (malformed !== undefined) throw malformed; + return runTool(call.function.name, input); + }); + } catch (error) { + return `error: ${error instanceof Error ? error.message : String(error)}`; // let the model recover + } +} +``` + +The rules that make this correct: + +- **Emit both halves of the model pair.** A `modelRequest` with no `modelResponse` + is a span the dashboard shows as running forever — hence the `catch`. +- **Record what the adapters record.** `role`, `content`, the tool calls the model + asked for, tokens, stop reason and duration on the response. Leave them out and the + trace has the shape but not the substance. +- **Pair model calls on `requestId`**, generated per call, so overlapping calls + cannot cross-pair. +- **Model calls are not timed for you.** Pass `duration_ms` yourself — an integer + (`Date.now()` differences are); a float throws `TypeError`. +- **Reuse the model's tool-call id** as `toolCallId`, so a `tool_use` lines up with + the `tool_calls[]` entry that asked for it. `toolCall()` times the call and records + a throw as `tool_result.error`, then rethrows. +- **Services and workers.** Pass your own request or job id as `sessionId` + (`agent("assistant", { sessionId: jobId }, fn)`). A dashboard session and the + record in your own logs or database are then the same string. (Retries: see + *Naming the agent, and owning the session id*.) +- **Sub-agents.** Nest `agent()` calls — directly, or from inside a tool the outer + agent runs. The inner one joins the session, and its `parent_id` is the outer + agent's name. +- **Don't reach for a module-level session variable.** Two overlapping runs mix + their events. `AsyncLocalStorage` already does this correctly. +- **A failed tool does not fail the run.** It is an `error` on its `tool_result`, + and if the model recovers the run ends `success` with no `error` event — so an + evaluation that counts `error` events will not see it. Count `tool_result`s + with an `error` instead. +- **Types are the only guard on names in plain JavaScript.** A misspelt required + option (`toolCallID`) is not a runtime error: the event is written without it and + never pairs. Use TypeScript, or check the ids when you verify. +- **`input` is typed as an object.** The SDK does not reject a primitive at + runtime, so wrap it yourself: `{ query: q }`. +- **Watch the size of `messages`.** Each `model_request` carries whatever you pass, + and the history grows every turn — including provider blobs such as encrypted + reasoning. Send role/content, and trim what a reader does not need. +- **Don't emit your own `agent_end` inside `agent()`.** The scope emits one; a second + is accepted and duplicates it. + +The complete, runnable version is `sdk/typescript/examples/research-agent.ts` in the +FailproofAI/failproofai repo. CI runs it, as an ES module and as CommonJS, against +the real `openai` client. + +**`using`, when a callback will not fit.** Use it for a scope opened in a +constructor and closed in a teardown: + +```ts +{ + using span = failproofai.agent.open("planner", { goal }); + using call = failproofai.toolCall.open("search", { input: { q } }); + call.call.output = await search(q); +} // tool_result, then agent_end +``` + +Prefer the callback form. It has nothing to unwind. + +## Verify + +The spool is the same directory the Python SDK uses: +`${FAILPROOFAI_HOME:-~/.failproofai}/custom-agents/events/`, unless the app passed +`configure({ baseDir })`. + +**On a machine running `failproofaid`, that directory is empty seconds after a +healthy run** — the daemon ships each file and deletes it. An empty real spool +means nothing; check with a throwaway spool the daemon does not watch. Use a +directory of your own, so parallel checks on one machine cannot delete each other's: + +```bash +export FAILPROOFAI_HOME="$(mktemp -d)" +npx tsx your-agent.ts # Next.js: set it on `next start` +node -e ' + const fs = require("fs"), d = process.env.FAILPROOFAI_HOME + "/custom-agents/events"; + for (const f of fs.readdirSync(d).filter(f => f.endsWith(".jsonl")).sort()) + for (const l of fs.readFileSync(d + "/" + f, "utf8").split("\n").filter(Boolean)) { + const e = JSON.parse(l); + console.log(e.timestamp, e.session_id, e.agent_id, e.type, e.tool_name ?? "", + e.tool_call_id ?? e.request_id ?? "", e.input_tokens ?? "", e.output_tokens ?? "", + e.outcome ?? "", e.error ?? "", e.environment); + }' +``` + +Then walk the same checklist as `SKILL.md` §5 (its first item — a dead flush +thread — is Python's; the TypeScript SDK logs `[failproofai-sdk]` warnings to +stderr instead): + +1. **One `agent_start`/`agent_end` pair per agent per run** — two when a sub-agent + runs, each with the right `parent_id`. No files at all usually means one of these: + - The process was killed by a signal with no handler, or returned from a + serverless handler without `await failproofai.flush()`. + - `instrument()` found nothing, or was not awaited: look for the + `[failproofai-sdk]` warning on stderr. + - The framework is bundled. +2. **Every pair closed, with the ids you expect** — each + `model_request`/`model_response` on `request_id`, each `tool_use`/`tool_result` on + `tool_call_id` — and **token counts** on every `model_response` (missing on + streams: see *Token counts*). +3. **Two overlapping runs give two session ids, with no events crossing.** +4. **`environment` is the label you expect** on every event. +5. **The `agent_id`s are your names**, not `LangGraph` or `ai.generateText`. + +Then confirm the real thing arrived, from a separate environment, with the +`fp-cloud-cli` skill: `fp sessions --session-id --since 1h` (the id you bound +with `session()`, or read with `failproofai.current()`). A session's `status` says +whether it finished, not whether it succeeded — a killed run is `done` too; its +`agent_end` `outcome` (`fp events --session-id --full`) is what says `failed`. diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignore new file mode 100644 index 000000000..9df836c24 --- /dev/null +++ b/sdk/typescript/.gitignore @@ -0,0 +1,30 @@ +# Build output +dist/ +*.tgz + +# Dependencies +node_modules/ + +# Testing & coverage +coverage/ +.vitest/ + +# Editors +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store + +# Integration fixtures: installed frameworks and the transpiled agents +integration/fixtures/*/node_modules/ +integration/fixtures/*/.run/ +# ...but their lockfiles are the point: each pins the framework release under +# test. The repo root ignores package-lock.json (it is a bun repo), so re-include. +!integration/fixtures/*/package-lock.json + +# The Next.js fixture: its builds, and the tsconfig/next-env.d.ts `next build` +# writes (and rewrites, per distDir) on every run. +integration/fixtures/nextjs/.next*/ +integration/fixtures/nextjs/next-env.d.ts +integration/fixtures/nextjs/tsconfig.json diff --git a/sdk/typescript/CHANGELOG.md b/sdk/typescript/CHANGELOG.md new file mode 100644 index 000000000..50d8ae2b3 --- /dev/null +++ b/sdk/typescript/CHANGELOG.md @@ -0,0 +1,113 @@ +# Changelog — `@failproofai/sdk` + +The TypeScript telemetry SDK. Released independently of the `failproofai` npm +package, of its Python sibling `failproofai-sdk`, and of `fp-cloud-cli`, so the +versions here line up with none of them. + +Headings are `## — `, and the section matching the version +in `src/version.ts` becomes that release's GitHub Release body. A release whose +section is missing or empty is refused before anything is built. + +## 0.0.1-beta.0 — 2026-09-23 + +### Added + +- **First release.** `@failproofai/sdk` is the TypeScript counterpart to the + Python `failproofai-sdk`: the same 15 events, the same wire format, the same + spool directory, the same evaluator protocol. A process running a Node agent + and a process running a Python one now write into one pipe, and the dashboard + cannot tell which wrote what. + +- **Scopes** — `session()`, `agent()`, `toolCall()`. Identity rides on + `AsyncLocalStorage`, so two concurrent runs in one process never mix. Each + takes a callback (`await agent("planner", fn)`) and also has a `.open()` + returning a `using`-compatible handle for the cases a callback cannot express. + A synchronous body stays synchronous — the scopes do not wrap every call in a + promise, because a constructor or an `EventEmitter` listener cannot await one. + +- **Adapters** — `instrument()` wires LangChain.js / LangGraph.js + (`@langchain/core` 0.3 – 1.x), the Vercel AI SDK (`ai` 4 – 7), Mastra + (`@mastra/core` 0.20 – 1.x) and LlamaIndex.TS (`llamaindex` 0.11.4 – 0.x), + and they draw the Python SDK's trees: a construct is an agent only if it owns + an LLM decision loop, a LangGraph node or workflow step is a hook, model and + tool calls are pairs carrying token counts and the model's own tool call id, + and a failure is recorded once, where it happened. LangChain traces match the + Python adapter's golden output event for event, including interrupt/resume. + The AI SDK is served by `telemetry()` — one object carrying an OpenTelemetry + tracer for `ai` 4–6 and a telemetry integration for `ai` 7 — plus + `wrapModel()` and, for `ai` 7, `instrument("ai")`; using them together records + each call once. On `ai` 4–6 `instrument("ai")` never takes the global + OpenTelemetry slot unless asked (`registerGlobalTracer: true`), because + taking it silently refuses the application's own tracing set up afterwards. + `langchainHandler()` works without `instrument()`. + +- **Safe in a long-running server.** Nothing a finished run leaves behind is + kept: tracker links go when their run closes (a FIFO cap full of finished + runs used to evict live ones and drop their events), LangGraph runs paused on + a human and resumed by another worker are forgotten after 15 minutes, and a + streamed model call that is cancelled or errors still closes. Concurrent + requests on one shared LlamaIndex query engine or agent are kept apart, and + `uninstrument()` stops Mastra recording through models and tools it had + already wrapped. Frameworks are found from the entry script as well as the + working directory, so a service started from `/` or a monorepo app with its + own nested copy of a framework is instrumented correctly. + +- **Tested against the real frameworks, not just in isolation.** `integration/` + installs real framework releases at both ends of every declared range from + per-fixture lockfiles, extracts the packed tarball into each, and runs one + agent as an ES module and as CommonJS — the dual-package case where an + adapter that patches the CommonJS copy of a framework records nothing at all + in an ES-module application. Adapters patch the copy the application loads, + and never load a second one. Runs in CI as `failproofai-ts-sdk-integrations`. + +- **Every commonly used surface, not just the headline API.** Beyond each + framework's main agent call the suite covers LangChain's v1 `createAgent`, + LCEL chains, retrievers, `.batch()` and nested `@langchain/core` copies; the + AI SDK's agent classes, embeddings, object generation, approval and + client-side tools and every stream-consumption style; Mastra instances, + networks, memory threads (the thread is the session), workflows with + suspend/resume, processors and MCP tools; LlamaIndex chat engines, query + engines, retrievers and `createWorkflow()` workflows — each under + concurrency as well. + +- **Next.js:** `withFailproofai(nextConfig)` from `@failproofai/sdk/next` + keeps the frameworks `instrument()` patches out of Next's server bundle, and + `instrument()` warns once per framework it cannot reach instead of recording + nothing silently. Importing the SDK in an Edge route is safe (a no-op build). + +- **Your own agent, no framework:** a guide to the three places every + hand-built agent already has (the run, the model call, the tool dispatcher) + and `examples/research-agent.ts`, a real OpenAI tool loop instrumented by hand + — the TypeScript twin of the Python SDK's `research_agent.py`. The integration + suite runs that exact file on every CI run, as ESM and CJS, against the real + `openai` client. + +- **Runtimes:** Node ≥ 20.9, Bun and Deno, every framework as ESM and CJS, + checked against Node's trace. + +- **Type declarations for every consumer setup** — ESM and CommonJS + `nodenext`, CommonJS `node16`, `moduleResolution: node` (every subpath, via + `typesVersions`) and `bundler` — on TypeScript ≥ 5.4. CommonJS consumers get + CommonJS declarations; `@arethetypeswrong/cli` reports no problems. + +- **Evaluator** — `@failproofai/sdk/evaluator` implements Evaluator v2: the wire + protocol, the worker state machine, the authoring API, and a + `failproofai-evaluator` command to run one. + +- **A sandbox that is a real one.** Server-authored evaluation source runs + through a restricted expression language that is **parsed and interpreted** + here — never `eval`'d, never handed to `node:vm`. That is not + belt-and-braces: JavaScript has a reachable path from any value to arbitrary + code (`x["constructor"]["constructor"]("…")()`), a static allowlist cannot + close it because the key is computed at runtime, and a `vm` context has its + own `Function` to reach. Every property read goes through one function that + checks the actual key at the moment of the read. A `worker_threads` sandbox + with V8 heap limits, a wall-clock `terminate()` and a bounded result sits + around that as the RESOURCE bound, and an evaluation that cannot be sandboxed + is refused rather than run. + +- **Zero runtime dependencies**, checked by the build and by a test. This + package installs into other people's agent processes; every dependency it + declared would be a version constraint they inherit. + +- Dual ESM + CommonJS build, Node ≥ 20.9. diff --git a/sdk/typescript/LICENSE b/sdk/typescript/LICENSE new file mode 100644 index 000000000..9802e634b --- /dev/null +++ b/sdk/typescript/LICENSE @@ -0,0 +1,42 @@ +MIT License + +Copyright (c) 2025 ExosphereHost Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Commons Clause License Condition v1.0 + +The Software is provided to you by the Licensor under the License, as defined +below, subject to the following condition. + +Without limiting other conditions in the License, the grant of rights under +the License will not include, and the License does not grant to you, the right +to Sell the Software. + +For purposes of the foregoing, "Sell" means practicing any or all of the +rights granted to you under the License to provide to third parties, for a +fee or other consideration (including without limitation fees for hosting or +consulting/support services related to the Software), a product or service +whose value derives, entirely or substantially, from the functionality of the +Software. Any license notice or attribution required by the License must also +include this Commons Clause License Condition notice. + +Software: failproofai +License: MIT +Licensor: ExosphereHost Inc diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md new file mode 100644 index 000000000..8c00d453c --- /dev/null +++ b/sdk/typescript/README.md @@ -0,0 +1,552 @@ +# @failproofai/sdk + +Telemetry for TypeScript and JavaScript AI agents. Emit events, spool them to +disk, let the daemon ship them. + +Zero runtime dependencies. Node ≥ 20.9, ESM and CommonJS. + +```bash +npm install @failproofai/sdk +``` + +```ts +import * as failproofai from "@failproofai/sdk"; + +await failproofai.agent("planner", { goal: question }, async () => { + const hits = await failproofai.toolCall("web_search", { input: { q } }, () => search(q)); + failproofai.event.modelResponse({ model, inputTokens: 1_200, outputTokens: 340 }); +}); +``` + +That is the whole setup. Events land in `~/.failproofai/custom-agents/events/`, +and `failproofaid` — the daemon the `failproofai` CLI installs — picks them up. +Nothing here opens a socket, and nothing blocks your agent loop on the network. + +This is the TypeScript counterpart to the Python +[`failproofai-sdk`](https://pypi.org/project/failproofai-sdk/). Same 15 events, +same wire format, same spool directory: a fleet running both writes into one +pipe, and the dashboard cannot tell which language wrote what. + +--- + +## Three surfaces + +### 1. Scopes + +`session()`, `agent()` and `toolCall()` bind run identity and — for the latter +two — bracket the work with its own events. + +```ts +await failproofai.session({ sessionId: requestId }, async () => { + await failproofai.agent("supervisor", { goal: "answer the question" }, async () => { + await failproofai.agent("researcher", async () => { + // parentId is "supervisor", sessionId is requestId — nothing was passed. + const docs = await failproofai.toolCall("search", { input: { q } }, () => search(q)); + }); + }); +}); +``` + +Identity rides on `AsyncLocalStorage`, so it follows `await`, `.then()`, timers +and callbacks created inside the scope — and two concurrent runs in one process +never mix. + +| what happened | events | `outcome` | +|---|---|---| +| the block returned | `agent_end` | `"success"` (or your `outcome`) | +| the block threw | `error`, then `agent_end` | `"failed"` | +| an `AbortError` | `agent_end` only | `"cancelled"` | + +The error is always re-thrown. A tool failure is recorded on the leaf +(`tool_result` with an `error`) and emits **no** run-level `error`: one the +agent loop catches is not a run failure, and one that propagates is reported +exactly once, by the enclosing `agent()`. + +A synchronous body stays synchronous — `agent("x", () => 1)` returns `1`, not a +promise — because a constructor or an `EventEmitter` listener cannot await one. + +**`using`, when a callback will not fit.** A scope opened in a constructor and +closed in a teardown, or one that straddles existing control flow: + +```ts +{ + using span = failproofai.agent.open("planner", { goal }); + using call = failproofai.toolCall.open("search", { input: { q } }); + call.call.output = await search(q); +} // tool_result, then agent_end +``` + +Prefer the callback form. It runs inside `AsyncLocalStorage.run()`, so there is +nothing to unwind and the whole class of "opened here, closed over there" bugs +is unreachable. + +### 2. Adapters + +```ts +await failproofai.instrument(); // whatever it can find +await failproofai.instrument("langchain"); // exactly one +failproofai.uninstrument(); // put everything back +``` + +| framework | supported | how it attaches | +|---|---|---| +| **LangChain.js / LangGraph.js** | `@langchain/core` 0.3 – 1.x, LangGraph.js 0.4 – 1.x | `CallbackManager.configure`, so every `invoke`/`stream`/`batch` is covered without passing `callbacks:` anywhere — or pass `langchainHandler()` yourself and patch nothing. | +| **Vercel AI SDK** | `ai` 4 – 7 | `telemetry()` at the call site, or `instrument("ai")` for the whole process on `ai` 7 (on 4–6 that is opt-in) — see below. | +| **Mastra** | `@mastra/core` 0.20 – 1.x | `Agent.generate`/`.stream`, the agent's model and tool resolution, and the workflow run/step engine. Tools built before `instrument()` are covered. | +| **LlamaIndex.TS** | `llamaindex` 0.11.4 – 0.x | `Settings.callbackManager` (subscribed) plus `AgentWorkflow.runStream`, for workflow runs and their steps. | + +**Every range in that table is tested, not declared.** `integration/` installs +real framework releases at both ends of each range, extracts the packed tarball +into each project, and runs one agent as an ES module and again as CommonJS on +every CI run — the same bar the Python SDK's framework job holds. + +**The mapping is the Python SDK's**, so a TypeScript agent and a Python one on +the same framework draw the same tree. A construct is an **agent** if and only +if it owns an LLM decision loop: a graph or chain run, an AI SDK +`generateText`/`streamText` call, a Mastra agent, a LlamaIndex agent run. +Machinery around it — a LangGraph node, a Mastra or LlamaIndex workflow step — +is a **hook** (`hook_triggered`/`hook_completed` with a `trigger_event`), never +a nested agent, so `agent_id` stays a small set of real names. Model calls are +`model_request`/`model_response` pairs with token counts; tool calls are +`tool_use`/`tool_result` carrying the model's own tool call id. A failure is +recorded once, on the event it happened in — not once per layer it unwound +through. + +**ES modules and CommonJS both work.** Most of these frameworks ship two builds, +and Node loads them as two unrelated copies: patching one does nothing to the +other. The adapters patch the copy your application loads, plus the CommonJS +copy if something has already `require`d it, and never load a second copy +nobody uses. What no adapter can reach is a framework **bundled into your own +output** (esbuild, webpack): the copy in `node_modules` is not the one running. +On Next.js, `withFailproofai()` fixes that (see below); elsewhere use the +call-site helpers — `langchainHandler()`, `telemetry()`, `wrapTool()`. + +**Naming the agent and owning the session id.** `agent_id` comes from the +graph's name (`createReactAgent({ name })`, `.withConfig({ runName })`), the AI +SDK's `functionId`, the Mastra agent's `name`, or LlamaIndex's `agent({ name })`. +An adapter mints a session id when none is bound; to use your own request or job +id, bind it around the call — +`await failproofai.session({ sessionId: requestId }, () => graph.invoke(input))`. +An `agent()` wrapper of the same name as the framework agent becomes that agent +rather than nesting a copy of it; a differently named one is its parent. + +Everything an adapter records is namespaced `fw_*`, bounded per field and per +event, and tagged with `framework` / `framework_version`. An adapter that fails +is logged and skipped; the others still install, because a broken LlamaIndex +should not cost you LangGraph. `FAILPROOFAI_SDK_STRICT=1` turns every swallowed +failure into a throw. + +**The Vercel AI SDK** exports plain functions from an ES module, and an ES +module namespace is immutable by specification — there is nowhere to stand. So +it uses the extension points the SDK itself documents: + +```ts +import { telemetry } from "@failproofai/sdk/ai"; + +const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + // on ai 7, `telemetry: telemetry({ … })` — the same object, the new name +}); +``` + +That is the complete integration: an agent span named by `functionId`, a +model request/response pair per step with token counts, and every tool call. +One call site works on every major: `ai` 4–6 read the tracer it carries, `ai` 7 +reads the telemetry integration it carries. + +`instrument("ai")` does the same for the whole process **on `ai` 7**: every +call, through the AI SDK's global telemetry-integration list, which is additive +and takes nothing from anybody else's. + +**On `ai` 4–6, `instrument("ai")` records nothing by itself, and logs one +warning saying so.** The only process-wide hook those majors have is the global +OpenTelemetry tracer provider — a single slot that OpenTelemetry refuses to hand +over once taken. Registering ours would silently refuse your own +`NodeSDK.start()` later in startup and send your http/database spans to a tracer +that exports nothing. Use `telemetry()` at the call site (above) or `wrapModel` +(below) there. If the process runs no OpenTelemetry of its own, you can opt in: + +```ts +await instrument("ai", { registerGlobalTracer: true }); +``` + +— it then records every call that passes `experimental_telemetry: { isEnabled: +true }` (the AI SDK only consults the global tracer for those), and only takes +the slot if it is still empty. `registerGlobalTracer: false` keeps the default +and silences the warning. + +If you would rather wrap the model once, `wrapModel` sees model calls only, +because tool calls happen above the model layer. A wrapped model called with +nothing around it is recorded as its own run, named after the model. A streamed +call closes however the stream stops: `stop_reason: "cancelled"` when the +consumer cancels it, `"error"` with the error when it fails part-way: + +```ts +import { wrapModel } from "@failproofai/sdk/ai"; +const model = await wrapModel(openai("gpt-4o")); +``` + +Using both is fine: the middleware notices the call is already being recorded +and defers, so each call is recorded once. + +**LangChain without patching**, for hosts where patching is not wanted — the +handler works with or without `instrument()`, and never double-records: + +```ts +import { langchainHandler } from "@failproofai/sdk/langchain"; +await graph.invoke(input, { callbacks: [langchainHandler()] }); +``` + +`instrument("langchain")` also takes `sessionId`, `captureContent`, +`includeChains`, `graphCallbacks` and `captureLimit`, as the Python adapter +does; a per-call `metadata: { failproofai_sdk_session_id }` picks the session +for one invocation. + +**Mastra** tool calls made outside any agent can be wrapped by hand: + +```ts +import { wrapTool } from "@failproofai/sdk/mastra"; +const lookup = wrapTool(createTool({ id: "lookup", /* … */ })); +``` + +#### What each adapter records, framework by framework + +- **LangChain / LangGraph** — also `createAgent` from the v1 `langchain` + package (its middleware hooks are steps), plain LCEL chains, retrievers + (`tool_use`/`tool_result` summarised as `{ n, sources }`), `.batch()` (one + session per input), `.stream()`/`.streamEvents()`, structured output, and + `interrupt()`/`Command` resumes (`human_wait` + `agent_pause`, then + `agent_resume` + `human_input`, across processes too). A provider package + that pins its own nested `@langchain/core` is instrumented as well. +- **Vercel AI SDK** — `generateText`/`streamText`/`generateObject`/ + `streamObject`, the agent classes (`ToolLoopAgent`/`Agent`; named by + `functionId`, since the SDK does not pass the agent's `id` through), parallel, + client-side and approval-gated tools. An `embed` inside an agent is that + agent's model call; a bare one is its own run. A stream that stops early ends + `cancelled`. On `ai` 7, pass `abortSignal: request.signal` in a route handler + so a client that disconnects closes the run. A tool approval produces two + runs — wrap them in one `failproofai.session()` to keep one session. +- **Mastra** — agents on a `Mastra` instance, agent networks (`.network()` is + one agent, with delegates nested under it), agent-as-tool, workflows + (branch, parallel, loops, nested, agent steps; steps are hooks), MCP tools, + structured output (a separate structuring model appears as a nested agent), + processors. A memory **thread** is the session, and a suspended workflow's run + id keeps its session through `resume()`, with the human-in-the-loop events. A + run blocked by a processor tripwire ends `rejected`. +- **LlamaIndex.TS** — `agent()`/`multiAgent()` workflows, `createWorkflow()` + workflows (llamaindex 0.12+), chat engines, query engines and retrievers + (named after the retriever class), the legacy `LLMAgent`, bare LLM calls. + Concurrent requests on one shared engine or agent stay in separate sessions. + LlamaIndex.TS has no embedding events, so `embeddings: true` records nothing. + +#### Token counts on streamed calls + +OpenAI-compatible APIs only report usage on a **stream** when the client asks +for it, and two frameworks don't ask by default — so their streamed model calls +arrive with no token counts, and there is nothing to record: + +```ts +// LlamaIndex +new OpenAI({ model, additionalChatOptions: { stream_options: { include_usage: true } } }); +// Mastra: build the model with usage on, e.g. createOpenAICompatible({ …, includeUsage: true }) +``` + +LangChain and the Vercel AI SDK already request it. + +#### Next.js + +`next build` bundles your server's dependencies by default, and a framework +bundled into the build is a copy `instrument()` cannot reach. Wrap the config +once, and call `instrument()` from Next's startup hook: + +```ts +// next.config.ts +import { withFailproofai } from "@failproofai/sdk/next"; +export default withFailproofai({ /* your config */ }); +``` + +```ts +// instrumentation.ts +export async function register() { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + const failproofai = await import("@failproofai/sdk"); + await failproofai.instrument(); +} +``` + +`withFailproofai` adds LangChain, Mastra, LlamaIndex and the SDK itself to +`serverExternalPackages`, keeping your own list. Without it, `instrument()` +warns once per framework it cannot reach — it never fails silently. If you list +the packages by hand, set `FAILPROOFAI_NEXT_EXTERNALS=1` to silence the warning. +The Vercel AI SDK and every call-site helper work either way. An **Edge** +route gets a no-op build: importing the SDK is safe, and nothing is recorded +there. + +#### Runtimes + +Node ≥ 20.9, Bun and Deno (including `npm:` imports) — every framework, as an ES +module and as CommonJS, is tested on each and must record the same trace Node +does. The SDK runs beside the `failproofaid` daemon, which ships what it writes. + +### 3. `event.*` + +The 15 event methods, for anything the adapters do not cover: + +`toolUse` · `toolResult` · `modelRequest` · `modelResponse` · `agentStart` · +`agentEnd` · `agentPause` · `agentResume` · `hookTriggered` · `hookCompleted` · +`error` · `humanWait` · `humanInput` · `humanPause` · `humanInterrupt` + +```ts +failproofai.event.humanWait({ inputId: "approval-1", prompt: "Ship it?" }); +// …later, from anywhere in the same session: +failproofai.event.humanInput({ inputId: "approval-1", response: "yes" }); +``` + +`sessionId` and `agentId` are optional on every one — omitted, they resolve from +the enclosing scope. Nothing bound and nothing passed is an **error**, never a +silent drop: ingest skips an event with no session and answers `200 OK`, so the +run would simply never appear. + +The paired events (`toolResult`, `agentResume`, `hookCompleted`, `humanInput`) +compute `duration_ms` from the matching start. You cannot pass it yourself — a +reported duration is unfalsifiable. + +Any key you add that the method does not name becomes a custom payload field. +Namespace anything framework-specific `fw_*`; a name that collides with a +declared field is refused rather than silently overwriting a promoted column. + +--- + +## Your own agent — no framework + +For an agent loop you wrote yourself, or a framework without an adapter. There is +nothing to instrument: you emit the events, with the same API the adapters use +underneath — so the trace is the same shape and the same quality. + +You do not need to know anything about how the agent is organised beyond this: +**every hand-built agent already has three places** — whatever its functions are +called — and those three are the whole integration. + +| where | what to add | emits | +|---|---|---| +| where **one run** starts and ends | `failproofai.agent("name", { goal }, async () => …)` | `agent_start` / `agent_end` | +| the **one function that calls the model** | `event.modelRequest` before, `event.modelResponse` after — both halves, even on failure | one pair per model turn | +| the **one function that runs tools** | `failproofai.toolCall(name, { toolCallId, input }, () => run())` | `tool_use` / `tool_result` | + +```ts +// 1. the run +await failproofai.agent("inventory", { goal: question }, async () => { + for (;;) { + const message = await callModel(messages); // 2. + if (!message.tool_calls?.length) return message.content; + for (const call of message.tool_calls) await dispatch(call); // 3. + } +}); + +// 2. the model call — pair on requestId, time it yourself, close it on failure +async function callModel(messages) { + const requestId = randomUUID(); + const started = Date.now(); + failproofai.event.modelRequest({ model: MODEL, requestId, + messages: messages.map((m) => ({ role: m.role, content: m.content })) }); + let reply; + try { + reply = await client.chat.completions.create({ model: MODEL, messages, tools }); + } catch (error) { + failproofai.event.modelResponse({ model: MODEL, requestId, stopReason: "error", + error: String(error), duration_ms: Date.now() - started }); + throw error; + } + const { message, finish_reason } = reply.choices[0]; + failproofai.event.modelResponse({ + model: reply.model, requestId, role: message.role, content: message.content ?? "", + stopReason: finish_reason, duration_ms: Date.now() - started, + inputTokens: reply.usage?.prompt_tokens, outputTokens: reply.usage?.completion_tokens, + fw_tool_calls: (message.tool_calls ?? []).map((c) => + ({ toolCallId: c.id, toolName: c.function.name, input: c.function.arguments })), + }); + return message; +} + +// 3. the tool dispatcher — reuse the model's own tool-call id +async function dispatch(call) { + const input = JSON.parse(call.function.arguments); + return failproofai.toolCall(call.function.name, { toolCallId: call.id, input }, + () => runTool(call.function.name, input)); +} +``` + +Identity is ambient: everything inside `agent()` — the model wrapper, the +dispatcher, any function they call — lands on that run's session without taking an +id. Nothing else in the program changes, including whatever the agent already +writes to its own database. + +- **A service or a worker:** pass your own request or job id as `sessionId` + (`agent("assistant", { sessionId: requestId }, …)`), so a session on the + dashboard and the record in your own logs or database are the same string. +- **Sub-agents:** nest `agent()` calls. The inner one joins the session and takes + the outer as its `parent_id`. +- **Emit the pairs.** A `modelRequest` with no `modelResponse` is a span the + dashboard shows as running forever — hence the `catch` above. + +[`examples/research-agent.ts`](./examples/research-agent.ts) is the complete, +runnable version: a real OpenAI tool loop instrumented exactly like this. The +integration suite runs that file on every CI run, as an ES module and as +CommonJS, against the real `openai` client — the example is proven, not just +documented. + +## Configuration + +```ts +failproofai.configure({ + environment: "production", // or AGENTEYE_ENVIRONMENT; defaults to "dev" + flushInterval: 0.5, // seconds + baseDir: undefined, // the ONLY way to move the spool +}); +``` + +Call it once at startup, before any `event.*` call. Nothing is applied unless +all of it validates, so a rejected call leaves the SDK exactly as it was. + +| variable | effect | +|---|---| +| `AGENTEYE_ENVIRONMENT` | the `environment` label on every event | +| `FAILPROOFAI_HOME` | moves the umbrella; the spool is always `/custom-agents` | +| `FAILPROOFAI_SDK_LOG_LEVEL` | `debug` \| `info` \| `warn` (default) \| `error` \| `silent` | +| `FAILPROOFAI_SDK_STRICT` | `1` makes a swallowed adapter failure throw | +| `FAILPROOFAI_SDK_STRICT_INTEGRATIONS` | `1` makes a framework-version warning throw | + +No environment variable can move the spool off the umbrella. A redirect with no +confirmation and no error means batches land where nothing reads them, and an +unread spool is indistinguishable from an idle one. + +Route the SDK's own log lines into your logger with +`failproofai.setLogger({ debug, info, warn, error })`. + +## Shutdown + +Buffered events are flushed on `process.on("exit")` automatically. + +A process killed by a signal never reaches that — Node's default for `SIGTERM` +is to terminate without running exit handlers — so a containerised agent loses +whatever the last interval had not yet written. **This package will not install +a signal handler for you**: registering one changes the process's behaviour (a +listener suppresses Node's default termination), and a library that silently +stopped Ctrl-C from working would be worse than the lost events. Two lines, at +your own startup: + +```ts +for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]] as const) { + process.once(signal, () => { + failproofai.flushSync(); + process.exit(code); + }); +} +``` + +On the way out, whatever is still open is closed rather than stranded: an +interrupted tool gets its `tool_result` with a `ProcessExit` error, and each open +agent — hand-written or opened by an adapter — gets an `error` and `agent_end` +with `outcome: "failed"`, innermost first. A deploy never leaves a run showing +as running forever. A `flushSync()` while the process carries on closes nothing. + +A short-lived script or a serverless handler should `await failproofai.flush()` +before returning: the interval alone does not guarantee delivery, and a function +that returns right after its last event routinely exits before the next cycle. + +--- + +## Evaluations + +```ts +import { Evaluator, EvalResult, Score } from "@failproofai/sdk/evaluator"; + +export const app = new Evaluator({ name: "my-evals", version: "1" }); + +app.eval("tool_success_rate", { version: "1" }, (session) => { + const results = session.eventsOfType("tool_result"); + const failures = results.filter((event) => event.payload.error != null).length; + return new EvalResult({ + score: new Score(results.length === 0 ? 1 : 1 - failures / results.length), + reasoning: `${failures} of ${results.length} tool calls failed`, + }); +}); +``` + +```bash +FAILPROOFAI_EVALUATOR_URL=https://… \ +FAILPROOFAI_EVALUATOR_TOKEN=… \ +npx failproofai-evaluator ./my-evals.js +``` + +The worker claims assignments, runs each definition's condition, submits a plan, +runs the planned evaluations under a heartbeat, and submits each result. + +**An evaluation must yield.** A synchronous function that never returns blocks +the one thread there is, and no timeout can fire while it does. Write `async` +evaluations, or let the sandbox run them. + +### The sandbox + +Server-authored ("managed") evaluations arrive as source. This package runs them +through a restricted expression language that is **parsed and interpreted** — +never `eval`'d, never handed to `node:vm`. + +That is not belt-and-braces. JavaScript has a reachable path from any value to +arbitrary code: + +```js +(() => {}).constructor("return process")() +x["constructor"]["constructor"]("…")() +``` + +A static allowlist cannot close the second one, because the key is computed at +runtime and no source check can see it. `node:vm` does not close it either — a +vm context has its own `Function`. Interpreting removes the question: every +property read goes through one function that checks the actual key at the moment +of the read, and the interpreter never constructs a function. + +A `worker_threads` sandbox sits around that with V8 heap limits, a wall-clock +`terminate()`, a bounded result and a cap on concurrent sandboxes. That is the +RESOURCE bound — it stops a permitted expression from eating the worker even +though every operation in it is individually legal. If the sandbox cannot be +established, managed source is **refused**, never run unbounded. + +--- + +## What it will not do to your process + +* **It will not block your agent loop.** Events go into an in-memory queue; a + timer writes them. The timer is `unref`'d, so importing this package never + stops a script exiting. +* **It will not grow without bound.** The queue is capped by count *and* by + measured bytes. Past either, the oldest events are discarded and a warning + says so — a telemetry outage must not become an OOM kill. +* **It will not take the process down.** One unencodable event is dropped + alone, not the batch around it. A throwing getter, a circular reference, a + `BigInt`, a lone surrogate: each is handled rather than propagated. +* **It will not leave a half-written batch.** Content is `fsync`ed before an + atomic rename, the directory is `fsync`ed after, and a failed write cleans up + its temporary file. +* **It will not leave transcripts world-readable.** Batches are `0600` inside a + `0700` directory. They carry goals, prompts, tool arguments and tool output. +* **It will not ship credentials.** API keys, tokens, JWTs, bearer headers and + secret-shaped assignments are redacted before the bytes reach disk. The daemon + redacts again before upload. + +## Zero dependencies, on purpose + +This package installs into other people's agent processes. Every dependency it +declared would be a version constraint they inherit, in the process whose +reliability it exists to improve. It imports nothing outside Node's standard +library; the framework packages are peer dependencies, all optional, imported +dynamically and only when you ask for them. + +A test fails if a runtime dependency is ever added, and the build refuses to +produce a tarball that declares one. + +## License + +MIT. See [LICENSE](./LICENSE). diff --git a/sdk/typescript/eslint.config.mjs b/sdk/typescript/eslint.config.mjs new file mode 100644 index 000000000..067031e82 --- /dev/null +++ b/sdk/typescript/eslint.config.mjs @@ -0,0 +1,59 @@ +import js from "@eslint/js"; +import tseslint from "typescript-eslint"; + +export default tseslint.config( + // examples/ imports packages this project does not install (openai); each is + // typechecked as a real consumer by its integration fixture (vanilla.test.ts). + { ignores: ["dist/**", "node_modules/**", "eslint.config.mjs", "integration/fixtures/**", "examples/**"] }, + js.configs.recommended, + ...tseslint.configs.recommendedTypeChecked, + { + languageOptions: { + parserOptions: { + projectService: { allowDefaultProject: ["scripts/*.mjs"] }, + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + // This package talks to four frameworks and one wire protocol, all of + // which hand it `unknown`. Narrowing every one of those at the boundary + // is exactly what the adapters and the `fromWire` readers already do by + // hand; the rules below would flag that work rather than the mistakes. + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "@typescript-eslint/no-unsafe-call": "off", + "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/no-unsafe-return": "off", + "@typescript-eslint/restrict-template-expressions": [ + "error", + { allowNumber: true, allowBoolean: true }, + ], + "@typescript-eslint/no-explicit-any": "error", + "@typescript-eslint/no-non-null-assertion": "off", + }, + }, + { + files: ["test/**/*.ts", "integration/*.ts", "scripts/**/*.mjs"], + rules: { + // A test's whole job is to feed the wrong shape in and watch what + // happens, so the casts it needs are the point rather than a smell. + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "@typescript-eslint/no-confusing-void-expression": "off", + "@typescript-eslint/require-await": "off", + // `await agent("x", () => 1)` is exactly what a caller writes, and the + // scopes deliberately stay synchronous for a synchronous body — so the + // value awaited here is sometimes a promise and sometimes not, which is + // the behaviour under test rather than a mistake. + "@typescript-eslint/await-thenable": "off", + }, + }, + { + files: ["scripts/**/*.mjs"], + ...tseslint.configs.disableTypeChecked, + languageOptions: { + // Build scripts run under plain Node, outside the typed project. + globals: { process: "readonly", console: "readonly", URL: "readonly" }, + }, + }, +); diff --git a/sdk/typescript/examples/research-agent.ts b/sdk/typescript/examples/research-agent.ts new file mode 100644 index 000000000..6b01130e1 --- /dev/null +++ b/sdk/typescript/examples/research-agent.ts @@ -0,0 +1,197 @@ +/** + * No framework — a real agent loop, hand-instrumented. + * + * npm install @failproofai/sdk openai + * OPENAI_API_KEY=… npx tsx examples/research-agent.ts + * + * A working tool-calling loop against the OpenAI API with no agent framework at + * all, instrumented by hand. The reference for "my agent is bespoke" — and the + * TypeScript twin of the Python SDK's `docs/manual/examples/research_agent.py`. + * + * Every hand-built agent already has three places this touches, whatever its + * functions are called: + * + * 1. where ONE RUN starts and ends → failproofai.agent() agent_start / agent_end + * 2. the ONE FUNCTION that calls a model → event.modelRequest/Response one pair per model turn + * 3. the ONE FUNCTION that runs a tool → failproofai.toolCall() tool_use / tool_result + * + * Identity is ambient: everything inside `agent()` lands on its session, so + * nothing else in the program changes. The one rule: emit the pairs — a + * `model_request` with no `model_response` is a span the dashboard shows as + * running forever. + * + * Environment: OPENAI_API_KEY; optionally OPENAI_BASE_URL (any OpenAI-compatible + * endpoint) and MODEL (default gpt-4o-mini). + */ +import { randomUUID } from "node:crypto"; + +import * as failproofai from "@failproofai/sdk"; +import OpenAI from "openai"; +import type { ChatCompletionMessageParam, ChatCompletionTool } from "openai/resources/chat/completions"; + +failproofai.configure({ environment: "examples" }); + +const client = new OpenAI(); // reads OPENAI_API_KEY / OPENAI_BASE_URL +const MODEL = process.env.MODEL ?? "gpt-4o-mini"; + +// ---------------------------------------------------------------- the tools + +const PRICE: Record = { widget: 42.0, gadget: 17.5 }; +const STOCK: Record = { widget: 120, gadget: 0 }; + +const TOOLS: ChatCompletionTool[] = [ + { + type: "function", + function: { + name: "price_of", + description: "Unit price of an item. Valid: widget, gadget.", + parameters: { type: "object", properties: { item: { type: "string" } }, required: ["item"] }, + }, + }, + { + type: "function", + function: { + name: "stock_of", + description: "Units in stock. Valid: widget, gadget.", + parameters: { type: "object", properties: { item: { type: "string" } }, required: ["item"] }, + }, + }, +]; + +function runTool(name: string, args: { item?: string }): string { + const item = String(args.item ?? "").toLowerCase().trim(); + const table = name === "price_of" ? PRICE : name === "stock_of" ? STOCK : null; + if (table === null) throw new Error(`unknown tool ${name}`); + if (!(item in table)) throw new Error(`unknown item ${JSON.stringify(item)}`); + return String(table[item]); +} + +// ---------------------------------------------------------- edit site 2 of 3 +// The one function that calls the model. `requestId` pairs the two halves even +// when calls overlap; `duration_ms` is yours to set — model events are not +// timed for you. A failed call still closes its pair, with the error, before +// rethrowing: the enclosing agent() then ends "failed". Only the provider call +// sits in the `try`, so nothing but a failed call can reach the error path. + +async function callModel(messages: ChatCompletionMessageParam[]) { + const requestId = randomUUID(); + const started = Date.now(); + failproofai.event.modelRequest({ + model: MODEL, + requestId, + // Role and content, plus the ids that link a tool result to the call that + // asked for it: what a reader of the trace needs, not the provider's full + // message objects. + messages: messages.map((m) => ({ + role: m.role, + content: typeof m.content === "string" ? m.content : m.content == null ? "" : JSON.stringify(m.content), + ...(m.role === "tool" ? { tool_call_id: m.tool_call_id } : {}), + ...(m.role === "assistant" && m.tool_calls + ? { tool_calls: m.tool_calls.map((c) => ({ id: c.id, name: c.type === "function" ? c.function.name : c.type })) } + : {}), + })), + tools: TOOLS.flatMap((t) => (t.type === "function" ? [{ name: t.function.name, description: t.function.description ?? "" }] : [])), + }); + let reply: OpenAI.Chat.Completions.ChatCompletion; + try { + reply = await client.chat.completions.create({ model: MODEL, messages, tools: TOOLS }); + } catch (error) { + failproofai.event.modelResponse({ + model: MODEL, + requestId, + stopReason: "error", + error: error instanceof Error ? `${error.constructor.name}: ${error.message}` : String(error), + duration_ms: Date.now() - started, + }); + throw error; + } + const choice = reply.choices[0]!; + const calls = (choice.message.tool_calls ?? []).filter((c) => c.type === "function"); + failproofai.event.modelResponse({ + model: reply.model, + requestId, + role: choice.message.role, + content: choice.message.content ?? "", + stopReason: choice.finish_reason, + inputTokens: reply.usage?.prompt_tokens ?? null, + outputTokens: reply.usage?.completion_tokens ?? null, + duration_ms: Date.now() - started, + // What the model asked for, so a turn that is only tool calls is not blank — + // the field and shape the framework adapters write. + fw_tool_calls: calls.map((c) => ({ toolCallId: c.id, toolName: c.function.name, input: c.function.arguments })), + }); + return choice.message; +} + +// ---------------------------------------------------------- edit site 3 of 3 +// The one function that runs tools. Reuse the model's own tool-call id, so a +// tool_use lines up with the tool_calls[] entry that asked for it. toolCall() +// times it and records a throw as tool_result.error — then rethrows, and here +// the loop turns that into a tool message so the model can recover. + +async function dispatch(call: { id: string; function: { name: string; arguments: string } }): Promise { + // Malformed arguments from the model are still a tool call: recorded with the + // raw text as input and failed inside toolCall(), so the trace shows it and + // the model gets an error it can recover from — not a crashed run, and not a + // call that silently never appears. + let args: { item?: string } = {}; + let malformed: unknown; + try { + const parsed: unknown = JSON.parse(call.function.arguments || "{}"); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new TypeError("tool arguments must be a JSON object"); + } + args = parsed as { item?: string }; + } catch (error) { + malformed = error; + } + try { + const input = malformed === undefined ? args : { arguments: call.function.arguments }; + return await failproofai.toolCall(call.function.name, { toolCallId: call.id, input }, async () => { + if (malformed !== undefined) throw malformed; + return runTool(call.function.name, args); + }); + } catch (error) { + return `error: ${error instanceof Error ? error.message : String(error)}`; + } +} + +// ---------------------------------------------------------- edit site 1 of 3 +// Where one run starts and ends. In a service, pass your own request or job id +// as `sessionId`, so a session on the dashboard and a record in your own +// database are the same string. + +async function main(): Promise { + const question = process.argv[2] || "Price and stock for widget and gadget?"; + const messages: ChatCompletionMessageParam[] = [ + { role: "system", content: "Use the tools for every number. Be terse." }, + { role: "user", content: question }, + ]; + + let sessionId = ""; + const answer = await failproofai.agent("inventory", { goal: question }, async (identity) => { + sessionId = identity.sessionId ?? ""; + for (let turn = 0; turn < 6; turn++) { + // bounded: an unbounded agent loop is its own bug + const message = await callModel(messages); + const calls = (message.tool_calls ?? []).filter((c) => c.type === "function"); + if (calls.length === 0) return message.content ?? ""; + messages.push(message); + for (const call of calls) { + messages.push({ role: "tool", tool_call_id: call.id, content: await dispatch(call) }); + } + } + return "(gave up after 6 turns)"; + }); + + console.log(answer); + console.log(`session ${sessionId}`); +} + +main() + .catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }) + // A short script must flush before it exits; a server flushes on its own. + .finally(() => failproofai.flush()); diff --git a/sdk/typescript/integration/ai.test.ts b/sdk/typescript/integration/ai.test.ts new file mode 100644 index 000000000..3ed9106a7 --- /dev/null +++ b/sdk/typescript/integration/ai.test.ts @@ -0,0 +1,920 @@ +import { describe, expect, it } from "vitest"; + +import { + FORMATS, + count, + describeTrace, + ofType, + runAgent, + traceViolations, + typecheck, + type Event, +} from "./harness.js"; + +/** + * The Vercel AI SDK (`ai`), against real releases of every supported major. + * + * The mapping (the adapter's module comment has the reasoning): + * + * * one `generateText` / `streamText` / `generateObject` / `streamObject` + * call is ONE agent, named by its `functionId` — never a span or call id; + * * each model step is a `model_request` / `model_response` pair on + * `request_id`, with integer token counts and a STRING stop reason; + * * each tool is a `tool_use` / `tool_result` pair carrying the MODEL's own + * tool call id; + * * a bare model call through `wrapModel` is its own run, named after the + * model — the LangChain precedent for a bare chat-model call — unless an + * enclosing `failproofai.agent()` already owns it; + * * a failure is recorded once, where it happened — no stack of `error` + * events. + * + * Four fixtures because the AI SDK changed its extension points at every + * major: v4 speaks LanguageModelV1 (`promptTokens`), v5 V2, v6 V3 (usage and + * finish reason became objects), and v7 dropped the OpenTelemetry `tracer` + * option for its own `Telemetry` integration interface. + */ + +const ALL = ["ai-4", "ai-5", "ai-6", "ai-7"] as const; +const only = process.env.FAILPROOFAI_IT_FIXTURES?.split(",").filter(Boolean); +const FIXTURES = ALL.filter((name) => !only || only.includes(name)); + +/** One line per event: `agent_id type [tool]`. */ +const shape = (events: Event[]): string[] => + events.map((e) => [e.agent_id, e.type, (e.tool_name ?? "") as string].join(" ").trim()); + +const LOOP = (agent: string): string[] => [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} tool_use weather`, + `${agent} tool_result weather`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, +]; +const GEN = LOOP("weather-agent"); + +/** + * A stream on v4–v6. The SDK runs a tool the moment its `tool-call` part + * arrives and holds the stream's `finish` part until the tool returns, so the + * model step's span — the only place its usage and finish reason exist — ends + * AFTER the tool. Events are stamped when they happen, never backdated, so + * that is the order they land in; `request_id` still pairs them. v7 closes + * the model call before it executes tools, so there a stream reads like GEN. + */ +const LEGACY_STREAM = [ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent tool_use weather", + "weather-agent tool_result weather", + "weather-agent model_response", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", +]; + +const tokens = (events: Event[]) => ofType(events, "model_response").map((e) => [e.input_tokens, e.output_tokens]); + +describe.each(FIXTURES)("%s", (fixture) => { + const major = Number(fixture.split("-")[1]); + const STREAM = major >= 7 ? GEN : LEGACY_STREAM; + + it("typechecks the README call sites as a customer's nodenext project", () => { + expect(typecheck(fixture)).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string, { expectNote = false } = {}) => { + const result = runAgent(fixture, format, scenario); + expect(result.status, describeTrace(result)).toBe(0); + // A warning from the SDK is how a silently-inert adapter shows itself + // ("could not resolve a session", "outside the supported range"). + const lines = result.stderr.split("\n").filter((line) => line.includes("[failproofai-sdk]")); + if (expectNote) { + // The one deliberate exception: instrument("ai") on ai 4–6 says, once, + // that by itself it records nothing there — and nothing else is said. + expect(lines, describeTrace(result)).toHaveLength(1); + expect(lines[0], describeTrace(result)).toContain("registerGlobalTracer"); + } else { + expect(lines, describeTrace(result)).toEqual([]); + } + return result; + }; + /** instrument("ai") without the opt-in: inert, and noted, on ai 4–6. */ + const noted = major < 7; + + it("records generateText with a tool loop as one agent", () => { + const result = run("generate"); + const { events } = result; + expect(shape(events), describeTrace(result)).toEqual(GEN); + expect(traceViolations(events), describeTrace(result)).toEqual([]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + expect(tokens(events)).toEqual([ + [11, 7], + [23, 9], + ]); + const responses = ofType(events, "model_response"); + expect(responses.map((e) => e.stop_reason)).toEqual(["tool-calls", "stop"]); + for (const e of [...responses, ...ofType(events, "model_request")]) expect(e.model).toBe("mock-model"); + for (const e of responses) expect(Number.isInteger(e.duration_ms)).toBe(true); + // The first step answered with a tool call and no text: its content is + // the call, not an empty string. + expect(responses[0]!.content).not.toBe(""); + expect(JSON.stringify(responses[0]!.content)).toContain("weather"); + expect(responses[1]!.content).toBe("It is 20C in Paris."); + const [use] = ofType(events, "tool_use"); + expect(use!.tool_call_id).toBe("call-1"); + expect(use!.input).toEqual({ city: "Paris" }); + expect(ofType(events, "tool_result")[0]!.output).toEqual({ city: "Paris", celsius: 20 }); + expect(ofType(events, "agent_end")[0]!.outcome).toBe("success"); + expect(count(events, "error")).toBe(0); + for (const event of events) { + expect(event.framework).toBe("ai"); + expect(String(event.framework_version)).toMatch(new RegExp(`^${major}\\.`)); + } + expect(result.stdout).toContain("It is 20C in Paris."); + }); + + it("records streamText like generateText, and stamps the request when it is made", () => { + const result = run("stream"); + const { events } = result; + expect(shape(events), describeTrace(result)).toEqual(STREAM); + expect(traceViolations(events), describeTrace(result)).toEqual([]); + expect(tokens(events)).toEqual([ + [13, 4], + [30, 6], + ]); + const responses = ofType(events, "model_response"); + expect(responses.map((e) => e.stop_reason)).toEqual(["tool-calls", "stop"]); + expect(responses[0]!.content).not.toBe(""); + expect(responses[1]!.content).toBe("Rome is 25C."); + expect(ofType(events, "tool_use")[0]!.tool_call_id).toBe("call-s1"); + const request = ofType(events, "model_request")[0]!; + const use = ofType(events, "tool_use")[0]!; + expect(String(request.timestamp) <= String(use.timestamp)).toBe(true); + expect(result.stdout).toContain("Rome is 25C."); + }); + + it.each(["object", "stream-object"])("records %s as one agent with one model call", (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual([ + "extractor agent_start", + "extractor model_request", + "extractor model_response", + "extractor agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(tokens(result.events)).toEqual([[5, 3]]); + expect(ofType(result.events, "model_response")[0]!.stop_reason).toBe("stop"); + expect(result.stdout).toContain('"city":"Paris"'); + }); + + it("records a bare wrapModel call as its own run, named after the model", () => { + const result = run("wrap"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "mock-model agent_start", + "mock-model model_request", + "mock-model model_response", + "mock-model agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([23, 9]); + expect(response.stop_reason).toBe("stop"); + expect(response.content).toBe("It is 20C in Paris."); + }); + + it("records a streamed wrapModel call with its usage", () => { + const result = run("wrap-stream"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "mock-model agent_start", + "mock-model model_request", + "mock-model model_response", + "mock-model agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([30, 6]); + expect(response.stop_reason).toBe("stop"); + expect(response.content).toBe("Rome is 25C."); + }); + + it("records wrapModel calls under an enclosing agent() rather than as runs of their own", () => { + const result = run("wrap-in-agent"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + "planner model_request", + "planner model_response", + "planner model_request", + "planner model_response", + "planner agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + expect(tokens(result.events)).toEqual([ + [11, 7], + [23, 9], + ]); + }); + + it("records each call once when telemetry() and wrapModel are both in use", () => { + const result = run("wrap-and-telemetry"); + expect(shape(result.events), describeTrace(result)).toEqual(GEN); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it.each(["instrument", "instrument-stream"])( + "%s: instrument('ai') records the same trace as telemetry() on ai 7, and on ai 4–6 records nothing and says so", + (scenario) => { + const result = run(scenario, { expectNote: noted }); + expect(result.stdout).toContain('"instrumented":["ai"]'); + if (major >= 7) { + expect(shape(result.events), describeTrace(result)).toEqual(scenario === "instrument" ? GEN : STREAM); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + } else { + // v4–v6's only process-wide hook is the global OpenTelemetry slot, + // which instrument("ai") no longer takes by default. + expect(result.events, describeTrace(result)).toEqual([]); + expect(result.stdout).toContain(scenario === "instrument" ? "It is 20C in Paris." : "Rome is 25C."); + } + }, + ); + + it.each(["instrument-global", "instrument-global-stream"])( + "%s: instrument('ai', { registerGlobalTracer: true }) records the same trace as telemetry()", + (scenario) => { + const result = run(scenario); + expect(result.stdout).toContain('"instrumented":["ai"]'); + expect(shape(result.events), describeTrace(result)).toEqual(scenario === "instrument-global" ? GEN : STREAM); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }, + ); + + it("leaves the global OpenTelemetry slot to a customer provider registered after instrument('ai')", () => { + const result = run("instrument-then-otel", { expectNote: noted }); + const report = JSON.parse(result.stdout.trim().split("\n").pop()!) as { + text: string; + customer: "absent" | { registered: boolean; ended: string[] }; + }; + expect(report.text).toBe("It is 20C in Paris."); + if (major >= 7) { + // v7 has no OpenTelemetry dependency; the integration records the call. + expect(report.customer).toBe("absent"); + expect(shape(result.events), describeTrace(result)).toEqual(GEN); + return; + } + // Their registration is accepted, and their tracer gets every span — + // the AI SDK's own and their service's. + expect(report.customer, describeTrace(result)).toEqual({ + registered: true, + ended: expect.arrayContaining(["ai.generateText", "ai.generateText.doGenerate", "ai.toolCall", "http.request"]), + }); + expect(result.events, describeTrace(result)).toEqual([]); + }); + + it("closes a streamed wrapModel call the reader cancels, as cancelled", () => { + const result = run("wrap-stream-cancel"); + expect(result.stdout).toContain('"cancelled":true'); + expect(shape(result.events), describeTrace(result)).toEqual([ + "mock-model agent_start", + "mock-model model_request", + "mock-model model_response", + "mock-model agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect(response.stop_reason).toBe("cancelled"); + expect(response.error).toBeUndefined(); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(count(result.events, "error")).toBe(0); + }); + + it("closes a streamed wrapModel call whose stream errors, with the error", () => { + const result = run("wrap-stream-error"); + expect(result.stdout).toContain('"threw":"connection reset"'); + expect(shape(result.events), describeTrace(result)).toEqual([ + "mock-model agent_start", + "mock-model model_request", + "mock-model model_response", + "mock-model agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect(response.stop_reason).toBe("error"); + expect(response.error).toMatch(/connection reset/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(count(result.events, "error")).toBe(0); + }); + + it("records nothing after uninstrument()", () => { + const result = run("uninstrument", { expectNote: noted }); + expect(result.stdout).toContain('"removed":["ai"]'); + expect(result.events, describeTrace(result)).toEqual([]); + }); + + it("records a failing tool once, on its tool_result", () => { + const result = run("tool-error"); + const { events } = result; + const toolResult = ofType(events, "tool_result")[0]!; + expect(toolResult.error, describeTrace(result)).toMatch(/weather service down/); + expect(count(events, "error"), describeTrace(result)).toBe(0); + expect(traceViolations(events), describeTrace(result)).toEqual([]); + if (major >= 5) { + // v5+ hands the failure back to the model as a tool-error result and + // the loop carries on — the agent recovered, so the run succeeded. + expect(shape(events), describeTrace(result)).toEqual(GEN); + expect(ofType(events, "agent_end")[0]!.outcome).toBe("success"); + } else { + // v4 throws a ToolExecutionError out of generateText: the run failed. + expect(shape(events), describeTrace(result)).toEqual([...GEN.slice(0, 5), "weather-agent agent_end"]); + expect(ofType(events, "agent_end")[0]!.outcome).toBe("failed"); + expect(result.stdout).toContain("threw"); + } + }); + + it("records a failing model once, on its model_response", () => { + const result = run("model-error"); + const { events } = result; + expect(result.stdout).toContain("model exploded"); + expect(shape(events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + expect(traceViolations(events), describeTrace(result)).toEqual([]); + const response = ofType(events, "model_response")[0]!; + expect(response.error).toMatch(/model exploded/); + expect(response.stop_reason).toBe("error"); + expect(ofType(events, "agent_end")[0]!.outcome).toBe("failed"); + expect(count(events, "error")).toBe(0); + }); + + it("nests the call under an enclosing agent() scope", () => { + const result = run("scope"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + ...GEN, + "planner agent_end", + ]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("planner"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("runs the README call sites cleanly", () => { + const result = run("readme"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const agents = ofType(result.events, "agent_start").map((e) => e.agent_id); + expect(agents).toEqual(expect.arrayContaining(["answer-question", "tagged", "mock-model"])); + }); + }); +}); + +/** + * Every commonly used surface of the SDK, on every major where it exists — + * `surfaces.ts` in each fixture. The cases above pin the README call sites + * and the adapter's core paths; these walk the rest of what customers call: + * the agent classes, embeddings, structured output, tool features, every way + * of consuming a stream, reasoning models and concurrency. + * + * Where the SDK itself makes a surface unrecordable, the case pins what IS + * recorded, with the reason beside it, so a change in either direction is + * seen. + */ +describe.each(FIXTURES)("%s: every surface", (fixture) => { + const major = Number(fixture.split("-")[1]); + /** The tool loop the scripted model runs: one tool call, then the answer. */ + const loop = (agent: string, tool = "weather"): string[] => [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} tool_use ${tool}`, + `${agent} tool_result ${tool}`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, + ]; + /** The same loop streamed: on v4–v6 the step's span ends after the tool it caused (see LEGACY_STREAM). */ + const streamed = (agent: string): string[] => + major >= 7 + ? loop(agent) + : [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} tool_use weather`, + `${agent} tool_result weather`, + `${agent} model_response`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, + ]; + const single = (agent: string): string[] => [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, + ]; + const hasAgentClass = major >= 5; + const hasApproval = major >= 6; + + it("typechecks every surface program as a customer's nodenext project", () => { + expect(typecheck(fixture, "tsconfig.surfaces.json")).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string, { expectNote = false } = {}) => { + const result = runAgent(fixture, format, scenario, {}, "surfaces"); + expect(result.status, describeTrace(result)).toBe(0); + const lines = result.stderr.split("\n").filter((line) => line.includes("[failproofai-sdk]")); + if (expectNote) { + expect(lines, describeTrace(result)).toHaveLength(1); + expect(lines[0], describeTrace(result)).toContain("registerGlobalTracer"); + } else { + expect(lines, describeTrace(result)).toEqual([]); + } + return result; + }; + /** Every JSON line the program reported, merged. */ + const reported = (result: { stdout: string }): Record => + Object.assign( + {}, + ...result.stdout + .trim() + .split("\n") + .filter((line) => line.startsWith("{")) + .map((line) => JSON.parse(line) as Record), + ) as Record; + const clean = (result: ReturnType) => + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + + // ---- 1. the agent classes (ai 5 Experimental_Agent, ai 6/7 ToolLoopAgent) + + it.each(["agent-generate", "agent-stream"])("%s: one agent per call, named by the agent's functionId", (scenario) => { + const result = run(scenario); + if (!hasAgentClass) { + expect(reported(result).skipped).toBe("no agent class"); + expect(result.events).toEqual([]); + return; + } + expect(shape(result.events), describeTrace(result)).toEqual( + scenario === "agent-generate" ? loop("support-agent") : streamed("support-agent"), + ); + clean(result); + expect(tokens(result.events)).toEqual([ + [11, 7], + [23, 9], + ]); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call-1"); + expect(reported(result).text).toBe("It is 20C in Paris."); + }); + + it("agent-id: an agent's own `id` never reaches telemetry, so without a functionId it is named after the operation", () => { + // ToolLoopAgent spreads its settings into generateText, which drops `id`; + // no span attribute (v4–v6) and no v7 event carries it. `functionId` in + // the agent's telemetry settings is what names it. + const result = run("agent-id"); + if (!hasAgentClass) return; + expect(shape(result.events), describeTrace(result)).toEqual(loop("ai.generateText")); + clean(result); + }); + + it.each(["agent-instrument", "agent-instrument-stream"])( + "%s: instrument('ai') records an agent on ai 7, and on ai 5–6 records nothing and says so", + (scenario) => { + const result = run(scenario, { expectNote: hasAgentClass && major < 7 }); + if (!hasAgentClass) return; + if (major >= 7) { + expect(shape(result.events), describeTrace(result)).toEqual(loop("support-agent")); + clean(result); + } else { + expect(result.events, describeTrace(result)).toEqual([]); + } + expect(reported(result).text).toBe("It is 20C in Paris."); + }, + ); + + it("agent-instrument-bare: with no telemetry setting at all, ai 7 still records the agent", () => { + const result = run("agent-instrument-bare"); + if (!hasAgentClass) return; + if (major >= 7) { + expect(shape(result.events), describeTrace(result)).toEqual(loop("ai.generateText")); + clean(result); + } else { + // v5–v6 consult no tracer unless the call sets isEnabled. + expect(result.events, describeTrace(result)).toEqual([]); + } + }); + + it("agent-wrap: a wrapped model under an agent with no enclosing scope records each step as its own run", () => { + const result = run("agent-wrap"); + if (!hasAgentClass) return; + // wrapModel sees model calls only: the tool loop above it is invisible. + expect(shape(result.events), describeTrace(result)).toEqual([...single("mock-model"), ...single("mock-model")]); + clean(result); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(2); + expect(tokens(result.events)).toEqual([ + [11, 7], + [23, 9], + ]); + }); + + it("agent-wrap-in-scope: inside agent(), the agent's wrapped model calls are steps of that agent", () => { + const result = run("agent-wrap-in-scope"); + if (!hasAgentClass) return; + expect(shape(result.events), describeTrace(result)).toEqual([ + "support-agent agent_start", + "support-agent model_request", + "support-agent model_response", + "support-agent model_request", + "support-agent model_response", + "support-agent agent_end", + ]); + clean(result); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + }); + + // ---- 2. embeddings: an agent only when nothing encloses them + + it("embed: a bare embed() is its own run, named by functionId, with its token count", () => { + const result = run("embed"); + expect(shape(result.events), describeTrace(result)).toEqual(single("indexer")); + clean(result); + const response = ofType(result.events, "model_response")[0]!; + expect(response.input_tokens).toBe(3); + expect(response.model).toBe("mock-embedder"); + }); + + it("embed-many: a bare embedMany() is one run with a model pair per provider call", () => { + const result = run("embed-many"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "ai.embedMany agent_start", + "ai.embedMany model_request", + "ai.embedMany model_response", + "ai.embedMany model_request", + "ai.embedMany model_response", + "ai.embedMany agent_end", + ]); + clean(result); + // maxEmbeddingsPerCall: 2 → three values are two provider calls. + expect(ofType(result.events, "model_response").map((e) => e.input_tokens)).toEqual([6, 3]); + }); + + it("embed-in-agent: inside agent(), embeddings are model calls of that agent — no nested agents", () => { + const result = run("embed-in-agent"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "rag agent_start", + "rag model_request", + "rag model_response", + "rag model_request", + "rag model_response", + "rag model_request", + "rag model_response", + "rag agent_end", + ]); + clean(result); + }); + + it("embed-in-tool: an embedding inside a tool is a model call of the agent that ran the tool", () => { + const result = run("embed-in-tool"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_use weather", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_result weather", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + clean(result); + expect(ofType(result.events, "model_request").map((e) => e.model)).toEqual(["mock-model", "mock-embedder", "mock-model"]); + }); + + // ---- 3. structured output + + it("object-partial: streamObject's partialObjectStream, consumed to the end, is one agent", () => { + const result = run("object-partial"); + expect(shape(result.events), describeTrace(result)).toEqual(single("extractor")); + clean(result); + expect(reported(result).object).toEqual({ city: "Paris" }); + expect(reported(result).partials).toContainEqual({ city: "Paris" }); + expect(tokens(result.events)).toEqual([[5, 3]]); + }); + + it("object-invalid: a schema failure is one error on the failed agent, after the model's answer", () => { + const result = run("object-invalid"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "extractor agent_start", + "extractor model_request", + "extractor model_response", + "extractor error", + "extractor agent_end", + ]); + clean(result); + expect(ofType(result.events, "error")[0]!.message).toMatch(/did not match schema/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(reported(result).threw).toBe("AI_NoObjectGeneratedError"); + }); + + it.each(["text-output", "stream-output"])("%s: generateText / streamText with Output.object is one agent", (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual(single("extractor")); + clean(result); + expect(tokens(result.events)).toEqual([[5, 3]]); + if (scenario === "text-output") expect(reported(result).output).toEqual({ city: "Paris" }); + else expect(reported(result).last).toEqual({ city: "Paris" }); + }); + + // ---- 4. tool features + + it("parallel-tools: two calls in one step pair on the model's own ids, whatever order they finish in", () => { + const result = run("parallel-tools"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_use weather", + "weather-agent tool_use weather", + "weather-agent tool_result weather", + "weather-agent tool_result weather", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + clean(result); + const uses = ofType(result.events, "tool_use"); + expect(uses.map((e) => [e.tool_call_id, e.input])).toEqual([ + ["call-p1", { city: "Paris" }], + ["call-p2", { city: "Rome" }], + ]); + // Rome returned first. + expect(ofType(result.events, "tool_result").map((e) => [e.tool_call_id, e.output])).toEqual([ + ["call-p2", { city: "Rome", celsius: 16 }], + ["call-p1", { city: "Paris", celsius: 20 }], + ]); + // The step's content lists both calls, inputs parsed, on every major. + expect(ofType(result.events, "model_response")[0]!.content).toEqual([ + { toolCallId: "call-p1", toolName: "weather", input: { city: "Paris" } }, + { toolCallId: "call-p2", toolName: "weather", input: { city: "Rome" } }, + ]); + }); + + it("client-tool: a tool with no execute is a call in the model's answer, never a tool_use left open", () => { + const result = run("client-tool"); + expect(shape(result.events), describeTrace(result)).toEqual(single("weather-agent")); + clean(result); + const response = ofType(result.events, "model_response")[0]!; + expect(response.stop_reason).toBe("tool-calls"); + expect(response.content).toEqual([{ toolCallId: "call-c1", toolName: "ask", input: { city: "Paris" } }]); + expect(reported(result).toolCalls).toBe(1); + }); + + it("tool-choice-required: a forced tool call is recorded like any other", () => { + const result = run("tool-choice-required"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_use weather", + "weather-agent tool_result weather", + "weather-agent agent_end", + ]); + clean(result); + }); + + it("repair: the tool_use carries the REPAIRED input", () => { + const result = run("repair"); + expect(shape(result.events), describeTrace(result)).toEqual(loop("weather-agent")); + clean(result); + // v4–v6 record the step as the model produced it; v7's + // onLanguageModelCallEnd already carries the repaired call. + expect(ofType(result.events, "model_response")[0]!.content).toEqual([ + { toolCallId: "call-r1", toolName: "weather", input: major >= 7 ? { city: "Paris" } : { town: "Paris" } }, + ]); + const use = ofType(result.events, "tool_use")[0]!; + expect([use.tool_call_id, use.input]).toEqual(["call-r1", { city: "Paris" }]); + }); + + it("prepare-step: a model switched by prepareStep is named on its own step", () => { + const result = run("prepare-step"); + expect(shape(result.events), describeTrace(result)).toEqual(loop("weather-agent")); + clean(result); + expect(ofType(result.events, "model_response").map((e) => e.model)).toEqual(["mock-model", "mock-model-large"]); + }); + + it("unknown-tool: a call to a tool that does not exist", () => { + const result = run("unknown-tool"); + clean(result); + if (major >= 5) { + // v5+ hands the model a tool error and the loop carries on; the SDK + // opens no tool span / execution for a tool it does not have. + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + } else { + // v4 throws NoSuchToolError out of generateText. + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent error", + "weather-agent agent_end", + ]); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(reported(result).threw).toBe("AI_NoSuchToolError"); + } + }); + + it("approval: a tool that needs approval runs in the call that carries the approval", () => { + const result = run("approval"); + if (!hasApproval) { + expect(reported(result).skipped).toBe("no tool approval"); + return; + } + // Two calls, two runs: the first ends with the model's request for + // approval (a tool call in its content, nothing executed); the second + // executes the approved tool, then answers. Put both in one + // failproofai.session() to see them in one session. + expect(shape(result.events), describeTrace(result)).toEqual([ + ...single("travel-agent"), + "travel-agent agent_start", + "travel-agent tool_use book", + "travel-agent tool_result book", + "travel-agent model_request", + "travel-agent model_response", + "travel-agent agent_end", + ]); + clean(result); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call-a1"); + expect(reported(result)).toMatchObject({ pending: 1, text: "Booked Paris." }); + }); + + // ---- 5. consuming a stream + + it.each(["stream-full", "stream-response", "stream-on-finish"])("%s: records the whole loop", (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual(streamed("weather-agent")); + clean(result); + expect(tokens(result.events)).toEqual([ + [11, 7], + [23, 9], + ]); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + const report = reported(result); + if (scenario === "stream-response") expect(report).toMatchObject({ status: 200, containsAnswer: true }); + if (scenario === "stream-on-finish") expect(report.finished).toBe("It is 20C in Paris."); + }); + + it("stream-abort: an AbortSignal mid-stream closes the model call and the agent as cancelled", () => { + const result = run("stream-abort"); + expect(shape(result.events), describeTrace(result)).toEqual(single("counter")); + clean(result); + expect(ofType(result.events, "model_response")[0]!.stop_reason).toBe("cancelled"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(count(result.events, "error")).toBe(0); + expect(reported(result).after).toEqual({ openCalls: 0, stats: { runs: 0, links: 0 } }); + }); + + it("stream-response-cancel-signal: a client that disconnects from a route passing abortSignal ends cancelled", () => { + const result = run("stream-response-cancel-signal"); + expect(shape(result.events), describeTrace(result)).toEqual(single("counter")); + clean(result); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(reported(result).after).toEqual({ openCalls: 0, stats: { runs: 0, links: 0 } }); + }); + + it.each(["stream-response-cancel", "stream-unconsumed"])( + "%s: a stream nobody finishes reading is closed once it is garbage (ai 4–6); ai 7 cannot see it", + (scenario) => { + const result = run(scenario); + const report = reported(result); + if (major >= 7) { + // v7 fires no telemetry callback when the consumer cancels or never + // reads, and hands the integration no per-call object whose + // collection could stand in for one: the agent stays open while the + // process lives. Bounded (MAX_OPEN_CALLS), and avoidable by passing + // the request's abortSignal — see stream-response-cancel-signal. When + // the process exits, the exit hook closes it as failed rather than + // leaving it rendered as running forever. + expect(count(result.events, "agent_start"), describeTrace(result)).toBe(1); + expect(report.after).toMatchObject({ openCalls: 1 }); + const ends = ofType(result.events, "agent_end"); + expect(ends.map((e) => e.outcome), describeTrace(result)).toEqual(["failed"]); + expect(ends[0]!.summary).toMatch(/process exited/); + return; + } + // v4–v6: the SDK ends a stream's root span from a flush() that never + // runs; the adapter ends it when the span is collected. + clean(result); + const end = ofType(result.events, "agent_end")[0]!; + expect(end.outcome, describeTrace(result)).toBe("cancelled"); + expect(end.fw_abandoned).toBe(true); + // (`report.before` is not asserted: an ordinary collection may already + // have closed it by then.) + expect(report.after).toEqual({ openCalls: 0, stats: { runs: 0, links: 0 } }); + }, + ); + + it("stream-error: a provider stream that breaks mid-way leaves nothing open", () => { + const result = run("stream-error"); + expect(shape(result.events), describeTrace(result)).toEqual(single("counter")); + clean(result); + expect(reported(result).threw).toBe("connection reset"); + expect(reported(result).after).toEqual({ openCalls: 0, stats: { runs: 0, links: 0 } }); + const response = ofType(result.events, "model_response")[0]!; + const end = ofType(result.events, "agent_end")[0]!; + if (major >= 7) { + expect(response.stop_reason).toBe("error"); + expect(response.error).toMatch(/connection reset/); + expect(end.outcome).toBe("failed"); + } else { + // v4–v6 never end either span nor report the error to the tracer; the + // operation is closed when it is collected, and cannot say why. + expect(response.stop_reason).toBe("cancelled"); + expect(end.outcome).toBe("cancelled"); + expect(end.fw_abandoned).toBe(true); + } + }); + + // ---- 6. reasoning models + + it.each(["reasoning-generate", "reasoning-stream", "reasoning-wrap"])( + "%s: reasoning parts leave the token count and the answer intact", + (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual(single(scenario === "reasoning-wrap" ? "mock-model" : "thinker")); + clean(result); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([12, 20]); + expect(response.stop_reason).toBe("stop"); + expect(response.content).toBe("Paris."); + expect(reported(result).text).toBe("Paris."); + }, + ); + + // ---- 7. concurrency + + it.each(["concurrent", "concurrent-stream"])("%s: 10 concurrent calls in one session are 10 agents with no cross-talk", (scenario) => { + const result = run(scenario); + clean(result); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["busy"])); + const starts = ofType(result.events, "agent_start").map((e) => e.agent_id).sort(); + expect(starts).toEqual(Array.from({ length: 10 }, (_, i) => `worker-${i}`).sort()); + for (let i = 0; i < 10; i += 1) { + const mine = result.events.filter((e) => e.agent_id === `worker-${i}`); + expect(mine.map((e) => e.type).sort(), describeTrace(result)).toEqual( + ["agent_end", "agent_start", "model_request", "model_request", "model_response", "model_response", "tool_result", "tool_use"], + ); + expect(ofType(mine, "tool_use")[0]!.tool_call_id).toBe(`call-${i}`); + expect(ofType(mine, "tool_use")[0]!.input).toEqual({ city: `city-${i}` }); + expect(ofType(mine, "tool_result")[0]!.tool_call_id).toBe(`call-${i}`); + expect(tokens(mine)).toEqual([ + [100 + i, 1], + [200 + i, 2], + ]); + // Each response pairs with a request of the SAME agent. + const requests = new Set(ofType(mine, "model_request").map((e) => e.request_id)); + for (const response of ofType(mine, "model_response")) expect(requests.has(response.request_id)).toBe(true); + } + expect(reported(result).texts).toEqual(Array.from({ length: 10 }, (_, i) => `answer ${i}`)); + }); + + it("concurrent-unscoped: 10 concurrent same-named calls outside any scope are 10 sessions of one agent each", () => { + const result = run("concurrent-unscoped"); + clean(result); + const sessions = new Map(); + for (const e of result.events) sessions.set(e.session_id, [...(sessions.get(e.session_id) ?? []), e]); + expect(sessions.size).toBe(10); + for (const events of sessions.values()) { + expect(shape(events)).toEqual(loop("worker")); + const i = Number(String(ofType(events, "tool_use")[0]!.tool_call_id).split("-")[1]); + expect(tokens(events)).toEqual([ + [100 + i, 1], + [200 + i, 2], + ]); + } + }); + + it("concurrent-wrap: 10 concurrent bare wrapped-model calls are 10 runs, each its own model's", () => { + const result = run("concurrent-wrap"); + clean(result); + const sessions = new Map(); + for (const e of result.events) sessions.set(e.session_id, [...(sessions.get(e.session_id) ?? []), e]); + expect(sessions.size).toBe(10); + for (const events of sessions.values()) { + const name = events[0]!.agent_id; + expect(shape(events)).toEqual(single(name)); + expect(tokens(events)).toEqual([[100 + Number(name.split("-")[1]), 1]]); + } + }); + }); +}); diff --git a/sdk/typescript/integration/fixtures/ai-4/agent.ts b/sdk/typescript/integration/fixtures/ai-4/agent.ts new file mode 100644 index 000000000..2c556f23d --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/agent.ts @@ -0,0 +1,337 @@ +// Vercel AI SDK 4.x consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import `ai`, pass the adapter at +// the call site (or wrap the model, or instrument), run. The scripted mock +// model makes it deterministic and offline — the tool loop's first call asks +// for `weather`, the next one answers — and never touches a real provider. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; only the model mock and the tool/loop +// spelling above it change between majors. +import { createRequire } from "node:module"; +import { join } from "node:path"; + +import * as failproofai from "@failproofai/sdk"; +import { middleware, telemetry, tracer, wrapModel } from "@failproofai/sdk/ai"; +import { + generateObject, + generateText, + simulateReadableStream, + streamObject, + streamText, + tool, + wrapLanguageModel, +} from "ai"; +import { MockLanguageModelV1 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: LanguageModelV1 (promptTokens/completionTokens, `text` + `toolCalls`) + +type Script = "loop" | "answer" | "object" | "fail"; +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const usage = (input: number, output: number) => ({ promptTokens: input, completionTokens: output }); +const rawCall = { rawPrompt: null, rawSettings: {} }; + +function scripted(script: Script = "loop") { + let generated = 0; + let streamed = 0; + return new MockLanguageModelV1({ + provider: "mock-provider", + modelId: "mock-model", + // v4 asks the model how to produce an object; v5+ always use JSON mode. + defaultObjectGenerationMode: "json", + doGenerate: async () => { + generated += 1; + if (script === "fail") throw new Error("model exploded"); + if (script === "object") return { text: '{"city":"Paris"}', finishReason: "stop", usage: usage(5, 3), rawCall }; + if (script === "loop" && generated === 1) { + return { + toolCalls: [{ toolCallType: "function", toolCallId: "call-1", toolName: "weather", args: '{"city":"Paris"}' }], + finishReason: "tool-calls", + usage: usage(11, 7), + rawCall, + }; + } + return { text: "It is 20C in Paris.", finishReason: "stop", usage: usage(23, 9), rawCall }; + }, + doStream: async () => { + streamed += 1; + if (script === "fail") throw new Error("model exploded"); + const text = (...deltas: string[]): StreamPart[] => deltas.map((textDelta): StreamPart => ({ type: "text-delta", textDelta })); + const chunks: StreamPart[] = + script === "object" + ? [...text('{"city":', '"Paris"}'), { type: "finish", finishReason: "stop", usage: usage(5, 3) }] + : script === "loop" && streamed === 1 + ? [ + { type: "tool-call", toolCallType: "function", toolCallId: "call-s1", toolName: "weather", args: '{"city":"Rome"}' }, + { type: "finish", finishReason: "tool-calls", usage: usage(13, 4) }, + ] + : [...text("Rome is ", "25C."), { type: "finish", finishReason: "stop", usage: usage(30, 6) }]; + return { stream: simulateReadableStream({ chunks }), rawCall }; + }, + }); +} + +const makeTools = (fail = false) => ({ + weather: tool({ + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => { + if (fail) throw new Error("weather service down"); + return { city, celsius: city === "Paris" ? 20 : 25 }; + }, + }), +}); +const loop = { maxSteps: 4 }; + +/** The bare tracer, as `experimental_telemetry.tracer` (v4–v6 only: v7 has no such option). */ +async function viaTracer(prompt: string): Promise { + const { text } = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: { isEnabled: true, tracer: tracer() }, + }); + return text; +} + +// --- the same in every fixture + +type Telemetry = ReturnType | { isEnabled: true; functionId: string }; + +async function ask(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean; failTool?: boolean } = {}) { + const result = await generateText({ + model, + prompt: "Weather in Paris?", + ...(options.tools === false ? {} : { tools: makeTools(options.failTool), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return result.text; +} + +async function askStreaming(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean } = {}) { + const result = streamText({ + model, + prompt: "Weather in Rome?", + ...(options.tools === false ? {} : { tools: makeTools(), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; +} + +const schema = z.object({ city: z.string() }); +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** The README's call sites, verbatim apart from the mock standing in for a provider. */ +async function readme(): Promise { + const model = scripted("answer"); + const prompt = "Weather in Paris?"; + const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + }); + const wrapped = await wrapModel(scripted("answer")); + const viaMiddleware = wrapLanguageModel({ model: scripted("answer"), middleware: middleware() }); + const withMetadata = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: telemetry({ functionId: "tagged", metadata: { tenant: "acme", attempt: 1 } }), + }); + report({ text, wrapped: (await generateText({ model: wrapped, prompt })).text, viaMiddleware: viaMiddleware.modelId, withMetadata: withMetadata.text, withTracer: await viaTracer(prompt) }); +} + +/** + * The customer's own OpenTelemetry, set up AFTER `instrument("ai")` — the + * usual order when tracing starts in a module loaded later (a `NodeSDK` + * started from an instrumentation file). Null when `@opentelemetry/api` is not + * installed (ai 7 dropped the dependency). + */ +function customerTracing(): { registered: boolean; ended: () => string[] } | null { + interface Api { + trace: { + setGlobalTracerProvider(provider: unknown): boolean; + getTracer(name: string): { startSpan(name: string): { end(): void } }; + }; + } + let api: Api; + try { + api = createRequire(join(process.cwd(), "agent.js"))("@opentelemetry/api") as Api; + } catch { + return null; + } + const ended: string[] = []; + const span = (name: string) => ({ + setAttribute() { return this; }, + setAttributes() { return this; }, + addEvent() { return this; }, + addLink() { return this; }, + addLinks() { return this; }, + setStatus() { return this; }, + updateName() { return this; }, + recordException() {}, + isRecording: () => true, + spanContext: () => ({ traceId: "0".repeat(31) + "1", spanId: "0".repeat(15) + "1", traceFlags: 1 }), + end: () => void ended.push(name), + }); + const theirs = { + startSpan: (name: string) => span(name), + startActiveSpan: (name: string, ...rest: unknown[]) => (rest[rest.length - 1] as (s: unknown) => unknown)(span(name)), + }; + const registered = api.trace.setGlobalTracerProvider({ getTracer: () => theirs }); + // What an http / pg / Next.js instrumentation does with the global API. + api.trace.getTracer("my-service").startSpan("http.request").end(); + return { registered, ended: () => [...new Set(ended)].sort() }; +} + +/** A model whose stream breaks part-way: the provider connection dropped. */ +function breakMidStream(model: M): M { + const target = model as unknown as { doStream: (options: unknown) => PromiseLike<{ stream: ReadableStream }> }; + const original = target.doStream.bind(target); + target.doStream = async (options: unknown) => { + const result = await original(options); + const reader = result.stream.getReader(); + let parts = 0; + return { + ...result, + stream: new ReadableStream({ + async pull(controller) { + if (parts++ === 2) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + }), + }; + }; + return model; +} + +/** Call a model's `doStream` directly, as a provider-level consumer does. */ +async function openStream(model: unknown): Promise> { + const call = { + inputFormat: "prompt", + mode: { type: "regular" }, + prompt: [{ role: "user", content: [{ type: "text", text: "Weather in Rome?" }] }], + }; + const { stream } = await (model as { doStream(options: unknown): PromiseLike<{ stream: ReadableStream }> }).doStream(call); + return stream.getReader(); +} + +async function main(scenario: string): Promise { + switch (scenario) { + case "generate": + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "stream": + report({ text: await askStreaming(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "object": { + const { object } = await generateObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ object }); + break; + } + case "stream-object": { + const result = streamObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + for await (const _ of result.partialObjectStream) void _; + report({ object: await result.object }); + break; + } + case "wrap": + report({ text: await ask(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-stream": + report({ text: await askStreaming(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, async () => report({ text: await ask(await wrapModel(scripted())) })), + ); + break; + case "wrap-and-telemetry": + report({ text: await ask(await wrapModel(scripted()), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "instrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-stream": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global-stream": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-then-otel": { + report({ instrumented: await failproofai.instrument("ai") }); + const customer = customerTracing(); + const text = await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }); + report({ text, customer: customer === null ? "absent" : { registered: customer.registered, ended: customer.ended() } }); + break; + } + case "wrap-stream-cancel": { + const reader = await openStream(await wrapModel(scripted("answer"))); + await reader.read(); + await reader.cancel("client disconnected"); + report({ cancelled: true }); + break; + } + case "wrap-stream-error": { + const reader = await openStream(await wrapModel(breakMidStream(scripted("answer")))); + try { + while (!(await reader.read()).done) { + // drain + } + report({ drained: true }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "uninstrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ removed: failproofai.uninstrument() }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "tool-error": + try { + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }), failTool: true }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "model-error": + try { + report({ text: await ask(scripted("fail"), { telemetry: telemetry({ functionId: "weather-agent" }), tools: false }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "scope": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) })), + ); + break; + case "readme": + await readme(); + break; + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-4/package-lock.json b/sdk/typescript/integration/fixtures/ai-4/package-lock.json new file mode 100644 index 000000000..030682357 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/package-lock.json @@ -0,0 +1,281 @@ +{ + "name": "failproofai-it-ai-4", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-ai-4", + "dependencies": { + "ai": "4.3.19", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/ai": { + "version": "4.3.19", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.19.tgz", + "integrity": "sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/ai-4/package.json b/sdk/typescript/integration/fixtures/ai-4/package.json new file mode 100644 index 000000000..eda95205c --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/package.json @@ -0,0 +1,13 @@ +{ + "name": "failproofai-it-ai-4", + "private": true, + "type": "module", + "description": "Integration fixture: Vercel AI SDK 4.x against the packed @failproofai/sdk.", + "dependencies": { + "ai": "4.3.19", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/ai-4/surfaces.ts b/sdk/typescript/integration/fixtures/ai-4/surfaces.ts new file mode 100644 index 000000000..f6dfb9872 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/surfaces.ts @@ -0,0 +1,605 @@ +// Vercel AI SDK 4.x: every commonly used surface. Run as `node surfaces.{mjs,cjs} `. +// +// The coverage companion to agent.ts. agent.ts pins the README call sites +// and the adapter's core paths; this program walks the rest of the SDK a +// customer actually uses — embeddings, structured output, every tool feature, +// every way of consuming a stream, reasoning models and concurrency — against +// the same scripted, offline mock. (ai 4 has no agent class.) +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; the section above it spells one major's API. +import * as failproofai from "@failproofai/sdk"; +import * as adapter from "@failproofai/sdk/ai"; +import { telemetry, wrapModel } from "@failproofai/sdk/ai"; +import { + Output, + embed, + embedMany, + generateObject, + generateText, + simulateReadableStream, + streamObject, + streamText, + tool, +} from "ai"; +import { MockEmbeddingModelV1, MockLanguageModelV1 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: ai 4 (LanguageModelV1, maxSteps, no agent class, no tool approval) + +const MAJOR = 4; + +interface Call { + id: string; + name: string; + input: Record; +} +/** One scripted model step: what the model "says" on its n-th call. */ +interface Step { + text?: string; + reasoning?: string; + calls?: Call[]; + usage: [number, number]; + reasoningTokens?: number; + fail?: string; + delayMs?: number; +} +interface ModelOptions { + modelId?: string; + chunkDelayMs?: number; + /** Break the provider stream after this many parts: the connection dropped. */ + breakAfterParts?: number; +} + +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** A string as two stream deltas. */ +const halves = (text: string): string[] => [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]; + +const usageOf = (step: Step) => ({ promptTokens: step.usage[0], completionTokens: step.usage[1] }); +const finishOf = (step: Step) => (step.calls?.length ? "tool-calls" : "stop"); +const rawCall = { rawPrompt: null, rawSettings: {} }; + +function model(steps: Step[], options: ModelOptions = {}) { + let n = 0; + const next = async (): Promise => { + const step = steps[Math.min(n, steps.length - 1)]!; + n += 1; + if (step.delayMs) await sleep(step.delayMs); + if (step.fail) throw new Error(step.fail); + return step; + }; + const toolCalls = (step: Step) => + (step.calls ?? []).map((call) => ({ toolCallType: "function" as const, toolCallId: call.id, toolName: call.name, args: JSON.stringify(call.input) })); + return new MockLanguageModelV1({ + provider: "mock-provider", + modelId: options.modelId ?? "mock-model", + // v4 asks the model how to produce an object; v5+ always use JSON mode. + defaultObjectGenerationMode: "json", + doGenerate: async () => { + const step = await next(); + return { + ...(step.text ? { text: step.text } : {}), + ...(step.reasoning ? { reasoning: step.reasoning } : {}), + ...(step.calls?.length ? { toolCalls: toolCalls(step) } : {}), + finishReason: finishOf(step), + usage: usageOf(step), + rawCall, + }; + }, + doStream: async ({ abortSignal }) => { + const step = await next(); + const chunks: StreamPart[] = []; + if (step.reasoning) for (const textDelta of halves(step.reasoning)) chunks.push({ type: "reasoning", textDelta }); + if (step.text) for (const textDelta of halves(step.text)) chunks.push({ type: "text-delta", textDelta }); + for (const call of toolCalls(step)) chunks.push({ type: "tool-call", ...call }); + chunks.push({ type: "finish", finishReason: finishOf(step), usage: usageOf(step) }); + return { stream: abortable(simulateReadableStream({ chunks, chunkDelayInMs: options.chunkDelayMs ?? 0 }), abortSignal, options.breakAfterParts), rawCall }; + }, + }); +} + +function embedder() { + return new MockEmbeddingModelV1({ + provider: "mock-provider", + modelId: "mock-embedder", + maxEmbeddingsPerCall: 2, + doEmbed: async ({ values }) => ({ embeddings: values.map((_, i) => [i, 0.5]), usage: { tokens: values.length * 3 } }), + }); +} + +const citySchema = z.object({ city: z.string() }); +type Weather = { city: string; celsius: number }; + +function weatherTool(options: { fail?: boolean; delayMs?: (city: string) => number; onRun?: (city: string) => Promise } = {}) { + return tool({ + description: "Current weather for a city", + parameters: citySchema, + execute: async ({ city }: { city: string }): Promise => { + if (options.delayMs) await sleep(options.delayMs(city)); + if (options.onRun) await options.onRun(city); + if (options.fail) throw new Error("weather service down"); + return { city, celsius: city.length * 4 }; + }, + }); +} +/** A client-side tool: no `execute`, so the SDK hands the call back to the caller. */ +const clientTool = () => tool({ description: "Ask the user", parameters: citySchema }); +/** ai 4 has no tool approval (`needsApproval` arrived in 6). */ +const HAS_APPROVAL = false; + +const steps = (n: number) => ({ maxSteps: n }); +/** ai 4 has no agent class (`Experimental_Agent` arrived in 5). */ +const HAS_AGENT = false; + +type Settings = Parameters[0]; +type Telemetry = NonNullable; + +function makeAgent(_options: { model: Settings["model"]; telemetry?: Telemetry; id?: string; tools?: Settings["tools"] }): { + generate(prompt: string): Promise; + stream(prompt: string): Promise; +} { + throw new Error("ai 4 has no agent class"); +} + +/** `generateText` / `streamText` with `experimental_output: Output.object(...)`. */ +const HAS_OUTPUT = true; +async function textWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = await generateText({ model: m, prompt: "Where?", experimental_output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + return result.experimental_output; +} +async function streamWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = streamText({ model: m, prompt: "Where?", experimental_output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.experimental_partialOutputStream) partials.push(partial); + return partials; +} + +/** What a Next.js route handler returns (ai 4: the data stream protocol). */ +const toResponse = (result: { toDataStreamResponse(): Response }): Response => result.toDataStreamResponse(); + +const repairOption = (fixed: Record) => ({ + experimental_repairToolCall: async ({ toolCall }: { toolCall: T }) => ({ ...toolCall, args: JSON.stringify(fixed) }), +}); +const prepareStepOption = (second: Settings["model"]) => ({ + experimental_prepareStep: async ({ stepNumber }: { stepNumber: number }) => (stepNumber === 1 ? { model: second } : undefined), +}); + +async function approveAndResume(_m: Settings["model"], _tel: Telemetry): Promise<{ pending: number; text: string }> { + throw new Error("ai 4 has no tool approval"); +} + +const partialObjects = async (m: Settings["model"], tel: Telemetry) => { + const result = streamObject({ model: m, schema: citySchema, prompt: "Where?", experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialObjectStream) partials.push(partial); + return { partials, object: await result.object }; +}; + +// --- the same in every fixture + +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** + * A provider stream that honours the call's abort signal, as a real provider's + * `fetch` does — once aborted, the next read fails with an AbortError — and + * that can drop its connection part-way (`breakAfter`). + */ +function abortable(stream: ReadableStream, signal: AbortSignal | undefined, breakAfter?: number): ReadableStream { + const reader = stream.getReader(); + let parts = 0; + return new ReadableStream({ + async pull(controller) { + if (signal?.aborted) { + controller.error(signal.reason ?? new DOMException("aborted", "AbortError")); + return; + } + if (breakAfter !== undefined && parts++ === breakAfter) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + cancel: (reason) => reader.cancel(reason), + }); +} + +/** + * Run the garbage collector until it has had a real chance at everything + * unreachable. `--expose-gc` switched on at runtime, so the harness needs no + * special flags. + */ +async function collectGarbage(): Promise { + const v8 = await import("node:v8"); + const vm = await import("node:vm"); + v8.setFlagsFromString("--expose-gc"); + const gc = vm.runInNewContext("gc") as () => void; + for (let i = 0; i < 10; i += 1) { + gc(); + await sleep(20); + } +} + +/** + * A route's streamed Response whose client went away after three chunks. + * Its own function, so nothing of the stream is left on `main`'s frame + * afterwards — the garbage collector may take all of it. + */ +async function disconnectedClient(withSignal = false): Promise<{ chunks: number }> { + // A Next.js route can pass `abortSignal: request.signal`; the disconnect then + // aborts the call as well as cancelling the body. + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ + model: m, + prompt: "Count", + ...(withSignal ? { abortSignal: controller.signal } : {}), + experimental_telemetry: telemetry({ functionId: "counter" }), + }); + const reader = toResponse(result).body!.getReader(); + let chunks = 0; + while (chunks < 3 && !(await reader.read()).done) chunks += 1; + controller.abort(); + await reader.cancel("client disconnected"); + return { chunks }; +} + +/** An aborted stream, in a function of its own for the same reason. */ +async function abortedStream(): Promise<{ text: string; threw: string | false }> { + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ model: m, prompt: "Count", abortSignal: controller.signal, experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) { + text += delta; + controller.abort(); + } + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).name }; + } +} + +/** A stream started and never read. */ +function neverRead(): void { + streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); +} + +/** A stream whose provider connection drops part-way; the error is the caller's to see. */ +async function brokenStream(): Promise<{ text: string; threw: string | false }> { + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { breakAfterParts: 2 }); + const result = streamText({ model: m, prompt: "Count", experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) text += delta; + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).message }; + } +} + +/** The adapter's bookkeeping (internal, untyped): what it still holds open. */ +const held = () => { + const internals = (adapter as unknown as { _internals: { openCalls(): number; tracker(): { stats(): unknown } | null } })._internals; + return { openCalls: internals.openCalls(), stats: internals.tracker()?.stats() ?? null }; +}; + +const LOOP: Step[] = [ + { calls: [{ id: "call-1", name: "weather", input: { city: "Paris" } }], usage: [11, 7] }, + { text: "It is 20C in Paris.", usage: [23, 9] }, +]; + +async function drainText(stream: AsyncIterable): Promise { + let text = ""; + for await (const delta of stream) text += delta; + return text; +} + +async function main(scenario: string): Promise { + switch (scenario) { + // ---- 1. the agent classes + case "agent-generate": + case "agent-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), telemetry: telemetry({ functionId: "support-agent" }) }); + report({ text: scenario === "agent-generate" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-id": { + // The agent's own `id` never reaches telemetry: the SDK spreads it into + // generateText, which drops it. functionId is what names the agent. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), id: "support-agent", telemetry: telemetry() }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-instrument": + case "agent-instrument-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai") }); + const agent = makeAgent({ model: model(LOOP), telemetry: { isEnabled: true, functionId: "support-agent" } }); + report({ text: scenario === "agent-instrument" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-instrument-bare": { + // No telemetry setting at all: ai 7 records every call once an + // integration is registered; ai 4–6 record nothing without isEnabled. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: false }) }); + report({ text: await makeAgent({ model: model(LOOP) }).generate("Weather in Paris?") }); + break; + } + case "agent-wrap": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-wrap-in-scope": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("support-agent", { goal: "help" }, async () => report({ text: await agent.stream("Weather in Paris?") })), + ); + break; + } + + // ---- 2. embeddings + case "embed": + report({ embedding: (await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry({ functionId: "indexer" }) })).embedding }); + break; + case "embed-many": { + const { embeddings } = await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embeddings: embeddings.length }); + break; + } + case "embed-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("rag", { goal: "answer" }, async () => { + await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry() }); + await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embedded: true }); + }), + ); + break; + case "embed-in-tool": { + const lookup = weatherTool({ + onRun: async (city) => { + await embed({ model: embedder(), value: city, experimental_telemetry: telemetry() }); + }, + }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: lookup }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + + // ---- 3. structured output + case "object-partial": { + const m = model([{ text: '{"city":"Paris"}', usage: [5, 3] }]); + report(await partialObjects(m, telemetry({ functionId: "extractor" }))); + break; + } + case "object-invalid": + try { + await generateObject({ model: model([{ text: '{"town":"Paris"}', usage: [5, 3] }]), schema: citySchema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + case "text-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + report({ output: await textWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })) }); + break; + } + case "stream-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + const partials = await streamWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })); + report({ partials: partials.length, last: partials[partials.length - 1] }); + break; + } + + // ---- 4. tool features + case "parallel-tools": { + const m = model([ + { + calls: [ + { id: "call-p1", name: "weather", input: { city: "Paris" } }, + { id: "call-p2", name: "weather", input: { city: "Rome" } }, + ], + usage: [11, 7], + }, + { text: "Paris 20C, Rome 16C.", usage: [40, 9] }, + ]); + // Rome finishes first: results arrive out of call order. + const tools = { weather: weatherTool({ delayMs: (city) => (city === "Paris" ? 40 : 5) }) }; + const { text } = await generateText({ model: m, prompt: "Weather?", tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "client-tool": { + const m = model([{ calls: [{ id: "call-c1", name: "ask", input: { city: "Paris" } }], usage: [11, 7] }]); + const result = await generateText({ model: m, prompt: "Weather?", tools: { ask: clientTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ toolCalls: result.toolCalls.length }); + break; + } + case "tool-choice-required": { + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, toolChoice: "required", ...steps(1), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "repair": { + const m = model([{ calls: [{ id: "call-r1", name: "weather", input: { town: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...repairOption({ city: "Paris" }), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "prepare-step": { + const second = model([LOOP[1]!], { modelId: "mock-model-large" }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...prepareStepOption(second), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "unknown-tool": { + const m = model([{ calls: [{ id: "call-u1", name: "teleport", input: { city: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + try { + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + } + case "approval": { + if (!HAS_APPROVAL) return report({ skipped: "no tool approval" }); + const m = model([{ calls: [{ id: "call-a1", name: "book", input: { city: "Paris" } }], usage: [11, 7] }, { text: "Booked Paris.", usage: [30, 4] }]); + report(await approveAndResume(m, telemetry({ functionId: "travel-agent" }))); + break; + } + + // ---- 5. consuming a stream + case "stream-full": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const types = new Set(); + for await (const part of result.fullStream) types.add(part.type); + report({ parts: [...types].sort(), text: await result.text }); + break; + } + case "stream-response": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const response = toResponse(result); + const body = await response.text(); + report({ status: response.status, containsAnswer: body.includes("Paris"), bytes: body.length }); + break; + } + case "stream-response-cancel": + // The browser went away mid-stream. Nothing in the SDK ends the operation + // after that; the adapter closes it once the stream is garbage. + report(await disconnectedClient()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-response-cancel-signal": + report(await disconnectedClient(true)); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-error": + report(await brokenStream()); + await sleep(50); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-on-finish": { + let finished = ""; + const result = streamText({ + model: model(LOOP), + prompt: "Weather?", + tools: { weather: weatherTool() }, + ...steps(4), + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + onFinish: ({ text }) => { + finished = text; + }, + }); + await result.consumeStream(); + report({ finished }); + break; + } + case "stream-abort": + report(await abortedStream()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-unconsumed": + // Never read. Nothing waits on it, so nothing should be held open. + neverRead(); + await sleep(200); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + + // ---- 6. reasoning models + case "reasoning-generate": + case "reasoning-stream": { + const m = model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }]); + const options = { model: m, prompt: "Capital of France?", providerOptions: { mock: { reasoningEffort: "high" } }, experimental_telemetry: telemetry({ functionId: "thinker" }) }; + if (scenario === "reasoning-generate") { + const result = await generateText(options); + report({ text: result.text }); + } else { + const result = streamText(options); + report({ text: await drainText(result.textStream) }); + } + break; + } + case "reasoning-wrap": { + const m = await wrapModel(model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }])); + const result = streamText({ model: m, prompt: "Capital of France?" }); + report({ text: await drainText(result.textStream) }); + break; + } + + // ---- 7. concurrency + case "concurrent": + case "concurrent-stream": { + const one = async (i: number) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2], delayMs: (i * 3) % 5 }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + const options = { model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: `worker-${i}` }) }; + if (scenario === "concurrent") return (await generateText(options)).text; + return await drainText(streamText(options).textStream); + }; + const texts = await failproofai.session({ sessionId: "busy" }, () => Promise.all(Array.from({ length: 10 }, (_, i) => one(i)))); + report({ texts }); + break; + } + case "concurrent-unscoped": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2] }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + return (await generateText({ model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "worker" }) })).text; + }), + ); + report({ texts }); + break; + } + case "concurrent-wrap": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = await wrapModel(model([{ text: `answer ${i}`, usage: [100 + i, 1], delayMs: (i * 7) % 11 }], { modelId: `model-${i}` })); + return (await generateText({ model: m, prompt: `q${i}` })).text; + }), + ); + report({ texts }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + report({ major: MAJOR }); + await failproofai.flush(); +} + +main(process.argv[2] ?? "agent-generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-4/tsconfig.json b/sdk/typescript/integration/fixtures/ai-4/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-4/tsconfig.surfaces.json b/sdk/typescript/integration/fixtures/ai-4/tsconfig.surfaces.json new file mode 100644 index 000000000..f827f9292 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-4/tsconfig.surfaces.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "files": ["surfaces.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-5/agent.ts b/sdk/typescript/integration/fixtures/ai-5/agent.ts new file mode 100644 index 000000000..247d4fd99 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/agent.ts @@ -0,0 +1,342 @@ +// Vercel AI SDK 5.x consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import `ai`, pass the adapter at +// the call site (or wrap the model, or instrument), run. The scripted mock +// model makes it deterministic and offline — the tool loop's first call asks +// for `weather`, the next one answers — and never touches a real provider. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; only the model mock and the tool/loop +// spelling above it change between majors. +import { createRequire } from "node:module"; +import { join } from "node:path"; + +import * as failproofai from "@failproofai/sdk"; +import { middleware, telemetry, tracer, wrapModel } from "@failproofai/sdk/ai"; +import { + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, + wrapLanguageModel, +} from "ai"; +import { MockLanguageModelV2 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: LanguageModelV2 (flat usage numbers, string finish reasons) + +type Script = "loop" | "answer" | "object" | "fail"; +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const usage = (input: number, output: number) => ({ inputTokens: input, outputTokens: output, totalTokens: input + output }); + +function scripted(script: Script = "loop") { + let generated = 0; + let streamed = 0; + return new MockLanguageModelV2({ + provider: "mock-provider", + modelId: "mock-model", + doGenerate: async () => { + generated += 1; + if (script === "fail") throw new Error("model exploded"); + if (script === "object") { + return { content: [{ type: "text", text: '{"city":"Paris"}' }], finishReason: "stop", usage: usage(5, 3), warnings: [] }; + } + if (script === "loop" && generated === 1) { + return { + content: [{ type: "tool-call", toolCallId: "call-1", toolName: "weather", input: '{"city":"Paris"}' }], + finishReason: "tool-calls", + usage: usage(11, 7), + warnings: [], + }; + } + return { content: [{ type: "text", text: "It is 20C in Paris." }], finishReason: "stop", usage: usage(23, 9), warnings: [] }; + }, + doStream: async () => { + streamed += 1; + if (script === "fail") throw new Error("model exploded"); + const text = (id: string, ...deltas: string[]): StreamPart[] => [ + { type: "text-start", id }, + ...deltas.map((delta): StreamPart => ({ type: "text-delta", id, delta })), + { type: "text-end", id }, + ]; + const chunks: StreamPart[] = + script === "object" + ? [{ type: "stream-start", warnings: [] }, ...text("o", '{"city":', '"Paris"}'), { type: "finish", finishReason: "stop", usage: usage(5, 3) }] + : script === "loop" && streamed === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "tool-call", toolCallId: "call-s1", toolName: "weather", input: '{"city":"Rome"}' }, + { type: "finish", finishReason: "tool-calls", usage: usage(13, 4) }, + ] + : [{ type: "stream-start", warnings: [] }, ...text("t", "Rome is ", "25C."), { type: "finish", finishReason: "stop", usage: usage(30, 6) }]; + return { stream: simulateReadableStream({ chunks }) }; + }, + }); +} + +const makeTools = (fail = false) => ({ + weather: tool({ + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => { + if (fail) throw new Error("weather service down"); + return { city, celsius: city === "Paris" ? 20 : 25 }; + }, + }), +}); +const loop = { stopWhen: stepCountIs(4) }; + +/** The bare tracer, as `experimental_telemetry.tracer` (v4–v6 only: v7 has no such option). */ +async function viaTracer(prompt: string): Promise { + const { text } = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: { isEnabled: true, tracer: tracer() }, + }); + return text; +} + +// --- the same in every fixture + +type Telemetry = ReturnType | { isEnabled: true; functionId: string }; + +async function ask(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean; failTool?: boolean } = {}) { + const result = await generateText({ + model, + prompt: "Weather in Paris?", + ...(options.tools === false ? {} : { tools: makeTools(options.failTool), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return result.text; +} + +async function askStreaming(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean } = {}) { + const result = streamText({ + model, + prompt: "Weather in Rome?", + ...(options.tools === false ? {} : { tools: makeTools(), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; +} + +const schema = z.object({ city: z.string() }); +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** The README's call sites, verbatim apart from the mock standing in for a provider. */ +async function readme(): Promise { + const model = scripted("answer"); + const prompt = "Weather in Paris?"; + const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + }); + const wrapped = await wrapModel(scripted("answer")); + const viaMiddleware = wrapLanguageModel({ model: scripted("answer"), middleware: middleware() }); + const withMetadata = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: telemetry({ functionId: "tagged", metadata: { tenant: "acme", attempt: 1 } }), + }); + report({ text, wrapped: (await generateText({ model: wrapped, prompt })).text, viaMiddleware: viaMiddleware.modelId, withMetadata: withMetadata.text, withTracer: await viaTracer(prompt) }); +} + +/** + * The customer's own OpenTelemetry, set up AFTER `instrument("ai")` — the + * usual order when tracing starts in a module loaded later (a `NodeSDK` + * started from an instrumentation file). Null when `@opentelemetry/api` is not + * installed (ai 7 dropped the dependency). + */ +function customerTracing(): { registered: boolean; ended: () => string[] } | null { + interface Api { + trace: { + setGlobalTracerProvider(provider: unknown): boolean; + getTracer(name: string): { startSpan(name: string): { end(): void } }; + }; + } + let api: Api; + try { + api = createRequire(join(process.cwd(), "agent.js"))("@opentelemetry/api") as Api; + } catch { + return null; + } + const ended: string[] = []; + const span = (name: string) => ({ + setAttribute() { return this; }, + setAttributes() { return this; }, + addEvent() { return this; }, + addLink() { return this; }, + addLinks() { return this; }, + setStatus() { return this; }, + updateName() { return this; }, + recordException() {}, + isRecording: () => true, + spanContext: () => ({ traceId: "0".repeat(31) + "1", spanId: "0".repeat(15) + "1", traceFlags: 1 }), + end: () => void ended.push(name), + }); + const theirs = { + startSpan: (name: string) => span(name), + startActiveSpan: (name: string, ...rest: unknown[]) => (rest[rest.length - 1] as (s: unknown) => unknown)(span(name)), + }; + const registered = api.trace.setGlobalTracerProvider({ getTracer: () => theirs }); + // What an http / pg / Next.js instrumentation does with the global API. + api.trace.getTracer("my-service").startSpan("http.request").end(); + return { registered, ended: () => [...new Set(ended)].sort() }; +} + +/** A model whose stream breaks part-way: the provider connection dropped. */ +function breakMidStream(model: M): M { + const target = model as unknown as { doStream: (options: unknown) => PromiseLike<{ stream: ReadableStream }> }; + const original = target.doStream.bind(target); + target.doStream = async (options: unknown) => { + const result = await original(options); + const reader = result.stream.getReader(); + let parts = 0; + return { + ...result, + stream: new ReadableStream({ + async pull(controller) { + if (parts++ === 2) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + }), + }; + }; + return model; +} + +/** Call a model's `doStream` directly, as a provider-level consumer does. */ +async function openStream(model: unknown): Promise> { + const call = { + inputFormat: "prompt", + mode: { type: "regular" }, + prompt: [{ role: "user", content: [{ type: "text", text: "Weather in Rome?" }] }], + }; + const { stream } = await (model as { doStream(options: unknown): PromiseLike<{ stream: ReadableStream }> }).doStream(call); + return stream.getReader(); +} + +async function main(scenario: string): Promise { + switch (scenario) { + case "generate": + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "stream": + report({ text: await askStreaming(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "object": { + const { object } = await generateObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ object }); + break; + } + case "stream-object": { + const result = streamObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + for await (const _ of result.partialObjectStream) void _; + report({ object: await result.object }); + break; + } + case "wrap": + report({ text: await ask(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-stream": + report({ text: await askStreaming(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, async () => report({ text: await ask(await wrapModel(scripted())) })), + ); + break; + case "wrap-and-telemetry": + report({ text: await ask(await wrapModel(scripted()), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "instrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-stream": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global-stream": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-then-otel": { + report({ instrumented: await failproofai.instrument("ai") }); + const customer = customerTracing(); + const text = await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }); + report({ text, customer: customer === null ? "absent" : { registered: customer.registered, ended: customer.ended() } }); + break; + } + case "wrap-stream-cancel": { + const reader = await openStream(await wrapModel(scripted("answer"))); + await reader.read(); + await reader.cancel("client disconnected"); + report({ cancelled: true }); + break; + } + case "wrap-stream-error": { + const reader = await openStream(await wrapModel(breakMidStream(scripted("answer")))); + try { + while (!(await reader.read()).done) { + // drain + } + report({ drained: true }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "uninstrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ removed: failproofai.uninstrument() }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "tool-error": + try { + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }), failTool: true }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "model-error": + try { + report({ text: await ask(scripted("fail"), { telemetry: telemetry({ functionId: "weather-agent" }), tools: false }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "scope": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) })), + ); + break; + case "readme": + await readme(); + break; + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-5/package-lock.json b/sdk/typescript/integration/fixtures/ai-5/package-lock.json new file mode 100644 index 000000000..44715939c --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/package-lock.json @@ -0,0 +1,168 @@ +{ + "name": "failproofai-it-ai-5", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-ai-5", + "dependencies": { + "ai": "5.0.263", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "2.0.155", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-2.0.155.tgz", + "integrity": "sha512-bPy/+zFCfmkfzizBPHDhhyHUVpcGirFKkGx/SYeeWcDqQtch3CQhXFYN52oWHb66EwAo8djMSGrOJ5BTBo37Zw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.4", + "@ai-sdk/provider-utils": "3.0.37", + "@vercel/oidc": "3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.4.tgz", + "integrity": "sha512-B0M50w0W43jTzZGyZrNwwaK1M79aKPw6PD/dQhkBR7vkR68yCINWBo2CbBLJrbrqYc46TaiSfsTiT+S9EmVl0w==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "3.0.37", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.37.tgz", + "integrity": "sha512-mLx1SgE20xKQ87xME+9uG6pFlYPTqGDrMFrZ6XcytDnmbWGwmCIh2Xb9zFrfXUrKycSFRDet4Zovj388L9b1dg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.4", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6", + "undici": "^5.29.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", + "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/ai": { + "version": "5.0.263", + "resolved": "https://registry.npmjs.org/ai/-/ai-5.0.263.tgz", + "integrity": "sha512-ncQZ81dMXXpbOAuqaIxWLaqUyRHeYLvYmPO5b+Z8+LA2jaqYIohzNkF3OOlE8kZHDz4gkQwf3vO6hCVowMBPsw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "2.0.155", + "@ai-sdk/provider": "2.0.4", + "@ai-sdk/provider-utils": "3.0.37", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/ai-5/package.json b/sdk/typescript/integration/fixtures/ai-5/package.json new file mode 100644 index 000000000..a6a1c2139 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/package.json @@ -0,0 +1,13 @@ +{ + "name": "failproofai-it-ai-5", + "private": true, + "type": "module", + "description": "Integration fixture: Vercel AI SDK 5.x against the packed @failproofai/sdk.", + "dependencies": { + "ai": "5.0.263", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/ai-5/surfaces.ts b/sdk/typescript/integration/fixtures/ai-5/surfaces.ts new file mode 100644 index 000000000..4fc899697 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/surfaces.ts @@ -0,0 +1,628 @@ +// Vercel AI SDK 5.x: every commonly used surface. Run as `node surfaces.{mjs,cjs} `. +// +// The coverage companion to agent.ts. agent.ts pins the README call sites +// and the adapter's core paths; this program walks the rest of the SDK a +// customer actually uses — the agent classes, embeddings, structured output, +// every tool feature, every way of consuming a stream, reasoning models and +// concurrency — against the same scripted, offline mock. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; the section above it spells one major's API. +import * as failproofai from "@failproofai/sdk"; +import * as adapter from "@failproofai/sdk/ai"; +import { telemetry, wrapModel } from "@failproofai/sdk/ai"; +import { + Experimental_Agent, + Output, + embed, + embedMany, + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, +} from "ai"; +import { MockEmbeddingModelV2, MockLanguageModelV2 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: ai 5 (LanguageModelV2, Experimental_Agent, no tool approval) + +const MAJOR = 5; + +interface Call { + id: string; + name: string; + input: Record; +} +/** One scripted model step: what the model "says" on its n-th call. */ +interface Step { + text?: string; + reasoning?: string; + calls?: Call[]; + usage: [number, number]; + reasoningTokens?: number; + fail?: string; + delayMs?: number; +} +interface ModelOptions { + modelId?: string; + chunkDelayMs?: number; + /** Break the provider stream after this many parts: the connection dropped. */ + breakAfterParts?: number; +} + +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; +type Content = Awaited>["content"][number]; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** A string as two stream deltas. */ +const halves = (text: string): string[] => [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]; + +const usageOf = (step: Step) => ({ + inputTokens: step.usage[0], + outputTokens: step.usage[1], + totalTokens: step.usage[0] + step.usage[1], + ...(step.reasoningTokens === undefined ? {} : { reasoningTokens: step.reasoningTokens }), +}); +const finishOf = (step: Step) => (step.calls?.length ? "tool-calls" : "stop"); + +function model(steps: Step[], options: ModelOptions = {}) { + let n = 0; + const next = async (): Promise => { + const step = steps[Math.min(n, steps.length - 1)]!; + n += 1; + if (step.delayMs) await sleep(step.delayMs); + if (step.fail) throw new Error(step.fail); + return step; + }; + return new MockLanguageModelV2({ + provider: "mock-provider", + modelId: options.modelId ?? "mock-model", + doGenerate: async () => { + const step = await next(); + const content: Content[] = []; + if (step.reasoning) content.push({ type: "reasoning", text: step.reasoning }); + if (step.text) content.push({ type: "text", text: step.text }); + for (const call of step.calls ?? []) { + content.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + return { content, finishReason: finishOf(step), usage: usageOf(step), warnings: [] }; + }, + doStream: async ({ abortSignal }) => { + const step = await next(); + const chunks: StreamPart[] = [{ type: "stream-start", warnings: [] }]; + if (step.reasoning) { + chunks.push({ type: "reasoning-start", id: "r" }); + for (const delta of halves(step.reasoning)) chunks.push({ type: "reasoning-delta", id: "r", delta }); + chunks.push({ type: "reasoning-end", id: "r" }); + } + if (step.text) { + chunks.push({ type: "text-start", id: "t" }); + for (const delta of halves(step.text)) chunks.push({ type: "text-delta", id: "t", delta }); + chunks.push({ type: "text-end", id: "t" }); + } + for (const call of step.calls ?? []) { + chunks.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + chunks.push({ type: "finish", finishReason: finishOf(step), usage: usageOf(step) }); + return { stream: abortable(simulateReadableStream({ chunks, chunkDelayInMs: options.chunkDelayMs ?? 0 }), abortSignal, options.breakAfterParts) }; + }, + }); +} + +function embedder() { + return new MockEmbeddingModelV2({ + provider: "mock-provider", + modelId: "mock-embedder", + maxEmbeddingsPerCall: 2, + doEmbed: async ({ values }) => ({ embeddings: values.map((_, i) => [i, 0.5]), usage: { tokens: values.length * 3 } }), + }); +} + +const citySchema = z.object({ city: z.string() }); +type Weather = { city: string; celsius: number }; + +function weatherTool(options: { fail?: boolean; delayMs?: (city: string) => number; onRun?: (city: string) => Promise } = {}) { + return tool({ + description: "Current weather for a city", + inputSchema: citySchema, + execute: async ({ city }: { city: string }): Promise => { + if (options.delayMs) await sleep(options.delayMs(city)); + if (options.onRun) await options.onRun(city); + if (options.fail) throw new Error("weather service down"); + return { city, celsius: city.length * 4 }; + }, + }); +} +/** A client-side tool: no `execute`, so the SDK hands the call back to the caller. */ +const clientTool = () => tool({ description: "Ask the user", inputSchema: citySchema }); +/** ai 5 has no tool approval (`needsApproval` arrived in 6). */ +const HAS_APPROVAL = false; + +const steps = (n: number) => ({ stopWhen: stepCountIs(n) }); +const HAS_AGENT = true; + +type Settings = Parameters[0]; +type Telemetry = NonNullable; + +/** ai 5's agent class has no `id`; the option is accepted and ignored here. */ +function makeAgent(options: { model: Settings["model"]; telemetry?: Telemetry; id?: string; tools?: Settings["tools"] }) { + const agent = new Experimental_Agent({ + model: options.model, + tools: options.tools ?? { weather: weatherTool() }, + ...steps(4), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return { + generate: async (prompt: string) => (await agent.generate({ prompt })).text, + stream: async (prompt: string) => { + const result = agent.stream({ prompt }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; + }, + }; +} + +/** `generateText` / `streamText` with `experimental_output: Output.object(...)`. */ +const HAS_OUTPUT = true; +async function textWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = await generateText({ model: m, prompt: "Where?", experimental_output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + return result.experimental_output; +} +async function streamWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = streamText({ model: m, prompt: "Where?", experimental_output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.experimental_partialOutputStream) partials.push(partial); + return partials; +} + +/** What a Next.js route handler returns. */ +const toResponse = (result: { toUIMessageStreamResponse(): Response }): Response => result.toUIMessageStreamResponse(); + +const repairOption = (fixed: Record) => ({ + experimental_repairToolCall: async ({ toolCall }: { toolCall: T }) => ({ ...toolCall, input: JSON.stringify(fixed) }), +}); +const prepareStepOption = (second: Settings["model"]) => ({ + prepareStep: async ({ stepNumber }: { stepNumber: number }) => (stepNumber === 1 ? { model: second } : {}), +}); + +async function approveAndResume(_m: Settings["model"], _tel: Telemetry): Promise<{ pending: number; text: string }> { + throw new Error("ai 5 has no tool approval"); +} + +const partialObjects = async (m: Settings["model"], tel: Telemetry) => { + const result = streamObject({ model: m, schema: citySchema, prompt: "Where?", experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialObjectStream) partials.push(partial); + return { partials, object: await result.object }; +}; + +// --- the same in every fixture + +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** + * A provider stream that honours the call's abort signal, as a real provider's + * `fetch` does — once aborted, the next read fails with an AbortError — and + * that can drop its connection part-way (`breakAfter`). + */ +function abortable(stream: ReadableStream, signal: AbortSignal | undefined, breakAfter?: number): ReadableStream { + const reader = stream.getReader(); + let parts = 0; + return new ReadableStream({ + async pull(controller) { + if (signal?.aborted) { + controller.error(signal.reason ?? new DOMException("aborted", "AbortError")); + return; + } + if (breakAfter !== undefined && parts++ === breakAfter) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + cancel: (reason) => reader.cancel(reason), + }); +} + +/** + * Run the garbage collector until it has had a real chance at everything + * unreachable. `--expose-gc` switched on at runtime, so the harness needs no + * special flags. + */ +async function collectGarbage(): Promise { + const v8 = await import("node:v8"); + const vm = await import("node:vm"); + v8.setFlagsFromString("--expose-gc"); + const gc = vm.runInNewContext("gc") as () => void; + for (let i = 0; i < 10; i += 1) { + gc(); + await sleep(20); + } +} + +/** + * A route's streamed Response whose client went away after three chunks. + * Its own function, so nothing of the stream is left on `main`'s frame + * afterwards — the garbage collector may take all of it. + */ +async function disconnectedClient(withSignal = false): Promise<{ chunks: number }> { + // A Next.js route can pass `abortSignal: request.signal`; the disconnect then + // aborts the call as well as cancelling the body. + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ + model: m, + prompt: "Count", + ...(withSignal ? { abortSignal: controller.signal } : {}), + experimental_telemetry: telemetry({ functionId: "counter" }), + }); + const reader = toResponse(result).body!.getReader(); + let chunks = 0; + while (chunks < 3 && !(await reader.read()).done) chunks += 1; + controller.abort(); + await reader.cancel("client disconnected"); + return { chunks }; +} + +/** An aborted stream, in a function of its own for the same reason. */ +async function abortedStream(): Promise<{ text: string; threw: string | false }> { + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ model: m, prompt: "Count", abortSignal: controller.signal, experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) { + text += delta; + controller.abort(); + } + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).name }; + } +} + +/** A stream started and never read. */ +function neverRead(): void { + streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); +} + +/** A stream whose provider connection drops part-way; the error is the caller's to see. */ +async function brokenStream(): Promise<{ text: string; threw: string | false }> { + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { breakAfterParts: 2 }); + const result = streamText({ model: m, prompt: "Count", experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) text += delta; + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).message }; + } +} + +/** The adapter's bookkeeping (internal, untyped): what it still holds open. */ +const held = () => { + const internals = (adapter as unknown as { _internals: { openCalls(): number; tracker(): { stats(): unknown } | null } })._internals; + return { openCalls: internals.openCalls(), stats: internals.tracker()?.stats() ?? null }; +}; + +const LOOP: Step[] = [ + { calls: [{ id: "call-1", name: "weather", input: { city: "Paris" } }], usage: [11, 7] }, + { text: "It is 20C in Paris.", usage: [23, 9] }, +]; + +async function drainText(stream: AsyncIterable): Promise { + let text = ""; + for await (const delta of stream) text += delta; + return text; +} + +async function main(scenario: string): Promise { + switch (scenario) { + // ---- 1. the agent classes + case "agent-generate": + case "agent-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), telemetry: telemetry({ functionId: "support-agent" }) }); + report({ text: scenario === "agent-generate" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-id": { + // The agent's own `id` never reaches telemetry: the SDK spreads it into + // generateText, which drops it. functionId is what names the agent. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), id: "support-agent", telemetry: telemetry() }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-instrument": + case "agent-instrument-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai") }); + const agent = makeAgent({ model: model(LOOP), telemetry: { isEnabled: true, functionId: "support-agent" } }); + report({ text: scenario === "agent-instrument" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-instrument-bare": { + // No telemetry setting at all: ai 7 records every call once an + // integration is registered; ai 4–6 record nothing without isEnabled. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: false }) }); + report({ text: await makeAgent({ model: model(LOOP) }).generate("Weather in Paris?") }); + break; + } + case "agent-wrap": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-wrap-in-scope": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("support-agent", { goal: "help" }, async () => report({ text: await agent.stream("Weather in Paris?") })), + ); + break; + } + + // ---- 2. embeddings + case "embed": + report({ embedding: (await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry({ functionId: "indexer" }) })).embedding }); + break; + case "embed-many": { + const { embeddings } = await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embeddings: embeddings.length }); + break; + } + case "embed-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("rag", { goal: "answer" }, async () => { + await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry() }); + await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embedded: true }); + }), + ); + break; + case "embed-in-tool": { + const lookup = weatherTool({ + onRun: async (city) => { + await embed({ model: embedder(), value: city, experimental_telemetry: telemetry() }); + }, + }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: lookup }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + + // ---- 3. structured output + case "object-partial": { + const m = model([{ text: '{"city":"Paris"}', usage: [5, 3] }]); + report(await partialObjects(m, telemetry({ functionId: "extractor" }))); + break; + } + case "object-invalid": + try { + await generateObject({ model: model([{ text: '{"town":"Paris"}', usage: [5, 3] }]), schema: citySchema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + case "text-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + report({ output: await textWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })) }); + break; + } + case "stream-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + const partials = await streamWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })); + report({ partials: partials.length, last: partials[partials.length - 1] }); + break; + } + + // ---- 4. tool features + case "parallel-tools": { + const m = model([ + { + calls: [ + { id: "call-p1", name: "weather", input: { city: "Paris" } }, + { id: "call-p2", name: "weather", input: { city: "Rome" } }, + ], + usage: [11, 7], + }, + { text: "Paris 20C, Rome 16C.", usage: [40, 9] }, + ]); + // Rome finishes first: results arrive out of call order. + const tools = { weather: weatherTool({ delayMs: (city) => (city === "Paris" ? 40 : 5) }) }; + const { text } = await generateText({ model: m, prompt: "Weather?", tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "client-tool": { + const m = model([{ calls: [{ id: "call-c1", name: "ask", input: { city: "Paris" } }], usage: [11, 7] }]); + const result = await generateText({ model: m, prompt: "Weather?", tools: { ask: clientTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ toolCalls: result.toolCalls.length }); + break; + } + case "tool-choice-required": { + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, toolChoice: "required", ...steps(1), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "repair": { + const m = model([{ calls: [{ id: "call-r1", name: "weather", input: { town: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...repairOption({ city: "Paris" }), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "prepare-step": { + const second = model([LOOP[1]!], { modelId: "mock-model-large" }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...prepareStepOption(second), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "unknown-tool": { + const m = model([{ calls: [{ id: "call-u1", name: "teleport", input: { city: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + try { + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + } + case "approval": { + if (!HAS_APPROVAL) return report({ skipped: "no tool approval" }); + const m = model([{ calls: [{ id: "call-a1", name: "book", input: { city: "Paris" } }], usage: [11, 7] }, { text: "Booked Paris.", usage: [30, 4] }]); + report(await approveAndResume(m, telemetry({ functionId: "travel-agent" }))); + break; + } + + // ---- 5. consuming a stream + case "stream-full": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const types = new Set(); + for await (const part of result.fullStream) types.add(part.type); + report({ parts: [...types].sort(), text: await result.text }); + break; + } + case "stream-response": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const response = toResponse(result); + const body = await response.text(); + report({ status: response.status, containsAnswer: body.includes("Paris"), bytes: body.length }); + break; + } + case "stream-response-cancel": + // The browser went away mid-stream. Nothing in the SDK ends the operation + // after that; the adapter closes it once the stream is garbage. + report(await disconnectedClient()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-response-cancel-signal": + report(await disconnectedClient(true)); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-error": + report(await brokenStream()); + await sleep(50); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-on-finish": { + let finished = ""; + const result = streamText({ + model: model(LOOP), + prompt: "Weather?", + tools: { weather: weatherTool() }, + ...steps(4), + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + onFinish: ({ text }) => { + finished = text; + }, + }); + await result.consumeStream(); + report({ finished }); + break; + } + case "stream-abort": + report(await abortedStream()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-unconsumed": + // Never read. Nothing waits on it, so nothing should be held open. + neverRead(); + await sleep(200); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + + // ---- 6. reasoning models + case "reasoning-generate": + case "reasoning-stream": { + const m = model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }]); + const options = { model: m, prompt: "Capital of France?", providerOptions: { mock: { reasoningEffort: "high" } }, experimental_telemetry: telemetry({ functionId: "thinker" }) }; + if (scenario === "reasoning-generate") { + const result = await generateText(options); + report({ text: result.text }); + } else { + const result = streamText(options); + report({ text: await drainText(result.textStream) }); + } + break; + } + case "reasoning-wrap": { + const m = await wrapModel(model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }])); + const result = streamText({ model: m, prompt: "Capital of France?" }); + report({ text: await drainText(result.textStream) }); + break; + } + + // ---- 7. concurrency + case "concurrent": + case "concurrent-stream": { + const one = async (i: number) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2], delayMs: (i * 3) % 5 }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + const options = { model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: `worker-${i}` }) }; + if (scenario === "concurrent") return (await generateText(options)).text; + return await drainText(streamText(options).textStream); + }; + const texts = await failproofai.session({ sessionId: "busy" }, () => Promise.all(Array.from({ length: 10 }, (_, i) => one(i)))); + report({ texts }); + break; + } + case "concurrent-unscoped": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2] }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + return (await generateText({ model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "worker" }) })).text; + }), + ); + report({ texts }); + break; + } + case "concurrent-wrap": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = await wrapModel(model([{ text: `answer ${i}`, usage: [100 + i, 1], delayMs: (i * 7) % 11 }], { modelId: `model-${i}` })); + return (await generateText({ model: m, prompt: `q${i}` })).text; + }), + ); + report({ texts }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + report({ major: MAJOR }); + await failproofai.flush(); +} + +main(process.argv[2] ?? "agent-generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-5/tsconfig.json b/sdk/typescript/integration/fixtures/ai-5/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-5/tsconfig.surfaces.json b/sdk/typescript/integration/fixtures/ai-5/tsconfig.surfaces.json new file mode 100644 index 000000000..f827f9292 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-5/tsconfig.surfaces.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "files": ["surfaces.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-6/agent.ts b/sdk/typescript/integration/fixtures/ai-6/agent.ts new file mode 100644 index 000000000..83ec01a7b --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/agent.ts @@ -0,0 +1,346 @@ +// Vercel AI SDK 6.x consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import `ai`, pass the adapter at +// the call site (or wrap the model, or instrument), run. The scripted mock +// model makes it deterministic and offline — the tool loop's first call asks +// for `weather`, the next one answers — and never touches a real provider. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; only the model mock and the tool/loop +// spelling above it change between majors. +import { createRequire } from "node:module"; +import { join } from "node:path"; + +import * as failproofai from "@failproofai/sdk"; +import { middleware, telemetry, tracer, wrapModel } from "@failproofai/sdk/ai"; +import { + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, + wrapLanguageModel, +} from "ai"; +import { MockLanguageModelV3 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: LanguageModelV3 (usage objects, finishReason objects) + +type Script = "loop" | "answer" | "object" | "fail"; +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const usage = (input: number, output: number) => ({ + inputTokens: { total: input, noCache: input, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: output, text: output, reasoning: undefined }, +}); +const finish = (reason: "stop" | "tool-calls") => ({ unified: reason, raw: reason }); + +function scripted(script: Script = "loop") { + let generated = 0; + let streamed = 0; + return new MockLanguageModelV3({ + provider: "mock-provider", + modelId: "mock-model", + doGenerate: async () => { + generated += 1; + if (script === "fail") throw new Error("model exploded"); + if (script === "object") { + return { content: [{ type: "text", text: '{"city":"Paris"}' }], finishReason: finish("stop"), usage: usage(5, 3), warnings: [] }; + } + if (script === "loop" && generated === 1) { + return { + content: [{ type: "tool-call", toolCallId: "call-1", toolName: "weather", input: '{"city":"Paris"}' }], + finishReason: finish("tool-calls"), + usage: usage(11, 7), + warnings: [], + }; + } + return { content: [{ type: "text", text: "It is 20C in Paris." }], finishReason: finish("stop"), usage: usage(23, 9), warnings: [] }; + }, + doStream: async () => { + streamed += 1; + if (script === "fail") throw new Error("model exploded"); + const text = (id: string, ...deltas: string[]): StreamPart[] => [ + { type: "text-start", id }, + ...deltas.map((delta): StreamPart => ({ type: "text-delta", id, delta })), + { type: "text-end", id }, + ]; + const chunks: StreamPart[] = + script === "object" + ? [{ type: "stream-start", warnings: [] }, ...text("o", '{"city":', '"Paris"}'), { type: "finish", finishReason: finish("stop"), usage: usage(5, 3) }] + : script === "loop" && streamed === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "tool-call", toolCallId: "call-s1", toolName: "weather", input: '{"city":"Rome"}' }, + { type: "finish", finishReason: finish("tool-calls"), usage: usage(13, 4) }, + ] + : [{ type: "stream-start", warnings: [] }, ...text("t", "Rome is ", "25C."), { type: "finish", finishReason: finish("stop"), usage: usage(30, 6) }]; + return { stream: simulateReadableStream({ chunks }) }; + }, + }); +} + +const makeTools = (fail = false) => ({ + weather: tool({ + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => { + if (fail) throw new Error("weather service down"); + return { city, celsius: city === "Paris" ? 20 : 25 }; + }, + }), +}); +const loop = { stopWhen: stepCountIs(4) }; + +/** The bare tracer, as `experimental_telemetry.tracer` (v4–v6 only: v7 has no such option). */ +async function viaTracer(prompt: string): Promise { + const { text } = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: { isEnabled: true, tracer: tracer() }, + }); + return text; +} + +// --- the same in every fixture + +type Telemetry = ReturnType | { isEnabled: true; functionId: string }; + +async function ask(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean; failTool?: boolean } = {}) { + const result = await generateText({ + model, + prompt: "Weather in Paris?", + ...(options.tools === false ? {} : { tools: makeTools(options.failTool), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return result.text; +} + +async function askStreaming(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean } = {}) { + const result = streamText({ + model, + prompt: "Weather in Rome?", + ...(options.tools === false ? {} : { tools: makeTools(), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; +} + +const schema = z.object({ city: z.string() }); +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** The README's call sites, verbatim apart from the mock standing in for a provider. */ +async function readme(): Promise { + const model = scripted("answer"); + const prompt = "Weather in Paris?"; + const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + }); + const wrapped = await wrapModel(scripted("answer")); + const viaMiddleware = wrapLanguageModel({ model: scripted("answer"), middleware: middleware() }); + const withMetadata = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: telemetry({ functionId: "tagged", metadata: { tenant: "acme", attempt: 1 } }), + }); + report({ text, wrapped: (await generateText({ model: wrapped, prompt })).text, viaMiddleware: viaMiddleware.modelId, withMetadata: withMetadata.text, withTracer: await viaTracer(prompt) }); +} + +/** + * The customer's own OpenTelemetry, set up AFTER `instrument("ai")` — the + * usual order when tracing starts in a module loaded later (a `NodeSDK` + * started from an instrumentation file). Null when `@opentelemetry/api` is not + * installed (ai 7 dropped the dependency). + */ +function customerTracing(): { registered: boolean; ended: () => string[] } | null { + interface Api { + trace: { + setGlobalTracerProvider(provider: unknown): boolean; + getTracer(name: string): { startSpan(name: string): { end(): void } }; + }; + } + let api: Api; + try { + api = createRequire(join(process.cwd(), "agent.js"))("@opentelemetry/api") as Api; + } catch { + return null; + } + const ended: string[] = []; + const span = (name: string) => ({ + setAttribute() { return this; }, + setAttributes() { return this; }, + addEvent() { return this; }, + addLink() { return this; }, + addLinks() { return this; }, + setStatus() { return this; }, + updateName() { return this; }, + recordException() {}, + isRecording: () => true, + spanContext: () => ({ traceId: "0".repeat(31) + "1", spanId: "0".repeat(15) + "1", traceFlags: 1 }), + end: () => void ended.push(name), + }); + const theirs = { + startSpan: (name: string) => span(name), + startActiveSpan: (name: string, ...rest: unknown[]) => (rest[rest.length - 1] as (s: unknown) => unknown)(span(name)), + }; + const registered = api.trace.setGlobalTracerProvider({ getTracer: () => theirs }); + // What an http / pg / Next.js instrumentation does with the global API. + api.trace.getTracer("my-service").startSpan("http.request").end(); + return { registered, ended: () => [...new Set(ended)].sort() }; +} + +/** A model whose stream breaks part-way: the provider connection dropped. */ +function breakMidStream(model: M): M { + const target = model as unknown as { doStream: (options: unknown) => PromiseLike<{ stream: ReadableStream }> }; + const original = target.doStream.bind(target); + target.doStream = async (options: unknown) => { + const result = await original(options); + const reader = result.stream.getReader(); + let parts = 0; + return { + ...result, + stream: new ReadableStream({ + async pull(controller) { + if (parts++ === 2) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + }), + }; + }; + return model; +} + +/** Call a model's `doStream` directly, as a provider-level consumer does. */ +async function openStream(model: unknown): Promise> { + const call = { + inputFormat: "prompt", + mode: { type: "regular" }, + prompt: [{ role: "user", content: [{ type: "text", text: "Weather in Rome?" }] }], + }; + const { stream } = await (model as { doStream(options: unknown): PromiseLike<{ stream: ReadableStream }> }).doStream(call); + return stream.getReader(); +} + +async function main(scenario: string): Promise { + switch (scenario) { + case "generate": + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "stream": + report({ text: await askStreaming(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "object": { + const { object } = await generateObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ object }); + break; + } + case "stream-object": { + const result = streamObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + for await (const _ of result.partialObjectStream) void _; + report({ object: await result.object }); + break; + } + case "wrap": + report({ text: await ask(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-stream": + report({ text: await askStreaming(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, async () => report({ text: await ask(await wrapModel(scripted())) })), + ); + break; + case "wrap-and-telemetry": + report({ text: await ask(await wrapModel(scripted()), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "instrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-stream": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global-stream": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-then-otel": { + report({ instrumented: await failproofai.instrument("ai") }); + const customer = customerTracing(); + const text = await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }); + report({ text, customer: customer === null ? "absent" : { registered: customer.registered, ended: customer.ended() } }); + break; + } + case "wrap-stream-cancel": { + const reader = await openStream(await wrapModel(scripted("answer"))); + await reader.read(); + await reader.cancel("client disconnected"); + report({ cancelled: true }); + break; + } + case "wrap-stream-error": { + const reader = await openStream(await wrapModel(breakMidStream(scripted("answer")))); + try { + while (!(await reader.read()).done) { + // drain + } + report({ drained: true }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "uninstrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ removed: failproofai.uninstrument() }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "tool-error": + try { + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }), failTool: true }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "model-error": + try { + report({ text: await ask(scripted("fail"), { telemetry: telemetry({ functionId: "weather-agent" }), tools: false }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "scope": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) })), + ); + break; + case "readme": + await readme(); + break; + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-6/package-lock.json b/sdk/typescript/integration/fixtures/ai-6/package-lock.json new file mode 100644 index 000000000..cc30af2cf --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/package-lock.json @@ -0,0 +1,156 @@ +{ + "name": "failproofai-it-ai-6", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-ai-6", + "dependencies": { + "ai": "6.0.288", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.198", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.198.tgz", + "integrity": "sha512-Ji11GLswEsPkDZHMrG1s/Yq7NhmSUZGHM0IJB3lfpjE0MnGrYk1egmWzPL3GmDWJCw2wab8BgerSi68JiabaXQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.52", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.16.tgz", + "integrity": "sha512-9Av6kg0t/IN/dcYAAmEJ4B9OPhcEJqwSR+GfHEA8olRkindXpUHadc3p3cyvgFUQFTVm1G5thtsmgZ9yVb2w3A==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.52", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.52.tgz", + "integrity": "sha512-cUimyIz1jjwYoF9n6HO7sU+UBW7Nu3T4lW7562yRvEhM9/8JqlW4cecxSpkiafjRzAphZG2NfJ3/Ch5cHoCIhg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" + }, + "engines": { + "node": ">=18.17" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/ai": { + "version": "6.0.288", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.288.tgz", + "integrity": "sha512-p7N1TMAPkTWlapALDVQQY3+QbUKP1nKatz/yg9SArWWjUdUyzahyCN43WG+cTQBlN5mj5sysRTjDvZWRezpZeA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.198", + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.52", + "@opentelemetry/api": "^1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/undici": { + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/ai-6/package.json b/sdk/typescript/integration/fixtures/ai-6/package.json new file mode 100644 index 000000000..a4ea7c036 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/package.json @@ -0,0 +1,13 @@ +{ + "name": "failproofai-it-ai-6", + "private": true, + "type": "module", + "description": "Integration fixture: Vercel AI SDK 6.x against the packed @failproofai/sdk.", + "dependencies": { + "ai": "6.0.288", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/ai-6/surfaces.ts b/sdk/typescript/integration/fixtures/ai-6/surfaces.ts new file mode 100644 index 000000000..89298c5dc --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/surfaces.ts @@ -0,0 +1,651 @@ +// Vercel AI SDK 6.x: every commonly used surface. Run as `node surfaces.{mjs,cjs} `. +// +// The coverage companion to agent.ts. agent.ts pins the README call sites +// and the adapter's core paths; this program walks the rest of the SDK a +// customer actually uses — the agent classes, embeddings, structured output, +// every tool feature, every way of consuming a stream, reasoning models and +// concurrency — against the same scripted, offline mock. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; the section above it spells one major's API. +import * as failproofai from "@failproofai/sdk"; +import * as adapter from "@failproofai/sdk/ai"; +import { telemetry, wrapModel } from "@failproofai/sdk/ai"; +import { + Output, + ToolLoopAgent, + embed, + embedMany, + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, + type ModelMessage, +} from "ai"; +import { MockEmbeddingModelV3, MockLanguageModelV3 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: ai 6 (LanguageModelV3, ToolLoopAgent, needsApproval) + +const MAJOR = 6; + +interface Call { + id: string; + name: string; + input: Record; +} +/** One scripted model step: what the model "says" on its n-th call. */ +interface Step { + text?: string; + reasoning?: string; + calls?: Call[]; + usage: [number, number]; + reasoningTokens?: number; + fail?: string; + delayMs?: number; +} +interface ModelOptions { + modelId?: string; + chunkDelayMs?: number; + /** Break the provider stream after this many parts: the connection dropped. */ + breakAfterParts?: number; +} + +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; +type Content = Awaited>["content"][number]; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** A string as two stream deltas. */ +const halves = (text: string): string[] => [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]; + +const usageOf = (step: Step) => ({ + inputTokens: { total: step.usage[0], noCache: step.usage[0], cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: step.usage[1], text: step.usage[1] - (step.reasoningTokens ?? 0), reasoning: step.reasoningTokens }, +}); +const finishOf = (step: Step) => { + const reason = step.calls?.length ? "tool-calls" : "stop"; + return { unified: reason, raw: reason } as const; +}; + +function model(steps: Step[], options: ModelOptions = {}) { + let n = 0; + const next = async (): Promise => { + const step = steps[Math.min(n, steps.length - 1)]!; + n += 1; + if (step.delayMs) await sleep(step.delayMs); + if (step.fail) throw new Error(step.fail); + return step; + }; + return new MockLanguageModelV3({ + provider: "mock-provider", + modelId: options.modelId ?? "mock-model", + doGenerate: async () => { + const step = await next(); + const content: Content[] = []; + if (step.reasoning) content.push({ type: "reasoning", text: step.reasoning }); + if (step.text) content.push({ type: "text", text: step.text }); + for (const call of step.calls ?? []) { + content.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + return { content, finishReason: finishOf(step), usage: usageOf(step), warnings: [] }; + }, + doStream: async ({ abortSignal }) => { + const step = await next(); + const chunks: StreamPart[] = [{ type: "stream-start", warnings: [] }]; + if (step.reasoning) { + chunks.push({ type: "reasoning-start", id: "r" }); + for (const delta of halves(step.reasoning)) chunks.push({ type: "reasoning-delta", id: "r", delta }); + chunks.push({ type: "reasoning-end", id: "r" }); + } + if (step.text) { + chunks.push({ type: "text-start", id: "t" }); + for (const delta of halves(step.text)) chunks.push({ type: "text-delta", id: "t", delta }); + chunks.push({ type: "text-end", id: "t" }); + } + for (const call of step.calls ?? []) { + chunks.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + chunks.push({ type: "finish", finishReason: finishOf(step), usage: usageOf(step) }); + return { stream: abortable(simulateReadableStream({ chunks, chunkDelayInMs: options.chunkDelayMs ?? 0 }), abortSignal, options.breakAfterParts) }; + }, + }); +} + +function embedder() { + return new MockEmbeddingModelV3({ + provider: "mock-provider", + modelId: "mock-embedder", + maxEmbeddingsPerCall: 2, + doEmbed: async ({ values }) => ({ embeddings: values.map((_, i) => [i, 0.5]), usage: { tokens: values.length * 3 }, warnings: [] }), + }); +} + +const citySchema = z.object({ city: z.string() }); +type Weather = { city: string; celsius: number }; + +function weatherTool(options: { fail?: boolean; delayMs?: (city: string) => number; onRun?: (city: string) => Promise } = {}) { + return tool({ + description: "Current weather for a city", + inputSchema: citySchema, + execute: async ({ city }: { city: string }): Promise => { + if (options.delayMs) await sleep(options.delayMs(city)); + if (options.onRun) await options.onRun(city); + if (options.fail) throw new Error("weather service down"); + return { city, celsius: city.length * 4 }; + }, + }); +} +/** A client-side tool: no `execute`, so the SDK hands the call back to the caller. */ +const clientTool = () => tool({ description: "Ask the user", inputSchema: citySchema }); +const HAS_APPROVAL = true; +const approvalTool = () => + tool({ + description: "Book a trip", + inputSchema: citySchema, + needsApproval: true, + execute: async ({ city }: { city: string }) => ({ booked: city }), + }); + +const steps = (n: number) => ({ stopWhen: stepCountIs(n) }); +const HAS_AGENT = true; + +type Settings = Parameters[0]; +type Telemetry = NonNullable; + +function makeAgent(options: { model: Settings["model"]; telemetry?: Telemetry; id?: string; tools?: Settings["tools"] }) { + const agent = new ToolLoopAgent({ + model: options.model, + ...(options.id ? { id: options.id } : {}), + tools: options.tools ?? { weather: weatherTool() }, + ...steps(4), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return { + generate: async (prompt: string) => (await agent.generate({ prompt })).text, + stream: async (prompt: string) => { + const result = await agent.stream({ prompt }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; + }, + }; +} + +/** `generateText` / `streamText` with `output: Output.object(...)`. */ +const HAS_OUTPUT = true; +async function textWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = await generateText({ model: m, prompt: "Where?", output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + return result.output; +} +async function streamWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = streamText({ model: m, prompt: "Where?", output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialOutputStream) partials.push(partial); + return partials; +} + +/** What a Next.js route handler returns. */ +const toResponse = (result: { toUIMessageStreamResponse(): Response }): Response => result.toUIMessageStreamResponse(); + +const repairOption = (fixed: Record) => ({ + experimental_repairToolCall: async ({ toolCall }: { toolCall: T }) => ({ ...toolCall, input: JSON.stringify(fixed) }), +}); +const prepareStepOption = (second: Settings["model"]) => ({ + prepareStep: async ({ stepNumber }: { stepNumber: number }) => (stepNumber === 1 ? { model: second } : {}), +}); + +/** Run a tool that needs approval: ask, approve, resume. Returns the final text. */ +async function approveAndResume(m: Settings["model"], tel: Telemetry): Promise<{ pending: number; text: string }> { + const prompt: ModelMessage[] = [{ role: "user", content: "Book Paris" }]; + const first = await generateText({ model: m, messages: prompt, tools: { book: approvalTool() }, ...steps(4), experimental_telemetry: tel }); + const requests = first.content.filter((part) => part.type === "tool-approval-request"); + const approvals: ModelMessage = { + role: "tool", + content: requests.map((request) => ({ type: "tool-approval-response" as const, approvalId: request.approvalId, approved: true })), + }; + const second = await generateText({ + model: m, + messages: [...prompt, ...first.response.messages, approvals], + tools: { book: approvalTool() }, + ...steps(4), + experimental_telemetry: tel, + }); + return { pending: requests.length, text: second.text }; +} + +const partialObjects = async (m: Settings["model"], tel: Telemetry) => { + const result = streamObject({ model: m, schema: citySchema, prompt: "Where?", experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialObjectStream) partials.push(partial); + return { partials, object: await result.object }; +}; + +// --- the same in every fixture + +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** + * A provider stream that honours the call's abort signal, as a real provider's + * `fetch` does — once aborted, the next read fails with an AbortError — and + * that can drop its connection part-way (`breakAfter`). + */ +function abortable(stream: ReadableStream, signal: AbortSignal | undefined, breakAfter?: number): ReadableStream { + const reader = stream.getReader(); + let parts = 0; + return new ReadableStream({ + async pull(controller) { + if (signal?.aborted) { + controller.error(signal.reason ?? new DOMException("aborted", "AbortError")); + return; + } + if (breakAfter !== undefined && parts++ === breakAfter) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + cancel: (reason) => reader.cancel(reason), + }); +} + +/** + * Run the garbage collector until it has had a real chance at everything + * unreachable. `--expose-gc` switched on at runtime, so the harness needs no + * special flags. + */ +async function collectGarbage(): Promise { + const v8 = await import("node:v8"); + const vm = await import("node:vm"); + v8.setFlagsFromString("--expose-gc"); + const gc = vm.runInNewContext("gc") as () => void; + for (let i = 0; i < 10; i += 1) { + gc(); + await sleep(20); + } +} + +/** + * A route's streamed Response whose client went away after three chunks. + * Its own function, so nothing of the stream is left on `main`'s frame + * afterwards — the garbage collector may take all of it. + */ +async function disconnectedClient(withSignal = false): Promise<{ chunks: number }> { + // A Next.js route can pass `abortSignal: request.signal`; the disconnect then + // aborts the call as well as cancelling the body. + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ + model: m, + prompt: "Count", + ...(withSignal ? { abortSignal: controller.signal } : {}), + experimental_telemetry: telemetry({ functionId: "counter" }), + }); + const reader = toResponse(result).body!.getReader(); + let chunks = 0; + while (chunks < 3 && !(await reader.read()).done) chunks += 1; + controller.abort(); + await reader.cancel("client disconnected"); + return { chunks }; +} + +/** An aborted stream, in a function of its own for the same reason. */ +async function abortedStream(): Promise<{ text: string; threw: string | false }> { + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ model: m, prompt: "Count", abortSignal: controller.signal, experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) { + text += delta; + controller.abort(); + } + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).name }; + } +} + +/** A stream started and never read. */ +function neverRead(): void { + streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); +} + +/** A stream whose provider connection drops part-way; the error is the caller's to see. */ +async function brokenStream(): Promise<{ text: string; threw: string | false }> { + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { breakAfterParts: 2 }); + const result = streamText({ model: m, prompt: "Count", experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) text += delta; + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).message }; + } +} + +/** The adapter's bookkeeping (internal, untyped): what it still holds open. */ +const held = () => { + const internals = (adapter as unknown as { _internals: { openCalls(): number; tracker(): { stats(): unknown } | null } })._internals; + return { openCalls: internals.openCalls(), stats: internals.tracker()?.stats() ?? null }; +}; + +const LOOP: Step[] = [ + { calls: [{ id: "call-1", name: "weather", input: { city: "Paris" } }], usage: [11, 7] }, + { text: "It is 20C in Paris.", usage: [23, 9] }, +]; + +async function drainText(stream: AsyncIterable): Promise { + let text = ""; + for await (const delta of stream) text += delta; + return text; +} + +async function main(scenario: string): Promise { + switch (scenario) { + // ---- 1. the agent classes + case "agent-generate": + case "agent-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), telemetry: telemetry({ functionId: "support-agent" }) }); + report({ text: scenario === "agent-generate" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-id": { + // The agent's own `id` never reaches telemetry: the SDK spreads it into + // generateText, which drops it. functionId is what names the agent. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), id: "support-agent", telemetry: telemetry() }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-instrument": + case "agent-instrument-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai") }); + const agent = makeAgent({ model: model(LOOP), telemetry: { isEnabled: true, functionId: "support-agent" } }); + report({ text: scenario === "agent-instrument" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-instrument-bare": { + // No telemetry setting at all: ai 7 records every call once an + // integration is registered; ai 4–6 record nothing without isEnabled. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: false }) }); + report({ text: await makeAgent({ model: model(LOOP) }).generate("Weather in Paris?") }); + break; + } + case "agent-wrap": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-wrap-in-scope": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("support-agent", { goal: "help" }, async () => report({ text: await agent.stream("Weather in Paris?") })), + ); + break; + } + + // ---- 2. embeddings + case "embed": + report({ embedding: (await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry({ functionId: "indexer" }) })).embedding }); + break; + case "embed-many": { + const { embeddings } = await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embeddings: embeddings.length }); + break; + } + case "embed-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("rag", { goal: "answer" }, async () => { + await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry() }); + await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embedded: true }); + }), + ); + break; + case "embed-in-tool": { + const lookup = weatherTool({ + onRun: async (city) => { + await embed({ model: embedder(), value: city, experimental_telemetry: telemetry() }); + }, + }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: lookup }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + + // ---- 3. structured output + case "object-partial": { + const m = model([{ text: '{"city":"Paris"}', usage: [5, 3] }]); + report(await partialObjects(m, telemetry({ functionId: "extractor" }))); + break; + } + case "object-invalid": + try { + await generateObject({ model: model([{ text: '{"town":"Paris"}', usage: [5, 3] }]), schema: citySchema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + case "text-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + report({ output: await textWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })) }); + break; + } + case "stream-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + const partials = await streamWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })); + report({ partials: partials.length, last: partials[partials.length - 1] }); + break; + } + + // ---- 4. tool features + case "parallel-tools": { + const m = model([ + { + calls: [ + { id: "call-p1", name: "weather", input: { city: "Paris" } }, + { id: "call-p2", name: "weather", input: { city: "Rome" } }, + ], + usage: [11, 7], + }, + { text: "Paris 20C, Rome 16C.", usage: [40, 9] }, + ]); + // Rome finishes first: results arrive out of call order. + const tools = { weather: weatherTool({ delayMs: (city) => (city === "Paris" ? 40 : 5) }) }; + const { text } = await generateText({ model: m, prompt: "Weather?", tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "client-tool": { + const m = model([{ calls: [{ id: "call-c1", name: "ask", input: { city: "Paris" } }], usage: [11, 7] }]); + const result = await generateText({ model: m, prompt: "Weather?", tools: { ask: clientTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ toolCalls: result.toolCalls.length }); + break; + } + case "tool-choice-required": { + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, toolChoice: "required", ...steps(1), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "repair": { + const m = model([{ calls: [{ id: "call-r1", name: "weather", input: { town: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...repairOption({ city: "Paris" }), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "prepare-step": { + const second = model([LOOP[1]!], { modelId: "mock-model-large" }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...prepareStepOption(second), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "unknown-tool": { + const m = model([{ calls: [{ id: "call-u1", name: "teleport", input: { city: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + try { + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + } + case "approval": { + if (!HAS_APPROVAL) return report({ skipped: "no tool approval" }); + const m = model([{ calls: [{ id: "call-a1", name: "book", input: { city: "Paris" } }], usage: [11, 7] }, { text: "Booked Paris.", usage: [30, 4] }]); + report(await approveAndResume(m, telemetry({ functionId: "travel-agent" }))); + break; + } + + // ---- 5. consuming a stream + case "stream-full": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const types = new Set(); + for await (const part of result.fullStream) types.add(part.type); + report({ parts: [...types].sort(), text: await result.text }); + break; + } + case "stream-response": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const response = toResponse(result); + const body = await response.text(); + report({ status: response.status, containsAnswer: body.includes("Paris"), bytes: body.length }); + break; + } + case "stream-response-cancel": + // The browser went away mid-stream. Nothing in the SDK ends the operation + // after that; the adapter closes it once the stream is garbage. + report(await disconnectedClient()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-response-cancel-signal": + report(await disconnectedClient(true)); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-error": + report(await brokenStream()); + await sleep(50); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-on-finish": { + let finished = ""; + const result = streamText({ + model: model(LOOP), + prompt: "Weather?", + tools: { weather: weatherTool() }, + ...steps(4), + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + onFinish: ({ text }) => { + finished = text; + }, + }); + await result.consumeStream(); + report({ finished }); + break; + } + case "stream-abort": + report(await abortedStream()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-unconsumed": + // Never read. Nothing waits on it, so nothing should be held open. + neverRead(); + await sleep(200); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + + // ---- 6. reasoning models + case "reasoning-generate": + case "reasoning-stream": { + const m = model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }]); + const options = { model: m, prompt: "Capital of France?", providerOptions: { mock: { reasoningEffort: "high" } }, experimental_telemetry: telemetry({ functionId: "thinker" }) }; + if (scenario === "reasoning-generate") { + const result = await generateText(options); + report({ text: result.text }); + } else { + const result = streamText(options); + report({ text: await drainText(result.textStream) }); + } + break; + } + case "reasoning-wrap": { + const m = await wrapModel(model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }])); + const result = streamText({ model: m, prompt: "Capital of France?" }); + report({ text: await drainText(result.textStream) }); + break; + } + + // ---- 7. concurrency + case "concurrent": + case "concurrent-stream": { + const one = async (i: number) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2], delayMs: (i * 3) % 5 }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + const options = { model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: `worker-${i}` }) }; + if (scenario === "concurrent") return (await generateText(options)).text; + return await drainText(streamText(options).textStream); + }; + const texts = await failproofai.session({ sessionId: "busy" }, () => Promise.all(Array.from({ length: 10 }, (_, i) => one(i)))); + report({ texts }); + break; + } + case "concurrent-unscoped": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2] }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + return (await generateText({ model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "worker" }) })).text; + }), + ); + report({ texts }); + break; + } + case "concurrent-wrap": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = await wrapModel(model([{ text: `answer ${i}`, usage: [100 + i, 1], delayMs: (i * 7) % 11 }], { modelId: `model-${i}` })); + return (await generateText({ model: m, prompt: `q${i}` })).text; + }), + ); + report({ texts }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + report({ major: MAJOR }); + await failproofai.flush(); +} + +main(process.argv[2] ?? "agent-generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-6/tsconfig.json b/sdk/typescript/integration/fixtures/ai-6/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-6/tsconfig.surfaces.json b/sdk/typescript/integration/fixtures/ai-6/tsconfig.surfaces.json new file mode 100644 index 000000000..f827f9292 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-6/tsconfig.surfaces.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "files": ["surfaces.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-7/agent.ts b/sdk/typescript/integration/fixtures/ai-7/agent.ts new file mode 100644 index 000000000..c8049f764 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/agent.ts @@ -0,0 +1,350 @@ +// Vercel AI SDK 7.x consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import `ai`, pass the adapter at +// the call site (or wrap the model, or instrument), run. The scripted mock +// model makes it deterministic and offline — the tool loop's first call asks +// for `weather`, the next one answers — and never touches a real provider. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; only the model mock and the tool/loop +// spelling above it change between majors. v7 is ESM-only: the CommonJS run +// of this file loads it through Node's require(esm). +import { createRequire } from "node:module"; +import { join } from "node:path"; + +import * as failproofai from "@failproofai/sdk"; +import { middleware, telemetry, wrapModel } from "@failproofai/sdk/ai"; +import { + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, + wrapLanguageModel, +} from "ai"; +import { MockLanguageModelV4 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: LanguageModelV4 (usage objects, finishReason objects) + +type Script = "loop" | "answer" | "object" | "fail"; +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const usage = (input: number, output: number) => ({ + inputTokens: { total: input, noCache: input, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: output, text: output, reasoning: undefined }, +}); +const finish = (reason: "stop" | "tool-calls") => ({ unified: reason, raw: reason }); + +function scripted(script: Script = "loop") { + let generated = 0; + let streamed = 0; + return new MockLanguageModelV4({ + provider: "mock-provider", + modelId: "mock-model", + doGenerate: async () => { + generated += 1; + if (script === "fail") throw new Error("model exploded"); + if (script === "object") { + return { content: [{ type: "text", text: '{"city":"Paris"}' }], finishReason: finish("stop"), usage: usage(5, 3), warnings: [] }; + } + if (script === "loop" && generated === 1) { + return { + content: [{ type: "tool-call", toolCallId: "call-1", toolName: "weather", input: '{"city":"Paris"}' }], + finishReason: finish("tool-calls"), + usage: usage(11, 7), + warnings: [], + }; + } + return { content: [{ type: "text", text: "It is 20C in Paris." }], finishReason: finish("stop"), usage: usage(23, 9), warnings: [] }; + }, + doStream: async () => { + streamed += 1; + if (script === "fail") throw new Error("model exploded"); + const text = (id: string, ...deltas: string[]): StreamPart[] => [ + { type: "text-start", id }, + ...deltas.map((delta): StreamPart => ({ type: "text-delta", id, delta })), + { type: "text-end", id }, + ]; + const chunks: StreamPart[] = + script === "object" + ? [{ type: "stream-start", warnings: [] }, ...text("o", '{"city":', '"Paris"}'), { type: "finish", finishReason: finish("stop"), usage: usage(5, 3) }] + : script === "loop" && streamed === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "tool-call", toolCallId: "call-s1", toolName: "weather", input: '{"city":"Rome"}' }, + { type: "finish", finishReason: finish("tool-calls"), usage: usage(13, 4) }, + ] + : [{ type: "stream-start", warnings: [] }, ...text("t", "Rome is ", "25C."), { type: "finish", finishReason: finish("stop"), usage: usage(30, 6) }]; + return { stream: simulateReadableStream({ chunks }) }; + }, + }); +} + +const makeTools = (fail = false) => ({ + weather: tool({ + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => { + if (fail) throw new Error("weather service down"); + return { city, celsius: city === "Paris" ? 20 : 25 }; + }, + }), +}); +const loop = { stopWhen: stepCountIs(4) }; + +/** + * v7 has no `tracer` option; its per-call equivalent is the `telemetry` + * option (the non-deprecated spelling of `experimental_telemetry`). + */ +async function viaTracer(prompt: string): Promise { + const { text } = await generateText({ + model: scripted("answer"), + prompt, + telemetry: telemetry({ functionId: "answer-question" }), + }); + return text; +} + +// --- the same in every fixture + +type Telemetry = ReturnType | { isEnabled: true; functionId: string }; + +async function ask(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean; failTool?: boolean } = {}) { + const result = await generateText({ + model, + prompt: "Weather in Paris?", + ...(options.tools === false ? {} : { tools: makeTools(options.failTool), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return result.text; +} + +async function askStreaming(model: Parameters[0]["model"], options: { telemetry?: Telemetry; tools?: boolean } = {}) { + const result = streamText({ + model, + prompt: "Weather in Rome?", + ...(options.tools === false ? {} : { tools: makeTools(), ...loop }), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; +} + +const schema = z.object({ city: z.string() }); +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** The README's call sites, verbatim apart from the mock standing in for a provider. */ +async function readme(): Promise { + const model = scripted("answer"); + const prompt = "Weather in Paris?"; + const { text } = await generateText({ + model, + prompt, + experimental_telemetry: telemetry({ functionId: "answer-question" }), + }); + const wrapped = await wrapModel(scripted("answer")); + const viaMiddleware = wrapLanguageModel({ model: scripted("answer"), middleware: middleware() }); + const withMetadata = await generateText({ + model: scripted("answer"), + prompt, + experimental_telemetry: telemetry({ functionId: "tagged", metadata: { tenant: "acme", attempt: 1 } }), + }); + report({ text, wrapped: (await generateText({ model: wrapped, prompt })).text, viaMiddleware: viaMiddleware.modelId, withMetadata: withMetadata.text, withTracer: await viaTracer(prompt) }); +} + +/** + * The customer's own OpenTelemetry, set up AFTER `instrument("ai")` — the + * usual order when tracing starts in a module loaded later (a `NodeSDK` + * started from an instrumentation file). Null when `@opentelemetry/api` is not + * installed (ai 7 dropped the dependency). + */ +function customerTracing(): { registered: boolean; ended: () => string[] } | null { + interface Api { + trace: { + setGlobalTracerProvider(provider: unknown): boolean; + getTracer(name: string): { startSpan(name: string): { end(): void } }; + }; + } + let api: Api; + try { + api = createRequire(join(process.cwd(), "agent.js"))("@opentelemetry/api") as Api; + } catch { + return null; + } + const ended: string[] = []; + const span = (name: string) => ({ + setAttribute() { return this; }, + setAttributes() { return this; }, + addEvent() { return this; }, + addLink() { return this; }, + addLinks() { return this; }, + setStatus() { return this; }, + updateName() { return this; }, + recordException() {}, + isRecording: () => true, + spanContext: () => ({ traceId: "0".repeat(31) + "1", spanId: "0".repeat(15) + "1", traceFlags: 1 }), + end: () => void ended.push(name), + }); + const theirs = { + startSpan: (name: string) => span(name), + startActiveSpan: (name: string, ...rest: unknown[]) => (rest[rest.length - 1] as (s: unknown) => unknown)(span(name)), + }; + const registered = api.trace.setGlobalTracerProvider({ getTracer: () => theirs }); + // What an http / pg / Next.js instrumentation does with the global API. + api.trace.getTracer("my-service").startSpan("http.request").end(); + return { registered, ended: () => [...new Set(ended)].sort() }; +} + +/** A model whose stream breaks part-way: the provider connection dropped. */ +function breakMidStream(model: M): M { + const target = model as unknown as { doStream: (options: unknown) => PromiseLike<{ stream: ReadableStream }> }; + const original = target.doStream.bind(target); + target.doStream = async (options: unknown) => { + const result = await original(options); + const reader = result.stream.getReader(); + let parts = 0; + return { + ...result, + stream: new ReadableStream({ + async pull(controller) { + if (parts++ === 2) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + }), + }; + }; + return model; +} + +/** Call a model's `doStream` directly, as a provider-level consumer does. */ +async function openStream(model: unknown): Promise> { + const call = { + inputFormat: "prompt", + mode: { type: "regular" }, + prompt: [{ role: "user", content: [{ type: "text", text: "Weather in Rome?" }] }], + }; + const { stream } = await (model as { doStream(options: unknown): PromiseLike<{ stream: ReadableStream }> }).doStream(call); + return stream.getReader(); +} + +async function main(scenario: string): Promise { + switch (scenario) { + case "generate": + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "stream": + report({ text: await askStreaming(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "object": { + const { object } = await generateObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ object }); + break; + } + case "stream-object": { + const result = streamObject({ model: scripted("object"), schema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + for await (const _ of result.partialObjectStream) void _; + report({ object: await result.object }); + break; + } + case "wrap": + report({ text: await ask(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-stream": + report({ text: await askStreaming(await wrapModel(scripted("answer")), { tools: false }) }); + break; + case "wrap-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, async () => report({ text: await ask(await wrapModel(scripted())) })), + ); + break; + case "wrap-and-telemetry": + report({ text: await ask(await wrapModel(scripted()), { telemetry: telemetry({ functionId: "weather-agent" }) }) }); + break; + case "instrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-stream": + report({ instrumented: await failproofai.instrument("ai") }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-global-stream": + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: true }) }); + report({ text: await askStreaming(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "instrument-then-otel": { + report({ instrumented: await failproofai.instrument("ai") }); + const customer = customerTracing(); + const text = await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }); + report({ text, customer: customer === null ? "absent" : { registered: customer.registered, ended: customer.ended() } }); + break; + } + case "wrap-stream-cancel": { + const reader = await openStream(await wrapModel(scripted("answer"))); + await reader.read(); + await reader.cancel("client disconnected"); + report({ cancelled: true }); + break; + } + case "wrap-stream-error": { + const reader = await openStream(await wrapModel(breakMidStream(scripted("answer")))); + try { + while (!(await reader.read()).done) { + // drain + } + report({ drained: true }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "uninstrument": + report({ instrumented: await failproofai.instrument("ai") }); + report({ removed: failproofai.uninstrument() }); + report({ text: await ask(scripted(), { telemetry: { isEnabled: true, functionId: "weather-agent" } }) }); + break; + case "tool-error": + try { + report({ text: await ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }), failTool: true }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "model-error": + try { + report({ text: await ask(scripted("fail"), { telemetry: telemetry({ functionId: "weather-agent" }), tools: false }) }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + case "scope": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => ask(scripted(), { telemetry: telemetry({ functionId: "weather-agent" }) })), + ); + break; + case "readme": + await readme(); + break; + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-7/package-lock.json b/sdk/typescript/integration/fixtures/ai-7/package-lock.json new file mode 100644 index 000000000..db079af7b --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/package-lock.json @@ -0,0 +1,153 @@ +{ + "name": "failproofai-it-ai-7", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-ai-7", + "dependencies": { + "ai": "7.0.111", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.89", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.89.tgz", + "integrity": "sha512-n0Q88UUASYQBGK3Ogs9pCe8h3ZXeTHFSuGOLBJAyH73MVGNKUv9w8EJX2f340WUF3eDWWOV305gO7YBpSCPblA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz", + "integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.45", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.45.tgz", + "integrity": "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.111", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.111.tgz", + "integrity": "sha512-jd35WisL5FT7lFB6wIsZ38G0EaL4p1o+bUuRfkZGTXqO21U3SWyEZ+muV7N191QQfNeExgoyaeFoa8Jx6Uo+0A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.89", + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/ai-7/package.json b/sdk/typescript/integration/fixtures/ai-7/package.json new file mode 100644 index 000000000..ca2a83694 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/package.json @@ -0,0 +1,13 @@ +{ + "name": "failproofai-it-ai-7", + "private": true, + "type": "module", + "description": "Integration fixture: Vercel AI SDK 7.x against the packed @failproofai/sdk.", + "dependencies": { + "ai": "7.0.111", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/ai-7/surfaces.ts b/sdk/typescript/integration/fixtures/ai-7/surfaces.ts new file mode 100644 index 000000000..29e880b21 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/surfaces.ts @@ -0,0 +1,651 @@ +// Vercel AI SDK 7.x: every commonly used surface. Run as `node surfaces.{mjs,cjs} `. +// +// The coverage companion to agent.ts. agent.ts pins the README call sites +// and the adapter's core paths; this program walks the rest of the SDK a +// customer actually uses — the agent classes, embeddings, structured output, +// every tool feature, every way of consuming a stream, reasoning models and +// concurrency — against the same scripted, offline mock. +// +// Everything below the `--- the same in every fixture` line is the same +// program in every ai-* fixture; the section above it spells one major's API. +import * as failproofai from "@failproofai/sdk"; +import * as adapter from "@failproofai/sdk/ai"; +import { telemetry, wrapModel } from "@failproofai/sdk/ai"; +import { + Output, + ToolLoopAgent, + embed, + embedMany, + generateObject, + generateText, + simulateReadableStream, + stepCountIs, + streamObject, + streamText, + tool, + type ModelMessage, +} from "ai"; +import { MockEmbeddingModelV4, MockLanguageModelV4 } from "ai/test"; +import { z } from "zod"; + +// --- version-specific: ai 7 (LanguageModelV4, ToolLoopAgent, needsApproval; no OpenTelemetry) + +const MAJOR = 7; + +interface Call { + id: string; + name: string; + input: Record; +} +/** One scripted model step: what the model "says" on its n-th call. */ +interface Step { + text?: string; + reasoning?: string; + calls?: Call[]; + usage: [number, number]; + reasoningTokens?: number; + fail?: string; + delayMs?: number; +} +interface ModelOptions { + modelId?: string; + chunkDelayMs?: number; + /** Break the provider stream after this many parts: the connection dropped. */ + breakAfterParts?: number; +} + +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; +type Content = Awaited>["content"][number]; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** A string as two stream deltas. */ +const halves = (text: string): string[] => [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]; + +const usageOf = (step: Step) => ({ + inputTokens: { total: step.usage[0], noCache: step.usage[0], cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: step.usage[1], text: step.usage[1] - (step.reasoningTokens ?? 0), reasoning: step.reasoningTokens }, +}); +const finishOf = (step: Step) => { + const reason = step.calls?.length ? "tool-calls" : "stop"; + return { unified: reason, raw: reason } as const; +}; + +function model(steps: Step[], options: ModelOptions = {}) { + let n = 0; + const next = async (): Promise => { + const step = steps[Math.min(n, steps.length - 1)]!; + n += 1; + if (step.delayMs) await sleep(step.delayMs); + if (step.fail) throw new Error(step.fail); + return step; + }; + return new MockLanguageModelV4({ + provider: "mock-provider", + modelId: options.modelId ?? "mock-model", + doGenerate: async () => { + const step = await next(); + const content: Content[] = []; + if (step.reasoning) content.push({ type: "reasoning", text: step.reasoning }); + if (step.text) content.push({ type: "text", text: step.text }); + for (const call of step.calls ?? []) { + content.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + return { content, finishReason: finishOf(step), usage: usageOf(step), warnings: [] }; + }, + doStream: async ({ abortSignal }) => { + const step = await next(); + const chunks: StreamPart[] = [{ type: "stream-start", warnings: [] }]; + if (step.reasoning) { + chunks.push({ type: "reasoning-start", id: "r" }); + for (const delta of halves(step.reasoning)) chunks.push({ type: "reasoning-delta", id: "r", delta }); + chunks.push({ type: "reasoning-end", id: "r" }); + } + if (step.text) { + chunks.push({ type: "text-start", id: "t" }); + for (const delta of halves(step.text)) chunks.push({ type: "text-delta", id: "t", delta }); + chunks.push({ type: "text-end", id: "t" }); + } + for (const call of step.calls ?? []) { + chunks.push({ type: "tool-call", toolCallId: call.id, toolName: call.name, input: JSON.stringify(call.input) }); + } + chunks.push({ type: "finish", finishReason: finishOf(step), usage: usageOf(step) }); + return { stream: abortable(simulateReadableStream({ chunks, chunkDelayInMs: options.chunkDelayMs ?? 0 }), abortSignal, options.breakAfterParts) }; + }, + }); +} + +function embedder() { + return new MockEmbeddingModelV4({ + provider: "mock-provider", + modelId: "mock-embedder", + maxEmbeddingsPerCall: 2, + doEmbed: async ({ values }) => ({ embeddings: values.map((_, i) => [i, 0.5]), usage: { tokens: values.length * 3 }, warnings: [] }), + }); +} + +const citySchema = z.object({ city: z.string() }); +type Weather = { city: string; celsius: number }; + +function weatherTool(options: { fail?: boolean; delayMs?: (city: string) => number; onRun?: (city: string) => Promise } = {}) { + return tool({ + description: "Current weather for a city", + inputSchema: citySchema, + execute: async ({ city }: { city: string }): Promise => { + if (options.delayMs) await sleep(options.delayMs(city)); + if (options.onRun) await options.onRun(city); + if (options.fail) throw new Error("weather service down"); + return { city, celsius: city.length * 4 }; + }, + }); +} +/** A client-side tool: no `execute`, so the SDK hands the call back to the caller. */ +const clientTool = () => tool({ description: "Ask the user", inputSchema: citySchema }); +const HAS_APPROVAL = true; +const approvalTool = () => + tool({ + description: "Book a trip", + inputSchema: citySchema, + needsApproval: true, + execute: async ({ city }: { city: string }) => ({ booked: city }), + }); + +const steps = (n: number) => ({ stopWhen: stepCountIs(n) }); +const HAS_AGENT = true; + +type Settings = Parameters[0]; +type Telemetry = NonNullable; + +function makeAgent(options: { model: Settings["model"]; telemetry?: Telemetry; id?: string; tools?: Settings["tools"] }) { + const agent = new ToolLoopAgent({ + model: options.model, + ...(options.id ? { id: options.id } : {}), + tools: options.tools ?? { weather: weatherTool() }, + ...steps(4), + ...(options.telemetry ? { experimental_telemetry: options.telemetry } : {}), + }); + return { + generate: async (prompt: string) => (await agent.generate({ prompt })).text, + stream: async (prompt: string) => { + const result = await agent.stream({ prompt }); + let text = ""; + for await (const delta of result.textStream) text += delta; + return text; + }, + }; +} + +/** `generateText` / `streamText` with `output: Output.object(...)`. */ +const HAS_OUTPUT = true; +async function textWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = await generateText({ model: m, prompt: "Where?", output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + return result.output; +} +async function streamWithOutput(m: Settings["model"], tel: Telemetry): Promise { + const result = streamText({ model: m, prompt: "Where?", output: Output.object({ schema: citySchema }), experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialOutputStream) partials.push(partial); + return partials; +} + +/** What a Next.js route handler returns. */ +const toResponse = (result: { toUIMessageStreamResponse(): Response }): Response => result.toUIMessageStreamResponse(); + +const repairOption = (fixed: Record) => ({ + experimental_repairToolCall: async ({ toolCall }: { toolCall: T }) => ({ ...toolCall, input: JSON.stringify(fixed) }), +}); +const prepareStepOption = (second: Settings["model"]) => ({ + prepareStep: async ({ stepNumber }: { stepNumber: number }) => (stepNumber === 1 ? { model: second } : {}), +}); + +/** Run a tool that needs approval: ask, approve, resume. Returns the final text. */ +async function approveAndResume(m: Settings["model"], tel: Telemetry): Promise<{ pending: number; text: string }> { + const prompt: ModelMessage[] = [{ role: "user", content: "Book Paris" }]; + const first = await generateText({ model: m, messages: prompt, tools: { book: approvalTool() }, ...steps(4), experimental_telemetry: tel }); + const requests = first.content.filter((part) => part.type === "tool-approval-request"); + const approvals: ModelMessage = { + role: "tool", + content: requests.map((request) => ({ type: "tool-approval-response" as const, approvalId: request.approvalId, approved: true })), + }; + const second = await generateText({ + model: m, + messages: [...prompt, ...first.response.messages, approvals], + tools: { book: approvalTool() }, + ...steps(4), + experimental_telemetry: tel, + }); + return { pending: requests.length, text: second.text }; +} + +const partialObjects = async (m: Settings["model"], tel: Telemetry) => { + const result = streamObject({ model: m, schema: citySchema, prompt: "Where?", experimental_telemetry: tel }); + const partials: unknown[] = []; + for await (const partial of result.partialObjectStream) partials.push(partial); + return { partials, object: await result.object }; +}; + +// --- the same in every fixture + +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** + * A provider stream that honours the call's abort signal, as a real provider's + * `fetch` does — once aborted, the next read fails with an AbortError — and + * that can drop its connection part-way (`breakAfter`). + */ +function abortable(stream: ReadableStream, signal: AbortSignal | undefined, breakAfter?: number): ReadableStream { + const reader = stream.getReader(); + let parts = 0; + return new ReadableStream({ + async pull(controller) { + if (signal?.aborted) { + controller.error(signal.reason ?? new DOMException("aborted", "AbortError")); + return; + } + if (breakAfter !== undefined && parts++ === breakAfter) { + controller.error(new Error("connection reset")); + return; + } + const next = await reader.read(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + cancel: (reason) => reader.cancel(reason), + }); +} + +/** + * Run the garbage collector until it has had a real chance at everything + * unreachable. `--expose-gc` switched on at runtime, so the harness needs no + * special flags. + */ +async function collectGarbage(): Promise { + const v8 = await import("node:v8"); + const vm = await import("node:vm"); + v8.setFlagsFromString("--expose-gc"); + const gc = vm.runInNewContext("gc") as () => void; + for (let i = 0; i < 10; i += 1) { + gc(); + await sleep(20); + } +} + +/** + * A route's streamed Response whose client went away after three chunks. + * Its own function, so nothing of the stream is left on `main`'s frame + * afterwards — the garbage collector may take all of it. + */ +async function disconnectedClient(withSignal = false): Promise<{ chunks: number }> { + // A Next.js route can pass `abortSignal: request.signal`; the disconnect then + // aborts the call as well as cancelling the body. + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ + model: m, + prompt: "Count", + ...(withSignal ? { abortSignal: controller.signal } : {}), + experimental_telemetry: telemetry({ functionId: "counter" }), + }); + const reader = toResponse(result).body!.getReader(); + let chunks = 0; + while (chunks < 3 && !(await reader.read()).done) chunks += 1; + controller.abort(); + await reader.cancel("client disconnected"); + return { chunks }; +} + +/** An aborted stream, in a function of its own for the same reason. */ +async function abortedStream(): Promise<{ text: string; threw: string | false }> { + const controller = new AbortController(); + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { chunkDelayMs: 30 }); + const result = streamText({ model: m, prompt: "Count", abortSignal: controller.signal, experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) { + text += delta; + controller.abort(); + } + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).name }; + } +} + +/** A stream started and never read. */ +function neverRead(): void { + streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); +} + +/** A stream whose provider connection drops part-way; the error is the caller's to see. */ +async function brokenStream(): Promise<{ text: string; threw: string | false }> { + const m = model([{ text: "one two three four five six seven eight", usage: [9, 8] }], { breakAfterParts: 2 }); + const result = streamText({ model: m, prompt: "Count", experimental_telemetry: telemetry({ functionId: "counter" }), onError: () => undefined }); + let text = ""; + try { + for await (const delta of result.textStream) text += delta; + return { text, threw: false }; + } catch (error) { + return { text, threw: (error as Error).message }; + } +} + +/** The adapter's bookkeeping (internal, untyped): what it still holds open. */ +const held = () => { + const internals = (adapter as unknown as { _internals: { openCalls(): number; tracker(): { stats(): unknown } | null } })._internals; + return { openCalls: internals.openCalls(), stats: internals.tracker()?.stats() ?? null }; +}; + +const LOOP: Step[] = [ + { calls: [{ id: "call-1", name: "weather", input: { city: "Paris" } }], usage: [11, 7] }, + { text: "It is 20C in Paris.", usage: [23, 9] }, +]; + +async function drainText(stream: AsyncIterable): Promise { + let text = ""; + for await (const delta of stream) text += delta; + return text; +} + +async function main(scenario: string): Promise { + switch (scenario) { + // ---- 1. the agent classes + case "agent-generate": + case "agent-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), telemetry: telemetry({ functionId: "support-agent" }) }); + report({ text: scenario === "agent-generate" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-id": { + // The agent's own `id` never reaches telemetry: the SDK spreads it into + // generateText, which drops it. functionId is what names the agent. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: model(LOOP), id: "support-agent", telemetry: telemetry() }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-instrument": + case "agent-instrument-stream": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai") }); + const agent = makeAgent({ model: model(LOOP), telemetry: { isEnabled: true, functionId: "support-agent" } }); + report({ text: scenario === "agent-instrument" ? await agent.generate("Weather in Paris?") : await agent.stream("Weather in Paris?") }); + break; + } + case "agent-instrument-bare": { + // No telemetry setting at all: ai 7 records every call once an + // integration is registered; ai 4–6 record nothing without isEnabled. + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + report({ instrumented: await failproofai.instrument("ai", { registerGlobalTracer: false }) }); + report({ text: await makeAgent({ model: model(LOOP) }).generate("Weather in Paris?") }); + break; + } + case "agent-wrap": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + report({ text: await agent.generate("Weather in Paris?") }); + break; + } + case "agent-wrap-in-scope": { + if (!HAS_AGENT) return report({ skipped: "no agent class" }); + const agent = makeAgent({ model: await wrapModel(model(LOOP)) }); + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("support-agent", { goal: "help" }, async () => report({ text: await agent.stream("Weather in Paris?") })), + ); + break; + } + + // ---- 2. embeddings + case "embed": + report({ embedding: (await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry({ functionId: "indexer" }) })).embedding }); + break; + case "embed-many": { + const { embeddings } = await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embeddings: embeddings.length }); + break; + } + case "embed-in-agent": + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("rag", { goal: "answer" }, async () => { + await embed({ model: embedder(), value: "Paris", experimental_telemetry: telemetry() }); + await embedMany({ model: embedder(), values: ["Paris", "Rome", "Oslo"], experimental_telemetry: telemetry() }); + report({ embedded: true }); + }), + ); + break; + case "embed-in-tool": { + const lookup = weatherTool({ + onRun: async (city) => { + await embed({ model: embedder(), value: city, experimental_telemetry: telemetry() }); + }, + }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: lookup }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + + // ---- 3. structured output + case "object-partial": { + const m = model([{ text: '{"city":"Paris"}', usage: [5, 3] }]); + report(await partialObjects(m, telemetry({ functionId: "extractor" }))); + break; + } + case "object-invalid": + try { + await generateObject({ model: model([{ text: '{"town":"Paris"}', usage: [5, 3] }]), schema: citySchema, prompt: "Where?", experimental_telemetry: telemetry({ functionId: "extractor" }) }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + case "text-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + report({ output: await textWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })) }); + break; + } + case "stream-output": { + if (!HAS_OUTPUT) return report({ skipped: "no Output" }); + const partials = await streamWithOutput(model([{ text: '{"city":"Paris"}', usage: [5, 3] }]), telemetry({ functionId: "extractor" })); + report({ partials: partials.length, last: partials[partials.length - 1] }); + break; + } + + // ---- 4. tool features + case "parallel-tools": { + const m = model([ + { + calls: [ + { id: "call-p1", name: "weather", input: { city: "Paris" } }, + { id: "call-p2", name: "weather", input: { city: "Rome" } }, + ], + usage: [11, 7], + }, + { text: "Paris 20C, Rome 16C.", usage: [40, 9] }, + ]); + // Rome finishes first: results arrive out of call order. + const tools = { weather: weatherTool({ delayMs: (city) => (city === "Paris" ? 40 : 5) }) }; + const { text } = await generateText({ model: m, prompt: "Weather?", tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "client-tool": { + const m = model([{ calls: [{ id: "call-c1", name: "ask", input: { city: "Paris" } }], usage: [11, 7] }]); + const result = await generateText({ model: m, prompt: "Weather?", tools: { ask: clientTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ toolCalls: result.toolCalls.length }); + break; + } + case "tool-choice-required": { + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, toolChoice: "required", ...steps(1), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "repair": { + const m = model([{ calls: [{ id: "call-r1", name: "weather", input: { town: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...repairOption({ city: "Paris" }), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "prepare-step": { + const second = model([LOOP[1]!], { modelId: "mock-model-large" }); + const { text } = await generateText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), ...prepareStepOption(second), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + break; + } + case "unknown-tool": { + const m = model([{ calls: [{ id: "call-u1", name: "teleport", input: { city: "Paris" } }], usage: [11, 7] }, LOOP[1]!]); + try { + const { text } = await generateText({ model: m, prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + report({ text }); + } catch (error) { + report({ threw: (error as Error).name }); + } + break; + } + case "approval": { + if (!HAS_APPROVAL) return report({ skipped: "no tool approval" }); + const m = model([{ calls: [{ id: "call-a1", name: "book", input: { city: "Paris" } }], usage: [11, 7] }, { text: "Booked Paris.", usage: [30, 4] }]); + report(await approveAndResume(m, telemetry({ functionId: "travel-agent" }))); + break; + } + + // ---- 5. consuming a stream + case "stream-full": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const types = new Set(); + for await (const part of result.fullStream) types.add(part.type); + report({ parts: [...types].sort(), text: await result.text }); + break; + } + case "stream-response": { + const result = streamText({ model: model(LOOP), prompt: "Weather?", tools: { weather: weatherTool() }, ...steps(4), experimental_telemetry: telemetry({ functionId: "weather-agent" }) }); + const response = toResponse(result); + const body = await response.text(); + report({ status: response.status, containsAnswer: body.includes("Paris"), bytes: body.length }); + break; + } + case "stream-response-cancel": + // The browser went away mid-stream. Nothing in the SDK ends the operation + // after that; the adapter closes it once the stream is garbage. + report(await disconnectedClient()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-response-cancel-signal": + report(await disconnectedClient(true)); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-error": + report(await brokenStream()); + await sleep(50); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-on-finish": { + let finished = ""; + const result = streamText({ + model: model(LOOP), + prompt: "Weather?", + tools: { weather: weatherTool() }, + ...steps(4), + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + onFinish: ({ text }) => { + finished = text; + }, + }); + await result.consumeStream(); + report({ finished }); + break; + } + case "stream-abort": + report(await abortedStream()); + await sleep(100); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + case "stream-unconsumed": + // Never read. Nothing waits on it, so nothing should be held open. + neverRead(); + await sleep(200); + report({ before: held() }); + await collectGarbage(); + report({ after: held() }); + break; + + // ---- 6. reasoning models + case "reasoning-generate": + case "reasoning-stream": { + const m = model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }]); + const options = { model: m, prompt: "Capital of France?", providerOptions: { mock: { reasoningEffort: "high" } }, experimental_telemetry: telemetry({ functionId: "thinker" }) }; + if (scenario === "reasoning-generate") { + const result = await generateText(options); + report({ text: result.text }); + } else { + const result = streamText(options); + report({ text: await drainText(result.textStream) }); + } + break; + } + case "reasoning-wrap": { + const m = await wrapModel(model([{ reasoning: "The user wants a city.", text: "Paris.", usage: [12, 20], reasoningTokens: 15 }])); + const result = streamText({ model: m, prompt: "Capital of France?" }); + report({ text: await drainText(result.textStream) }); + break; + } + + // ---- 7. concurrency + case "concurrent": + case "concurrent-stream": { + const one = async (i: number) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2], delayMs: (i * 3) % 5 }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + const options = { model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: `worker-${i}` }) }; + if (scenario === "concurrent") return (await generateText(options)).text; + return await drainText(streamText(options).textStream); + }; + const texts = await failproofai.session({ sessionId: "busy" }, () => Promise.all(Array.from({ length: 10 }, (_, i) => one(i)))); + report({ texts }); + break; + } + case "concurrent-unscoped": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = model([ + { calls: [{ id: `call-${i}`, name: "weather", input: { city: `city-${i}` } }], usage: [100 + i, 1], delayMs: (i * 7) % 11 }, + { text: `answer ${i}`, usage: [200 + i, 2] }, + ]); + const tools = { weather: weatherTool({ delayMs: () => (i * 13) % 17 }) }; + return (await generateText({ model: m, prompt: `q${i}`, tools, ...steps(4), experimental_telemetry: telemetry({ functionId: "worker" }) })).text; + }), + ); + report({ texts }); + break; + } + case "concurrent-wrap": { + const texts = await Promise.all( + Array.from({ length: 10 }, async (_, i) => { + const m = await wrapModel(model([{ text: `answer ${i}`, usage: [100 + i, 1], delayMs: (i * 7) % 11 }], { modelId: `model-${i}` })); + return (await generateText({ model: m, prompt: `q${i}` })).text; + }), + ); + report({ texts }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + report({ major: MAJOR }); + await failproofai.flush(); +} + +main(process.argv[2] ?? "agent-generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/ai-7/tsconfig.json b/sdk/typescript/integration/fixtures/ai-7/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/ai-7/tsconfig.surfaces.json b/sdk/typescript/integration/fixtures/ai-7/tsconfig.surfaces.json new file mode 100644 index 000000000..f827f9292 --- /dev/null +++ b/sdk/typescript/integration/fixtures/ai-7/tsconfig.surfaces.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "files": ["surfaces.ts"] +} diff --git a/sdk/typescript/integration/fixtures/langchain-0.3/agent.ts b/sdk/typescript/integration/fixtures/langchain-0.3/agent.ts new file mode 100644 index 000000000..935224f11 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-0.3/agent.ts @@ -0,0 +1,623 @@ +// LangChain.js / LangGraph.js consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The scripted model makes it deterministic and offline — first call asks +// for the `get_weather` tool, the next one answers — and mirrors the Python +// SDK's golden trace for the same graph, which is what the assertions compare +// against. +// +// The SAME file is the agent of every `langchain-*` fixture (it is copied, not +// shared, because the harness transpiles each fixture's own `agent.ts` beside +// its own `node_modules`). Keep it to APIs present across the whole declared +// peer range, so a failure is always the adapter's and never the fixture's. +import { spawnSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; + +import * as failproofai from "@failproofai/sdk"; +import { langchainHandler } from "@failproofai/sdk/langchain"; +import type { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; +import { Document } from "@langchain/core/documents"; +import { Embeddings } from "@langchain/core/embeddings"; +import { + BaseChatModel, + type BaseChatModelCallOptions, + type BaseChatModelParams, +} from "@langchain/core/language_models/chat_models"; +import { AIMessage, AIMessageChunk, HumanMessage, type BaseMessage } from "@langchain/core/messages"; +import { StringOutputParser } from "@langchain/core/output_parsers"; +import { ChatGenerationChunk, type ChatResult } from "@langchain/core/outputs"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { BaseRetriever } from "@langchain/core/retrievers"; +import { + RunnableLambda, + RunnableParallel, + RunnablePassthrough, + RunnableSequence, + type RunnableConfig, +} from "@langchain/core/runnables"; +import { tool } from "@langchain/core/tools"; +import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; +import { VectorStore } from "@langchain/core/vectorstores"; +import { Annotation, Command, END, MemorySaver, MessagesAnnotation, START, StateGraph, interrupt } from "@langchain/langgraph"; +import { ToolNode, createReactAgent } from "@langchain/langgraph/prebuilt"; +import { z } from "zod"; + +class ScriptedModel extends BaseChatModel { + calls = 0; + private readonly fail: boolean; + + constructor(fields: BaseChatModelParams & { fail?: boolean } = {}) { + super(fields); + this.fail = fields.fail ?? false; + } + + _llmType(): string { + return "scripted"; + } + + // The script decides the tool calls; binding only has to hand back a model. + override bindTools(): this { + return this; + } + + async _generate(messages: BaseMessage[]): Promise { + this.calls += 1; + if (this.fail) throw new Error("model exploded"); + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + response_metadata: { finish_reason: "tool_calls" }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +/** + * A model that streams, the way a real provider integration does: one chunk + * per piece, each reported through `handleLLMNewToken`. Five chunks, the same + * split the Python golden's `GenericFakeChatModel` makes of this sentence. + */ +class StreamingModel extends ScriptedModel { + override async *_streamResponseChunks( + _messages: BaseMessage[], + _options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun, + ): AsyncGenerator { + for (const text of ["hello", " ", "there", " ", "friend"]) { + const chunk = new ChatGenerationChunk({ text, message: new AIMessageChunk({ content: text }) }); + yield chunk; + await runManager?.handleLLMNewToken(text, undefined, undefined, undefined, undefined, { chunk }); + } + } +} + +const getWeather = tool(async ({ city }: { city: string }) => `sunny in ${city}`, { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), +}); + +// Same name, and it fails. `ToolNode` turns the throw into an error +// ToolMessage for the model, so the RUN carries on — the failure belongs to the +// tool call alone. +const brokenWeather = tool( + async ({ city }: { city: string }): Promise => { + throw new Error(`no weather for ${city}`); + }, + { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), + }, +); + +// --------------------------------------------------------------------------- +// Plain LangChain: LCEL, retrievers, structured output, bound tools +// --------------------------------------------------------------------------- + +const ANSWER = "It is sunny in Paris."; +/** How `AnswerModel` streams `ANSWER`: five chunks, usage on the last one. */ +const ANSWER_CHUNKS = ["It", " is", " sunny", " in", " Paris."]; + +/** + * Always answers, and streams like a provider does. The usage rides on the + * LAST chunk, where OpenAI and Anthropic put it, so a streamed call's tokens + * survive only if the adapter reads the aggregated generation. + */ +class AnswerModel extends BaseChatModel { + _llmType(): string { + return "answer"; + } + + async _generate(): Promise { + const message = new AIMessage({ + content: ANSWER, + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }); + return { generations: [{ text: ANSWER, message }] }; + } + + override async *_streamResponseChunks( + _messages: BaseMessage[], + _options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun, + ): AsyncGenerator { + for (const [index, text] of ANSWER_CHUNKS.entries()) { + const last = index === ANSWER_CHUNKS.length - 1; + const message = new AIMessageChunk({ + content: text, + ...(last ? { usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 } } : {}), + }); + const chunk = new ChatGenerationChunk({ text, message }); + yield chunk; + await runManager?.handleLLMNewToken(text, undefined, undefined, undefined, undefined, { chunk }); + } + } +} + +interface ToolCallingOptions extends BaseChatModelCallOptions { + tools?: Array<{ function?: { name?: string } }>; +} + +/** + * A model that binds tools the way a real provider integration does: the + * formatted tools and `tool_choice` ride on the call options, and come back + * out through `invocationParams` — which is where the adapter reads them. + * Asks for whichever tool it was bound to first, then answers — as a chunk, + * which is what a 1.x provider returns and what `withStructuredOutput` checks. + */ +class ToolCallingModel extends BaseChatModel { + _llmType(): string { + return "tool-calling"; + } + + override invocationParams(options?: this["ParsedCallOptions"]): Record { + return { model: "tool-calling-1", tools: options?.tools, tool_choice: options?.tool_choice }; + } + + override bindTools(tools: unknown[], kwargs?: Partial) { + return this.withConfig({ + tools: tools.map((t) => convertToOpenAITool(t as never)), + ...kwargs, + } as Partial) as never; + } + + async _generate(messages: BaseMessage[], options: this["ParsedCallOptions"]): Promise { + const wanted = options.tools?.[0]?.function?.name ?? "get_weather"; + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessageChunk({ + content: ANSWER, + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + }) + : new AIMessageChunk({ + content: "", + tool_calls: [ + wanted === "Weather" + ? { id: "call_s", name: "Weather", args: { city: "Paris", sky: "sunny" }, type: "tool_call" } + : { id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }, + ], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +const WEATHER_DOCS = [ + new Document({ pageContent: "Paris is sunny", metadata: { source: "wx.txt" } }), + new Document({ pageContent: "Rome is rainy", metadata: { source: "wx2.txt" } }), +]; + +/** The smallest custom retriever: what a hand-rolled RAG lookup is. */ +class Docs extends BaseRetriever { + lc_namespace = ["failproofai", "fixtures"]; + + async _getRelevantDocuments(_query: string): Promise { + return WEATHER_DOCS; + } +} + +/** Deterministic, offline embeddings: letter frequencies. */ +class LetterEmbeddings extends Embeddings { + constructor() { + super({}); + } + + private vector(text: string): number[] { + const out = new Array(26).fill(0); + for (const ch of text.toLowerCase()) { + const code = ch.charCodeAt(0) - 97; + if (code >= 0 && code < 26) out[code] += 1; + } + return out; + } + + async embedDocuments(texts: string[]): Promise { + return texts.map((text) => this.vector(text)); + } + + async embedQuery(text: string): Promise { + return this.vector(text); + } +} + +/** + * An in-memory vector store, so `.asRetriever()` — the retriever almost every + * RAG app actually runs — is exercised without a network or a native module. + */ +class TinyVectorStore extends VectorStore { + declare FilterType: never; + private rows: Array<{ vector: number[]; doc: Document }> = []; + + _vectorstoreType(): string { + return "tiny"; + } + + async addVectors(vectors: number[][], documents: Document[]): Promise { + vectors.forEach((vector, i) => this.rows.push({ vector, doc: documents[i]! })); + } + + async addDocuments(documents: Document[]): Promise { + await this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents); + } + + async similaritySearchVectorWithScore(query: number[], k: number): Promise> { + const dot = (a: number[], b: number[]) => a.reduce((sum, x, i) => sum + x * (b[i] ?? 0), 0); + return this.rows + .map(({ vector, doc }): [Document, number] => [doc, dot(vector, query)]) + .sort((a, b) => b[1] - a[1]) + .slice(0, k); + } +} + +const prompt = () => ChatPromptTemplate.fromMessages([["human", "{q}"]]); +const lcel = () => prompt().pipe(new AnswerModel({})).pipe(new StringOutputParser()); + +/** + * Every plain-LangChain surface, each runnable with or without an explicit + * handler: `` runs it under `instrument()`, `handler:` passes + * `callbacks: [langchainHandler()]` instead and never instruments. + */ +const SURFACES: Record Promise> = { + lcel: (config) => lcel().invoke({ q: "weather?" }, config), + sequence: (config) => + RunnableSequence.from([ + RunnableLambda.from((x: number) => x + 1), + RunnableLambda.from((x: number) => x * 2), + ]).invoke(1, config), + parallel: (config) => + RunnableParallel.from({ + a: RunnableLambda.from((x: number) => x + 1), + b: RunnableLambda.from((x: number) => x * 2), + }).invoke(1, config), + lambda: (config) => RunnableLambda.from((x: string) => x.toUpperCase()).withConfig({ runName: "shout" }).invoke("abc", config), + retriever: (config) => new Docs().invoke("weather in Paris", config), + vectorstore: async (config) => { + const store = new TinyVectorStore(new LetterEmbeddings(), {}); + await store.addDocuments([WEATHER_DOCS[0]!]); + return store.asRetriever({ k: 1 }).invoke("Paris", config); + }, + rag: (config) => + RunnableSequence.from([ + { + context: new Docs().pipe((docs: Document[]) => docs.map((d) => d.pageContent).join("\n")), + q: new RunnablePassthrough(), + }, + ChatPromptTemplate.fromMessages([["human", "{context}\n{q}"]]), + new AnswerModel({}), + new StringOutputParser(), + ]).invoke("weather?", config), + batch3: (config) => lcel().batch([{ q: "a" }, { q: "b" }, { q: "c" }], config), + "stream-events": async (config) => { + const kinds: string[] = []; + for await (const event of lcel().streamEvents({ q: "weather?" }, { ...config, version: "v2" })) { + kinds.push(event.event); + } + return kinds.includes("on_chat_model_stream"); + }, + "lcel-stream": async (config) => { + let text = ""; + for await (const chunk of await lcel().stream({ q: "weather?" }, config)) text += chunk; + return text; + }, + structured: (config) => + new ToolCallingModel({}) + .withStructuredOutput(z.object({ city: z.string(), sky: z.string() }), { name: "Weather" }) + .invoke("weather?", config), + "bind-tools": (config) => + (new ToolCallingModel({}).bindTools([getWeather], { tool_choice: "get_weather" }) as unknown as ToolCallingModel) + .invoke("weather?", config) + .then((message) => (message as AIMessage).tool_calls?.length), +}; + +function buildGraph(model: ScriptedModel, tools = [getWeather]) { + const callModel = async (state: typeof MessagesAnnotation.State) => ({ + messages: [await model.invoke(state.messages)], + }); + const route = (state: typeof MessagesAnnotation.State) => { + const last = state.messages[state.messages.length - 1] as AIMessage; + return last.tool_calls?.length ? "tools" : END; + }; + return new StateGraph(MessagesAnnotation) + .addNode("agent", callModel) + .addNode("tools", new ToolNode(tools)) + .addEdge(START, "agent") + .addConditionalEdges("agent", route, ["tools", END]) + .addEdge("tools", "agent") + .compile({ name: "weather_graph" }); +} + +const Trail = Annotation.Root({ + trail: Annotation({ reducer: (a, b) => a.concat(b), default: () => [] }), +}); + +/** plan -> approve (asks a human) -> act, checkpointed so it can be resumed. */ +function buildHitlGraph(checkpointer = new MemorySaver()) { + return new StateGraph(Trail) + .addNode("plan", async () => ({ trail: ["plan"] })) + .addNode("approve", async () => { + const answer = interrupt({ prompt: "ship it?", options: ["yes", "no"] }) as string; + return { trail: [`approve:${answer}`] }; + }) + .addNode("act", async () => ({ trail: ["act"] })) + .addEdge(START, "plan") + .addEdge("plan", "approve") + .addEdge("approve", "act") + .addEdge("act", END) + .compile({ name: "hitl_graph", checkpointer }); +} + +/** A compiled graph used as a node of another one. */ +function buildParentGraph() { + const child = new StateGraph(Trail) + .addNode("inner", async () => ({ trail: ["inner"] })) + .addEdge(START, "inner") + .addEdge("inner", END) + .compile({ name: "child_graph" }); + return new StateGraph(Trail) + .addNode("pre", async () => ({ trail: ["pre"] })) + .addNode("child", child) + .addEdge(START, "pre") + .addEdge("pre", "child") + .addEdge("child", END) + .compile({ name: "parent_graph" }); +} + +/** + * A `MemorySaver`'s contents as JSON, and back. The one piece of state two + * processes share in a real deployment is the checkpointer; this stands in for + * a database-backed one so the pause and the approval can happen in two + * genuinely separate processes with nothing else in common. + */ +interface SaverState { + storage: Record; + writes: Record; +} +const encodeBytes = (_key: string, value: unknown): unknown => + value instanceof Uint8Array ? { __u8: Buffer.from(value).toString("base64") } : value; +const decodeBytes = (_key: string, value: unknown): unknown => + value !== null && typeof value === "object" && typeof (value as { __u8?: unknown }).__u8 === "string" + ? new Uint8Array(Buffer.from((value as { __u8: string }).__u8, "base64")) + : value; + +function saverState(saver: MemorySaver): SaverState { + return saver as unknown as SaverState; +} + +const question = { messages: [new HumanMessage("weather?")] }; +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** `instrument()` options per case; every case not listed uses the defaults. */ +const OPTIONS: Record> = { + "include-chains": { includeChains: ["summarise"] }, + "session-option": { sessionId: "fixed-session" }, + "capture-off": { captureContent: false }, +}; + +async function main(scenario: string): Promise { + const explicit = scenario.startsWith("handler:"); + const surface = SURFACES[explicit ? scenario.slice("handler:".length) : scenario]; + if (surface !== undefined) { + if (!explicit) report({ instrumented: await failproofai.instrument("langchain") }); + const out = await surface(explicit ? { callbacks: [langchainHandler() as never] } : {}); + report({ out }); + await failproofai.flush(); + return; + } + + if (scenario !== "handler") { + report({ instrumented: await failproofai.instrument("langchain", OPTIONS[scenario] ?? {}) }); + } + + switch (scenario) { + case "graph": + case "session-option": + case "capture-off": { + const out = await buildGraph(new ScriptedModel()).invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "stream": { + let chunks = 0; + for await (const _ of await buildGraph(new ScriptedModel()).stream(question, { streamMode: "updates" })) { + chunks += 1; + } + report({ chunks }); + break; + } + case "react": { + const agent = createReactAgent({ llm: new ScriptedModel(), tools: [getWeather], name: "react_bot" }); + const out = await agent.invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "error": { + try { + await buildGraph(new ScriptedModel({ fail: true })).invoke({ messages: [new HumanMessage("x")] }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "tool-error": { + const out = await buildGraph(new ScriptedModel(), [brokenWeather]).invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "model": { + const out = await new ScriptedModel().invoke([new HumanMessage("hi")]); + report({ toolCalls: (out as AIMessage).tool_calls?.length }); + break; + } + case "tool": { + report({ out: await getWeather.invoke({ city: "Rome" }) }); + break; + } + case "batch": { + const out = await new ScriptedModel().batch([[new HumanMessage("a")], [new HumanMessage("b")]]); + report({ batch: out.length }); + break; + } + case "stream-model": { + let text = ""; + for await (const chunk of await new StreamingModel().stream([new HumanMessage("hi")])) { + text += String(chunk.content); + } + report({ text }); + break; + } + case "include-chains": { + const chain = RunnableLambda.from((x: string) => x) + .pipe(RunnableLambda.from((x: string) => x.toUpperCase()).withConfig({ runName: "summarise" })) + .withConfig({ runName: "pipeline" }); + report({ out: await chain.invoke("abc") }); + break; + } + case "metadata-session": { + await buildGraph(new ScriptedModel()).invoke(question, { + metadata: { failproofai_sdk_session_id: "meta-sid" }, + }); + break; + } + case "thread-session": { + await buildGraph(new ScriptedModel()).invoke(question, { configurable: { thread_id: "thread-9" } }); + break; + } + case "hitl": + case "hitl-uninstrument": { + const graph = buildHitlGraph(); + const config = { configurable: { thread_id: "t-1" } }; + const first = await graph.invoke({ trail: [] }, config); + report({ interrupted: "__interrupt__" in first }); + if (scenario === "hitl") { + const done = await graph.invoke(new Command({ resume: "yes" }), config); + report({ trail: done.trail }); + } else { + report({ removed: failproofai.uninstrument() }); + } + break; + } + case "remote-resume-pause": { + // Process A: serve the request that interrupts, persist the checkpoint, + // exit with the pause still open. + const saver = new MemorySaver(); + // `durability: "sync"`: the default persists the checkpoint in the + // background, and this process is about to exit. + await buildHitlGraph(saver).invoke({ trail: [] }, { configurable: { thread_id: "t-1" }, durability: "sync" }); + // The two tables, not the saver: 1.x gives MemorySaver a `toJSON`. + const { storage, writes } = saverState(saver); + writeFileSync(process.argv[3]!, JSON.stringify({ storage, writes }, encodeBytes)); + break; + } + case "remote-resume": { + // Process B: a different process, sharing only the checkpointer, takes + // the human's answer. + const file = `${process.argv[1]}.${process.pid}.checkpoint.json`; + const child = spawnSync(process.execPath, [process.argv[1]!, "remote-resume-pause", file], { + env: process.env, + encoding: "utf8", + }); + if (child.status !== 0) throw new Error(`the pausing process failed: ${child.stderr}`); + const saver = new MemorySaver(); + const saved = JSON.parse(readFileSync(file, "utf8"), decodeBytes) as SaverState; + rmSync(file, { force: true }); + Object.assign(saverState(saver).storage, saved.storage); + Object.assign(saverState(saver).writes, saved.writes); + const done = await buildHitlGraph(saver).invoke(new Command({ resume: "yes" }), { + configurable: { thread_id: "t-1" }, + }); + report({ trail: done.trail }); + break; + } + case "abort": { + // The caller gives up: the signal fires while a node is running. + const controller = new AbortController(); + const graph = new StateGraph(Trail) + .addNode("slow", async () => { + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { trail: ["slow"] }; + }) + .addEdge(START, "slow") + .addEdge("slow", END) + .compile({ name: "abort_graph" }); + try { + await graph.invoke({ trail: [] }, { signal: controller.signal }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + // LangGraph.js 1.x never closes an aborted graph's own run; the adapter + // does, once the run has been silent for its grace period. Outlive it. + await new Promise((resolve) => setTimeout(resolve, 3_300)); + break; + } + case "subgraph": { + const out = await buildParentGraph().invoke({ trail: [] }); + report({ trail: out.trail }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => buildGraph(new ScriptedModel()).invoke(question)), + ); + break; + } + case "handler": { + // No instrument(): the explicit handler is the patch-free path. + const out = await buildGraph(new ScriptedModel()).invoke(question, { + callbacks: [langchainHandler() as never], + }); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "handler-and-instrument": { + report({ instrumented: await failproofai.instrument("langchain") }); + await buildGraph(new ScriptedModel()).invoke(question, { callbacks: [langchainHandler() as never] }); + break; + } + case "uninstrument": { + report({ removed: failproofai.uninstrument() }); + await buildGraph(new ScriptedModel()).invoke(question); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "graph").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/langchain-0.3/package-lock.json b/sdk/typescript/integration/fixtures/langchain-0.3/package-lock.json new file mode 100644 index 000000000..74966101c --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-0.3/package-lock.json @@ -0,0 +1,463 @@ +{ + "name": "failproofai-it-langchain-0-3", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-langchain-0-3", + "dependencies": { + "@langchain/core": "0.3.80", + "@langchain/langgraph": "0.4.10", + "zod": "3.25.76", + "zod-to-json-schema": "3.25.2" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, + "node_modules/@langchain/core": { + "version": "0.3.80", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.80.tgz", + "integrity": "sha512-vcJDV2vk1AlCwSh3aBm/urQ1ZrlXFFBocv11bz/NBUfLWD5/UDNMzwPdaAd2dKvNmTWa9FM2lirLU3+JCf4cRA==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.3.67", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.25.32", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/langgraph": { + "version": "0.4.10", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-0.4.10.tgz", + "integrity": "sha512-4gS6Y7aKu82ZhY3y5QVJ0TNaXGNHgsiBm7JNy4eUP42z1/73l2an8lcaQrpHtEF/Lpopwkp8Lz3j3rCnTJNUTg==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^0.1.3", + "@langchain/langgraph-sdk": "~0.1.10", + "uuid": "^10.0.0", + "zod": "^3.25.32" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.58 < 0.4.0", + "zod-to-json-schema": "^3.x" + }, + "peerDependenciesMeta": { + "zod-to-json-schema": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-0.1.3.tgz", + "integrity": "sha512-hCfCqvbAS5iAbKxEtSEQVSQSEkeoAOvDumVZ1RwCFGi5CxV1/3dTnHpQn2DcvKm+M4fchhV7mq++Ej2GB/hSdQ==", + "license": "MIT", + "dependencies": { + "uuid": "^10.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.31 <0.4.0 || ^1.0.0-alpha" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-0.1.10.tgz", + "integrity": "sha512-9srSCb2bSvcvehMgjA2sMMwX0o1VUgPN6ghwm5Fwc9JGAKsQa6n1S4eCwy1h4abuYxwajH5n3spBw+4I2WYbgw==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.15", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^9.0.0" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.31 <0.4.0 || ^1.0.0-alpha", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "license": "MIT" + }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/console-table-printer": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.16.1.tgz", + "integrity": "sha512-Sc9FRJ4O9xKGNrvulNdPfK5SyBcZ6lcaRnDE4AQ/uw6IDtjHhsqyzzqcnMikjyGaiOOF2tNOKoBhbVjRvFy9Lw==", + "license": "MIT", + "dependencies": { + "simple-wcswidth": "^1.1.2" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/langsmith": { + "version": "0.3.87", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.87.tgz", + "integrity": "sha512-XXR1+9INH8YX96FKWc5tie0QixWz6tOqAsAKfcJyPkE0xPep+NDz0IQLR32q4bn10QK3LqD2HN6T3n6z1YLW7Q==", + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "chalk": "^4.1.2", + "console-table-printer": "^2.12.1", + "p-queue": "^6.6.2", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-wcswidth": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", + "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-0.3/package.json b/sdk/typescript/integration/fixtures/langchain-0.3/package.json new file mode 100644 index 000000000..c5366dac1 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-0.3/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-langchain-0-3", + "private": true, + "type": "module", + "description": "Integration fixture: LangChain.js 0.3 + LangGraph.js 0.4 (the floor of the declared peer range) against the packed @failproofai/sdk.", + "dependencies": { + "@langchain/core": "0.3.80", + "@langchain/langgraph": "0.4.10", + "zod": "3.25.76", + "zod-to-json-schema": "3.25.2" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-0.3/tsconfig.json b/sdk/typescript/integration/fixtures/langchain-0.3/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-0.3/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/langchain-1/agent-v1.ts b/sdk/typescript/integration/fixtures/langchain-1/agent-v1.ts new file mode 100644 index 000000000..5804517bb --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-1/agent-v1.ts @@ -0,0 +1,99 @@ +// The v1 `langchain` package's `createAgent`. Run as `node agent-v1.{mjs,cjs} `. +// +// Its own program, beside `agent.ts`, because `langchain` 1.x exists only on +// the 1.x line: `agent.ts` is shared verbatim with the 0.3 fixture and must +// keep to APIs present across the whole declared peer range. +// +// `createAgent` compiles to a LangGraph `StateGraph` — a `model` node and a +// `tools` node, plus one node per middleware hook (`.before_model`, …) — +// so the expected trace is the Python SDK's for `langchain.agents.create_agent` +// with the same scripted model (langchain 1.4.2, langchain-core 1.6.3, +// langgraph 1.2.11): the agent named after the agent, its nodes as hooks. +import * as failproofai from "@failproofai/sdk"; +import { langchainHandler } from "@failproofai/sdk/langchain"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; +import { AIMessage, HumanMessage, type BaseMessage } from "@langchain/core/messages"; +import type { ChatResult } from "@langchain/core/outputs"; +import type { RunnableConfig } from "@langchain/core/runnables"; +import { tool } from "@langchain/core/tools"; +import { createAgent, createMiddleware } from "langchain"; +import { z } from "zod"; + +/** First call asks for `get_weather`, the next one answers. */ +class ScriptedModel extends BaseChatModel { + _llmType(): string { + return "scripted"; + } + + // The script decides the tool calls; binding only has to hand back a model. + override bindTools(): this { + return this; + } + + async _generate(messages: BaseMessage[]): Promise { + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + response_metadata: { finish_reason: "tool_calls" }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +const getWeather = tool(async ({ city }: { city: string }) => `sunny in ${city}`, { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), +}); + +/** + * One node-shaped hook (`beforeModel`, which becomes a graph node) and one + * wrapping hook (`wrapModelCall`, which runs INSIDE the model node and so must + * add nothing of its own) — the two ways middleware reaches the callbacks. + */ +const audit = createMiddleware({ + name: "Audit", + beforeModel: () => undefined, + wrapModelCall: (request, handler) => handler(request), +}); + +const question = { messages: [new HumanMessage("weather?")] }; + +const CASES: Record Promise> = { + "create-agent": (config) => + createAgent({ model: new ScriptedModel({}), tools: [getWeather], name: "weather_agent" }).invoke(question, config), + "create-agent-mw": (config) => + createAgent({ model: new ScriptedModel({}), tools: [getWeather], name: "weather_agent", middleware: [audit] }).invoke( + question, + config, + ), + "create-agent-stream": async (config) => { + const agent = createAgent({ model: new ScriptedModel({}), tools: [getWeather], name: "weather_agent" }); + let chunks = 0; + for await (const _ of await agent.stream(question, { ...config, streamMode: "updates" })) chunks += 1; + return chunks; + }, +}; + +async function main(scenario: string): Promise { + const explicit = scenario.startsWith("handler:"); + const run = CASES[explicit ? scenario.slice("handler:".length) : scenario]; + if (run === undefined) throw new Error(`unknown scenario ${scenario}`); + if (!explicit) console.log(JSON.stringify({ instrumented: await failproofai.instrument("langchain") })); + const out = (await run(explicit ? { callbacks: [langchainHandler() as never] } : {})) as { messages?: BaseMessage[] }; + console.log(JSON.stringify({ answer: out.messages?.at(-1)?.content ?? out })); + await failproofai.flush(); +} + +main(process.argv[2] ?? "create-agent").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/langchain-1/agent.ts b/sdk/typescript/integration/fixtures/langchain-1/agent.ts new file mode 100644 index 000000000..935224f11 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-1/agent.ts @@ -0,0 +1,623 @@ +// LangChain.js / LangGraph.js consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The scripted model makes it deterministic and offline — first call asks +// for the `get_weather` tool, the next one answers — and mirrors the Python +// SDK's golden trace for the same graph, which is what the assertions compare +// against. +// +// The SAME file is the agent of every `langchain-*` fixture (it is copied, not +// shared, because the harness transpiles each fixture's own `agent.ts` beside +// its own `node_modules`). Keep it to APIs present across the whole declared +// peer range, so a failure is always the adapter's and never the fixture's. +import { spawnSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; + +import * as failproofai from "@failproofai/sdk"; +import { langchainHandler } from "@failproofai/sdk/langchain"; +import type { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; +import { Document } from "@langchain/core/documents"; +import { Embeddings } from "@langchain/core/embeddings"; +import { + BaseChatModel, + type BaseChatModelCallOptions, + type BaseChatModelParams, +} from "@langchain/core/language_models/chat_models"; +import { AIMessage, AIMessageChunk, HumanMessage, type BaseMessage } from "@langchain/core/messages"; +import { StringOutputParser } from "@langchain/core/output_parsers"; +import { ChatGenerationChunk, type ChatResult } from "@langchain/core/outputs"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { BaseRetriever } from "@langchain/core/retrievers"; +import { + RunnableLambda, + RunnableParallel, + RunnablePassthrough, + RunnableSequence, + type RunnableConfig, +} from "@langchain/core/runnables"; +import { tool } from "@langchain/core/tools"; +import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; +import { VectorStore } from "@langchain/core/vectorstores"; +import { Annotation, Command, END, MemorySaver, MessagesAnnotation, START, StateGraph, interrupt } from "@langchain/langgraph"; +import { ToolNode, createReactAgent } from "@langchain/langgraph/prebuilt"; +import { z } from "zod"; + +class ScriptedModel extends BaseChatModel { + calls = 0; + private readonly fail: boolean; + + constructor(fields: BaseChatModelParams & { fail?: boolean } = {}) { + super(fields); + this.fail = fields.fail ?? false; + } + + _llmType(): string { + return "scripted"; + } + + // The script decides the tool calls; binding only has to hand back a model. + override bindTools(): this { + return this; + } + + async _generate(messages: BaseMessage[]): Promise { + this.calls += 1; + if (this.fail) throw new Error("model exploded"); + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + response_metadata: { finish_reason: "tool_calls" }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +/** + * A model that streams, the way a real provider integration does: one chunk + * per piece, each reported through `handleLLMNewToken`. Five chunks, the same + * split the Python golden's `GenericFakeChatModel` makes of this sentence. + */ +class StreamingModel extends ScriptedModel { + override async *_streamResponseChunks( + _messages: BaseMessage[], + _options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun, + ): AsyncGenerator { + for (const text of ["hello", " ", "there", " ", "friend"]) { + const chunk = new ChatGenerationChunk({ text, message: new AIMessageChunk({ content: text }) }); + yield chunk; + await runManager?.handleLLMNewToken(text, undefined, undefined, undefined, undefined, { chunk }); + } + } +} + +const getWeather = tool(async ({ city }: { city: string }) => `sunny in ${city}`, { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), +}); + +// Same name, and it fails. `ToolNode` turns the throw into an error +// ToolMessage for the model, so the RUN carries on — the failure belongs to the +// tool call alone. +const brokenWeather = tool( + async ({ city }: { city: string }): Promise => { + throw new Error(`no weather for ${city}`); + }, + { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), + }, +); + +// --------------------------------------------------------------------------- +// Plain LangChain: LCEL, retrievers, structured output, bound tools +// --------------------------------------------------------------------------- + +const ANSWER = "It is sunny in Paris."; +/** How `AnswerModel` streams `ANSWER`: five chunks, usage on the last one. */ +const ANSWER_CHUNKS = ["It", " is", " sunny", " in", " Paris."]; + +/** + * Always answers, and streams like a provider does. The usage rides on the + * LAST chunk, where OpenAI and Anthropic put it, so a streamed call's tokens + * survive only if the adapter reads the aggregated generation. + */ +class AnswerModel extends BaseChatModel { + _llmType(): string { + return "answer"; + } + + async _generate(): Promise { + const message = new AIMessage({ + content: ANSWER, + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }); + return { generations: [{ text: ANSWER, message }] }; + } + + override async *_streamResponseChunks( + _messages: BaseMessage[], + _options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun, + ): AsyncGenerator { + for (const [index, text] of ANSWER_CHUNKS.entries()) { + const last = index === ANSWER_CHUNKS.length - 1; + const message = new AIMessageChunk({ + content: text, + ...(last ? { usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 } } : {}), + }); + const chunk = new ChatGenerationChunk({ text, message }); + yield chunk; + await runManager?.handleLLMNewToken(text, undefined, undefined, undefined, undefined, { chunk }); + } + } +} + +interface ToolCallingOptions extends BaseChatModelCallOptions { + tools?: Array<{ function?: { name?: string } }>; +} + +/** + * A model that binds tools the way a real provider integration does: the + * formatted tools and `tool_choice` ride on the call options, and come back + * out through `invocationParams` — which is where the adapter reads them. + * Asks for whichever tool it was bound to first, then answers — as a chunk, + * which is what a 1.x provider returns and what `withStructuredOutput` checks. + */ +class ToolCallingModel extends BaseChatModel { + _llmType(): string { + return "tool-calling"; + } + + override invocationParams(options?: this["ParsedCallOptions"]): Record { + return { model: "tool-calling-1", tools: options?.tools, tool_choice: options?.tool_choice }; + } + + override bindTools(tools: unknown[], kwargs?: Partial) { + return this.withConfig({ + tools: tools.map((t) => convertToOpenAITool(t as never)), + ...kwargs, + } as Partial) as never; + } + + async _generate(messages: BaseMessage[], options: this["ParsedCallOptions"]): Promise { + const wanted = options.tools?.[0]?.function?.name ?? "get_weather"; + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessageChunk({ + content: ANSWER, + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + }) + : new AIMessageChunk({ + content: "", + tool_calls: [ + wanted === "Weather" + ? { id: "call_s", name: "Weather", args: { city: "Paris", sky: "sunny" }, type: "tool_call" } + : { id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }, + ], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +const WEATHER_DOCS = [ + new Document({ pageContent: "Paris is sunny", metadata: { source: "wx.txt" } }), + new Document({ pageContent: "Rome is rainy", metadata: { source: "wx2.txt" } }), +]; + +/** The smallest custom retriever: what a hand-rolled RAG lookup is. */ +class Docs extends BaseRetriever { + lc_namespace = ["failproofai", "fixtures"]; + + async _getRelevantDocuments(_query: string): Promise { + return WEATHER_DOCS; + } +} + +/** Deterministic, offline embeddings: letter frequencies. */ +class LetterEmbeddings extends Embeddings { + constructor() { + super({}); + } + + private vector(text: string): number[] { + const out = new Array(26).fill(0); + for (const ch of text.toLowerCase()) { + const code = ch.charCodeAt(0) - 97; + if (code >= 0 && code < 26) out[code] += 1; + } + return out; + } + + async embedDocuments(texts: string[]): Promise { + return texts.map((text) => this.vector(text)); + } + + async embedQuery(text: string): Promise { + return this.vector(text); + } +} + +/** + * An in-memory vector store, so `.asRetriever()` — the retriever almost every + * RAG app actually runs — is exercised without a network or a native module. + */ +class TinyVectorStore extends VectorStore { + declare FilterType: never; + private rows: Array<{ vector: number[]; doc: Document }> = []; + + _vectorstoreType(): string { + return "tiny"; + } + + async addVectors(vectors: number[][], documents: Document[]): Promise { + vectors.forEach((vector, i) => this.rows.push({ vector, doc: documents[i]! })); + } + + async addDocuments(documents: Document[]): Promise { + await this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents); + } + + async similaritySearchVectorWithScore(query: number[], k: number): Promise> { + const dot = (a: number[], b: number[]) => a.reduce((sum, x, i) => sum + x * (b[i] ?? 0), 0); + return this.rows + .map(({ vector, doc }): [Document, number] => [doc, dot(vector, query)]) + .sort((a, b) => b[1] - a[1]) + .slice(0, k); + } +} + +const prompt = () => ChatPromptTemplate.fromMessages([["human", "{q}"]]); +const lcel = () => prompt().pipe(new AnswerModel({})).pipe(new StringOutputParser()); + +/** + * Every plain-LangChain surface, each runnable with or without an explicit + * handler: `` runs it under `instrument()`, `handler:` passes + * `callbacks: [langchainHandler()]` instead and never instruments. + */ +const SURFACES: Record Promise> = { + lcel: (config) => lcel().invoke({ q: "weather?" }, config), + sequence: (config) => + RunnableSequence.from([ + RunnableLambda.from((x: number) => x + 1), + RunnableLambda.from((x: number) => x * 2), + ]).invoke(1, config), + parallel: (config) => + RunnableParallel.from({ + a: RunnableLambda.from((x: number) => x + 1), + b: RunnableLambda.from((x: number) => x * 2), + }).invoke(1, config), + lambda: (config) => RunnableLambda.from((x: string) => x.toUpperCase()).withConfig({ runName: "shout" }).invoke("abc", config), + retriever: (config) => new Docs().invoke("weather in Paris", config), + vectorstore: async (config) => { + const store = new TinyVectorStore(new LetterEmbeddings(), {}); + await store.addDocuments([WEATHER_DOCS[0]!]); + return store.asRetriever({ k: 1 }).invoke("Paris", config); + }, + rag: (config) => + RunnableSequence.from([ + { + context: new Docs().pipe((docs: Document[]) => docs.map((d) => d.pageContent).join("\n")), + q: new RunnablePassthrough(), + }, + ChatPromptTemplate.fromMessages([["human", "{context}\n{q}"]]), + new AnswerModel({}), + new StringOutputParser(), + ]).invoke("weather?", config), + batch3: (config) => lcel().batch([{ q: "a" }, { q: "b" }, { q: "c" }], config), + "stream-events": async (config) => { + const kinds: string[] = []; + for await (const event of lcel().streamEvents({ q: "weather?" }, { ...config, version: "v2" })) { + kinds.push(event.event); + } + return kinds.includes("on_chat_model_stream"); + }, + "lcel-stream": async (config) => { + let text = ""; + for await (const chunk of await lcel().stream({ q: "weather?" }, config)) text += chunk; + return text; + }, + structured: (config) => + new ToolCallingModel({}) + .withStructuredOutput(z.object({ city: z.string(), sky: z.string() }), { name: "Weather" }) + .invoke("weather?", config), + "bind-tools": (config) => + (new ToolCallingModel({}).bindTools([getWeather], { tool_choice: "get_weather" }) as unknown as ToolCallingModel) + .invoke("weather?", config) + .then((message) => (message as AIMessage).tool_calls?.length), +}; + +function buildGraph(model: ScriptedModel, tools = [getWeather]) { + const callModel = async (state: typeof MessagesAnnotation.State) => ({ + messages: [await model.invoke(state.messages)], + }); + const route = (state: typeof MessagesAnnotation.State) => { + const last = state.messages[state.messages.length - 1] as AIMessage; + return last.tool_calls?.length ? "tools" : END; + }; + return new StateGraph(MessagesAnnotation) + .addNode("agent", callModel) + .addNode("tools", new ToolNode(tools)) + .addEdge(START, "agent") + .addConditionalEdges("agent", route, ["tools", END]) + .addEdge("tools", "agent") + .compile({ name: "weather_graph" }); +} + +const Trail = Annotation.Root({ + trail: Annotation({ reducer: (a, b) => a.concat(b), default: () => [] }), +}); + +/** plan -> approve (asks a human) -> act, checkpointed so it can be resumed. */ +function buildHitlGraph(checkpointer = new MemorySaver()) { + return new StateGraph(Trail) + .addNode("plan", async () => ({ trail: ["plan"] })) + .addNode("approve", async () => { + const answer = interrupt({ prompt: "ship it?", options: ["yes", "no"] }) as string; + return { trail: [`approve:${answer}`] }; + }) + .addNode("act", async () => ({ trail: ["act"] })) + .addEdge(START, "plan") + .addEdge("plan", "approve") + .addEdge("approve", "act") + .addEdge("act", END) + .compile({ name: "hitl_graph", checkpointer }); +} + +/** A compiled graph used as a node of another one. */ +function buildParentGraph() { + const child = new StateGraph(Trail) + .addNode("inner", async () => ({ trail: ["inner"] })) + .addEdge(START, "inner") + .addEdge("inner", END) + .compile({ name: "child_graph" }); + return new StateGraph(Trail) + .addNode("pre", async () => ({ trail: ["pre"] })) + .addNode("child", child) + .addEdge(START, "pre") + .addEdge("pre", "child") + .addEdge("child", END) + .compile({ name: "parent_graph" }); +} + +/** + * A `MemorySaver`'s contents as JSON, and back. The one piece of state two + * processes share in a real deployment is the checkpointer; this stands in for + * a database-backed one so the pause and the approval can happen in two + * genuinely separate processes with nothing else in common. + */ +interface SaverState { + storage: Record; + writes: Record; +} +const encodeBytes = (_key: string, value: unknown): unknown => + value instanceof Uint8Array ? { __u8: Buffer.from(value).toString("base64") } : value; +const decodeBytes = (_key: string, value: unknown): unknown => + value !== null && typeof value === "object" && typeof (value as { __u8?: unknown }).__u8 === "string" + ? new Uint8Array(Buffer.from((value as { __u8: string }).__u8, "base64")) + : value; + +function saverState(saver: MemorySaver): SaverState { + return saver as unknown as SaverState; +} + +const question = { messages: [new HumanMessage("weather?")] }; +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** `instrument()` options per case; every case not listed uses the defaults. */ +const OPTIONS: Record> = { + "include-chains": { includeChains: ["summarise"] }, + "session-option": { sessionId: "fixed-session" }, + "capture-off": { captureContent: false }, +}; + +async function main(scenario: string): Promise { + const explicit = scenario.startsWith("handler:"); + const surface = SURFACES[explicit ? scenario.slice("handler:".length) : scenario]; + if (surface !== undefined) { + if (!explicit) report({ instrumented: await failproofai.instrument("langchain") }); + const out = await surface(explicit ? { callbacks: [langchainHandler() as never] } : {}); + report({ out }); + await failproofai.flush(); + return; + } + + if (scenario !== "handler") { + report({ instrumented: await failproofai.instrument("langchain", OPTIONS[scenario] ?? {}) }); + } + + switch (scenario) { + case "graph": + case "session-option": + case "capture-off": { + const out = await buildGraph(new ScriptedModel()).invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "stream": { + let chunks = 0; + for await (const _ of await buildGraph(new ScriptedModel()).stream(question, { streamMode: "updates" })) { + chunks += 1; + } + report({ chunks }); + break; + } + case "react": { + const agent = createReactAgent({ llm: new ScriptedModel(), tools: [getWeather], name: "react_bot" }); + const out = await agent.invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "error": { + try { + await buildGraph(new ScriptedModel({ fail: true })).invoke({ messages: [new HumanMessage("x")] }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "tool-error": { + const out = await buildGraph(new ScriptedModel(), [brokenWeather]).invoke(question); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "model": { + const out = await new ScriptedModel().invoke([new HumanMessage("hi")]); + report({ toolCalls: (out as AIMessage).tool_calls?.length }); + break; + } + case "tool": { + report({ out: await getWeather.invoke({ city: "Rome" }) }); + break; + } + case "batch": { + const out = await new ScriptedModel().batch([[new HumanMessage("a")], [new HumanMessage("b")]]); + report({ batch: out.length }); + break; + } + case "stream-model": { + let text = ""; + for await (const chunk of await new StreamingModel().stream([new HumanMessage("hi")])) { + text += String(chunk.content); + } + report({ text }); + break; + } + case "include-chains": { + const chain = RunnableLambda.from((x: string) => x) + .pipe(RunnableLambda.from((x: string) => x.toUpperCase()).withConfig({ runName: "summarise" })) + .withConfig({ runName: "pipeline" }); + report({ out: await chain.invoke("abc") }); + break; + } + case "metadata-session": { + await buildGraph(new ScriptedModel()).invoke(question, { + metadata: { failproofai_sdk_session_id: "meta-sid" }, + }); + break; + } + case "thread-session": { + await buildGraph(new ScriptedModel()).invoke(question, { configurable: { thread_id: "thread-9" } }); + break; + } + case "hitl": + case "hitl-uninstrument": { + const graph = buildHitlGraph(); + const config = { configurable: { thread_id: "t-1" } }; + const first = await graph.invoke({ trail: [] }, config); + report({ interrupted: "__interrupt__" in first }); + if (scenario === "hitl") { + const done = await graph.invoke(new Command({ resume: "yes" }), config); + report({ trail: done.trail }); + } else { + report({ removed: failproofai.uninstrument() }); + } + break; + } + case "remote-resume-pause": { + // Process A: serve the request that interrupts, persist the checkpoint, + // exit with the pause still open. + const saver = new MemorySaver(); + // `durability: "sync"`: the default persists the checkpoint in the + // background, and this process is about to exit. + await buildHitlGraph(saver).invoke({ trail: [] }, { configurable: { thread_id: "t-1" }, durability: "sync" }); + // The two tables, not the saver: 1.x gives MemorySaver a `toJSON`. + const { storage, writes } = saverState(saver); + writeFileSync(process.argv[3]!, JSON.stringify({ storage, writes }, encodeBytes)); + break; + } + case "remote-resume": { + // Process B: a different process, sharing only the checkpointer, takes + // the human's answer. + const file = `${process.argv[1]}.${process.pid}.checkpoint.json`; + const child = spawnSync(process.execPath, [process.argv[1]!, "remote-resume-pause", file], { + env: process.env, + encoding: "utf8", + }); + if (child.status !== 0) throw new Error(`the pausing process failed: ${child.stderr}`); + const saver = new MemorySaver(); + const saved = JSON.parse(readFileSync(file, "utf8"), decodeBytes) as SaverState; + rmSync(file, { force: true }); + Object.assign(saverState(saver).storage, saved.storage); + Object.assign(saverState(saver).writes, saved.writes); + const done = await buildHitlGraph(saver).invoke(new Command({ resume: "yes" }), { + configurable: { thread_id: "t-1" }, + }); + report({ trail: done.trail }); + break; + } + case "abort": { + // The caller gives up: the signal fires while a node is running. + const controller = new AbortController(); + const graph = new StateGraph(Trail) + .addNode("slow", async () => { + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { trail: ["slow"] }; + }) + .addEdge(START, "slow") + .addEdge("slow", END) + .compile({ name: "abort_graph" }); + try { + await graph.invoke({ trail: [] }, { signal: controller.signal }); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).name }); + } + // LangGraph.js 1.x never closes an aborted graph's own run; the adapter + // does, once the run has been silent for its grace period. Outlive it. + await new Promise((resolve) => setTimeout(resolve, 3_300)); + break; + } + case "subgraph": { + const out = await buildParentGraph().invoke({ trail: [] }); + report({ trail: out.trail }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => buildGraph(new ScriptedModel()).invoke(question)), + ); + break; + } + case "handler": { + // No instrument(): the explicit handler is the patch-free path. + const out = await buildGraph(new ScriptedModel()).invoke(question, { + callbacks: [langchainHandler() as never], + }); + report({ answer: out.messages.at(-1)?.content }); + break; + } + case "handler-and-instrument": { + report({ instrumented: await failproofai.instrument("langchain") }); + await buildGraph(new ScriptedModel()).invoke(question, { callbacks: [langchainHandler() as never] }); + break; + } + case "uninstrument": { + report({ removed: failproofai.uninstrument() }); + await buildGraph(new ScriptedModel()).invoke(question); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "graph").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/langchain-1/package-lock.json b/sdk/typescript/integration/fixtures/langchain-1/package-lock.json new file mode 100644 index 000000000..50a1ed97f --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-1/package-lock.json @@ -0,0 +1,336 @@ +{ + "name": "failproofai-it-langchain-1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-langchain-1", + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "langchain": "1.5.12", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, + "node_modules/@langchain/core": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.12.tgz", + "integrity": "sha512-DvNsrN5Gz+bgEy/B/kK1AvxxcrwdrflI8hZJxqo/DtA17FMUeWM5k0o1c6G8ChH8+sIakV2UsDudFmNZJp1ugQ==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.17", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.17.tgz", + "integrity": "sha512-gkd34M42D5SdRaaHS6NSD+zTuZBgmEqecld3BenVlql11WshvarKH/2ZQbNmQQ0knbHgtaQEg3VeH0bUFt2B1Q==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.11.2", + "@langchain/protocol": "^0.0.19", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.11.2.tgz", + "integrity": "sha512-b2s4qdFKePgZudPJZfDCtQOmVfHoiKCTgSSdOxXtMYZzg9A+g2kI/ll1ZPMYIydGE3QIgGLIhyFCz5ULedi5Nw==", + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.19", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.2.tgz", + "integrity": "sha512-prbX4Z3YszrFNgH+MW5Zoeq3baXrMtP/MQnFeET90UB/GtGcGDQ5Usg9OCy6ETjTTntOw1SL2z9fMPUppN3Guw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/protocol": { + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", + "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/langchain": { + "version": "1.5.12", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.12.tgz", + "integrity": "sha512-cWNsKiRLyv0NN1e9IR+0CUyS4cBn1LSJ7OhqTa0RQGK735cb4l1+wd4/lQJz4imJ4sojOl+VfHw6kKZwWFNYlQ==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph": "^1.4.13", + "@langchain/langgraph-checkpoint": "^1.1.5", + "langsmith": ">=0.5.0 <1.0.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.12" + } + }, + "node_modules/langsmith": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.10.5.tgz", + "integrity": "sha512-VUh6LQYGb+wjmMz5Ulazqx1rUHLbWi2XZvVkSHfxDjz6V7u9TJOB6jW5CC84xcpffu7Jr1MZpBbEz3gg972HbA==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-1/package.json b/sdk/typescript/integration/fixtures/langchain-1/package.json new file mode 100644 index 000000000..a2945d7ec --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-1/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-langchain-1", + "private": true, + "type": "module", + "description": "Integration fixture: LangChain.js 1.x + LangGraph.js 1.x against the packed @failproofai/sdk.", + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "langchain": "1.5.12", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-1/tsconfig.json b/sdk/typescript/integration/fixtures/langchain-1/tsconfig.json new file mode 100644 index 000000000..c3658e63a --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-1/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts", "agent-v1.ts"] +} diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/.npmrc b/sdk/typescript/integration/fixtures/langchain-dup-core/.npmrc new file mode 100644 index 000000000..37c7e9fc0 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/.npmrc @@ -0,0 +1,4 @@ +# Install the vendored provider as a real copy (node_modules/lc-weather-provider, +# its own core nested beneath it) rather than a symlink: the layout npm gives a +# provider fetched from the registry. +install-links=true diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/agent.ts b/sdk/typescript/integration/fixtures/langchain-dup-core/agent.ts new file mode 100644 index 000000000..b093cc6d3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/agent.ts @@ -0,0 +1,96 @@ +// Two copies of @langchain/core in one process. Run as `node agent.{mjs,cjs} `. +// +// The app is on @langchain/core 1.2.12 + LangGraph.js 1.4.17. Its provider, +// `lc-weather-provider` (vendor/), pins @langchain/core 0.3.80 as a hard +// dependency — the way @langchain/community and many third-party integrations +// did — so npm cannot dedupe it and nests the second core at +// node_modules/lc-weather-provider/node_modules/@langchain/core. Everything the +// provider exports is built on THAT copy: its own Runnable, its own +// CallbackManager, which `instrument()` resolving from the app never sees. +// +// `` runs under `instrument()`; `handler:` passes +// `callbacks: [langchainHandler()]` instead and never instruments. +import * as failproofai from "@failproofai/sdk"; +import { langchainHandler } from "@failproofai/sdk/langchain"; +import { AIMessage, HumanMessage } from "@langchain/core/messages"; +import { StringOutputParser } from "@langchain/core/output_parsers"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { RunnableLambda, type RunnableConfig } from "@langchain/core/runnables"; +import { tool } from "@langchain/core/tools"; +import { END, MessagesAnnotation, START, StateGraph } from "@langchain/langgraph"; +import { ToolNode } from "@langchain/langgraph/prebuilt"; +import { ChatWeather, WeatherRetriever, forecast, weatherTool } from "lc-weather-provider"; +import { z } from "zod"; + +const getWeather = tool(async ({ city }: { city: string }) => `sunny in ${city}`, { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), +}); + +/** The app's graph (app core), whose model node calls the PROVIDER's model (nested core). */ +function buildGraph() { + const model = new ChatWeather({}); + // No config threaded through, as customers write it: the model finds its + // parent run through the AsyncLocalStorage both copies share on globalThis. + const callModel = async (state: typeof MessagesAnnotation.State) => ({ + messages: [(await model.invoke(state.messages)) as unknown as AIMessage], + }); + const route = (state: typeof MessagesAnnotation.State) => { + const last = state.messages[state.messages.length - 1] as AIMessage; + return last.tool_calls?.length ? "tools" : END; + }; + return new StateGraph(MessagesAnnotation) + .addNode("agent", callModel) + .addNode("tools", new ToolNode([getWeather])) + .addEdge(START, "agent") + .addConditionalEdges("agent", route, ["tools", END]) + .addEdge("tools", "agent") + .compile({ name: "weather_graph" }); +} + +const CASES: Record Promise> = { + // Roots created by the NESTED copy: nothing of the app's core is involved. + "nested-model": (config) => new ChatWeather({}).invoke([new HumanMessage("hi")], config).then((m) => m.tool_calls?.length), + "nested-tool": (config) => weatherTool.invoke("Rome", config), + "nested-retriever": (config) => new WeatherRetriever({}).invoke("Paris", config).then((docs) => docs.length), + "nested-runnable": (config) => forecast.invoke("Rome", config), + // The nested copy's runs as CHILDREN of the app copy's. + "app-chain": (config) => + ChatPromptTemplate.fromMessages([["human", "{q}"]]) + .pipe(new ChatWeather({}) as never) + .pipe(new StringOutputParser()) + .invoke({ q: "weather?" }, config), + "app-graph": (config) => + buildGraph() + .invoke({ messages: [new HumanMessage("weather?")] }, config) + .then((out) => out.messages.at(-1)?.content), + // An app-copy runnable wrapping a provider model call with NO config + // threaded through — the child finds its parent only through the shared + // AsyncLocalStorage, which both copies read from `globalThis`. + "app-lambda": (config) => + RunnableLambda.from(async (q: string) => (await new ChatWeather({}).invoke(q)).tool_calls?.length) + .withConfig({ runName: "outer" }) + .invoke("weather?", config), +}; + +/** + * ``: under `instrument()`. `handler:`: `callbacks: + * [langchainHandler()]` and no instrument. `both:`: the two together. + * `uninstrument:`: instrument, uninstrument, then run. + */ +async function main(scenario: string): Promise { + const [mode, name] = scenario.includes(":") ? scenario.split(":", 2) : ["instrument", scenario]; + const run = CASES[name!]; + if (run === undefined) throw new Error(`unknown scenario ${scenario}`); + if (mode !== "handler") console.log(JSON.stringify({ instrumented: await failproofai.instrument("langchain") })); + if (mode === "uninstrument") console.log(JSON.stringify({ removed: failproofai.uninstrument() })); + const explicit = mode === "handler" || mode === "both"; + console.log(JSON.stringify({ out: await run(explicit ? { callbacks: [langchainHandler() as never] } : {}) })); + await failproofai.flush(); +} + +main(process.argv[2] ?? "nested-model").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/package-lock.json b/sdk/typescript/integration/fixtures/langchain-dup-core/package-lock.json new file mode 100644 index 000000000..230930bf1 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/package-lock.json @@ -0,0 +1,579 @@ +{ + "name": "failproofai-it-langchain-dup-core", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-langchain-dup-core", + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "lc-weather-provider": "file:vendor/lc-weather-provider", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, + "node_modules/@langchain/core": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.12.tgz", + "integrity": "sha512-DvNsrN5Gz+bgEy/B/kK1AvxxcrwdrflI8hZJxqo/DtA17FMUeWM5k0o1c6G8ChH8+sIakV2UsDudFmNZJp1ugQ==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.17", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.17.tgz", + "integrity": "sha512-gkd34M42D5SdRaaHS6NSD+zTuZBgmEqecld3BenVlql11WshvarKH/2ZQbNmQQ0knbHgtaQEg3VeH0bUFt2B1Q==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.11.2", + "@langchain/protocol": "^0.0.19", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.11.2.tgz", + "integrity": "sha512-b2s4qdFKePgZudPJZfDCtQOmVfHoiKCTgSSdOxXtMYZzg9A+g2kI/ll1ZPMYIydGE3QIgGLIhyFCz5ULedi5Nw==", + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.19", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.2.tgz", + "integrity": "sha512-prbX4Z3YszrFNgH+MW5Zoeq3baXrMtP/MQnFeET90UB/GtGcGDQ5Usg9OCy6ETjTTntOw1SL2z9fMPUppN3Guw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/protocol": { + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", + "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "license": "MIT" + }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/console-table-printer": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.16.1.tgz", + "integrity": "sha512-Sc9FRJ4O9xKGNrvulNdPfK5SyBcZ6lcaRnDE4AQ/uw6IDtjHhsqyzzqcnMikjyGaiOOF2tNOKoBhbVjRvFy9Lw==", + "license": "MIT", + "dependencies": { + "simple-wcswidth": "^1.1.2" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/langsmith": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.10.5.tgz", + "integrity": "sha512-VUh6LQYGb+wjmMz5Ulazqx1rUHLbWi2XZvVkSHfxDjz6V7u9TJOB6jW5CC84xcpffu7Jr1MZpBbEz3gg972HbA==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/lc-weather-provider": { + "version": "1.0.0", + "resolved": "file:vendor/lc-weather-provider", + "license": "MIT", + "dependencies": { + "@langchain/core": "0.3.80" + } + }, + "node_modules/lc-weather-provider/node_modules/@langchain/core": { + "version": "0.3.80", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.80.tgz", + "integrity": "sha512-vcJDV2vk1AlCwSh3aBm/urQ1ZrlXFFBocv11bz/NBUfLWD5/UDNMzwPdaAd2dKvNmTWa9FM2lirLU3+JCf4cRA==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.3.67", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.25.32", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/lc-weather-provider/node_modules/langsmith": { + "version": "0.3.87", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.87.tgz", + "integrity": "sha512-XXR1+9INH8YX96FKWc5tie0QixWz6tOqAsAKfcJyPkE0xPep+NDz0IQLR32q4bn10QK3LqD2HN6T3n6z1YLW7Q==", + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "chalk": "^4.1.2", + "console-table-printer": "^2.12.1", + "p-queue": "^6.6.2", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/lc-weather-provider/node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lc-weather-provider/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-wcswidth": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", + "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/package.json b/sdk/typescript/integration/fixtures/langchain-dup-core/package.json new file mode 100644 index 000000000..3a8e3edf2 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-langchain-dup-core", + "private": true, + "type": "module", + "description": "Integration fixture: an app on @langchain/core 1.x + LangGraph.js 1.x whose provider package (vendor/lc-weather-provider) pins @langchain/core 0.3, so npm nests a SECOND core under it — two CallbackManager classes in one process.", + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "lc-weather-provider": "file:vendor/lc-weather-provider", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/tsconfig.json b/sdk/typescript/integration/fixtures/langchain-dup-core/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.cjs b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.cjs new file mode 100644 index 000000000..9d37467cd --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.cjs @@ -0,0 +1,53 @@ +"use strict"; +// The `require` build of index.mjs — the same module; keep the two in step. +const { Document } = require("@langchain/core/documents"); +const { BaseChatModel } = require("@langchain/core/language_models/chat_models"); +const { AIMessage } = require("@langchain/core/messages"); +const { BaseRetriever } = require("@langchain/core/retrievers"); +const { RunnableLambda } = require("@langchain/core/runnables"); +const { DynamicTool } = require("@langchain/core/tools"); + +const typeOf = (message) => (typeof message._getType === "function" ? message._getType() : message.getType()); + +class ChatWeather extends BaseChatModel { + _llmType() { + return "weather"; + } + + bindTools() { + return this; + } + + async _generate(messages) { + const answered = messages.some((m) => typeOf(m) === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +class WeatherRetriever extends BaseRetriever { + lc_namespace = ["lc_weather_provider"]; + + async _getRelevantDocuments() { + return [new Document({ pageContent: "Paris is sunny", metadata: { source: "wx.txt" } })]; + } +} + +const weatherTool = new DynamicTool({ + name: "get_weather", + description: "Current weather for a city", + func: async (city) => `sunny in ${city}`, +}); + +const forecast = RunnableLambda.from(async (city) => `sunny in ${city}`).withConfig({ runName: "forecast" }); + +module.exports = { ChatWeather, WeatherRetriever, weatherTool, forecast }; diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.d.ts b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.d.ts new file mode 100644 index 000000000..cd160bc87 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.d.ts @@ -0,0 +1,17 @@ +// Deliberately loose: these classes come from a DIFFERENT @langchain/core than +// the app's, so typing them against the app's copy would be a lie. +export declare class ChatWeather { + constructor(fields?: Record); + invoke(input: unknown, config?: object): Promise<{ content: unknown; tool_calls?: unknown[] }>; +} +export declare class WeatherRetriever { + constructor(fields?: Record); + invoke(input: string, config?: object): Promise; +} +export declare const weatherTool: { + name: string; + invoke(input: unknown, config?: object): Promise; +}; +export declare const forecast: { + invoke(input: string, config?: object): Promise; +}; diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.mjs b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.mjs new file mode 100644 index 000000000..7c016b4f8 --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/index.mjs @@ -0,0 +1,55 @@ +// A provider package built on ITS OWN copy of @langchain/core (0.3.80, nested +// under this package by npm because the app runs 1.x). Every class here comes +// from that nested copy — its own Runnable, its own CallbackManager. +// index.cjs is the same module for `require`; keep the two in step. +import { Document } from "@langchain/core/documents"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; +import { AIMessage } from "@langchain/core/messages"; +import { BaseRetriever } from "@langchain/core/retrievers"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { DynamicTool } from "@langchain/core/tools"; + +const typeOf = (message) => (typeof message._getType === "function" ? message._getType() : message.getType()); + +/** First call asks for `get_weather`, the next one answers — the fixture's usual script. */ +export class ChatWeather extends BaseChatModel { + _llmType() { + return "weather"; + } + + bindTools() { + return this; + } + + async _generate(messages) { + const answered = messages.some((m) => typeOf(m) === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +export class WeatherRetriever extends BaseRetriever { + lc_namespace = ["lc_weather_provider"]; + + async _getRelevantDocuments() { + return [new Document({ pageContent: "Paris is sunny", metadata: { source: "wx.txt" } })]; + } +} + +export const weatherTool = new DynamicTool({ + name: "get_weather", + description: "Current weather for a city", + func: async (city) => `sunny in ${city}`, +}); + +/** A provider-built runnable the app invokes directly. */ +export const forecast = RunnableLambda.from(async (city) => `sunny in ${city}`).withConfig({ runName: "forecast" }); diff --git a/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/package.json b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/package.json new file mode 100644 index 000000000..32e3f80ab --- /dev/null +++ b/sdk/typescript/integration/fixtures/langchain-dup-core/vendor/lc-weather-provider/package.json @@ -0,0 +1,18 @@ +{ + "name": "lc-weather-provider", + "version": "1.0.0", + "description": "Integration fixture: a LangChain provider package that pins its OWN @langchain/core as a hard dependency, the way @langchain/community and many third-party integrations did, so npm nests a second core under it.", + "license": "MIT", + "main": "./index.cjs", + "types": "./index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.mjs", + "require": "./index.cjs" + } + }, + "dependencies": { + "@langchain/core": "0.3.80" + } +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.11/agent.ts b/sdk/typescript/integration/fixtures/llamaindex-0.11/agent.ts new file mode 100644 index 000000000..72266cd41 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.11/agent.ts @@ -0,0 +1,659 @@ +// LlamaIndex.TS consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The scripted model makes it deterministic and offline, and it is built +// the way LlamaIndex's own provider packages build theirs (`@llamaindex/openai` +// decorates `chat` with exactly `@wrapEventCaller @wrapLLMEvent`), so the +// callback events it produces are the ones a real provider produces. The +// assertions compare against the Python SDK's golden trace for the equivalent +// `FunctionAgent` program. +import * as failproofai from "@failproofai/sdk"; +import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator"; +import { + ToolCallLLM, + type ChatMessage, + type ChatResponse, + type ChatResponseChunk, + type LLMChatParamsNonStreaming, + type LLMChatParamsStreaming, + type LLMMetadata, + type ToolCallLLMMessageOptions, +} from "@llamaindex/core/llms"; +import { RetrieverQueryEngine, type QueryBundle } from "@llamaindex/core/query-engine"; +import { getResponseSynthesizer } from "@llamaindex/core/response-synthesizers"; +import { BaseRetriever } from "@llamaindex/core/retriever"; +import { TextNode, type NodeWithScore } from "@llamaindex/core/schema"; +import { extractText } from "@llamaindex/core/utils"; +import { agent, createWorkflow, multiAgent, workflowEvent } from "@llamaindex/workflow"; +import { + BaseEmbedding, + CondenseQuestionChatEngine, + ContextChatEngine, + Document, + FunctionTool, + LLMAgent, + QueryEngineTool, + Settings, + SimpleChatEngine, + VectorStoreIndex, + tool, +} from "llamaindex"; +import { z } from "zod"; + +type Options = ToolCallLLMMessageOptions; + +/** One scripted turn: either a tool call or a final answer, with its usage. */ +interface Turn { + tool?: { name: string; input: Record; id: string }; + /** Several tool calls in ONE assistant message (parallel tool calling). */ + tools?: Array<{ name: string; input: Record; id: string }>; + text?: string; + /** The provider fails: before answering, or (streaming) after the first chunk. */ + fail?: string; + usage: { prompt_tokens: number; completion_tokens: number }; +} + +const weatherTurns = (): Turn[] => [ + { tool: { name: "get_weather", input: { city: "Paris" }, id: "call_1" }, usage: { prompt_tokens: 12, completion_tokens: 5 } }, + { text: "It is sunny in Paris.", usage: { prompt_tokens: 30, completion_tokens: 7 } }, +]; + +class ScriptedLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "scripted-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + private readonly turns: Turn[]; + + constructor(turns: Turn[] = weatherTurns()) { + super(); + this.turns = turns; + } + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const turn = this.turns.shift() ?? { text: "done", usage: { prompt_tokens: 1, completion_tokens: 1 } }; + const calls = turn.tools ?? (turn.tool ? [turn.tool] : []); + const options: Options = calls.length > 0 ? { toolCall: calls } : {}; + const message: ChatMessage = { role: "assistant", content: turn.text ?? "", options }; + if (params.stream) { + // OpenAI's stream shape: content chunks, then a content-less chunk whose + // `raw` carries the usage (what `stream_options.include_usage` sends). + return (async function* (): AsyncGenerator> { + yield { delta: turn.text ?? "", raw: { choices: [{ delta: {} }] }, options }; + if (turn.fail) throw new Error(turn.fail); + yield { delta: "", raw: { choices: [], usage: turn.usage }, options: {} }; + })(); + } + if (turn.fail) throw new Error(turn.fail); + return { message, raw: { choices: [{ finish_reason: calls.length > 0 ? "tool_calls" : "stop" }], usage: turn.usage } }; + } +} + +// --------------------------------------------------------------------------- +// One object built once and shared by concurrent requests — the way a server +// holds its query engine or agent. Everything below is stateless per call, so +// two calls in flight at once cannot disturb each other: whatever the trace +// mixes up is the adapter's doing. +// --------------------------------------------------------------------------- + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const textOf = (content: unknown): string => (typeof content === "string" ? content : JSON.stringify(content)); +/** The city a request is about. Paris is the SLOW request, so Rome starts and ends inside it. */ +const cityIn = (text: string): "Paris" | "Rome" => (text.includes("Rome") ? "Rome" : "Paris"); +const delay = (city: string) => sleep(city === "Paris" ? 40 : 5); +/** Per-city usage, so a model_response carries which request it answered. */ +const USAGE = { Paris: { prompt_tokens: 11, completion_tokens: 2 }, Rome: { prompt_tokens: 21, completion_tokens: 3 } }; + +/** A model that answers from its input alone: a tool call first, then the answer. */ +class EchoLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "echo-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const city = cityIn(params.messages.map((m) => textOf(m.content)).join("\n")); + await delay(city); + const answered = params.messages.some((m) => m.options !== undefined && "toolResult" in m.options); + const call = params.tools?.length && !answered ? { name: "get_weather", input: { city }, id: `call_${city}` } : null; + const options: Options = call ? { toolCall: [call] } : {}; + const text = call ? "" : `It is sunny in ${city}.`; + const usage = USAGE[city]; + if (params.stream) { + return (async function* (): AsyncGenerator> { + yield { delta: text, raw: { choices: [{ delta: {} }] }, options }; + await delay(city); + yield { delta: "", raw: { choices: [], usage }, options: {} }; + })(); + } + const message: ChatMessage = { role: "assistant", content: text, options }; + return { message, raw: { choices: [{ finish_reason: call ? "tool_calls" : "stop" }], usage } }; + } +} + +/** A retriever with no index and no embeddings: one note about the question. */ +class NotesRetriever extends BaseRetriever { + // BaseRetriever's constructor is protected; a subclass's is public. + constructor() { + super(); + } + + async _retrieve(params: QueryBundle): Promise { + const question = extractText(params.query); + await delay(cityIn(question)); + return [{ node: new TextNode({ text: `notes on ${question}` }), score: 1 }]; + } +} + +const getWeather = tool({ + name: "get_weather", + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), + execute: ({ city }) => `sunny in ${city}`, +}); + +const broken = tool({ + name: "broken", + description: "Always fails", + parameters: z.object({ city: z.string() }), + execute: (): string => { + throw new Error("tool exploded"); + }, +}); + +// --------------------------------------------------------------------------- +// The coverage sweep: every commonly used LlamaIndex.TS surface, offline. +// --------------------------------------------------------------------------- + +/** + * An embedding model with no network: one dimension per city, so a question + * about Paris retrieves the Paris note and nothing else. + */ +const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Delhi", "Tokyo", "Quito", "Accra", "Hanoi"] as const; +class FakeEmbedding extends BaseEmbedding { + // BaseEmbedding's constructor is protected; a subclass's is public. + constructor() { + super(); + } + + async getTextEmbedding(text: string): Promise { + return [...CITIES.map((city) => (text.includes(city) ? 1 : 0)), 0.01]; + } +} + +const HISTORY: ChatMessage[] = [ + { role: "user", content: "hi" }, + { role: "assistant", content: "hello" }, +]; + +/** Every city's note, indexed with the fake embedding model. */ +const cityIndex = () => + VectorStoreIndex.fromDocuments(CITIES.map((city) => new Document({ text: `${city} is sunny.`, id_: `doc-${city}` }))); + +/** The city a question is about. */ +const cityOf = (text: string): string => { + const hits = CITIES.filter((city) => text.includes(city)); + return hits[hits.length - 1] ?? "Paris"; +}; + +/** + * `EchoLLM` for ten requests: answers from its input alone (a tool call, then + * the answer), with a per-city delay and per-city usage so every event says + * which request it belongs to. + */ +class CityLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "city-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const city = cityOf(params.messages.map((m) => textOf(m.content)).join("\n")); + const n = CITIES.indexOf(city as (typeof CITIES)[number]); + await sleep(((n * 7) % 5) * 6); + const answered = params.messages.some((m) => m.options !== undefined && "toolResult" in m.options); + const call = params.tools?.length && !answered ? { name: "get_weather", input: { city }, id: `call_${city}` } : null; + const options: Options = call ? { toolCall: [call] } : {}; + const text = call ? "" : `It is sunny in ${city}.`; + const usage = { prompt_tokens: 100 + n, completion_tokens: n }; + if (params.stream) { + return (async function* (): AsyncGenerator> { + yield { delta: text, raw: { choices: [{ delta: {} }] }, options }; + await sleep(((n * 3) % 4) * 5); + yield { delta: "", raw: { choices: [], usage }, options: {} }; + })(); + } + const message: ChatMessage = { role: "assistant", content: text, options }; + return { message, raw: { choices: [{ finish_reason: call ? "tool_calls" : "stop" }], usage } }; + } +} + +const say = (text: string, prompt = 5, completion = 2): Turn => ({ + text, + usage: { prompt_tokens: prompt, completion_tokens: completion }, +}); + +/** Drain whatever a streaming chat or query returned. */ +async function drain(stream: AsyncIterable): Promise { + let chunks = 0; + for await (const _ of stream) chunks += 1; + return chunks; +} + +const lookup = FunctionTool.from(({ city }: { city: string }) => `sunny in ${city}`, { + name: "lookup_weather", + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), +}); + +const getTime = tool({ + name: "get_time", + description: "Current time in a city", + parameters: z.object({ city: z.string() }), + execute: ({ city }) => `noon in ${city}`, +}); + +/** A plain `createWorkflow()` workflow with two steps; the second asks the model. */ +function plainWorkflow(llm: ScriptedLLM) { + const start = workflowEvent({ debugLabel: "start" }); + const researched = workflowEvent({ debugLabel: "researched" }); + const stop = workflowEvent({ debugLabel: "stop" }); + const workflow = createWorkflow(); + // Named functions: a step's name is its handler's name. `...args` because the + // floor's runtime calls `handler(event)` and later ones `handler(context, event)`. + workflow.handle([start], async function research(...args: unknown[]) { + const event = args[args.length - 1] as { data: string }; + await sleep(1); + return researched.with(`notes on ${event.data}`); + }); + workflow.handle([researched], async function answer(...args: unknown[]) { + const event = args[args.length - 1] as { data: string }; + const response = await llm.chat({ messages: [{ role: "user", content: event.data }] }); + return stop.with(String(response.message.content)); + }); + return { workflow, start, stop }; +} + +async function runPlainWorkflow(llm: ScriptedLLM, input: string): Promise { + const { workflow, start, stop } = plainWorkflow(llm); + const { stream, sendEvent } = workflow.createContext(); + sendEvent(start.with(input)); + const events = await stream.until(stop).toArray(); + return String((events.at(-1) as { data: unknown }).data); +} + +const report = (value: unknown) => console.log(JSON.stringify(value)); +const QUESTION = "weather in Paris?"; + +async function main(scenario: string): Promise { + // Built BEFORE instrument(): the adapter must not depend on construction order. + const early = agent({ llm: new ScriptedLLM(), tools: [getWeather] }); + // Shared objects, also built at startup, before instrument(). + const sharedEngine = new RetrieverQueryEngine( + new NotesRetriever(), + getResponseSynthesizer("compact", { llm: new EchoLLM() }), + ); + const sharedLegacy = new LLMAgent({ llm: new EchoLLM(), tools: [getWeather] }); + const sharedAgent = agent({ llm: new EchoLLM(), tools: [getWeather] }); + // Adapter options, for the cases that exercise one (`embeddings: true`). + const options = JSON.parse(process.env.FAILPROOFAI_IT_OPTIONS ?? "{}") as Record; + report({ instrumented: await failproofai.instrument("llamaindex", options) }); + const QUESTIONS = ["weather in Paris?", "weather in Rome?"]; + + switch (scenario) { + case "workflow": { + const out = await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "named": { + const out = await agent({ name: "weather_bot", llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "early": { + const out = await early.run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "stream": { + let events = 0; + for await (const _ of agent({ llm: new ScriptedLLM(), tools: [getWeather] }).runStream(QUESTION)) events += 1; + report({ events }); + break; + } + case "legacy": { + const out = await new LLMAgent({ llm: new ScriptedLLM(), tools: [getWeather] }).chat({ message: QUESTION }); + report({ answer: String(out.message.content) }); + break; + } + case "model": { + const out = await new ScriptedLLM([{ text: "hello", usage: { prompt_tokens: 3, completion_tokens: 1 } }]).chat({ + messages: [{ role: "user", content: "hi" }], + }); + report({ answer: out.message.content }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => + agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION), + ), + ); + break; + } + case "tool-error": { + const turns = weatherTurns(); + turns[0]!.tool = { name: "broken", input: { city: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [broken] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "handoff": { + const triage = agent({ + name: "triage", + description: "Routes questions", + llm: new ScriptedLLM([ + { + tool: { name: "handOff", input: { toAgent: "forecaster", reason: "weather question" }, id: "call_h" }, + usage: { prompt_tokens: 8, completion_tokens: 4 }, + }, + ]), + tools: [getWeather], + canHandoffTo: ["forecaster"], + }); + const forecaster = agent({ + name: "forecaster", + description: "Knows the weather", + llm: new ScriptedLLM(), + tools: [getWeather], + }); + const out = await multiAgent({ agents: [triage, forecaster], rootAgent: triage }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "shared-query": { + const answers = await Promise.all(QUESTIONS.map((query) => sharedEngine.query({ query }))); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + case "shared-legacy": { + const answers = await Promise.all(QUESTIONS.map((message) => sharedLegacy.chat({ message }))); + report({ answers: answers.map((a) => String(a.message.content)) }); + break; + } + case "shared-workflow": { + const answers = await Promise.all(QUESTIONS.map((question) => sharedAgent.run(question))); + report({ answers: answers.map((a) => a.data.result) }); + break; + } + case "uninstrument": { + // One recorded run, then nothing: the second workflow and the legacy agent + // run after uninstrument() and must leave no trace. + await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ removed: failproofai.uninstrument() }); + await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + await new LLMAgent({ llm: new ScriptedLLM(), tools: [getWeather] }).chat({ message: QUESTION }); + break; + } + // -- chat engines -------------------------------------------------------- + case "chat-simple": { + const engine = new SimpleChatEngine({ llm: new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]) }); + const out = await engine.chat({ message: QUESTION, chatHistory: HISTORY }); + report({ answer: out.message.content }); + break; + } + case "chat-simple-stream": { + const engine = new SimpleChatEngine({ llm: new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]) }); + report({ chunks: await drain(await engine.chat({ message: QUESTION, chatHistory: HISTORY, stream: true })) }); + break; + } + case "chat-context": + case "chat-context-stream": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + // The floor's chat memory reads Settings.llm even when a chatModel is given. + const llm = new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]); + Settings.llm = llm; + const engine = new ContextChatEngine({ + retriever: index.asRetriever({ similarityTopK: 1 }), + chatModel: llm, + chatHistory: HISTORY, + }); + if (scenario === "chat-context") { + report({ answer: (await engine.chat({ message: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.chat({ message: QUESTION, stream: true })) }); + } + break; + } + case "chat-condense": + case "chat-condense-stream": { + Settings.embedModel = new FakeEmbedding(); + // CondenseQuestionChatEngine takes its model from Settings, and so does + // the query engine's synthesizer: first the condensed question, then the answer. + Settings.llm = new ScriptedLLM([say("What is the weather in Paris?", 6, 3), say("It is sunny in Paris.", 9, 4)]); + const index = await cityIndex(); + const engine = new CondenseQuestionChatEngine({ + queryEngine: index.asQueryEngine({ similarityTopK: 1 }), + chatHistory: HISTORY, + }); + if (scenario === "chat-condense") { + report({ answer: (await engine.chat({ message: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.chat({ message: QUESTION, stream: true })) }); + } + break; + } + // -- retrieval and indexes ----------------------------------------------- + case "index-build": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + report({ built: typeof index.asRetriever }); + break; + } + case "retriever": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + const nodes = await index.asRetriever({ similarityTopK: 1 }).retrieve({ query: QUESTION }); + report({ nodes: nodes.map((n) => n.node.id_) }); + break; + } + case "index-query": + case "index-query-stream": { + Settings.embedModel = new FakeEmbedding(); + Settings.llm = new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]); + const engine = (await cityIndex()).asQueryEngine({ similarityTopK: 1 }); + if (scenario === "index-query") { + report({ answer: (await engine.query({ query: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.query({ query: QUESTION, stream: true })) }); + } + break; + } + // -- plain workflows ----------------------------------------------------- + case "custom-workflow": { + report({ answer: await runPlainWorkflow(new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]), QUESTION) }); + break; + } + case "custom-workflow-scoped": { + const answer = await failproofai.agent("forecast_flow", { goal: QUESTION }, () => + runPlainWorkflow(new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]), QUESTION), + ); + report({ answer }); + break; + } + // -- agents -------------------------------------------------------------- + case "handoff3": { + const handOff = (to: string, id: string): Turn => ({ + tool: { name: "handOff", input: { toAgent: to, reason: "next" }, id }, + usage: { prompt_tokens: 8, completion_tokens: 4 }, + }); + const triage = agent({ + name: "triage", + description: "Routes questions", + llm: new ScriptedLLM([handOff("researcher", "call_h1")]), + tools: [getWeather], + canHandoffTo: ["researcher"], + }); + const researcher = agent({ + name: "researcher", + description: "Finds facts", + llm: new ScriptedLLM([handOff("forecaster", "call_h2")]), + tools: [getWeather], + canHandoffTo: ["forecaster"], + }); + const forecaster = agent({ + name: "forecaster", + description: "Knows the weather", + llm: new ScriptedLLM(), + tools: [getWeather], + }); + const out = await multiAgent({ agents: [triage, researcher, forecaster], rootAgent: triage }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "structured": { + const llm = new ScriptedLLM([ + say("It is sunny in Paris.", 9, 4), + { + tool: { name: "format_output", input: { city: "Paris", sky: "sunny" }, id: "call_s" }, + usage: { prompt_tokens: 4, completion_tokens: 3 }, + }, + ]); + // `responseFormat` arrived in @llamaindex/workflow after the floor, so it + // is passed untyped: the floor's types (and runtime) ignore it. + const params = { responseFormat: z.object({ city: z.string(), sky: z.string() }) } as Record; + const out = await agent({ llm, tools: [getWeather] }).run(QUESTION, params as never); + report({ answer: out.data.result, object: (out.data as { object?: unknown }).object }); + break; + } + // -- tool variants ------------------------------------------------------- + case "function-tool": { + const turns = weatherTurns(); + turns[0]!.tool = { name: "lookup_weather", input: { city: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [lookup] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "query-engine-tool": { + Settings.embedModel = new FakeEmbedding(); + // The query engine's synthesizer answers from Settings.llm, the agent from its own. + Settings.llm = new ScriptedLLM([say("Paris is sunny.", 7, 3)]); + const notes = new QueryEngineTool({ + queryEngine: (await cityIndex()).asQueryEngine({ similarityTopK: 1 }), + metadata: { name: "city_notes", description: "Notes about cities" }, + }); + const turns = weatherTurns(); + turns[0]!.tool = { name: "city_notes", input: { query: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [notes] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "parallel-tools": { + const turns = weatherTurns(); + turns[0]!.tool = undefined; + turns[0]!.tools = [ + { name: "get_weather", input: { city: "Paris" }, id: "call_1" }, + { name: "get_time", input: { city: "Paris" }, id: "call_2" }, + ]; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [getWeather, getTime] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "stream-error": { + // LlamaIndex's workflow runtime lets a provider error that happens + // mid-stream escape as an UNHANDLED rejection (reproducible without the + // SDK), and `run()` never settles. A server survives that with a handler; + // so does this case, and the run must still close as failed. + const unhandled: string[] = []; + process.on("unhandledRejection", (error) => unhandled.push(String(error))); + const turns: Turn[] = [ + { text: "It is", fail: "provider dropped the stream", usage: { prompt_tokens: 1, completion_tokens: 1 } }, + ]; + const running = agent({ llm: new ScriptedLLM(turns), tools: [getWeather] }).run(QUESTION); + const settled = await Promise.race([ + running.then( + () => "resolved", + (error: unknown) => `rejected: ${String(error)}`, + ), + sleep(300).then(() => "pending"), + ]); + report({ settled, unhandled }); + break; + } + // -- concurrency --------------------------------------------------------- + case "concurrent-agents": { + const shared = agent({ llm: new CityLLM(), tools: [getWeather] }); + const answers = await Promise.all(CITIES.map((city) => shared.run(`weather in ${city}?`))); + report({ answers: answers.map((a) => a.data.result) }); + break; + } + case "concurrent-queries": { + Settings.embedModel = new FakeEmbedding(); + // The synthesizer answers from Settings.llm: one model object shared by all ten. + Settings.llm = new CityLLM(); + const engine = (await cityIndex()).asQueryEngine({ similarityTopK: 1 }); + const answers = await Promise.all(CITIES.map((city) => engine.query({ query: `weather in ${city}?` }))); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + case "concurrent-chats": { + Settings.embedModel = new FakeEmbedding(); + const llm = new CityLLM(); + Settings.llm = llm; + const engine = new ContextChatEngine({ + retriever: (await cityIndex()).asRetriever({ similarityTopK: 1 }), + chatModel: llm, + }); + // Per-call history: a shared engine's own memory would mix the requests' + // messages in the FRAMEWORK, which is not what this case is about. + const answers = await Promise.all( + CITIES.map((city) => engine.chat({ message: `weather in ${city}?`, chatHistory: [] })), + ); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "workflow").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.11/package-lock.json b/sdk/typescript/integration/fixtures/llamaindex-0.11/package-lock.json new file mode 100644 index 000000000..e9dcfcc79 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.11/package-lock.json @@ -0,0 +1,635 @@ +{ + "name": "failproofai-it-llamaindex-0.11", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-llamaindex-0.11", + "dependencies": { + "@llamaindex/core": "0.6.8", + "@llamaindex/workflow": "1.1.5", + "llamaindex": "0.11.4", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz", + "integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@llama-flow/core": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@llama-flow/core/-/core-0.4.4.tgz", + "integrity": "sha512-hwK1EQ+atUG/E7XcDV3KsTaA8op29pb8gbpVurpsqbLnGFkdTT4F/6V7Hy1cC2o/yOY+DKc/rxoIsH1uJS0cZg==", + "license": "MIT", + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.7.0", + "hono": "^4.7.4", + "next": "^15.2.2", + "p-retry": "^6.2.1", + "rxjs": "^7.8.2", + "zod": "^3.24.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + }, + "hono": { + "optional": true + }, + "next": { + "optional": true + }, + "p-retry": { + "optional": true + }, + "rxjs": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@llamaindex/cloud": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/@llamaindex/cloud/-/cloud-4.0.12.tgz", + "integrity": "sha512-8AEdPp/RyG0kkzUMqleWvv0olPXeMt8iGKD3LSQ1T4VE/IyCxemS0XMStXJTi1PSN91Gm5/GABKWXzGqBPbSSA==", + "license": "MIT", + "dependencies": { + "p-retry": "^6.2.1", + "zod": "^3.25.7" + }, + "peerDependencies": { + "@llama-flow/core": "^0.4.1", + "@llamaindex/core": "0.6.8", + "@llamaindex/env": "0.1.30" + } + }, + "node_modules/@llamaindex/core": { + "version": "0.6.8", + "resolved": "https://registry.npmjs.org/@llamaindex/core/-/core-0.6.8.tgz", + "integrity": "sha512-0PWrHI4YQTD6ZgRDjGFZoJ2x96PLd0ajcMjvlYKZQRQo07MZwbS9Eh/sVsrCmLq+ptZfoE64QoA29b2QEWXr8A==", + "dependencies": { + "@llamaindex/env": "0.1.30", + "@types/node": "^22.9.0", + "magic-bytes.js": "^1.10.0", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.3" + } + }, + "node_modules/@llamaindex/env": { + "version": "0.1.30", + "resolved": "https://registry.npmjs.org/@llamaindex/env/-/env-0.1.30.tgz", + "integrity": "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "js-tiktoken": "^1.0.12", + "pathe": "^1.1.2" + }, + "peerDependencies": { + "@huggingface/transformers": "^3.5.0", + "gpt-tokenizer": "^2.5.0" + }, + "peerDependenciesMeta": { + "@huggingface/transformers": { + "optional": true + }, + "gpt-tokenizer": { + "optional": true + } + } + }, + "node_modules/@llamaindex/node-parser": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/@llamaindex/node-parser/-/node-parser-2.0.8.tgz", + "integrity": "sha512-XCJr611xpx0Ubz9B+96hunzg0w/2rmxT5lCxQ8BoVz18xr8yr9Pq4l3jaKZ7sJoXUwnSs7gLz/rwpGgTkLLmsQ==", + "dependencies": { + "html-to-text": "^9.0.5" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.8", + "@llamaindex/env": "0.1.30", + "tree-sitter": "^0.22.0", + "web-tree-sitter": "^0.24.3" + } + }, + "node_modules/@llamaindex/workflow": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@llamaindex/workflow/-/workflow-1.1.5.tgz", + "integrity": "sha512-uy1SUYwZoEv23ZduBPZ+EjFewtgVGIUkLcE1GrnfeWqmWDESCzJPr1V4NHaaWTr5CQryXkYuezNbL+xSCtQ06g==", + "dependencies": { + "@llama-flow/core": "^0.4.1" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.8", + "@llamaindex/env": "0.1.30", + "zod": "^3.23.8" + } + }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@types/lodash": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.25.tgz", + "integrity": "sha512-+K1NIO8I+F9/wNulfVvu23QYd0Pe9/OCqRrim4NoYIf1VoEDL90Ve4ClzpyqBLc7NpGGWRvYNCKZ1BE/Jpf8dQ==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "license": "MIT" + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/llamaindex": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/llamaindex/-/llamaindex-0.11.4.tgz", + "integrity": "sha512-mU0wd0jGRQpYRq+vRrnvc3CZdgnkSWAAjRrnqX6wpb0ED6yOVUEbHzlmdGus5kgC0J8rpmW8BTXC1voszjmCxQ==", + "license": "MIT", + "dependencies": { + "@llamaindex/cloud": "4.0.12", + "@llamaindex/core": "0.6.8", + "@llamaindex/env": "0.1.30", + "@llamaindex/node-parser": "2.0.8", + "@llamaindex/workflow": "1.1.5", + "@types/lodash": "^4.17.7", + "@types/node": "^22.9.0", + "ajv": "^8.17.1", + "lodash": "^4.17.21", + "magic-bytes.js": "^1.10.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/magic-bytes.js": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/magic-bytes.js/-/magic-bytes.js-1.13.1.tgz", + "integrity": "sha512-x5sn4UX2k5gCWlcfmoFwG4TPie8+dctESyqOBdhB5p6MsgWXdBKGmt9nXPObj/JI50TTL928lc5Yt1WntMn1bw==", + "license": "MIT" + }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "license": "MIT", + "peer": true, + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "peer": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "license": "MIT", + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "license": "MIT" + }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "license": "MIT", + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/tree-sitter": { + "version": "0.22.4", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.22.4.tgz", + "integrity": "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg==", + "hasInstallScript": true, + "license": "MIT", + "peer": true, + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/web-tree-sitter": { + "version": "0.24.7", + "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.24.7.tgz", + "integrity": "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ==", + "license": "MIT", + "peer": true + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.11/package.json b/sdk/typescript/integration/fixtures/llamaindex-0.11/package.json new file mode 100644 index 000000000..3df486663 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.11/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-llamaindex-0.11", + "private": true, + "type": "module", + "description": "Integration fixture: LlamaIndex.TS 0.11.4, the supported floor (the first release on @llamaindex/workflow 1.1), against the packed @failproofai/sdk.", + "dependencies": { + "@llamaindex/core": "0.6.8", + "@llamaindex/workflow": "1.1.5", + "llamaindex": "0.11.4", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.11/tsconfig.json b/sdk/typescript/integration/fixtures/llamaindex-0.11/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.11/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.12/agent.ts b/sdk/typescript/integration/fixtures/llamaindex-0.12/agent.ts new file mode 100644 index 000000000..72266cd41 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.12/agent.ts @@ -0,0 +1,659 @@ +// LlamaIndex.TS consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The scripted model makes it deterministic and offline, and it is built +// the way LlamaIndex's own provider packages build theirs (`@llamaindex/openai` +// decorates `chat` with exactly `@wrapEventCaller @wrapLLMEvent`), so the +// callback events it produces are the ones a real provider produces. The +// assertions compare against the Python SDK's golden trace for the equivalent +// `FunctionAgent` program. +import * as failproofai from "@failproofai/sdk"; +import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator"; +import { + ToolCallLLM, + type ChatMessage, + type ChatResponse, + type ChatResponseChunk, + type LLMChatParamsNonStreaming, + type LLMChatParamsStreaming, + type LLMMetadata, + type ToolCallLLMMessageOptions, +} from "@llamaindex/core/llms"; +import { RetrieverQueryEngine, type QueryBundle } from "@llamaindex/core/query-engine"; +import { getResponseSynthesizer } from "@llamaindex/core/response-synthesizers"; +import { BaseRetriever } from "@llamaindex/core/retriever"; +import { TextNode, type NodeWithScore } from "@llamaindex/core/schema"; +import { extractText } from "@llamaindex/core/utils"; +import { agent, createWorkflow, multiAgent, workflowEvent } from "@llamaindex/workflow"; +import { + BaseEmbedding, + CondenseQuestionChatEngine, + ContextChatEngine, + Document, + FunctionTool, + LLMAgent, + QueryEngineTool, + Settings, + SimpleChatEngine, + VectorStoreIndex, + tool, +} from "llamaindex"; +import { z } from "zod"; + +type Options = ToolCallLLMMessageOptions; + +/** One scripted turn: either a tool call or a final answer, with its usage. */ +interface Turn { + tool?: { name: string; input: Record; id: string }; + /** Several tool calls in ONE assistant message (parallel tool calling). */ + tools?: Array<{ name: string; input: Record; id: string }>; + text?: string; + /** The provider fails: before answering, or (streaming) after the first chunk. */ + fail?: string; + usage: { prompt_tokens: number; completion_tokens: number }; +} + +const weatherTurns = (): Turn[] => [ + { tool: { name: "get_weather", input: { city: "Paris" }, id: "call_1" }, usage: { prompt_tokens: 12, completion_tokens: 5 } }, + { text: "It is sunny in Paris.", usage: { prompt_tokens: 30, completion_tokens: 7 } }, +]; + +class ScriptedLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "scripted-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + private readonly turns: Turn[]; + + constructor(turns: Turn[] = weatherTurns()) { + super(); + this.turns = turns; + } + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const turn = this.turns.shift() ?? { text: "done", usage: { prompt_tokens: 1, completion_tokens: 1 } }; + const calls = turn.tools ?? (turn.tool ? [turn.tool] : []); + const options: Options = calls.length > 0 ? { toolCall: calls } : {}; + const message: ChatMessage = { role: "assistant", content: turn.text ?? "", options }; + if (params.stream) { + // OpenAI's stream shape: content chunks, then a content-less chunk whose + // `raw` carries the usage (what `stream_options.include_usage` sends). + return (async function* (): AsyncGenerator> { + yield { delta: turn.text ?? "", raw: { choices: [{ delta: {} }] }, options }; + if (turn.fail) throw new Error(turn.fail); + yield { delta: "", raw: { choices: [], usage: turn.usage }, options: {} }; + })(); + } + if (turn.fail) throw new Error(turn.fail); + return { message, raw: { choices: [{ finish_reason: calls.length > 0 ? "tool_calls" : "stop" }], usage: turn.usage } }; + } +} + +// --------------------------------------------------------------------------- +// One object built once and shared by concurrent requests — the way a server +// holds its query engine or agent. Everything below is stateless per call, so +// two calls in flight at once cannot disturb each other: whatever the trace +// mixes up is the adapter's doing. +// --------------------------------------------------------------------------- + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const textOf = (content: unknown): string => (typeof content === "string" ? content : JSON.stringify(content)); +/** The city a request is about. Paris is the SLOW request, so Rome starts and ends inside it. */ +const cityIn = (text: string): "Paris" | "Rome" => (text.includes("Rome") ? "Rome" : "Paris"); +const delay = (city: string) => sleep(city === "Paris" ? 40 : 5); +/** Per-city usage, so a model_response carries which request it answered. */ +const USAGE = { Paris: { prompt_tokens: 11, completion_tokens: 2 }, Rome: { prompt_tokens: 21, completion_tokens: 3 } }; + +/** A model that answers from its input alone: a tool call first, then the answer. */ +class EchoLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "echo-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const city = cityIn(params.messages.map((m) => textOf(m.content)).join("\n")); + await delay(city); + const answered = params.messages.some((m) => m.options !== undefined && "toolResult" in m.options); + const call = params.tools?.length && !answered ? { name: "get_weather", input: { city }, id: `call_${city}` } : null; + const options: Options = call ? { toolCall: [call] } : {}; + const text = call ? "" : `It is sunny in ${city}.`; + const usage = USAGE[city]; + if (params.stream) { + return (async function* (): AsyncGenerator> { + yield { delta: text, raw: { choices: [{ delta: {} }] }, options }; + await delay(city); + yield { delta: "", raw: { choices: [], usage }, options: {} }; + })(); + } + const message: ChatMessage = { role: "assistant", content: text, options }; + return { message, raw: { choices: [{ finish_reason: call ? "tool_calls" : "stop" }], usage } }; + } +} + +/** A retriever with no index and no embeddings: one note about the question. */ +class NotesRetriever extends BaseRetriever { + // BaseRetriever's constructor is protected; a subclass's is public. + constructor() { + super(); + } + + async _retrieve(params: QueryBundle): Promise { + const question = extractText(params.query); + await delay(cityIn(question)); + return [{ node: new TextNode({ text: `notes on ${question}` }), score: 1 }]; + } +} + +const getWeather = tool({ + name: "get_weather", + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), + execute: ({ city }) => `sunny in ${city}`, +}); + +const broken = tool({ + name: "broken", + description: "Always fails", + parameters: z.object({ city: z.string() }), + execute: (): string => { + throw new Error("tool exploded"); + }, +}); + +// --------------------------------------------------------------------------- +// The coverage sweep: every commonly used LlamaIndex.TS surface, offline. +// --------------------------------------------------------------------------- + +/** + * An embedding model with no network: one dimension per city, so a question + * about Paris retrieves the Paris note and nothing else. + */ +const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Delhi", "Tokyo", "Quito", "Accra", "Hanoi"] as const; +class FakeEmbedding extends BaseEmbedding { + // BaseEmbedding's constructor is protected; a subclass's is public. + constructor() { + super(); + } + + async getTextEmbedding(text: string): Promise { + return [...CITIES.map((city) => (text.includes(city) ? 1 : 0)), 0.01]; + } +} + +const HISTORY: ChatMessage[] = [ + { role: "user", content: "hi" }, + { role: "assistant", content: "hello" }, +]; + +/** Every city's note, indexed with the fake embedding model. */ +const cityIndex = () => + VectorStoreIndex.fromDocuments(CITIES.map((city) => new Document({ text: `${city} is sunny.`, id_: `doc-${city}` }))); + +/** The city a question is about. */ +const cityOf = (text: string): string => { + const hits = CITIES.filter((city) => text.includes(city)); + return hits[hits.length - 1] ?? "Paris"; +}; + +/** + * `EchoLLM` for ten requests: answers from its input alone (a tool call, then + * the answer), with a per-city delay and per-city usage so every event says + * which request it belongs to. + */ +class CityLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "city-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + @wrapEventCaller + @wrapLLMEvent + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const city = cityOf(params.messages.map((m) => textOf(m.content)).join("\n")); + const n = CITIES.indexOf(city as (typeof CITIES)[number]); + await sleep(((n * 7) % 5) * 6); + const answered = params.messages.some((m) => m.options !== undefined && "toolResult" in m.options); + const call = params.tools?.length && !answered ? { name: "get_weather", input: { city }, id: `call_${city}` } : null; + const options: Options = call ? { toolCall: [call] } : {}; + const text = call ? "" : `It is sunny in ${city}.`; + const usage = { prompt_tokens: 100 + n, completion_tokens: n }; + if (params.stream) { + return (async function* (): AsyncGenerator> { + yield { delta: text, raw: { choices: [{ delta: {} }] }, options }; + await sleep(((n * 3) % 4) * 5); + yield { delta: "", raw: { choices: [], usage }, options: {} }; + })(); + } + const message: ChatMessage = { role: "assistant", content: text, options }; + return { message, raw: { choices: [{ finish_reason: call ? "tool_calls" : "stop" }], usage } }; + } +} + +const say = (text: string, prompt = 5, completion = 2): Turn => ({ + text, + usage: { prompt_tokens: prompt, completion_tokens: completion }, +}); + +/** Drain whatever a streaming chat or query returned. */ +async function drain(stream: AsyncIterable): Promise { + let chunks = 0; + for await (const _ of stream) chunks += 1; + return chunks; +} + +const lookup = FunctionTool.from(({ city }: { city: string }) => `sunny in ${city}`, { + name: "lookup_weather", + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), +}); + +const getTime = tool({ + name: "get_time", + description: "Current time in a city", + parameters: z.object({ city: z.string() }), + execute: ({ city }) => `noon in ${city}`, +}); + +/** A plain `createWorkflow()` workflow with two steps; the second asks the model. */ +function plainWorkflow(llm: ScriptedLLM) { + const start = workflowEvent({ debugLabel: "start" }); + const researched = workflowEvent({ debugLabel: "researched" }); + const stop = workflowEvent({ debugLabel: "stop" }); + const workflow = createWorkflow(); + // Named functions: a step's name is its handler's name. `...args` because the + // floor's runtime calls `handler(event)` and later ones `handler(context, event)`. + workflow.handle([start], async function research(...args: unknown[]) { + const event = args[args.length - 1] as { data: string }; + await sleep(1); + return researched.with(`notes on ${event.data}`); + }); + workflow.handle([researched], async function answer(...args: unknown[]) { + const event = args[args.length - 1] as { data: string }; + const response = await llm.chat({ messages: [{ role: "user", content: event.data }] }); + return stop.with(String(response.message.content)); + }); + return { workflow, start, stop }; +} + +async function runPlainWorkflow(llm: ScriptedLLM, input: string): Promise { + const { workflow, start, stop } = plainWorkflow(llm); + const { stream, sendEvent } = workflow.createContext(); + sendEvent(start.with(input)); + const events = await stream.until(stop).toArray(); + return String((events.at(-1) as { data: unknown }).data); +} + +const report = (value: unknown) => console.log(JSON.stringify(value)); +const QUESTION = "weather in Paris?"; + +async function main(scenario: string): Promise { + // Built BEFORE instrument(): the adapter must not depend on construction order. + const early = agent({ llm: new ScriptedLLM(), tools: [getWeather] }); + // Shared objects, also built at startup, before instrument(). + const sharedEngine = new RetrieverQueryEngine( + new NotesRetriever(), + getResponseSynthesizer("compact", { llm: new EchoLLM() }), + ); + const sharedLegacy = new LLMAgent({ llm: new EchoLLM(), tools: [getWeather] }); + const sharedAgent = agent({ llm: new EchoLLM(), tools: [getWeather] }); + // Adapter options, for the cases that exercise one (`embeddings: true`). + const options = JSON.parse(process.env.FAILPROOFAI_IT_OPTIONS ?? "{}") as Record; + report({ instrumented: await failproofai.instrument("llamaindex", options) }); + const QUESTIONS = ["weather in Paris?", "weather in Rome?"]; + + switch (scenario) { + case "workflow": { + const out = await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "named": { + const out = await agent({ name: "weather_bot", llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "early": { + const out = await early.run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "stream": { + let events = 0; + for await (const _ of agent({ llm: new ScriptedLLM(), tools: [getWeather] }).runStream(QUESTION)) events += 1; + report({ events }); + break; + } + case "legacy": { + const out = await new LLMAgent({ llm: new ScriptedLLM(), tools: [getWeather] }).chat({ message: QUESTION }); + report({ answer: String(out.message.content) }); + break; + } + case "model": { + const out = await new ScriptedLLM([{ text: "hello", usage: { prompt_tokens: 3, completion_tokens: 1 } }]).chat({ + messages: [{ role: "user", content: "hi" }], + }); + report({ answer: out.message.content }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => + agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION), + ), + ); + break; + } + case "tool-error": { + const turns = weatherTurns(); + turns[0]!.tool = { name: "broken", input: { city: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [broken] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "handoff": { + const triage = agent({ + name: "triage", + description: "Routes questions", + llm: new ScriptedLLM([ + { + tool: { name: "handOff", input: { toAgent: "forecaster", reason: "weather question" }, id: "call_h" }, + usage: { prompt_tokens: 8, completion_tokens: 4 }, + }, + ]), + tools: [getWeather], + canHandoffTo: ["forecaster"], + }); + const forecaster = agent({ + name: "forecaster", + description: "Knows the weather", + llm: new ScriptedLLM(), + tools: [getWeather], + }); + const out = await multiAgent({ agents: [triage, forecaster], rootAgent: triage }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "shared-query": { + const answers = await Promise.all(QUESTIONS.map((query) => sharedEngine.query({ query }))); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + case "shared-legacy": { + const answers = await Promise.all(QUESTIONS.map((message) => sharedLegacy.chat({ message }))); + report({ answers: answers.map((a) => String(a.message.content)) }); + break; + } + case "shared-workflow": { + const answers = await Promise.all(QUESTIONS.map((question) => sharedAgent.run(question))); + report({ answers: answers.map((a) => a.data.result) }); + break; + } + case "uninstrument": { + // One recorded run, then nothing: the second workflow and the legacy agent + // run after uninstrument() and must leave no trace. + await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + report({ removed: failproofai.uninstrument() }); + await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + await new LLMAgent({ llm: new ScriptedLLM(), tools: [getWeather] }).chat({ message: QUESTION }); + break; + } + // -- chat engines -------------------------------------------------------- + case "chat-simple": { + const engine = new SimpleChatEngine({ llm: new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]) }); + const out = await engine.chat({ message: QUESTION, chatHistory: HISTORY }); + report({ answer: out.message.content }); + break; + } + case "chat-simple-stream": { + const engine = new SimpleChatEngine({ llm: new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]) }); + report({ chunks: await drain(await engine.chat({ message: QUESTION, chatHistory: HISTORY, stream: true })) }); + break; + } + case "chat-context": + case "chat-context-stream": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + // The floor's chat memory reads Settings.llm even when a chatModel is given. + const llm = new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]); + Settings.llm = llm; + const engine = new ContextChatEngine({ + retriever: index.asRetriever({ similarityTopK: 1 }), + chatModel: llm, + chatHistory: HISTORY, + }); + if (scenario === "chat-context") { + report({ answer: (await engine.chat({ message: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.chat({ message: QUESTION, stream: true })) }); + } + break; + } + case "chat-condense": + case "chat-condense-stream": { + Settings.embedModel = new FakeEmbedding(); + // CondenseQuestionChatEngine takes its model from Settings, and so does + // the query engine's synthesizer: first the condensed question, then the answer. + Settings.llm = new ScriptedLLM([say("What is the weather in Paris?", 6, 3), say("It is sunny in Paris.", 9, 4)]); + const index = await cityIndex(); + const engine = new CondenseQuestionChatEngine({ + queryEngine: index.asQueryEngine({ similarityTopK: 1 }), + chatHistory: HISTORY, + }); + if (scenario === "chat-condense") { + report({ answer: (await engine.chat({ message: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.chat({ message: QUESTION, stream: true })) }); + } + break; + } + // -- retrieval and indexes ----------------------------------------------- + case "index-build": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + report({ built: typeof index.asRetriever }); + break; + } + case "retriever": { + Settings.embedModel = new FakeEmbedding(); + const index = await cityIndex(); + const nodes = await index.asRetriever({ similarityTopK: 1 }).retrieve({ query: QUESTION }); + report({ nodes: nodes.map((n) => n.node.id_) }); + break; + } + case "index-query": + case "index-query-stream": { + Settings.embedModel = new FakeEmbedding(); + Settings.llm = new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]); + const engine = (await cityIndex()).asQueryEngine({ similarityTopK: 1 }); + if (scenario === "index-query") { + report({ answer: (await engine.query({ query: QUESTION })).message.content }); + } else { + report({ chunks: await drain(await engine.query({ query: QUESTION, stream: true })) }); + } + break; + } + // -- plain workflows ----------------------------------------------------- + case "custom-workflow": { + report({ answer: await runPlainWorkflow(new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]), QUESTION) }); + break; + } + case "custom-workflow-scoped": { + const answer = await failproofai.agent("forecast_flow", { goal: QUESTION }, () => + runPlainWorkflow(new ScriptedLLM([say("It is sunny in Paris.", 9, 4)]), QUESTION), + ); + report({ answer }); + break; + } + // -- agents -------------------------------------------------------------- + case "handoff3": { + const handOff = (to: string, id: string): Turn => ({ + tool: { name: "handOff", input: { toAgent: to, reason: "next" }, id }, + usage: { prompt_tokens: 8, completion_tokens: 4 }, + }); + const triage = agent({ + name: "triage", + description: "Routes questions", + llm: new ScriptedLLM([handOff("researcher", "call_h1")]), + tools: [getWeather], + canHandoffTo: ["researcher"], + }); + const researcher = agent({ + name: "researcher", + description: "Finds facts", + llm: new ScriptedLLM([handOff("forecaster", "call_h2")]), + tools: [getWeather], + canHandoffTo: ["forecaster"], + }); + const forecaster = agent({ + name: "forecaster", + description: "Knows the weather", + llm: new ScriptedLLM(), + tools: [getWeather], + }); + const out = await multiAgent({ agents: [triage, researcher, forecaster], rootAgent: triage }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "structured": { + const llm = new ScriptedLLM([ + say("It is sunny in Paris.", 9, 4), + { + tool: { name: "format_output", input: { city: "Paris", sky: "sunny" }, id: "call_s" }, + usage: { prompt_tokens: 4, completion_tokens: 3 }, + }, + ]); + // `responseFormat` arrived in @llamaindex/workflow after the floor, so it + // is passed untyped: the floor's types (and runtime) ignore it. + const params = { responseFormat: z.object({ city: z.string(), sky: z.string() }) } as Record; + const out = await agent({ llm, tools: [getWeather] }).run(QUESTION, params as never); + report({ answer: out.data.result, object: (out.data as { object?: unknown }).object }); + break; + } + // -- tool variants ------------------------------------------------------- + case "function-tool": { + const turns = weatherTurns(); + turns[0]!.tool = { name: "lookup_weather", input: { city: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [lookup] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "query-engine-tool": { + Settings.embedModel = new FakeEmbedding(); + // The query engine's synthesizer answers from Settings.llm, the agent from its own. + Settings.llm = new ScriptedLLM([say("Paris is sunny.", 7, 3)]); + const notes = new QueryEngineTool({ + queryEngine: (await cityIndex()).asQueryEngine({ similarityTopK: 1 }), + metadata: { name: "city_notes", description: "Notes about cities" }, + }); + const turns = weatherTurns(); + turns[0]!.tool = { name: "city_notes", input: { query: "Paris" }, id: "call_1" }; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [notes] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "parallel-tools": { + const turns = weatherTurns(); + turns[0]!.tool = undefined; + turns[0]!.tools = [ + { name: "get_weather", input: { city: "Paris" }, id: "call_1" }, + { name: "get_time", input: { city: "Paris" }, id: "call_2" }, + ]; + const out = await agent({ llm: new ScriptedLLM(turns), tools: [getWeather, getTime] }).run(QUESTION); + report({ answer: out.data.result }); + break; + } + case "stream-error": { + // LlamaIndex's workflow runtime lets a provider error that happens + // mid-stream escape as an UNHANDLED rejection (reproducible without the + // SDK), and `run()` never settles. A server survives that with a handler; + // so does this case, and the run must still close as failed. + const unhandled: string[] = []; + process.on("unhandledRejection", (error) => unhandled.push(String(error))); + const turns: Turn[] = [ + { text: "It is", fail: "provider dropped the stream", usage: { prompt_tokens: 1, completion_tokens: 1 } }, + ]; + const running = agent({ llm: new ScriptedLLM(turns), tools: [getWeather] }).run(QUESTION); + const settled = await Promise.race([ + running.then( + () => "resolved", + (error: unknown) => `rejected: ${String(error)}`, + ), + sleep(300).then(() => "pending"), + ]); + report({ settled, unhandled }); + break; + } + // -- concurrency --------------------------------------------------------- + case "concurrent-agents": { + const shared = agent({ llm: new CityLLM(), tools: [getWeather] }); + const answers = await Promise.all(CITIES.map((city) => shared.run(`weather in ${city}?`))); + report({ answers: answers.map((a) => a.data.result) }); + break; + } + case "concurrent-queries": { + Settings.embedModel = new FakeEmbedding(); + // The synthesizer answers from Settings.llm: one model object shared by all ten. + Settings.llm = new CityLLM(); + const engine = (await cityIndex()).asQueryEngine({ similarityTopK: 1 }); + const answers = await Promise.all(CITIES.map((city) => engine.query({ query: `weather in ${city}?` }))); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + case "concurrent-chats": { + Settings.embedModel = new FakeEmbedding(); + const llm = new CityLLM(); + Settings.llm = llm; + const engine = new ContextChatEngine({ + retriever: (await cityIndex()).asRetriever({ similarityTopK: 1 }), + chatModel: llm, + }); + // Per-call history: a shared engine's own memory would mix the requests' + // messages in the FRAMEWORK, which is not what this case is about. + const answers = await Promise.all( + CITIES.map((city) => engine.chat({ message: `weather in ${city}?`, chatHistory: [] })), + ); + report({ answers: answers.map((a) => a.message.content) }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "workflow").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.12/package-lock.json b/sdk/typescript/integration/fixtures/llamaindex-0.12/package-lock.json new file mode 100644 index 000000000..ea02a27a5 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.12/package-lock.json @@ -0,0 +1,553 @@ +{ + "name": "failproofai-it-llamaindex-0.12", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-llamaindex-0.12", + "dependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/workflow": "1.1.24", + "llamaindex": "0.12.1", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz", + "integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@finom/zod-to-json-schema": { + "version": "3.24.11", + "resolved": "https://registry.npmjs.org/@finom/zod-to-json-schema/-/zod-to-json-schema-3.24.11.tgz", + "integrity": "sha512-fL656yBPiWebtfGItvtXLWrFNGlF1NcDFS0WdMQXMs9LluVg0CfT5E2oXYp0pidl0vVG53XkW55ysijNkU5/hA==", + "deprecated": "Use https://www.npmjs.com/package/zod-v3-to-json-schema instead. See issue comment for details: https://github.com/StefanTerdell/zod-to-json-schema/issues/178#issuecomment-3533122539", + "license": "ISC", + "peerDependencies": { + "zod": "^4.0.14" + } + }, + "node_modules/@llamaindex/core": { + "version": "0.6.22", + "resolved": "https://registry.npmjs.org/@llamaindex/core/-/core-0.6.22.tgz", + "integrity": "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg==", + "dependencies": { + "@finom/zod-to-json-schema": "3.24.11", + "@llamaindex/env": "0.1.30", + "@types/node": "^24.0.13", + "magic-bytes.js": "^1.10.0", + "zod": "^4.1.5" + } + }, + "node_modules/@llamaindex/core/node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@llamaindex/core/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/@llamaindex/env": { + "version": "0.1.30", + "resolved": "https://registry.npmjs.org/@llamaindex/env/-/env-0.1.30.tgz", + "integrity": "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "js-tiktoken": "^1.0.12", + "pathe": "^1.1.2" + }, + "peerDependencies": { + "@huggingface/transformers": "^3.5.0", + "gpt-tokenizer": "^2.5.0" + }, + "peerDependenciesMeta": { + "@huggingface/transformers": { + "optional": true + }, + "gpt-tokenizer": { + "optional": true + } + } + }, + "node_modules/@llamaindex/node-parser": { + "version": "2.0.22", + "resolved": "https://registry.npmjs.org/@llamaindex/node-parser/-/node-parser-2.0.22.tgz", + "integrity": "sha512-uj5O89WShAAyiSZ8f8tU7hnLJ6pSmlY2a6hkAOs8odkUgT87dEqaPHpsK7w0iJdEFiob7GoLeRhv2K624FooXg==", + "dependencies": { + "html-to-text": "^9.0.5" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30", + "tree-sitter": "^0.22.0", + "web-tree-sitter": "^0.24.3" + } + }, + "node_modules/@llamaindex/workflow": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@llamaindex/workflow/-/workflow-1.1.24.tgz", + "integrity": "sha512-VyKsbRkFlnT5dRNKbgLXQV+ZpQ+CAFgmC9LaZv6hD/fIKo6wq1wQW/ZqLZgZt569xeHgxmrXPB6KHdqn/AhPbQ==", + "dependencies": { + "@llamaindex/workflow-core": "^1.3.2" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30" + } + }, + "node_modules/@llamaindex/workflow-core": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@llamaindex/workflow-core/-/workflow-core-1.3.3.tgz", + "integrity": "sha512-WJIcD4K2suGbNkwU5CC70jKKrA5tARba42nMs8Pou1RGzmoxqg+K+b7vyLBmiDtImR8P40YLmkayCIRVQPBmsg==", + "license": "MIT", + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.7.0", + "hono": "^4.7.4", + "next": "^15.2.2", + "p-retry": "^6.2.1", + "rxjs": "^7.8.2", + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + }, + "hono": { + "optional": true + }, + "next": { + "optional": true + }, + "p-retry": { + "optional": true + }, + "rxjs": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@types/lodash": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.25.tgz", + "integrity": "sha512-+K1NIO8I+F9/wNulfVvu23QYd0Pe9/OCqRrim4NoYIf1VoEDL90Ve4ClzpyqBLc7NpGGWRvYNCKZ1BE/Jpf8dQ==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/llamaindex": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/llamaindex/-/llamaindex-0.12.1.tgz", + "integrity": "sha512-/tXXITk/iVGBycOFaDhev6dgTBIr6Ycu4FoPIt6A5JcEAiB6ujONjiV36flVXUR8JdqwMtS767XMjV+36nV4yQ==", + "license": "MIT", + "dependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30", + "@llamaindex/node-parser": "2.0.22", + "@llamaindex/workflow": "1.1.24", + "@types/lodash": "^4.17.7", + "@types/node": "^24.0.13", + "lodash": "^4.17.21", + "magic-bytes.js": "^1.10.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/llamaindex/node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/llamaindex/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/magic-bytes.js": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/magic-bytes.js/-/magic-bytes.js-1.13.1.tgz", + "integrity": "sha512-x5sn4UX2k5gCWlcfmoFwG4TPie8+dctESyqOBdhB5p6MsgWXdBKGmt9nXPObj/JI50TTL928lc5Yt1WntMn1bw==", + "license": "MIT" + }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "license": "MIT", + "peer": true, + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "peer": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "license": "MIT", + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "license": "MIT" + }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "license": "MIT", + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/tree-sitter": { + "version": "0.22.4", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.22.4.tgz", + "integrity": "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg==", + "hasInstallScript": true, + "license": "MIT", + "peer": true, + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-tree-sitter": { + "version": "0.24.7", + "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.24.7.tgz", + "integrity": "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ==", + "license": "MIT", + "peer": true + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.12/package.json b/sdk/typescript/integration/fixtures/llamaindex-0.12/package.json new file mode 100644 index 000000000..cdfbb2772 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.12/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-llamaindex-0.12", + "private": true, + "type": "module", + "description": "Integration fixture: LlamaIndex.TS 0.12 (+ @llamaindex/workflow 1.1) against the packed @failproofai/sdk.", + "dependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/workflow": "1.1.24", + "llamaindex": "0.12.1", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/llamaindex-0.12/tsconfig.json b/sdk/typescript/integration/fixtures/llamaindex-0.12/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/llamaindex-0.12/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/mastra-0/agent.ts b/sdk/typescript/integration/fixtures/mastra-0/agent.ts new file mode 100644 index 000000000..503fb8c04 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-0/agent.ts @@ -0,0 +1,877 @@ +// Mastra 0.x consumer (the last 0.x line). Run as `node agent.{mjs,cjs} `. +// +// The same program as `../mastra-1/agent.ts`, differing only where the 0.x API +// does: a tool's `execute` receives `{ context }` rather than the input itself, +// a workflow run is created with `createRunAsync()`, and an MCPClient lists its +// tools with `getTools()` / `getToolsets()`. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The model is a hand-written AI SDK `LanguageModelV2` — Mastra's own +// model interface, so nothing here reaches a network — scripted per agent: +// the first call asks for a tool, the next one answers. Every agent gets its +// own script so the token counts in the trace say which call they came from. +import * as failproofai from "@failproofai/sdk"; +import { wrapTool } from "@failproofai/sdk/mastra"; +import { Agent } from "@mastra/core/agent"; +import { createTool } from "@mastra/core/tools"; +import { createStep, createWorkflow } from "@mastra/core/workflows"; +import { Mastra } from "@mastra/core/mastra"; +import { InMemoryStore } from "@mastra/core/storage"; +import { MCPClient } from "@mastra/mcp"; +import { Memory } from "@mastra/memory"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; +import { z } from "zod"; + +type Turn = { tool: string; input: Record; usage: [number, number] } | { text: string; usage: [number, number] }; + +/** + * Holds the FIRST streamed step open after its response metadata: `reached` + * fires once the stream is parked there, and it moves on when `wait` settles. + */ +type Gate = { reached: () => void; wait: Promise }; + +function barrier(): { wait: Promise; open: () => void } { + let open!: () => void; + const wait = new Promise((resolve) => (open = resolve)); + return { wait, open }; +} + +/** + * A scripted `LanguageModelV2`: `turns` alternate per call, so a two-step tool + * loop is turn 0 (tool call) then turn 1 (answer). `fail` makes every call + * throw, which is how a provider error reaches Mastra. Tool call ids are + * `_`, so two models in one session never reuse an id. + */ +function scriptedModel(turns: Turn[], options: { fail?: boolean; callPrefix?: string; gate?: Gate } = {}) { + let calls = 0; + const next = (): Turn => turns[calls++ % turns.length]!; + const usage = ([input, output]: [number, number]) => ({ + inputTokens: input, + outputTokens: output, + totalTokens: input + output, + }); + return { + specificationVersion: "v2" as const, + provider: "scripted", + modelId: "scripted-model", + supportedUrls: {}, + async doGenerate() { + if (options.fail) throw new Error("model exploded"); + const turn = next(); + return "tool" in turn + ? { + content: [ + { type: "tool-call" as const, toolCallId: `${options.callPrefix ?? "call"}_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }, + ], + finishReason: "tool-calls" as const, + usage: usage(turn.usage), + warnings: [], + } + : { + content: [{ type: "text" as const, text: turn.text }], + finishReason: "stop" as const, + usage: usage(turn.usage), + warnings: [], + }; + }, + async doStream() { + if (options.fail) throw new Error("model exploded"); + const turn = next(); + const parts: unknown[] = [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: `resp_${calls}`, modelId: "scripted-model", timestamp: new Date(0) }, + ]; + if ("tool" in turn) { + parts.push({ type: "tool-call", toolCallId: `${options.callPrefix ?? "call"}_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }); + parts.push({ type: "finish", finishReason: "tool-calls", usage: usage(turn.usage) }); + } else { + parts.push({ type: "text-start", id: "t" }); + // Two deltas, so a step's content has to be assembled from the stream. + // (Not one per word: Mastra 0.24's textStream drops the third of five + // such deltas on its own, instrumented or not.) + const cut = turn.text.indexOf(" ", turn.text.length / 2) + 1; + for (const delta of [turn.text.slice(0, cut), turn.text.slice(cut)]) { + parts.push({ type: "text-delta", id: "t", delta }); + } + parts.push({ type: "text-end", id: "t" }); + parts.push({ type: "finish", finishReason: "stop", usage: usage(turn.usage) }); + } + const gate = calls === 1 ? options.gate : undefined; + if (gate) { + let index = 0; + return { + stream: new ReadableStream({ + async pull(controller) { + if (index === 2) { + gate.reached(); + await gate.wait; + } + if (index >= parts.length) controller.close(); + else controller.enqueue(parts[index++]); + }, + }), + }; + } + return { + stream: new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }), + }; + }, + }; +} + +const WEATHER_TURNS: Turn[] = [ + { tool: "weather", input: { city: "Paris" }, usage: [11, 7] }, + { text: "It is sunny in Paris.", usage: [23, 9] }, +]; + +const weather = createTool({ + id: "weather", + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ context }: { context: { city: string } }) => ({ city: context.city, forecast: "sunny" }), +}); + +const brokenWeather = createTool({ + id: "weather", + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async (): Promise<{ city: string }> => { + throw new Error("tool exploded"); + }, +}); + +const weatherAgent = (options: { fail?: boolean; tool?: typeof weather; gate?: Gate } = {}) => + new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: scriptedModel(WEATHER_TURNS, options) as never, + tools: { weather: options.tool ?? weather }, + }); + +function buildWorkflow(fail = false) { + const fetchCity = createStep({ + id: "fetch-city", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ city: z.string() }), + execute: async () => ({ city: "Paris" }), + }); + const ask = createStep({ + id: "ask-agent", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ answer: z.string() }), + execute: async ({ inputData }) => { + if (fail) throw new Error("step exploded"); + const out = await weatherAgent().generate(`Weather in ${inputData.city}?`); + return { answer: out.text }; + }, + }); + return createWorkflow({ + id: "weather-flow", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ answer: z.string() }), + }) + .then(fetchCity) + .then(ask) + .commit(); +} + +const report = (value: unknown) => console.log(JSON.stringify(value)); +const question = "What is the weather in Paris?"; + +// --------------------------------------------------------------------------- +// The coverage cases below need a model that answers from what it is ASKED +// rather than from how often it has been called: concurrent runs share one +// model, and memory, networks and structured output change what a call sees. +// --------------------------------------------------------------------------- + +type Prompt = Array<{ role: string; content: unknown }>; +type CallOptions = { prompt: Prompt; responseFormat?: { type?: string; schema?: { properties?: Record } } }; + +/** The text of the LAST user message — the question this call answers. */ +function lastUserText(prompt: Prompt): string { + for (let i = prompt.length - 1; i >= 0; i -= 1) { + const message = prompt[i]!; + if (message.role !== "user") continue; + if (typeof message.content === "string") return message.content; + if (Array.isArray(message.content)) { + return message.content + .map((part: { type?: string; text?: string }) => (part.type === "text" ? (part.text ?? "") : "")) + .join(""); + } + } + return ""; +} + +const cityOf = (text: string): string => /in ([A-Z][a-z]+)/.exec(text)?.[1] ?? "Paris"; + +/** Every text a call's prompt carries, system included, for routing on. */ +const promptText = (prompt: Prompt): string => JSON.stringify(prompt); + +let decidedCalls = 0; + +/** + * A `LanguageModelV2` whose every answer is `decide(options)`. Tool call ids + * are `_` over one process-wide counter, so no two calls anywhere + * reuse one — the concurrency case needs that to tell runs apart. + */ +function decidingModel(modelId: string, decide: (options: CallOptions) => Turn, prefix = "call") { + const usage = ([input, output]: [number, number]) => ({ inputTokens: input, outputTokens: output, totalTokens: input + output }); + const answer = (options: CallOptions) => { + decidedCalls += 1; + return { turn: decide(options), id: `${prefix}_${decidedCalls}` }; + }; + return { + specificationVersion: "v2" as const, + provider: "scripted", + modelId, + supportedUrls: {}, + async doGenerate(options: CallOptions) { + const { turn, id } = answer(options); + return "tool" in turn + ? { + content: [{ type: "tool-call" as const, toolCallId: id, toolName: turn.tool, input: JSON.stringify(turn.input) }], + finishReason: "tool-calls" as const, + usage: usage(turn.usage), + warnings: [], + } + : { + content: [{ type: "text" as const, text: turn.text }], + finishReason: "stop" as const, + usage: usage(turn.usage), + warnings: [], + }; + }, + async doStream(options: CallOptions) { + const { turn, id } = answer(options); + const parts: unknown[] = [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: `resp_${id}`, modelId, timestamp: new Date(0) }, + ]; + if ("tool" in turn) { + parts.push({ type: "tool-call", toolCallId: id, toolName: turn.tool, input: JSON.stringify(turn.input) }); + parts.push({ type: "finish", finishReason: "tool-calls", usage: usage(turn.usage) }); + } else { + parts.push({ type: "text-start", id: "t" }); + parts.push({ type: "text-delta", id: "t", delta: turn.text }); + parts.push({ type: "text-end", id: "t" }); + parts.push({ type: "finish", finishReason: "stop", usage: usage(turn.usage) }); + } + return { + stream: new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }), + }; + }, + }; +} + +/** Ask `tool` for the city in the question, then answer from its result. */ +const toolThenAnswer = (tool: string) => (options: CallOptions): Turn => { + // Memory titling a new thread (on by default in 0.x) asks the agent's model. + if (promptText(options.prompt).includes("short title")) return { text: "Weather", usage: [5, 3] }; + const city = cityOf(lastUserText(options.prompt)); + return options.prompt.at(-1)?.role === "tool" + ? { text: `It is sunny in ${city}.`, usage: [23, 9] } + : { tool, input: { city }, usage: [11, 7] }; +}; + +const routedAgent = (name: string, tools: Record = { weather }, extra: Record = {}) => + new Agent({ + id: name, + name, + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer(Object.keys(tools)[0] ?? "weather")) as never, + tools: tools as never, + ...extra, + } as never); + +const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Tokyo", "Quito", "Dakar", "Hanoi", "Perth"]; + +const answerStep = createStep({ + id: "answer", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ answer: z.string() }), + execute: async ({ inputData }) => ({ answer: `sunny in ${inputData.city}` }), +}); + +/** A two-step workflow, used on its own and nested inside another. */ +function innerWorkflow() { + const pick = createStep({ + id: "pick-city", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ city: z.string() }), + execute: async ({ inputData }) => ({ city: cityOf(inputData.question) }), + }); + return createWorkflow({ + id: "inner-flow", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ answer: z.string() }), + }) + .then(pick) + .then(answerStep) + .commit(); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- one entry point for differently-typed workflows +function workflowCase(name: string): { createRunAsync(): Promise } { + const input = z.object({ question: z.string() }); + const out = z.object({ answer: z.string() }); + const city = createStep({ + id: "city", + inputSchema: input, + outputSchema: z.object({ city: z.string() }), + execute: async ({ inputData }) => ({ city: cityOf(inputData.question) }), + }); + switch (name) { + case "branch": { + const sunny = createStep({ id: "sunny", inputSchema: z.object({ city: z.string() }), outputSchema: out, execute: async () => ({ answer: "sunny" }) }); + const rainy = createStep({ id: "rainy", inputSchema: z.object({ city: z.string() }), outputSchema: out, execute: async () => ({ answer: "rainy" }) }); + return createWorkflow({ id: "branch-flow", inputSchema: input, outputSchema: z.any() }) + .then(city) + .branch([ + [async ({ inputData }) => inputData.city === "Paris", sunny], + [async ({ inputData }) => inputData.city !== "Paris", rainy], + ]) + .commit(); + } + case "parallel": { + const high = createStep({ id: "high", inputSchema: z.object({ city: z.string() }), outputSchema: z.object({ t: z.number() }), execute: async () => ({ t: 25 }) }); + const low = createStep({ id: "low", inputSchema: z.object({ city: z.string() }), outputSchema: z.object({ t: z.number() }), execute: async () => ({ t: 12 }) }); + return createWorkflow({ id: "parallel-flow", inputSchema: input, outputSchema: z.any() }).then(city).parallel([high, low]).commit(); + } + case "loop": { + const count = createStep({ + id: "count", + inputSchema: z.object({ n: z.number() }), + outputSchema: z.object({ n: z.number() }), + execute: async ({ inputData }) => ({ n: inputData.n + 1 }), + }); + const cities = createStep({ + id: "cities", + inputSchema: z.object({ n: z.number() }), + outputSchema: z.array(z.object({ city: z.string() })), + execute: async () => [{ city: "Paris" }, { city: "Rome" }], + }); + return createWorkflow({ id: "loop-flow", inputSchema: z.object({ n: z.number() }), outputSchema: z.any() }) + .dowhile(count, async ({ inputData }) => inputData.n < 3) + .then(cities) + .foreach(answerStep) + .commit(); + } + case "nested": + return createWorkflow({ id: "outer-flow", inputSchema: input, outputSchema: out }).then(innerWorkflow()).commit(); + case "agent-step": { + const toPrompt = createStep({ + id: "to-prompt", + inputSchema: input, + outputSchema: z.object({ prompt: z.string() }), + execute: async ({ inputData }) => ({ prompt: inputData.question }), + }); + return createWorkflow({ id: "agent-step-flow", inputSchema: input, outputSchema: z.any() }) + .then(toPrompt) + .then(createStep(routedAgent("weather-agent"))) + .commit(); + } + case "suspend": { + const approve = createStep({ + id: "approve", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ city: z.string(), approved: z.boolean() }), + suspendSchema: z.object({ prompt: z.string() }), + resumeSchema: z.object({ approved: z.boolean() }), + execute: async ({ inputData, resumeData, suspend }) => { + if (resumeData === undefined) return (await suspend({ prompt: `Look up ${inputData.city}?` })) as never; + return { city: inputData.city, approved: resumeData.approved }; + }, + }); + const done = createStep({ + id: "done", + inputSchema: z.object({ city: z.string(), approved: z.boolean() }), + outputSchema: out, + execute: async ({ inputData }) => ({ answer: inputData.approved ? `sunny in ${inputData.city}` : "declined" }), + }); + return createWorkflow({ id: "approval-flow", inputSchema: input, outputSchema: out }).then(city).then(approve).then(done).commit(); + } + default: + throw new Error(`unknown workflow ${name}`); + } +} + +const mcpServer = () => ({ command: process.execPath, args: [join(process.cwd(), "mcp-server.mjs")] }); + +async function main(scenario: string): Promise { + report({ instrumented: await failproofai.instrument("mastra") }); + + switch (scenario) { + case "generate": { + const out = await weatherAgent().generate(question); + report({ answer: out.text }); + break; + } + case "stream": { + const out = await weatherAgent().stream(question); + let text = ""; + for await (const chunk of out.textStream) text += chunk; + report({ answer: text }); + break; + } + case "subagent": { + const helper = new Agent({ + id: "helper", + name: "helper", + description: "Looks up the weather.", + instructions: "Answer weather questions.", + model: scriptedModel(WEATHER_TURNS) as never, + tools: { weather }, + }); + const boss = new Agent({ + id: "boss", + name: "boss", + instructions: "Delegate weather questions.", + model: scriptedModel([ + { tool: "agent-helper", input: { prompt: "Weather in Paris?" }, usage: [40, 12] }, + { text: "The helper says it is sunny.", usage: [60, 8] }, + ], { callPrefix: "delegate" }) as never, + agents: { helper }, + }); + const out = await boss.generate(question); + report({ answer: out.text }); + break; + } + case "workflow": { + const run = await buildWorkflow().createRunAsync(); + const out = await run.start({ inputData: { question } }); + report({ status: out.status }); + break; + } + case "workflow-error": { + const run = await buildWorkflow(true).createRunAsync(); + const out = await run.start({ inputData: { question } }); + report({ status: out.status }); + break; + } + case "wraptool": { + const tool = wrapTool(weather); + report({ out: await tool.execute!({ context: { city: "Rome" } } as never) }); + break; + } + case "tool-error": { + const out = await weatherAgent({ tool: brokenWeather as never }).generate(question); + report({ answer: out.text }); + break; + } + case "model-error": { + try { + await weatherAgent({ fail: true }).generate(question); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "stream-model-error": { + const out = await weatherAgent({ fail: true }).stream(question); + let chunks = 0; + for await (const _ of out.fullStream) chunks += 1; + report({ chunks, error: String(out.error) }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => weatherAgent().generate(question)), + ); + break; + } + case "uninstrument": { + report({ removed: failproofai.uninstrument() }); + await weatherAgent().generate(question); + break; + } + case "uninstrument-midstream": { + // uninstrument() lands while the first model step's stream is parked + // mid-flight; the caller then reads the run to the end regardless. + const reached = barrier(); + const gate = barrier(); + await failproofai.session({ sessionId: "req-1" }, async () => { + const out = await weatherAgent({ gate: { reached: reached.open, wait: gate.wait } }).stream(question); + const reading = (async () => { + let text = ""; + for await (const chunk of out.textStream) text += chunk; + return text; + })(); + await reached.wait; + report({ removed: failproofai.uninstrument() }); + gate.open(); + report({ answer: await reading }); + }); + break; + } + case "uninstrument-reuse": { + // One Agent instance: run while instrumented, then again (both ways) + // after uninstrument(). + await failproofai.session({ sessionId: "req-1" }, async () => { + const agent = weatherAgent(); + await agent.generate(question); + report({ removed: failproofai.uninstrument() }); + const again = await agent.generate(question); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + report({ answers: [again.text, text] }); + }); + break; + } + case "reinstrument": { + report({ removed: failproofai.uninstrument() }); + report({ instrumented: await failproofai.instrument("mastra") }); + const out = await weatherAgent().generate(question); + report({ answer: out.text }); + break; + } + case "reinstrument-reuse": { + // The same Agent instance across an uninstrument()/instrument() cycle. + const agent = weatherAgent(); + await agent.generate(question); + report({ removed: failproofai.uninstrument() }); + report({ instrumented: await failproofai.instrument("mastra") }); + const out = await agent.generate(question); + report({ answer: out.text }); + break; + } + case "mastra-instance": { + // Registered on a Mastra instance and fetched back through it — the + // shape `mastra dev` and every deployer use. + const mastra = new Mastra({ + agents: { weatherAgent: weatherAgent() }, + workflows: { weatherFlow: buildWorkflow() }, + logger: false, + }); + const out = await mastra.getAgent("weatherAgent").generate(question); + const run = await mastra.getWorkflow("weatherFlow").createRunAsync(); + const flow = await run.start({ inputData: { question } }); + report({ answer: out.text, status: flow.status }); + break; + } + case "concurrent": { + // Ten runs of ONE Agent instance at once, in one session. + const agent = routedAgent("weather-agent"); + await failproofai.session({ sessionId: "req-1" }, async () => { + const outs = await Promise.all(CITIES.map((city) => agent.generate(`What is the weather in ${city}?`))); + report({ answers: outs.map((out) => out.text) }); + }); + break; + } + case "concurrent-sessions": { + // The same, with no scope: every run is its own session, so a leaked + // event would land in the wrong one where the test can see it. + const agent = routedAgent("weather-agent"); + const outs = await Promise.all( + CITIES.map((city, i) => + i % 2 === 0 + ? agent.generate(`What is the weather in ${city}?`).then((out) => out.text) + : agent.stream(`What is the weather in ${city}?`).then(async (out) => { + let text = ""; + for await (const chunk of out.textStream) text += chunk; + return text; + }), + ), + ); + report({ answers: outs }); + break; + } + case "memory": { + // Two turns of one conversation (thread) through @mastra/memory. + const agent = routedAgent("weather-agent", { weather }, { + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + const memory = { thread: "thread-42", resource: "user-7" }; + const first = await agent.generate("What is the weather in Paris?", { memory }); + const second = await agent.stream("And what is the weather in Rome?", { memory }); + let text = ""; + for await (const chunk of second.textStream) text += chunk; + report({ answers: [first.text, text] }); + break; + } + case "memory-scoped": { + // An enclosing session scope wins over the thread. + const agent = routedAgent("weather-agent", { weather }, { + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + await failproofai.session({ sessionId: "req-1" }, () => + agent.generate(question, { memory: { thread: "thread-42", resource: "user-7" } }), + ); + break; + } + case "wf-branch": + case "wf-parallel": + case "wf-loop": + case "wf-nested": + case "wf-agent-step": { + const name = scenario.slice(3); + const run = await workflowCase(name).createRunAsync(); + const out = await run.start({ inputData: (name === "loop" ? { n: 0 } : { question }) as never }); + report({ status: out.status, result: out.status === "success" ? out.result : undefined }); + break; + } + case "wf-stream": { + // A workflow run streamed rather than awaited. + const run = await workflowCase("agent-step").createRunAsync(); + const streamed = await run.stream({ inputData: { question } }); + let chunks = 0; + for await (const _ of streamed.fullStream) chunks += 1; + report({ chunks, status: await streamed.status }); + break; + } + case "wf-suspend": { + // A resume reads the suspended run's snapshot back from storage. + const mastra = new Mastra({ workflows: { approval: workflowCase("suspend") as never }, storage: new InMemoryStore(), logger: false }); + const run = await (mastra.getWorkflow("approval") as unknown as ReturnType).createRunAsync(); + const first = await run.start({ inputData: { question } }); + report({ status: first.status }); + const second = await run.resume({ step: "approve", resumeData: { approved: true } }); + report({ status: second.status, result: second.status === "success" ? second.result : undefined }); + break; + } + case "processors": { + // An input processor that rewrites the prompt and an output processor + // that rewrites the answer: neither may add or lose an event. + const agent = routedAgent("weather-agent", { weather }, { + inputProcessors: [ + { + id: "tag-input", + processInput: ({ messages }: { messages: unknown[] }) => messages, + }, + ], + outputProcessors: [ + { + id: "shout", + processOutputResult: ({ messages }: { messages: unknown[] }) => messages, + }, + ], + }); + const out = await agent.generate(question); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + report({ answers: [out.text, text] }); + break; + } + case "tripwire": { + // A guardrail that blocks the prompt: no model call, the run rejected. + const agent = routedAgent("weather-agent", { weather }, { + inputProcessors: [ + { + id: "block-paris", + processInput: ({ messages, abort }: { messages: unknown[]; abort: (reason: string) => never }) => + JSON.stringify(messages).includes("Paris") ? abort("Paris is blocked") : messages, + }, + ], + }); + const out = await agent.generate(question); + const streamed = await agent.stream(question); + for await (const _ of streamed.fullStream) void _; + report({ tripwire: Boolean((out as { tripwire?: unknown }).tripwire), text: out.text }); + break; + } + case "tripwire-output": { + // A guardrail on the OUTPUT stream: the model answers, the processor + // blocks the answer mid-stream. + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", () => ({ text: "It is sunny in Paris.", usage: [9, 4] })) as never, + outputProcessors: [ + { + id: "block-answer", + processOutputStream: ({ part, abort }: { part: { type: string }; abort: (reason: string) => never }) => + part.type === "text-delta" ? abort("answer blocked") : part, + }, + ], + } as never); + const streamed = await agent.stream(question); + let chunks = 0; + for await (const _ of streamed.fullStream) chunks += 1; + report({ chunks, tripwire: Boolean((streamed as { tripwire?: unknown }).tripwire) }); + break; + } + case "usage-openai-compatible": { + // Mastra's own model for an OpenAI-compatible endpoint (`{ id, url }`), + // against a loopback server that — like OpenAI — sends a streamed + // step's usage only when the request asks for it, or always when + // USAGE_ALWAYS=1 (as some compatible servers do). + const requests: Array<{ stream?: boolean; streamOptions?: unknown }> = []; + const usage = { prompt_tokens: 17, completion_tokens: 5, total_tokens: 22 }; + const server = createServer((req, res) => { + let raw = ""; + req.on("data", (chunk: Buffer) => (raw += chunk.toString())); + req.on("end", () => { + const body = JSON.parse(raw || "{}") as { stream?: boolean; stream_options?: { include_usage?: boolean } }; + requests.push({ stream: body.stream, streamOptions: body.stream_options }); + const base = { id: "c1", created: 0, model: "compat-model" }; + if (!body.stream) { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ ...base, object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "Sunny." }, finish_reason: "stop" }], usage })); + return; + } + res.setHeader("content-type", "text/event-stream"); + const send = (chunk: unknown) => res.write(`data: ${JSON.stringify(chunk)}\n\n`); + send({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "Sunny." }, finish_reason: null }] }); + send({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }); + if (process.env.USAGE_ALWAYS === "1" || body.stream_options?.include_usage) { + send({ ...base, object: "chat.completion.chunk", choices: [], usage }); + } + res.end("data: [DONE]\n\n"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`; + const agent = new Agent({ id: "compat-agent", name: "compat-agent", instructions: "Be brief.", model: { id: "custom/compat-model", url, apiKey: "test-key" } as never }); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + const streamedUsage = await streamed.usage; + const generated = await agent.generate(question); + report({ text, mastraStreamTokens: streamedUsage.inputTokens ?? 0, mastraGenerateTokens: generated.usage.inputTokens ?? 0, requests }); + } finally { + // Deno's node:http keeps idle keep-alive sockets open through close(), + // so the process would outlive the test; Node closes idle ones itself. + server.closeAllConnections(); + server.close(); + } + break; + } + case "mcp": + case "mcp-toolsets": { + const mcp = new MCPClient({ id: `mcp-${process.pid}`, servers: { weatherServer: mcpServer() } }); + try { + if (scenario === "mcp") { + const tools = await mcp.getTools(); + const agent = routedAgent("weather-agent", tools); + const out = await agent.generate(question); + report({ tools: Object.keys(tools), answer: out.text }); + } else { + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer("forecast")) as never, + }); + const out = await agent.generate(question, { toolsets: await mcp.getToolsets() }); + report({ answer: out.text }); + } + } finally { + await mcp.disconnect(); + } + break; + } + case "structured": { + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", (options) => ({ + text: JSON.stringify({ city: cityOf(lastUserText(options.prompt)), forecast: "sunny" }), + usage: [15, 6], + })) as never, + }); + const schema = z.object({ city: z.string(), forecast: z.string() }); + const out = await agent.generate(question, { structuredOutput: { schema } }); + const streamed = await agent.stream(question, { structuredOutput: { schema } }); + report({ object: out.object, streamed: await streamed.object }); + break; + } + case "structured-model": { + // Structuring by a SECOND model: Mastra runs it through an agent of + // its own, after the main loop. + const agent = routedAgent("weather-agent"); + const structurer = decidingModel("structuring-model", () => ({ text: JSON.stringify({ city: "Paris", forecast: "sunny" }), usage: [30, 5] })); + const out = await agent.generate(question, { + structuredOutput: { schema: z.object({ city: z.string(), forecast: z.string() }), model: structurer as never }, + }); + report({ object: out.object }); + break; + } + case "maxsteps-tool-error": { + // One step only, and the tool it calls throws. + const agent = routedAgent("weather-agent", { weather: brokenWeather }); + const out = await agent.generate(question, { maxSteps: 1 }); + report({ finishReason: out.finishReason, text: out.text }); + break; + } + case "network": { + // An agent network: the planner's model routes to `helper` once, the + // helper answers with its tool, the planner's model judges it complete. + const helper = new Agent({ + id: "helper", + name: "helper", + description: "Looks up the weather.", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer("weather")) as never, + tools: { weather }, + }); + const planner = new Agent({ + id: "planner", + name: "planner", + instructions: "Coordinate the specialists.", + model: decidingModel("router-model", (options) => { + // Tell the calls apart by the schema they ask for, falling back to + // the prompt when a release injects the schema as text instead. The + // completion check comes first: its prompt quotes the routing answer. + const fields = options.responseFormat?.schema?.properties ?? {}; + const asked = lastUserText(options.prompt); + if ("isComplete" in fields || (asked.includes("evaluate") && asked.includes("complete"))) { + return { text: JSON.stringify({ isComplete: true, completionReason: "answered", finalResult: "It is sunny in Paris." }), usage: [40, 6] }; + } + if ("primitiveId" in fields || promptText(options.prompt).includes("primitiveId")) { + return { + text: JSON.stringify({ primitiveId: "helper", primitiveType: "agent", prompt: question, selectionReason: "weather" }), + usage: [50, 10], + }; + } + return { text: "Weather in Paris", usage: [5, 3] }; + }) as never, + agents: { helper }, + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + const stream = await planner.network(question, { memory: { thread: "thread-net", resource: "user-7" } }); + for await (const _ of stream) void _; + report({ status: await stream.status }); + break; + } + case "agent-in-tool": { + // Delegation by hand: a tool whose body runs another agent. + const helper = routedAgent("helper"); + const ask = createTool({ + id: "ask-helper", + description: "Ask the helper", + inputSchema: z.object({ city: z.string() }), + execute: async ({ context }: { context: { city: string } }) => ({ answer: (await helper.generate(`What is the weather in ${context.city}?`)).text }), + }); + const boss = routedAgent("boss", { "ask-helper": ask }); + const out = await boss.generate(question); + report({ answer: out.text }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/mastra-0/mcp-server.mjs b/sdk/typescript/integration/fixtures/mastra-0/mcp-server.mjs new file mode 100644 index 000000000..95632abb3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-0/mcp-server.mjs @@ -0,0 +1,66 @@ +// A minimal MCP server over stdio, for the `mcp` case of agent.ts. +// +// Hand-written JSON-RPC rather than the MCP SDK's server so the fixture does +// not depend on which SDK major `@mastra/mcp` happens to pull in: newline- +// delimited JSON on stdin/stdout, which is the whole stdio transport. It +// serves one tool, `forecast`, and nothing reaches a network. +import { createInterface } from "node:readline"; + +const TOOLS = [ + { + name: "forecast", + description: "Forecast for a city", + inputSchema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, +]; + +const send = (message) => process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", ...message })}\n`); + +function handle(request) { + switch (request.method) { + case "initialize": + return { + // Echo the client's version: this server speaks the subset every + // version shares. + protocolVersion: request.params?.protocolVersion ?? "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "weather-mcp", version: "1.0.0" }, + }; + case "ping": + return {}; + case "tools/list": + return { tools: TOOLS }; + case "tools/call": { + const city = request.params?.arguments?.city; + return { + content: [{ type: "text", text: `Forecast for ${String(city)}: sunny` }], + isError: false, + }; + } + case "resources/list": + return { resources: [] }; + case "prompts/list": + return { prompts: [] }; + default: + return undefined; + } +} + +createInterface({ input: process.stdin }).on("line", (line) => { + if (!line.trim()) return; + let request; + try { + request = JSON.parse(line); + } catch { + return; + } + // A notification (no id) needs no answer. + if (request.id === undefined || request.id === null) return; + const result = handle(request); + if (result === undefined) send({ id: request.id, error: { code: -32601, message: `unknown method ${request.method}` } }); + else send({ id: request.id, result }); +}); diff --git a/sdk/typescript/integration/fixtures/mastra-0/package-lock.json b/sdk/typescript/integration/fixtures/mastra-0/package-lock.json new file mode 100644 index 000000000..310412f40 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-0/package-lock.json @@ -0,0 +1,6417 @@ +{ + "name": "failproofai-it-mastra-0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-mastra-0", + "dependencies": { + "@mastra/core": "0.24.9", + "@mastra/mcp": "0.14.5", + "@mastra/memory": "0.15.13", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@a2a-js/sdk": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@a2a-js/sdk/-/sdk-0.2.5.tgz", + "integrity": "sha512-VTDuRS5V0ATbJ/LkaQlisMnTAeYKXAK6scMguVBstf+KIBQ7HIuKhiXLv+G/hvejkV+THoXzoNifInAkU81P1g==", + "dependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^4.17.23", + "body-parser": "^2.2.0", + "cors": "^2.8.5", + "express": "^4.21.2", + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/anthropic-v5": { + "name": "@ai-sdk/anthropic", + "version": "2.0.33", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-2.0.33.tgz", + "integrity": "sha512-egqr9PHqqX2Am5mn/Xs1C3+1/wphVKiAjpsVpW85eLc2WpW7AgiAg52DCBr4By9bw3UVVuMeR4uEO1X0dKDUDA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/anthropic-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-2.0.12.tgz", + "integrity": "sha512-W+cB1sOWvPcz9qiIsNtD+HxUrBUva2vWv2K1EFukuImX+HA0uZx3EyyOjhYQ9gtf/teqEG80M6OvJ7xx/VLV2A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.17", + "@vercel/oidc": "3.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/gateway/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/gateway/node_modules/@ai-sdk/provider-utils": { + "version": "3.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.17.tgz", + "integrity": "sha512-TR3Gs4I3Tym4Ll+EPdzRdvo/rc8Js6c4nVhFLuvGLX/Y4V9ZcQMa/HTiYsHEgmYrf1zVi6Q145UEZUfleOwOjw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/google-v5": { + "name": "@ai-sdk/google", + "version": "2.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-2.0.40.tgz", + "integrity": "sha512-E7MTVE6vhWXQJzXQDvojwA9t5xlhWpxttCH3R/kUyiE6y0tT8Ay2dmZLO+bLpFBQ5qrvBMrjKWpDVQMoo6TJZg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.17" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/google-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/google-v5/node_modules/@ai-sdk/provider-utils": { + "version": "3.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.17.tgz", + "integrity": "sha512-TR3Gs4I3Tym4Ll+EPdzRdvo/rc8Js6c4nVhFLuvGLX/Y4V9ZcQMa/HTiYsHEgmYrf1zVi6Q145UEZUfleOwOjw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mistral-v5": { + "name": "@ai-sdk/mistral", + "version": "2.0.23", + "resolved": "https://registry.npmjs.org/@ai-sdk/mistral/-/mistral-2.0.23.tgz", + "integrity": "sha512-np2bTlL5ZDi7iAOPCF5SZ5xKqls059iOvsigbgd9VNUCIrWSf6GYOaPvoWEgJ650TUOZitTfMo9MiEhLgutPfA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.16" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mistral-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/mistral-v5/node_modules/@ai-sdk/provider-utils": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.16.tgz", + "integrity": "sha512-lsWQY9aDXHitw7C1QRYIbVGmgwyT98TF3MfM8alNIXKpdJdi+W782Rzd9f1RyOfgRmZ08gJ2EYNDhWNK7RqpEA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-compatible": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-1.0.22.tgz", + "integrity": "sha512-Q+lwBIeMprc/iM+vg1yGjvzRrp74l316wDpqWdbmd4VXXlllblzGsUgBLTeKvcEapFTgqk0FRETvSb58Y6dsfA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-compatible-v5": { + "name": "@ai-sdk/openai-compatible", + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-1.0.22.tgz", + "integrity": "sha512-Q+lwBIeMprc/iM+vg1yGjvzRrp74l316wDpqWdbmd4VXXlllblzGsUgBLTeKvcEapFTgqk0FRETvSb58Y6dsfA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-compatible-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/openai-compatible/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/openai-v5": { + "name": "@ai-sdk/openai", + "version": "2.0.53", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-2.0.53.tgz", + "integrity": "sha512-GIkR3+Fyif516ftXv+YPSPstnAHhcZxNoR2s8uSHhQ1yBT7I7aQYTVwpjAuYoT3GR+TeP50q7onj2/nDRbT2FQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.12.tgz", + "integrity": "sha512-ZtbdvYxdMoria+2SlNarEk6Hlgyf+zzcznlD55EAl+7VZvJaSg2sqPvwArY7L6TfDEDJsnCq0fdhBSkYo0Xqdg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v5": { + "name": "@ai-sdk/provider-utils", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.12.tgz", + "integrity": "sha512-ZtbdvYxdMoria+2SlNarEk6Hlgyf+zzcznlD55EAl+7VZvJaSg2sqPvwArY7L6TfDEDJsnCq0fdhBSkYo0Xqdg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-v5": { + "name": "@ai-sdk/provider", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/xai-v5": { + "name": "@ai-sdk/xai", + "version": "2.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/xai/-/xai-2.0.26.tgz", + "integrity": "sha512-+VtaLZSxmoKnNeJGM9bbtbZ3QMkPFlBB4N8prngbrSnvU/hG8cNdvvSBW/rIk6/DHrc2R8nFntNIBQoIRuBdQw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai-compatible": "1.0.22", + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/xai-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser": { + "version": "11.9.3", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.9.3.tgz", + "integrity": "sha512-60vepv88RwcJtSHrD6MjIL6Ta3SOYbgfnkHb+ppAVK+o9mXprRtulx7VlRl3lN3bbvysAfCS7WMVfhUYemB0IQ==", + "license": "MIT", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.15", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.5", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.5.tgz", + "integrity": "sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@isaacs/ttlcache": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-1.4.1.tgz", + "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", + "license": "MIT" + }, + "node_modules/@mastra/core": { + "version": "0.24.9", + "resolved": "https://registry.npmjs.org/@mastra/core/-/core-0.24.9.tgz", + "integrity": "sha512-EjAPnX6pq7Y+7YN/kO92HIHqfk9Z/jtggE4Ww6wiL2Gvr01eFoNZSmsrIT4vTQAdD4oM41R2x1ndgtKFAJRH0w==", + "license": "Apache-2.0", + "dependencies": { + "@a2a-js/sdk": "~0.2.4", + "@ai-sdk/anthropic-v5": "npm:@ai-sdk/anthropic@2.0.33", + "@ai-sdk/google-v5": "npm:@ai-sdk/google@2.0.40", + "@ai-sdk/mistral-v5": "npm:@ai-sdk/mistral@2.0.23", + "@ai-sdk/openai-compatible-v5": "npm:@ai-sdk/openai-compatible@1.0.22", + "@ai-sdk/openai-v5": "npm:@ai-sdk/openai@2.0.53", + "@ai-sdk/provider": "^1.1.3", + "@ai-sdk/provider-utils": "^2.2.8", + "@ai-sdk/provider-utils-v5": "npm:@ai-sdk/provider-utils@3.0.12", + "@ai-sdk/provider-v5": "npm:@ai-sdk/provider@2.0.0", + "@ai-sdk/ui-utils": "^1.2.11", + "@ai-sdk/xai-v5": "npm:@ai-sdk/xai@2.0.26", + "@isaacs/ttlcache": "^1.4.1", + "@mastra/schema-compat": "0.11.9", + "@openrouter/ai-sdk-provider-v5": "npm:@openrouter/ai-sdk-provider@1.2.3", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/auto-instrumentations-node": "^0.62.1", + "@opentelemetry/core": "^2.0.1", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.203.0", + "@opentelemetry/otlp-exporter-base": "^0.203.0", + "@opentelemetry/otlp-transformer": "^0.203.0", + "@opentelemetry/resources": "^2.0.1", + "@opentelemetry/sdk-metrics": "^2.0.1", + "@opentelemetry/sdk-node": "^0.203.0", + "@opentelemetry/sdk-trace-base": "^2.0.1", + "@opentelemetry/sdk-trace-node": "^2.0.1", + "@opentelemetry/semantic-conventions": "^1.36.0", + "@sindresorhus/slugify": "^2.2.1", + "ai": "^4.3.19", + "ai-v5": "npm:ai@5.0.97", + "date-fns": "^3.6.0", + "dotenv": "^16.6.1", + "hono": "^4.9.7", + "hono-openapi": "^0.4.8", + "js-tiktoken": "^1.0.20", + "json-schema": "^0.4.0", + "lru-cache": "^11.2.2", + "p-map": "^7.0.3", + "p-retry": "^7.1.0", + "pino": "^9.7.0", + "pino-pretty": "^13.0.0", + "radash": "^12.1.1", + "sift": "^17.1.3", + "xstate": "^5.20.1", + "zod-to-json-schema": "^3.24.6" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/core/node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@mastra/core/node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@mastra/core/node_modules/@mastra/schema-compat": { + "version": "0.11.9", + "resolved": "https://registry.npmjs.org/@mastra/schema-compat/-/schema-compat-0.11.9.tgz", + "integrity": "sha512-LXEChx5n3bcuSFWQ5Wn9K2spLEpzHGf+DCnAeryuecpOo8VGLJ2QCK9Ugsnfjuc6hC0Ha73HvL1AD8zDhjmYOg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0", + "json-schema-to-zod": "^2.7.0", + "zod-from-json-schema": "^0.5.0", + "zod-from-json-schema-v3": "npm:zod-from-json-schema@^0.0.5", + "zod-to-json-schema": "^3.24.6" + }, + "peerDependencies": { + "ai": "^4.0.0 || ^5.0.0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/core/node_modules/@openrouter/ai-sdk-provider-v5": { + "name": "@openrouter/ai-sdk-provider", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@openrouter/ai-sdk-provider/-/ai-sdk-provider-1.2.3.tgz", + "integrity": "sha512-a6Nc8dPRHakRH9966YJ/HZJhLOds7DuPTscNZDoAr+Aw+tEFUlacSJMvb/b3gukn74mgbuaJRji9YOn62ipfVg==", + "license": "Apache-2.0", + "dependencies": { + "@openrouter/sdk": "^0.1.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "ai": "^5.0.0", + "zod": "^3.24.1 || ^v4" + } + }, + "node_modules/@mastra/core/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@mastra/core/node_modules/ai": { + "version": "4.3.19", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.19.tgz", + "integrity": "sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/@mastra/core/node_modules/ai/node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@mastra/core/node_modules/hono-openapi": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/hono-openapi/-/hono-openapi-0.4.8.tgz", + "integrity": "sha512-LYr5xdtD49M7hEAduV1PftOMzuT8ZNvkyWfh1DThkLsIr4RkvDb12UxgIiFbwrJB6FLtFXLoOZL9x4IeDk2+VA==", + "license": "MIT", + "dependencies": { + "json-schema-walker": "^2.0.0" + }, + "peerDependencies": { + "@hono/arktype-validator": "^2.0.0", + "@hono/effect-validator": "^1.2.0", + "@hono/typebox-validator": "^0.2.0 || ^0.3.0", + "@hono/valibot-validator": "^0.5.1", + "@hono/zod-validator": "^0.4.1", + "@sinclair/typebox": "^0.34.9", + "@valibot/to-json-schema": "^1.0.0-beta.3", + "arktype": "^2.0.0", + "effect": "^3.11.3", + "hono": "^4.6.13", + "openapi-types": "^12.1.3", + "valibot": "^1.0.0-beta.9", + "zod": "^3.23.8", + "zod-openapi": "^4.0.0" + }, + "peerDependenciesMeta": { + "@hono/arktype-validator": { + "optional": true + }, + "@hono/effect-validator": { + "optional": true + }, + "@hono/typebox-validator": { + "optional": true + }, + "@hono/valibot-validator": { + "optional": true + }, + "@hono/zod-validator": { + "optional": true + }, + "@sinclair/typebox": { + "optional": true + }, + "@valibot/to-json-schema": { + "optional": true + }, + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "hono": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-openapi": { + "optional": true + } + } + }, + "node_modules/@mastra/core/node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/@mastra/mcp": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/@mastra/mcp/-/mcp-0.14.5.tgz", + "integrity": "sha512-PNVLzD9XY2zs9fRnYOSGz1nf9hKlDU9dYo+EMJVy15wLdx9ZW72j0iuv4m3Aicdp5Y61l5LPSDPbDTSlO2Y3CA==", + "license": "Apache-2.0", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^14.2.1", + "@modelcontextprotocol/sdk": "^1.17.5", + "date-fns": "^4.1.0", + "exit-hook": "^4.0.0", + "fast-deep-equal": "^3.1.3", + "uuid": "^11.1.0", + "zod-from-json-schema": "^0.5.0", + "zod-from-json-schema-v3": "npm:zod-from-json-schema@^0.0.5" + }, + "peerDependencies": { + "@mastra/core": ">=0.20.1-0 <0.25.0-0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/mcp/node_modules/@apidevtools/json-schema-ref-parser": { + "version": "14.2.1", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-14.2.1.tgz", + "integrity": "sha512-HmdFw9CDYqM6B25pqGBpNeLCKvGPlIx1EbLrVL0zPvj50CJQUHyBNBw45Muk0kEIkogo1VZvOKHajdMuAzSxRg==", + "license": "MIT", + "dependencies": { + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">= 20" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" + }, + "peerDependencies": { + "@types/json-schema": "^7.0.15" + } + }, + "node_modules/@mastra/mcp/node_modules/date-fns": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.4.0.tgz", + "integrity": "sha512-+1UMbeh68lH1SegH83CGWwpb6OHHbpSgr3+s5Eww5M4CAgswBpoWS0AjTOfEJ33HiYKz1hdj/KTFprzXHmq/6w==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } + }, + "node_modules/@mastra/memory": { + "version": "0.15.13", + "resolved": "https://registry.npmjs.org/@mastra/memory/-/memory-0.15.13.tgz", + "integrity": "sha512-88RBgT1VIseyvKdkzVJFw88gtFmf/+6vOMwJaeLzSq++oHtoD+A0nkUvCtuX3aYsicpf5IPUBnvEgJ9pH6J/gg==", + "license": "Apache-2.0", + "dependencies": { + "@mastra/schema-compat": "0.11.9", + "@upstash/redis": "^1.35.5", + "ai": "^4.3.19", + "ai-v5": "npm:ai@5.0.60", + "async-mutex": "^0.5.0", + "js-tiktoken": "^1.0.20", + "json-schema": "^0.4.0", + "pg": "^8.16.3", + "pg-pool": "^3.10.1", + "postgres": "^3.4.7", + "redis": "^5.8.3", + "xxhash-wasm": "^1.1.0", + "zod-to-json-schema": "^3.24.6" + }, + "peerDependencies": { + "@mastra/core": ">=0.20.1-0 <0.25.0-0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/memory/node_modules/@ai-sdk/gateway": { + "version": "1.0.33", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-1.0.33.tgz", + "integrity": "sha512-v9i3GPEo4t3fGcSkQkc07xM6KJN75VUv7C1Mqmmsu2xD8lQwnQfsrgAXyNuWe20yGY0eHuheSPDZhiqsGKtH1g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.10", + "@vercel/oidc": "^3.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@mastra/memory/node_modules/@ai-sdk/gateway/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mastra/memory/node_modules/@ai-sdk/provider-utils": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.10.tgz", + "integrity": "sha512-T1gZ76gEIwffep6MWI0QNy9jgoybUHE7TRaHB5k54K8mF91ciGFlbtCGxDYhMH3nCRergKwYFIDeFF0hJSIQHQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@mastra/memory/node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mastra/memory/node_modules/@mastra/schema-compat": { + "version": "0.11.9", + "resolved": "https://registry.npmjs.org/@mastra/schema-compat/-/schema-compat-0.11.9.tgz", + "integrity": "sha512-LXEChx5n3bcuSFWQ5Wn9K2spLEpzHGf+DCnAeryuecpOo8VGLJ2QCK9Ugsnfjuc6hC0Ha73HvL1AD8zDhjmYOg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0", + "json-schema-to-zod": "^2.7.0", + "zod-from-json-schema": "^0.5.0", + "zod-from-json-schema-v3": "npm:zod-from-json-schema@^0.0.5", + "zod-to-json-schema": "^3.24.6" + }, + "peerDependencies": { + "ai": "^4.0.0 || ^5.0.0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/memory/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@mastra/memory/node_modules/ai": { + "version": "4.3.19", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.19.tgz", + "integrity": "sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/@mastra/memory/node_modules/ai-v5": { + "name": "ai", + "version": "5.0.60", + "resolved": "https://registry.npmjs.org/ai/-/ai-5.0.60.tgz", + "integrity": "sha512-80U/3kmdBW6g+JkLXpz/P2EwkyEaWlPlYtuLUpx/JYK9F7WZh9NnkYoh1KvUi1Sbpo0NyurBTvX0a2AG9mmbDA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "1.0.33", + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.10", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@mastra/memory/node_modules/ai-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mastra/memory/node_modules/ai/node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@mastra/memory/node_modules/ai/node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@mastra/memory/node_modules/ai/node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@mastra/memory/node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@openrouter/sdk": { + "version": "0.1.27", + "resolved": "https://registry.npmjs.org/@openrouter/sdk/-/sdk-0.1.27.tgz", + "integrity": "sha512-RH//L10bSmc81q25zAZudiI4kNkLgxF2E+WU42vghp3N6TEvZ6F0jK7uT3tOxkEn91gzmMw9YVmDENy7SJsajQ==", + "license": "Apache-2.0", + "dependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.203.0.tgz", + "integrity": "sha512-9B9RU0H7Ya1Dx/Rkyc4stuBZSGVQF27WigitInx2QQoj6KUpEFYPKoWjdFTunJYxmXmh17HeBvbMa1EhGyPmqQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node": { + "version": "0.62.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.62.2.tgz", + "integrity": "sha512-Ipe6X7ddrCiRsuewyTU83IvKiSFT4piqmv9z8Ovg1E7v98pdTj1pUE6sDrHV50zl7/ypd+cONBgt+EYSZu4u9Q==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/instrumentation-amqplib": "^0.50.0", + "@opentelemetry/instrumentation-aws-lambda": "^0.54.1", + "@opentelemetry/instrumentation-aws-sdk": "^0.58.0", + "@opentelemetry/instrumentation-bunyan": "^0.49.0", + "@opentelemetry/instrumentation-cassandra-driver": "^0.49.0", + "@opentelemetry/instrumentation-connect": "^0.47.0", + "@opentelemetry/instrumentation-cucumber": "^0.19.0", + "@opentelemetry/instrumentation-dataloader": "^0.21.1", + "@opentelemetry/instrumentation-dns": "^0.47.0", + "@opentelemetry/instrumentation-express": "^0.52.0", + "@opentelemetry/instrumentation-fastify": "^0.48.0", + "@opentelemetry/instrumentation-fs": "^0.23.0", + "@opentelemetry/instrumentation-generic-pool": "^0.47.0", + "@opentelemetry/instrumentation-graphql": "^0.51.0", + "@opentelemetry/instrumentation-grpc": "^0.203.0", + "@opentelemetry/instrumentation-hapi": "^0.50.0", + "@opentelemetry/instrumentation-http": "^0.203.0", + "@opentelemetry/instrumentation-ioredis": "^0.51.0", + "@opentelemetry/instrumentation-kafkajs": "^0.13.0", + "@opentelemetry/instrumentation-knex": "^0.48.0", + "@opentelemetry/instrumentation-koa": "^0.51.0", + "@opentelemetry/instrumentation-lru-memoizer": "^0.48.0", + "@opentelemetry/instrumentation-memcached": "^0.47.0", + "@opentelemetry/instrumentation-mongodb": "^0.56.0", + "@opentelemetry/instrumentation-mongoose": "^0.50.0", + "@opentelemetry/instrumentation-mysql": "^0.49.0", + "@opentelemetry/instrumentation-mysql2": "^0.50.0", + "@opentelemetry/instrumentation-nestjs-core": "^0.49.0", + "@opentelemetry/instrumentation-net": "^0.47.0", + "@opentelemetry/instrumentation-oracledb": "^0.29.0", + "@opentelemetry/instrumentation-pg": "^0.56.1", + "@opentelemetry/instrumentation-pino": "^0.50.1", + "@opentelemetry/instrumentation-redis": "^0.52.0", + "@opentelemetry/instrumentation-restify": "^0.49.0", + "@opentelemetry/instrumentation-router": "^0.48.0", + "@opentelemetry/instrumentation-runtime-node": "^0.17.1", + "@opentelemetry/instrumentation-socket.io": "^0.50.0", + "@opentelemetry/instrumentation-tedious": "^0.22.0", + "@opentelemetry/instrumentation-undici": "^0.14.0", + "@opentelemetry/instrumentation-winston": "^0.48.1", + "@opentelemetry/resource-detector-alibaba-cloud": "^0.31.3", + "@opentelemetry/resource-detector-aws": "^2.3.0", + "@opentelemetry/resource-detector-azure": "^0.10.0", + "@opentelemetry/resource-detector-container": "^0.7.3", + "@opentelemetry/resource-detector-gcp": "^0.37.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/sdk-node": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.4.1", + "@opentelemetry/core": "^2.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.11.0.tgz", + "integrity": "sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz", + "integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.203.0.tgz", + "integrity": "sha512-g/2Y2noc/l96zmM+g0LdeuyYKINyBwN6FJySoU15LHPLcMN/1a0wNk2SegwKcxrRdE7Xsm7fkIR5n6XFe3QpPw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/sdk-logs": "0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.203.0.tgz", + "integrity": "sha512-s0hys1ljqlMTbXx2XiplmMJg9wG570Z5lH7wMvrZX6lcODI56sG4HL03jklF63tBeyNwK2RV1/ntXGo3HgG4Qw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/sdk-logs": "0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.203.0.tgz", + "integrity": "sha512-nl/7S91MXn5R1aIzoWtMKGvqxgJgepB/sH9qW0rZvZtabnsjbf8OQ1uSx3yogtvLr0GzwD596nQKz2fV7q2RBw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-logs": "0.203.0", + "@opentelemetry/sdk-trace-base": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.203.0.tgz", + "integrity": "sha512-FCCj9nVZpumPQSEI57jRAA89hQQgONuoC35Lt+rayWY/mzCAc6BQT7RFyFaZKJ2B7IQ8kYjOCPsF/HGFWjdQkQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.203.0", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-metrics": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.203.0.tgz", + "integrity": "sha512-HFSW10y8lY6BTZecGNpV3GpoSy7eaO0Z6GATwZasnT4bEsILp8UJXNG5OmEsz4SdwCSYvyCbTJdNbZP3/8LGCQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-metrics": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.203.0.tgz", + "integrity": "sha512-OZnhyd9npU7QbyuHXFEPVm3LnjZYifuKpT3kTnF84mXeEQ84pJJZgyLBpU4FSkSwUkt/zbMyNAI7y5+jYTWGIg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.203.0", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-metrics": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.203.0.tgz", + "integrity": "sha512-2jLuNuw5m4sUj/SncDf/mFPabUxMZmmYetx5RKIMIQyPnl6G6ooFzfeE8aXNRf8YD1ZXNlCnRPcISxjveGJHNg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-metrics": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.203.0.tgz", + "integrity": "sha512-322coOTf81bm6cAA8+ML6A+m4r2xTCdmAZzGNTboPXRzhwPt4JEmovsFAs+grpdarObd68msOJ9FfH3jxM6wqA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.203.0.tgz", + "integrity": "sha512-ZDiaswNYo0yq/cy1bBLJFe691izEJ6IgNmkjm4C6kE9ub/OMQqDXORx2D2j8fzTBTxONyzusbaZlqtfmyqURPw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.203.0.tgz", + "integrity": "sha512-1xwNTJ86L0aJmWRwENCJlH4LULMG2sOXWIVw+Szta4fkqKVY50Eo4HoVKKq6U9QEytrWCr8+zjw0q/ZOeXpcAQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-2.0.1.tgz", + "integrity": "sha512-a9eeyHIipfdxzCfc2XPrE+/TI3wmrZUDFtG2RRXHSbZZULAny7SyybSvaDvS77a7iib5MPiAvluwVvbGTsHxsw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.203.0.tgz", + "integrity": "sha512-ke1qyM+3AK2zPuBPb6Hk/GCsc5ewbLvPNkEuELx/JmANeEp6ZjnZ+wypPAJSucTw0wvCGrUaibDSdcrGFoWxKQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "import-in-the-middle": "^1.8.1", + "require-in-the-middle": "^7.1.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-amqplib": { + "version": "0.50.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.50.0.tgz", + "integrity": "sha512-kwNs/itehHG/qaQBcVrLNcvXVPW0I4FCOVtw3LHMLdYIqD7GJ6Yv2nX+a4YHjzbzIeRYj8iyMp0Bl7tlkidq5w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-aws-lambda": { + "version": "0.54.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-lambda/-/instrumentation-aws-lambda-0.54.1.tgz", + "integrity": "sha512-qm8pGSAM1mXk7unbrGktWWGJc6IFI58ZsaHJ+i420Fp5VO3Vf7GglIgaXTS8CKBrVB4LHFj3NvzJg31PtsAQcA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/aws-lambda": "8.10.152" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-aws-sdk": { + "version": "0.58.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-sdk/-/instrumentation-aws-sdk-0.58.0.tgz", + "integrity": "sha512-9vFH7gU686dsAeLMCkqUj9y0MQZ1xrTtStSpNV2UaGWtDnRjJrAdJLu9Y545oKEaDTeVaob4UflyZvvpZnw3Xw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.34.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-bunyan": { + "version": "0.49.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-bunyan/-/instrumentation-bunyan-0.49.0.tgz", + "integrity": "sha512-ky5Am1y6s3Ex/3RygHxB/ZXNG07zPfg9Z6Ora+vfeKcr/+I6CJbWXWhSBJor3gFgKN3RvC11UWVURnmDpBS6Pg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "^0.203.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@types/bunyan": "1.8.11" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-cassandra-driver": { + "version": "0.49.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cassandra-driver/-/instrumentation-cassandra-driver-0.49.0.tgz", + "integrity": "sha512-BNIvqldmLkeikfI5w5Rlm9vG5NnQexfPoxOgEMzfDVOEF+vS6351I6DzWLLgWWR9CNF/jQJJi/lr6am2DLp0Rw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-connect": { + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.47.0.tgz", + "integrity": "sha512-pjenvjR6+PMRb6/4X85L4OtkQCootgb/Jzh/l/Utu3SJHBid1F+gk9sTGU2FWuhhEfV6P7MZ7BmCdHXQjgJ42g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/connect": "3.4.38" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-cucumber": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cucumber/-/instrumentation-cucumber-0.19.0.tgz", + "integrity": "sha512-99ms8kQWRuPt5lkDqbJJzD+7Tq5TMUlBZki4SA2h6CgK4ncX+tyep9XFY1e+XTBLJIWmuFMGbWqBLJ4fSKIQNQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/instrumentation-dataloader": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.21.1.tgz", + "integrity": "sha512-hNAm/bwGawLM8VDjKR0ZUDJ/D/qKR3s6lA5NV+btNaPVm2acqhPcT47l2uCVi+70lng2mywfQncor9v8/ykuyw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-dns": { + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.47.0.tgz", + "integrity": "sha512-775fOnewWkTF4iXMGKgwvOGqEmPrU1PZpXjjqvTrEErYBJe7Fz1WlEeUStHepyKOdld7Ghv7TOF/kE3QDctvrg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-express": { + "version": "0.52.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.52.0.tgz", + "integrity": "sha512-W7pizN0Wh1/cbNhhTf7C62NpyYw7VfCFTYg0DYieSTrtPBT1vmoSZei19wfKLnrMsz3sHayCg0HxCVL2c+cz5w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-fastify": { + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.48.0.tgz", + "integrity": "sha512-3zQlE/DoVfVH6/ycuTv7vtR/xib6WOa0aLFfslYcvE62z0htRu/ot8PV/zmMZfnzpTQj8S/4ULv36R6UIbpJIg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-fs": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.23.0.tgz", + "integrity": "sha512-Puan+QopWHA/KNYvDfOZN6M/JtF6buXEyD934vrb8WhsX1/FuM7OtoMlQyIqAadnE8FqqDL4KDPiEfCQH6pQcQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-generic-pool": { + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.47.0.tgz", + "integrity": "sha512-UfHqf3zYK+CwDwEtTjaD12uUqGGTswZ7ofLBEdQ4sEJp9GHSSJMQ2hT3pgBxyKADzUdoxQAv/7NqvL42ZI+Qbw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-graphql": { + "version": "0.51.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.51.0.tgz", + "integrity": "sha512-LchkOu9X5DrXAnPI1+Z06h/EH/zC7D6sA86hhPrk3evLlsJTz0grPrkL/yUJM9Ty0CL/y2HSvmWQCjbJEz/ADg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-grpc": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-grpc/-/instrumentation-grpc-0.203.0.tgz", + "integrity": "sha512-Qmjx2iwccHYRLoE4RFS46CvQE9JG9Pfeae4EPaNZjvIuJxb/pZa2R9VWzRlTehqQWpAvto/dGhtkw8Tv+o0LTg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "0.203.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-hapi": { + "version": "0.50.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.50.0.tgz", + "integrity": "sha512-5xGusXOFQXKacrZmDbpHQzqYD1gIkrMWuwvlrEPkYOsjUqGUjl1HbxCsn5Y9bUXOCgP1Lj6A4PcKt1UiJ2MujA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.203.0.tgz", + "integrity": "sha512-y3uQAcCOAwnO6vEuNVocmpVzG3PER6/YZqbPbbffDdJ9te5NkHEkfSMNzlC3+v7KlE+WinPGc3N7MR30G1HY2g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/instrumentation": "0.203.0", + "@opentelemetry/semantic-conventions": "^1.29.0", + "forwarded-parse": "2.1.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation-ioredis": { + "version": "0.51.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.51.0.tgz", + "integrity": "sha512-9IUws0XWCb80NovS+17eONXsw1ZJbHwYYMXiwsfR9TSurkLV5UNbRSKb9URHO+K+pIJILy9wCxvyiOneMr91Ig==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/redis-common": "^0.38.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-kafkajs": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.13.0.tgz", + "integrity": "sha512-FPQyJsREOaGH64hcxlzTsIEQC4DYANgTwHjiB7z9lldmvua1LRMVn3/FfBlzXoqF179B0VGYviz6rn75E9wsDw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.30.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-knex": { + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.48.0.tgz", + "integrity": "sha512-V5wuaBPv/lwGxuHjC6Na2JFRjtPgstw19jTFl1B1b6zvaX8zVDYUDaR5hL7glnQtUSCMktPttQsgK4dhXpddcA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.33.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-koa": { + "version": "0.51.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.51.0.tgz", + "integrity": "sha512-XNLWeMTMG1/EkQBbgPYzCeBD0cwOrfnn8ao4hWgLv0fNCFQu1kCsJYygz2cvKuCs340RlnG4i321hX7R8gj3Rg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-lru-memoizer": { + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.48.0.tgz", + "integrity": "sha512-KUW29wfMlTPX1wFz+NNrmE7IzN7NWZDrmFWHM/VJcmFEuQGnnBuTIdsP55CnBDxKgQ/qqYFp4udQFNtjeFosPw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-memcached": { + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-memcached/-/instrumentation-memcached-0.47.0.tgz", + "integrity": "sha512-vXDs/l4hlWy1IepPG1S6aYiIZn+tZDI24kAzwKKJmR2QEJRL84PojmALAEJGazIOLl/VdcCPZdMb0U2K0VzojA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/memcached": "^2.2.6" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongodb": { + "version": "0.56.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.56.0.tgz", + "integrity": "sha512-YG5IXUUmxX3Md2buVMvxm9NWlKADrnavI36hbJsihqqvBGsWnIfguf0rUP5Srr0pfPqhQjUP+agLMsvu0GmUpA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongoose": { + "version": "0.50.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.50.0.tgz", + "integrity": "sha512-Am8pk1Ct951r4qCiqkBcGmPIgGhoDiFcRtqPSLbJrUZqEPUsigjtMjoWDRLG1Ki1NHgOF7D0H7d+suWz1AAizw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mysql": { + "version": "0.49.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.49.0.tgz", + "integrity": "sha512-QU9IUNqNsrlfE3dJkZnFHqLjlndiU39ll/YAAEvWE40sGOCi9AtOF6rmEGzJ1IswoZ3oyePV7q2MP8SrhJfVAA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/mysql": "2.15.27" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mysql2": { + "version": "0.50.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.50.0.tgz", + "integrity": "sha512-PoOMpmq73rOIE3nlTNLf3B1SyNYGsp7QXHYKmeTZZnJ2Ou7/fdURuOhWOI0e6QZ5gSem18IR1sJi6GOULBQJ9g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@opentelemetry/sql-common": "^0.41.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-nestjs-core": { + "version": "0.49.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.49.0.tgz", + "integrity": "sha512-1R/JFwdmZIk3T/cPOCkVvFQeKYzbbUvDxVH3ShXamUwBlGkdEu5QJitlRMyVNZaHkKZKWgYrBarGQsqcboYgaw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.30.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-net": { + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-net/-/instrumentation-net-0.47.0.tgz", + "integrity": "sha512-csoJ++Njpf7C09JH+0HNGenuNbDZBqO1rFhMRo6s0rAmJwNh9zY3M/urzptmKlqbKnf4eH0s+CKHy/+M8fbFsQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-oracledb": { + "version": "0.29.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-oracledb/-/instrumentation-oracledb-0.29.0.tgz", + "integrity": "sha512-2aHLiJdkyiUbooIUm7FaZf+O4jyqEl+RfFpgud1dxT87QeeYM216wi+xaMNzsb5yKtRBqbA3qeHBCyenYrOZwA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/oracledb": "6.5.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-pg": { + "version": "0.56.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.56.1.tgz", + "integrity": "sha512-0/PiHDPVaLdcXNw6Gqb3JBdMxComMEwh444X8glwiynJKJHRTR49+l2cqJfoOVzB8Sl1XRl3Yaqw6aDi3s8e9w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.34.0", + "@opentelemetry/sql-common": "^0.41.0", + "@types/pg": "8.15.5", + "@types/pg-pool": "2.0.6" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-pino": { + "version": "0.50.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pino/-/instrumentation-pino-0.50.1.tgz", + "integrity": "sha512-pBbvuWiHA9iAumAuQ0SKYOXK7NRlbnVTf/qBV0nMdRnxBPrc/GZTbh0f7Y59gZfYsbCLhXLL1oRTEnS+PwS3CA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "^0.203.0", + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-redis": { + "version": "0.52.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.52.0.tgz", + "integrity": "sha512-R8Y7cCZlJ2Vl31S2i7bl5SqyC/aul54ski4wCFip/Tp9WGtLK1xVATi2rwy2wkc8ZCtjdEe9eEVR+QFG6gGZxg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/redis-common": "^0.38.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-restify": { + "version": "0.49.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-restify/-/instrumentation-restify-0.49.0.tgz", + "integrity": "sha512-tsGZZhS4mVZH7omYxw5jpsrD3LhWizqWc0PYtAnzpFUvL5ZINHE+cm57bssTQ2AK/GtZMxu9LktwCvIIf3dSmw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-router": { + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-router/-/instrumentation-router-0.48.0.tgz", + "integrity": "sha512-Wixrc8CchuJojXpaS/dCQjFOMc+3OEil1H21G+WLYQb8PcKt5kzW9zDBT19nyjjQOx/D/uHPfgbrT+Dc7cfJ9w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-runtime-node": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-runtime-node/-/instrumentation-runtime-node-0.17.1.tgz", + "integrity": "sha512-c1FlAk+bB2uF9a8YneGmNPTl7c/xVaan4mmWvbkWcOmH/ipKqR1LaKUlz/BMzLrJLjho1EJlG2NrS2w2Arg+nw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-socket.io": { + "version": "0.50.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-socket.io/-/instrumentation-socket.io-0.50.0.tgz", + "integrity": "sha512-6JN6lnKN9ZuZtZdMQIR+no1qHzQvXSZUsNe3sSWMgqmNRyEXuDUWBIyKKeG0oHRHtR4xE4QhJyD4D5kKRPWZFA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-tedious": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.22.0.tgz", + "integrity": "sha512-XrrNSUCyEjH1ax9t+Uo6lv0S2FCCykcF7hSxBMxKf7Xn0bPRxD3KyFUZy25aQXzbbbUHhtdxj3r2h88SfEM3aA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.203.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/tedious": "^4.0.14" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-undici": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.14.0.tgz", + "integrity": "sha512-2HN+7ztxAReXuxzrtA3WboAKlfP5OsPA57KQn2AdYZbJ3zeRPcLXyW4uO/jpLE6PLm0QRtmeGCmfYpqRlwgSwg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.7.0" + } + }, + "node_modules/@opentelemetry/instrumentation-winston": { + "version": "0.48.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-winston/-/instrumentation-winston-0.48.1.tgz", + "integrity": "sha512-XyOuVwdziirHHYlsw+BWrvdI/ymjwnexupKA787zQQ+D5upaE/tseZxjfQa7+t4+FdVLxHICaMTmkSD4yZHpzQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "^0.203.0", + "@opentelemetry/instrumentation": "^0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.203.0.tgz", + "integrity": "sha512-Wbxf7k+87KyvxFr5D7uOiSq/vHXWommvdnNE7vECO3tAhsA2GfOlpWINCMWUEPdHZ7tCXxw6Epp3vgx3jU7llQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-transformer": "0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.203.0.tgz", + "integrity": "sha512-te0Ze1ueJF+N/UOFl5jElJW4U0pZXQ8QklgSfJ2linHN0JJsuaHG8IabEUi2iqxY8ZBDlSiz1Trfv5JcjWWWwQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/otlp-exporter-base": "0.203.0", + "@opentelemetry/otlp-transformer": "0.203.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.203.0.tgz", + "integrity": "sha512-Y8I6GgoCna0qDQ2W6GCRtaF24SnvqvA8OfeTi7fqigD23u8Jpb4R5KFv/pRvrlGagcCLICMIyh9wiejp4TXu/A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-logs": "0.203.0", + "@opentelemetry/sdk-metrics": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1", + "protobufjs": "^7.3.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-b3": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-2.0.1.tgz", + "integrity": "sha512-Hc09CaQ8Tf5AGLmf449H726uRoBNGPBL4bjr7AnnUpzWMvhdn61F78z9qb6IqB737TffBsokGAK1XykFEZ1igw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-b3/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-2.0.1.tgz", + "integrity": "sha512-7PMdPBmGVH2eQNb/AtSJizQNgeNTfh6jQFqys6lfhd6P4r+m/nTh3gKPPpaCXVdRQ+z93vfKk+4UGty390283w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/redis-common": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz", + "integrity": "sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + } + }, + "node_modules/@opentelemetry/resource-detector-alibaba-cloud": { + "version": "0.31.11", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-alibaba-cloud/-/resource-detector-alibaba-cloud-0.31.11.tgz", + "integrity": "sha512-R/asn6dAOWMfkLeEwqHCUz0cNbb9oiHVyd11iwlypeT/p9bR1lCX5juu5g/trOwxo62dbuFcDbBdKCJd3O2Edg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/resources": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resource-detector-aws": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-aws/-/resource-detector-aws-2.22.0.tgz", + "integrity": "sha512-3kM+xfsSHs0hShUAx9IFhX5jniaXxJL3CTjbCMNCFS+EY7bswWbukV3au0jfqEgzeNxGxHEy8RLA1TOhAVLpTQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resource-detector-azure": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-azure/-/resource-detector-azure-0.10.0.tgz", + "integrity": "sha512-5cNAiyPBg53Uxe/CW7hsCq8HiKNAUGH+gi65TtgpzSR9bhJG4AEbuZhbJDFwe97tn2ifAD1JTkbc/OFuaaFWbA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resource-detector-container": { + "version": "0.7.11", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-container/-/resource-detector-container-0.7.11.tgz", + "integrity": "sha512-XUxnGuANa/EdxagipWMXKYFC7KURwed9/V0+NtYjFmwWHzV9/J4IYVGTK8cWDpyUvAQf/vE4sMa3rnS025ivXQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/resources": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resource-detector-gcp": { + "version": "0.37.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-gcp/-/resource-detector-gcp-0.37.0.tgz", + "integrity": "sha512-LGpJBECIMsVKhiulb4nxUw++m1oF4EiDDPmFGW2aqYaAF0oUvJNv8Z/55CAzcZ7SxvlTgUwzewXDBsuCup7iqw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "gcp-metadata": "^6.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz", + "integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.203.0.tgz", + "integrity": "sha512-vM2+rPq0Vi3nYA5akQD2f3QwossDnTDLvKbea6u/A2NZ3XDkPxMfo/PNrDoXhDUD/0pPo2CdH5ce/thn9K0kLw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.11.0.tgz", + "integrity": "sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node": { + "version": "0.203.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.203.0.tgz", + "integrity": "sha512-zRMvrZGhGVMvAbbjiNQW3eKzW/073dlrSiAKPVWmkoQzah9wfynpVPeL55f9fVIm0GaBxTLcPeukWGy0/Wj7KQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/exporter-logs-otlp-grpc": "0.203.0", + "@opentelemetry/exporter-logs-otlp-http": "0.203.0", + "@opentelemetry/exporter-logs-otlp-proto": "0.203.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "0.203.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.203.0", + "@opentelemetry/exporter-metrics-otlp-proto": "0.203.0", + "@opentelemetry/exporter-prometheus": "0.203.0", + "@opentelemetry/exporter-trace-otlp-grpc": "0.203.0", + "@opentelemetry/exporter-trace-otlp-http": "0.203.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.203.0", + "@opentelemetry/exporter-zipkin": "2.0.1", + "@opentelemetry/instrumentation": "0.203.0", + "@opentelemetry/propagator-b3": "2.0.1", + "@opentelemetry/propagator-jaeger": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-logs": "0.203.0", + "@opentelemetry/sdk-metrics": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1", + "@opentelemetry/sdk-trace-node": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/context-async-hooks": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.0.1.tgz", + "integrity": "sha512-XuY23lSI3d4PEqKA+7SLtAgwqIfc6E/E9eAQWLN1vlpC53ybO3o6jW4BsXo1xvz9lYyyWItfQDDLzezER01mCw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/resources": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.0.1", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.0.1.tgz", + "integrity": "sha512-UhdbPF19pMpBtCWYP5lHbTogLWx9N0EBxtdagvkn5YtsAnCBZzL7SjktG+ZmupRgifsHMjwUaCCaVmqGfSADmA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.0.1", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz", + "integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz", + "integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/sdk-trace": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.11.0.tgz", + "integrity": "sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.11.0", + "@opentelemetry/core": "2.11.0", + "@opentelemetry/sdk-trace-base": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sql-common": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", + "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0" + } + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@redis/bloom": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.12.1.tgz", + "integrity": "sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@redis/json": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-5.12.1.tgz", + "integrity": "sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/search": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-5.12.1.tgz", + "integrity": "sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/time-series": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-5.12.1.tgz", + "integrity": "sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@sindresorhus/slugify": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/slugify/-/slugify-2.2.1.tgz", + "integrity": "sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==", + "license": "MIT", + "dependencies": { + "@sindresorhus/transliterate": "^1.0.0", + "escape-string-regexp": "^5.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@sindresorhus/transliterate": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/transliterate/-/transliterate-1.6.0.tgz", + "integrity": "sha512-doH1gimEu3A46VX6aVxpHTeHrytJAG6HgdxntYnCFiIFHEM/ZGpG8KiZGBChchjQmG0XFIBL552kBTjVcMZXwQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^5.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/aws-lambda": { + "version": "8.10.152", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.152.tgz", + "integrity": "sha512-soT/c2gYBnT5ygwiHPmd9a1bftj462NWVk2tKCc1PYHSIacB2UwbTS2zYG4jzag1mRDuzg/OjtxQjQ2NKRB6Rw==", + "license": "MIT" + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bunyan": { + "version": "1.8.11", + "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.11.tgz", + "integrity": "sha512-758fRH7umIMk5qt5ELmRMff4mLDlN+xyYzC+dkPTdKwbSkJFvz6xwyScrytPU0QIBbRRwbiE8/BIg8bpajerNQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.19", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", + "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.9", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.9.tgz", + "integrity": "sha512-QP2ESEe/ImWY0HDwNAnK9PvEffUyhLTnWkk7KXzHfyeWAnlrDe1fN77bXl6ia8KT3wPlmA7t9/VPRpnf4Ex9sg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/memcached": { + "version": "2.2.10", + "resolved": "https://registry.npmjs.org/@types/memcached/-/memcached-2.2.10.tgz", + "integrity": "sha512-AM9smvZN55Gzs2wRrqeMHVP7KE8KWgCJO/XL5yCly2xF6EKa4YlbpK+cLSAH4NG/Ah64HrlegmGqW8kYws7Vxg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "license": "MIT" + }, + "node_modules/@types/mysql": { + "version": "2.15.27", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", + "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/oracledb": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/@types/oracledb/-/oracledb-6.5.2.tgz", + "integrity": "sha512-kK1eBS/Adeyis+3OlBDMeQQuasIDLUYXsi2T15ccNJ0iyUpQ4xDF7svFu3+bGVrI0CMBUclPciz+lsQR3JX3TQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/pg": { + "version": "8.15.5", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.5.tgz", + "integrity": "sha512-LF7lF6zWEKxuT3/OR8wAZGzkg4ENGXFNyiV/JeOt9z5B+0ZVwbql9McqX5c/WStFq1GaGso7H1AzP/qSzmlCKQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/pg-pool": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", + "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", + "license": "MIT", + "dependencies": { + "@types/pg": "*" + } + }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/tedious": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", + "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@upstash/redis": { + "version": "1.39.0", + "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.39.0.tgz", + "integrity": "sha512-VwkpYTdxSfjSulQS5h16NHtg/1UEheBfvez2AeGKinAujOoGXhXCcohQRzsQ/d2g/XzbjzTVGhG8Ds8uOI0Mxg==", + "license": "MIT", + "dependencies": { + "uncrypto": "^0.1.3" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.0.5.tgz", + "integrity": "sha512-fnYhv671l+eTTp48gB4zEsTW/YtRgRPnkI2nT7x6qw5rkI1Lq2hTmQIpHPgyThI0znLK+vX2n9XxKdXZ7BUbbw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ai-v5": { + "name": "ai", + "version": "5.0.97", + "resolved": "https://registry.npmjs.org/ai/-/ai-5.0.97.tgz", + "integrity": "sha512-8zBx0b/owis4eJI2tAlV8a1Rv0BANmLxontcAelkLNwEHhgfgXeKpDkhNB6OgV+BJSwboIUDkgd9312DdJnCOQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "2.0.12", + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.17", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ai-v5/node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ai-v5/node_modules/@ai-sdk/provider-utils": { + "version": "3.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.17.tgz", + "integrity": "sha512-TR3Gs4I3Tym4Ll+EPdzRdvo/rc8Js6c4nVhFLuvGLX/Y4V9ZcQMa/HTiYsHEgmYrf1zVi6Q145UEZUfleOwOjw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ai-v5/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "license": "MIT" + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/date-fns": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", + "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } + }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/exit-hook": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-4.0.0.tgz", + "integrity": "sha512-Fqs7ChZm72y40wKjOFXBKg7nJZvQJmewP5/7LtePDdnah/+FH9Hp5sgMujSCMPXlxOAW2//1jrW9pnsY7o20vQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/express": { + "version": "4.22.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz", + "integrity": "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.13", + "proxy-addr": "~2.0.7", + "qs": "~6.16.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/express/node_modules/body-parser": { + "version": "1.20.8", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.8.tgz", + "integrity": "sha512-JNcyFQ64OiijEkPzUBTCe+hyPXUD/3LEldGQ6iF5LR1w00mx9o7xtDWHXBY2iItjdCFGoilOLNQbH943ut7pHA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.16.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/express/node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/express/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/express/node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-copy": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.1.1.tgz", + "integrity": "sha512-A4QTJmuiztpGtr6AMeJts9R4hbj2ZBUwtOaKrG6rw2y7t6+IaJKjz5M3XDs8BUznxDH43FVc6A0y/gWlMl4UtA==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/forwarded-parse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", + "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", + "license": "MIT" + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "license": "MIT" + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/import-in-the-middle": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.15.0.tgz", + "integrity": "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.14.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^1.2.2", + "module-details-from-path": "^1.0.3" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-core-module": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", + "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/json-schema-to-zod": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/json-schema-to-zod/-/json-schema-to-zod-2.8.1.tgz", + "integrity": "sha512-fRr1mHgZ7hboLKBUdR428gd9dIHUFGivUqOeiDcSmyXkNZCtB1uGaZLvsjZ4GaN5pwBIs+TGIOf6s+Rp5/R/zA==", + "license": "ISC", + "bin": { + "json-schema-to-zod": "dist/cjs/cli.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/json-schema-walker": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/json-schema-walker/-/json-schema-walker-2.0.0.tgz", + "integrity": "sha512-nXN2cMky0Iw7Af28w061hmxaPDaML5/bQD9nwm1lOoIKEGjHcRGxqWe4MfrkYThYAPjSUhmsp4bJNoLAyVn9Xw==", + "license": "MIT", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^11.1.0", + "clone": "^2.1.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT", + "peer": true + }, + "node_modules/p-map": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.8.tgz", + "integrity": "sha512-MitaVsCuCFIvOLLPIU7NnfrZvS9H9h7kwMUkDo+T2pEISaJD48IV9S8iIdXB7PsvvdxyYcsSTTrr90XKsbulNw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/pg": { + "version": "8.23.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", + "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.0.tgz", + "integrity": "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pino": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", + "integrity": "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.1.3.tgz", + "integrity": "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^4.0.0", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^4.0.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^5.0.2" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-pretty/node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/postgres": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.9.tgz", + "integrity": "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw==", + "license": "Unlicense", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process-warning": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz", + "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/radash": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/radash/-/radash-12.1.1.tgz", + "integrity": "sha512-h36JMxKRqrAxVD8201FrCpyeNuUY9Y5zZwujr20fFO77tpUtGa6EZzfKw/3WaiBX95fq7+MpsuMLNdSnORAwSA==", + "license": "MIT", + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/redis": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-5.12.1.tgz", + "integrity": "sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==", + "license": "MIT", + "dependencies": { + "@redis/bloom": "5.12.1", + "@redis/client": "5.12.1", + "@redis/json": "5.12.1", + "@redis/search": "5.12.1", + "@redis/time-series": "5.12.1" + }, + "engines": { + "node": ">= 18.19.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.5.2.tgz", + "integrity": "sha512-gAZ+kLqBdHarXB64XpAe2VCjB7rIRv+mU8tfRWziHRJ5umKsIHN2tLLv6EtMw7WCdP19S0ERVMldNvxYCHnhSQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3", + "resolve": "^1.22.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sift": { + "version": "17.1.3", + "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz", + "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==", + "license": "MIT" + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz", + "integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/thread-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.2.0.tgz", + "integrity": "sha512-zLBvqpwr4Esa0kRjcrzGU6zL25lePWaCLMx0RQFrmteozIfeNdaMLpG5U7PeHzvlFkAWaRKA9/KVW4F60iB+qw==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/uncrypto": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", + "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xstate": { + "version": "5.33.2", + "resolved": "https://registry.npmjs.org/xstate/-/xstate-5.33.2.tgz", + "integrity": "sha512-8tC7yXgeCvpT8gKEeEje6ikJJG1wpnoLiXe+HfECW8m10ubMd3QxKOwWA8KxPJVBcLwfdRoMKYxIBGYKmo37/A==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/xstate" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/xxhash-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", + "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", + "license": "MIT" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-from-json-schema": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/zod-from-json-schema/-/zod-from-json-schema-0.5.6.tgz", + "integrity": "sha512-U33AJ7ZWS6y9XNSzMWcdy8hRAvZmWhTtpYJu0SXPT5AArbc9nq2ur7Magzmn5RF9KBV4b3FP0nCpmqqlfXlR9w==", + "license": "MIT", + "dependencies": { + "zod": "^4.0.17" + } + }, + "node_modules/zod-from-json-schema-v3": { + "name": "zod-from-json-schema", + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/zod-from-json-schema/-/zod-from-json-schema-0.0.5.tgz", + "integrity": "sha512-zYEoo86M1qpA1Pq6329oSyHLS785z/mTwfr9V1Xf/ZLhuuBGaMlDGu/pDVGVUe4H4oa1EFgWZT53DP0U3oT9CQ==", + "license": "MIT", + "dependencies": { + "zod": "^3.24.2" + } + }, + "node_modules/zod-from-json-schema-v3/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/mastra-0/package.json b/sdk/typescript/integration/fixtures/mastra-0/package.json new file mode 100644 index 000000000..0c49f27de --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-0/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-mastra-0", + "private": true, + "type": "module", + "description": "Integration fixture: Mastra 0.x (@mastra/core, last 0.x release) against the packed @failproofai/sdk.", + "dependencies": { + "@mastra/core": "0.24.9", + "@mastra/mcp": "0.14.5", + "@mastra/memory": "0.15.13", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/mastra-0/tsconfig.json b/sdk/typescript/integration/fixtures/mastra-0/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-0/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/mastra-1/agent.ts b/sdk/typescript/integration/fixtures/mastra-1/agent.ts new file mode 100644 index 000000000..faefc8a01 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-1/agent.ts @@ -0,0 +1,872 @@ +// Mastra 1.x consumer. Run as `node agent.{mjs,cjs} `. +// +// Deliberately the shape a customer writes: import the framework, instrument, +// run. The model is a hand-written AI SDK `LanguageModelV2` — Mastra's own +// model interface, so nothing here reaches a network — scripted per agent: +// the first call asks for a tool, the next one answers. Every agent gets its +// own script so the token counts in the trace say which call they came from. +import * as failproofai from "@failproofai/sdk"; +import { wrapTool } from "@failproofai/sdk/mastra"; +import { Agent } from "@mastra/core/agent"; +import { createTool } from "@mastra/core/tools"; +import { createStep, createWorkflow } from "@mastra/core/workflows"; +import { Mastra } from "@mastra/core/mastra"; +import { InMemoryStore } from "@mastra/core/storage"; +import { MCPClient } from "@mastra/mcp"; +import { Memory } from "@mastra/memory"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; +import { z } from "zod"; + +type Turn = { tool: string; input: Record; usage: [number, number] } | { text: string; usage: [number, number] }; + +/** + * Holds the FIRST streamed step open after its response metadata: `reached` + * fires once the stream is parked there, and it moves on when `wait` settles. + */ +type Gate = { reached: () => void; wait: Promise }; + +function barrier(): { wait: Promise; open: () => void } { + let open!: () => void; + const wait = new Promise((resolve) => (open = resolve)); + return { wait, open }; +} + +/** + * A scripted `LanguageModelV2`: `turns` alternate per call, so a two-step tool + * loop is turn 0 (tool call) then turn 1 (answer). `fail` makes every call + * throw, which is how a provider error reaches Mastra. Tool call ids are + * `_`, so two models in one session never reuse an id. + */ +function scriptedModel(turns: Turn[], options: { fail?: boolean; callPrefix?: string; gate?: Gate } = {}) { + let calls = 0; + const next = (): Turn => turns[calls++ % turns.length]!; + const usage = ([input, output]: [number, number]) => ({ + inputTokens: input, + outputTokens: output, + totalTokens: input + output, + }); + return { + specificationVersion: "v2" as const, + provider: "scripted", + modelId: "scripted-model", + supportedUrls: {}, + async doGenerate() { + if (options.fail) throw new Error("model exploded"); + const turn = next(); + return "tool" in turn + ? { + content: [ + { type: "tool-call" as const, toolCallId: `${options.callPrefix ?? "call"}_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }, + ], + finishReason: "tool-calls" as const, + usage: usage(turn.usage), + warnings: [], + } + : { + content: [{ type: "text" as const, text: turn.text }], + finishReason: "stop" as const, + usage: usage(turn.usage), + warnings: [], + }; + }, + async doStream() { + if (options.fail) throw new Error("model exploded"); + const turn = next(); + const parts: unknown[] = [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: `resp_${calls}`, modelId: "scripted-model", timestamp: new Date(0) }, + ]; + if ("tool" in turn) { + parts.push({ type: "tool-call", toolCallId: `${options.callPrefix ?? "call"}_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }); + parts.push({ type: "finish", finishReason: "tool-calls", usage: usage(turn.usage) }); + } else { + parts.push({ type: "text-start", id: "t" }); + // Two deltas, so a step's content has to be assembled from the stream. + // (Not one per word: Mastra 0.24's textStream drops the third of five + // such deltas on its own, instrumented or not.) + const cut = turn.text.indexOf(" ", turn.text.length / 2) + 1; + for (const delta of [turn.text.slice(0, cut), turn.text.slice(cut)]) { + parts.push({ type: "text-delta", id: "t", delta }); + } + parts.push({ type: "text-end", id: "t" }); + parts.push({ type: "finish", finishReason: "stop", usage: usage(turn.usage) }); + } + const gate = calls === 1 ? options.gate : undefined; + if (gate) { + let index = 0; + return { + stream: new ReadableStream({ + async pull(controller) { + if (index === 2) { + gate.reached(); + await gate.wait; + } + if (index >= parts.length) controller.close(); + else controller.enqueue(parts[index++]); + }, + }), + }; + } + return { + stream: new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }), + }; + }, + }; +} + +const WEATHER_TURNS: Turn[] = [ + { tool: "weather", input: { city: "Paris" }, usage: [11, 7] }, + { text: "It is sunny in Paris.", usage: [23, 9] }, +]; + +const weather = createTool({ + id: "weather", + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async (input: { city: string }) => ({ city: input.city, forecast: "sunny" }), +}); + +const brokenWeather = createTool({ + id: "weather", + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async (): Promise<{ city: string }> => { + throw new Error("tool exploded"); + }, +}); + +const weatherAgent = (options: { fail?: boolean; tool?: typeof weather; gate?: Gate } = {}) => + new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: scriptedModel(WEATHER_TURNS, options) as never, + tools: { weather: options.tool ?? weather }, + }); + +function buildWorkflow(fail = false) { + const fetchCity = createStep({ + id: "fetch-city", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ city: z.string() }), + execute: async () => ({ city: "Paris" }), + }); + const ask = createStep({ + id: "ask-agent", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ answer: z.string() }), + execute: async ({ inputData }) => { + if (fail) throw new Error("step exploded"); + const out = await weatherAgent().generate(`Weather in ${inputData.city}?`); + return { answer: out.text }; + }, + }); + return createWorkflow({ + id: "weather-flow", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ answer: z.string() }), + }) + .then(fetchCity) + .then(ask) + .commit(); +} + +const report = (value: unknown) => console.log(JSON.stringify(value)); +const question = "What is the weather in Paris?"; + +// --------------------------------------------------------------------------- +// The coverage cases below need a model that answers from what it is ASKED +// rather than from how often it has been called: concurrent runs share one +// model, and memory, networks and structured output change what a call sees. +// --------------------------------------------------------------------------- + +type Prompt = Array<{ role: string; content: unknown }>; +type CallOptions = { prompt: Prompt; responseFormat?: { type?: string; schema?: { properties?: Record } } }; + +/** The text of the LAST user message — the question this call answers. */ +function lastUserText(prompt: Prompt): string { + for (let i = prompt.length - 1; i >= 0; i -= 1) { + const message = prompt[i]!; + if (message.role !== "user") continue; + if (typeof message.content === "string") return message.content; + if (Array.isArray(message.content)) { + return message.content + .map((part: { type?: string; text?: string }) => (part.type === "text" ? (part.text ?? "") : "")) + .join(""); + } + } + return ""; +} + +const cityOf = (text: string): string => /in ([A-Z][a-z]+)/.exec(text)?.[1] ?? "Paris"; + +/** Every text a call's prompt carries, system included, for routing on. */ +const promptText = (prompt: Prompt): string => JSON.stringify(prompt); + +let decidedCalls = 0; + +/** + * A `LanguageModelV2` whose every answer is `decide(options)`. Tool call ids + * are `_` over one process-wide counter, so no two calls anywhere + * reuse one — the concurrency case needs that to tell runs apart. + */ +function decidingModel(modelId: string, decide: (options: CallOptions) => Turn, prefix = "call") { + const usage = ([input, output]: [number, number]) => ({ inputTokens: input, outputTokens: output, totalTokens: input + output }); + const answer = (options: CallOptions) => { + decidedCalls += 1; + return { turn: decide(options), id: `${prefix}_${decidedCalls}` }; + }; + return { + specificationVersion: "v2" as const, + provider: "scripted", + modelId, + supportedUrls: {}, + async doGenerate(options: CallOptions) { + const { turn, id } = answer(options); + return "tool" in turn + ? { + content: [{ type: "tool-call" as const, toolCallId: id, toolName: turn.tool, input: JSON.stringify(turn.input) }], + finishReason: "tool-calls" as const, + usage: usage(turn.usage), + warnings: [], + } + : { + content: [{ type: "text" as const, text: turn.text }], + finishReason: "stop" as const, + usage: usage(turn.usage), + warnings: [], + }; + }, + async doStream(options: CallOptions) { + const { turn, id } = answer(options); + const parts: unknown[] = [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: `resp_${id}`, modelId, timestamp: new Date(0) }, + ]; + if ("tool" in turn) { + parts.push({ type: "tool-call", toolCallId: id, toolName: turn.tool, input: JSON.stringify(turn.input) }); + parts.push({ type: "finish", finishReason: "tool-calls", usage: usage(turn.usage) }); + } else { + parts.push({ type: "text-start", id: "t" }); + parts.push({ type: "text-delta", id: "t", delta: turn.text }); + parts.push({ type: "text-end", id: "t" }); + parts.push({ type: "finish", finishReason: "stop", usage: usage(turn.usage) }); + } + return { + stream: new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }), + }; + }, + }; +} + +/** Ask `tool` for the city in the question, then answer from its result. */ +const toolThenAnswer = (tool: string) => (options: CallOptions): Turn => { + // Memory titling a new thread (on by default in 0.x) asks the agent's model. + if (promptText(options.prompt).includes("short title")) return { text: "Weather", usage: [5, 3] }; + const city = cityOf(lastUserText(options.prompt)); + return options.prompt.at(-1)?.role === "tool" + ? { text: `It is sunny in ${city}.`, usage: [23, 9] } + : { tool, input: { city }, usage: [11, 7] }; +}; + +const routedAgent = (name: string, tools: Record = { weather }, extra: Record = {}) => + new Agent({ + id: name, + name, + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer(Object.keys(tools)[0] ?? "weather")) as never, + tools: tools as never, + ...extra, + } as never); + +const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Tokyo", "Quito", "Dakar", "Hanoi", "Perth"]; + +const answerStep = createStep({ + id: "answer", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ answer: z.string() }), + execute: async ({ inputData }) => ({ answer: `sunny in ${inputData.city}` }), +}); + +/** A two-step workflow, used on its own and nested inside another. */ +function innerWorkflow() { + const pick = createStep({ + id: "pick-city", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ city: z.string() }), + execute: async ({ inputData }) => ({ city: cityOf(inputData.question) }), + }); + return createWorkflow({ + id: "inner-flow", + inputSchema: z.object({ question: z.string() }), + outputSchema: z.object({ answer: z.string() }), + }) + .then(pick) + .then(answerStep) + .commit(); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- one entry point for differently-typed workflows +function workflowCase(name: string): { createRun(): Promise } { + const input = z.object({ question: z.string() }); + const out = z.object({ answer: z.string() }); + const city = createStep({ + id: "city", + inputSchema: input, + outputSchema: z.object({ city: z.string() }), + execute: async ({ inputData }) => ({ city: cityOf(inputData.question) }), + }); + switch (name) { + case "branch": { + const sunny = createStep({ id: "sunny", inputSchema: z.object({ city: z.string() }), outputSchema: out, execute: async () => ({ answer: "sunny" }) }); + const rainy = createStep({ id: "rainy", inputSchema: z.object({ city: z.string() }), outputSchema: out, execute: async () => ({ answer: "rainy" }) }); + return createWorkflow({ id: "branch-flow", inputSchema: input, outputSchema: z.any() }) + .then(city) + .branch([ + [async ({ inputData }) => inputData.city === "Paris", sunny], + [async ({ inputData }) => inputData.city !== "Paris", rainy], + ]) + .commit(); + } + case "parallel": { + const high = createStep({ id: "high", inputSchema: z.object({ city: z.string() }), outputSchema: z.object({ t: z.number() }), execute: async () => ({ t: 25 }) }); + const low = createStep({ id: "low", inputSchema: z.object({ city: z.string() }), outputSchema: z.object({ t: z.number() }), execute: async () => ({ t: 12 }) }); + return createWorkflow({ id: "parallel-flow", inputSchema: input, outputSchema: z.any() }).then(city).parallel([high, low]).commit(); + } + case "loop": { + const count = createStep({ + id: "count", + inputSchema: z.object({ n: z.number() }), + outputSchema: z.object({ n: z.number() }), + execute: async ({ inputData }) => ({ n: inputData.n + 1 }), + }); + const cities = createStep({ + id: "cities", + inputSchema: z.object({ n: z.number() }), + outputSchema: z.array(z.object({ city: z.string() })), + execute: async () => [{ city: "Paris" }, { city: "Rome" }], + }); + return createWorkflow({ id: "loop-flow", inputSchema: z.object({ n: z.number() }), outputSchema: z.any() }) + .dowhile(count, async ({ inputData }) => inputData.n < 3) + .then(cities) + .foreach(answerStep) + .commit(); + } + case "nested": + return createWorkflow({ id: "outer-flow", inputSchema: input, outputSchema: out }).then(innerWorkflow()).commit(); + case "agent-step": { + const toPrompt = createStep({ + id: "to-prompt", + inputSchema: input, + outputSchema: z.object({ prompt: z.string() }), + execute: async ({ inputData }) => ({ prompt: inputData.question }), + }); + return createWorkflow({ id: "agent-step-flow", inputSchema: input, outputSchema: z.any() }) + .then(toPrompt) + .then(createStep(routedAgent("weather-agent"))) + .commit(); + } + case "suspend": { + const approve = createStep({ + id: "approve", + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ city: z.string(), approved: z.boolean() }), + suspendSchema: z.object({ prompt: z.string() }), + resumeSchema: z.object({ approved: z.boolean() }), + execute: async ({ inputData, resumeData, suspend }) => { + if (resumeData === undefined) return (await suspend({ prompt: `Look up ${inputData.city}?` })) as never; + return { city: inputData.city, approved: resumeData.approved }; + }, + }); + const done = createStep({ + id: "done", + inputSchema: z.object({ city: z.string(), approved: z.boolean() }), + outputSchema: out, + execute: async ({ inputData }) => ({ answer: inputData.approved ? `sunny in ${inputData.city}` : "declined" }), + }); + return createWorkflow({ id: "approval-flow", inputSchema: input, outputSchema: out }).then(city).then(approve).then(done).commit(); + } + default: + throw new Error(`unknown workflow ${name}`); + } +} + +const mcpServer = () => ({ command: process.execPath, args: [join(process.cwd(), "mcp-server.mjs")] }); + +async function main(scenario: string): Promise { + report({ instrumented: await failproofai.instrument("mastra") }); + + switch (scenario) { + case "generate": { + const out = await weatherAgent().generate(question); + report({ answer: out.text }); + break; + } + case "stream": { + const out = await weatherAgent().stream(question); + let text = ""; + for await (const chunk of out.textStream) text += chunk; + report({ answer: text }); + break; + } + case "subagent": { + const helper = new Agent({ + id: "helper", + name: "helper", + description: "Looks up the weather.", + instructions: "Answer weather questions.", + model: scriptedModel(WEATHER_TURNS) as never, + tools: { weather }, + }); + const boss = new Agent({ + id: "boss", + name: "boss", + instructions: "Delegate weather questions.", + model: scriptedModel([ + { tool: "agent-helper", input: { prompt: "Weather in Paris?" }, usage: [40, 12] }, + { text: "The helper says it is sunny.", usage: [60, 8] }, + ], { callPrefix: "delegate" }) as never, + agents: { helper }, + }); + const out = await boss.generate(question); + report({ answer: out.text }); + break; + } + case "workflow": { + const run = await buildWorkflow().createRun(); + const out = await run.start({ inputData: { question } }); + report({ status: out.status }); + break; + } + case "workflow-error": { + const run = await buildWorkflow(true).createRun(); + const out = await run.start({ inputData: { question } }); + report({ status: out.status }); + break; + } + case "wraptool": { + const tool = wrapTool(weather); + report({ out: await tool.execute!({ city: "Rome" }, {} as never) }); + break; + } + case "tool-error": { + const out = await weatherAgent({ tool: brokenWeather as never }).generate(question); + report({ answer: out.text }); + break; + } + case "model-error": { + try { + await weatherAgent({ fail: true }).generate(question); + report({ threw: false }); + } catch (error) { + report({ threw: (error as Error).message }); + } + break; + } + case "stream-model-error": { + const out = await weatherAgent({ fail: true }).stream(question); + let chunks = 0; + for await (const _ of out.fullStream) chunks += 1; + report({ chunks, error: String(out.error) }); + break; + } + case "scope": { + await failproofai.session({ sessionId: "req-1" }, () => + failproofai.agent("planner", { goal: "plan" }, () => weatherAgent().generate(question)), + ); + break; + } + case "uninstrument": { + report({ removed: failproofai.uninstrument() }); + await weatherAgent().generate(question); + break; + } + case "uninstrument-midstream": { + // uninstrument() lands while the first model step's stream is parked + // mid-flight; the caller then reads the run to the end regardless. + const reached = barrier(); + const gate = barrier(); + await failproofai.session({ sessionId: "req-1" }, async () => { + const out = await weatherAgent({ gate: { reached: reached.open, wait: gate.wait } }).stream(question); + const reading = (async () => { + let text = ""; + for await (const chunk of out.textStream) text += chunk; + return text; + })(); + await reached.wait; + report({ removed: failproofai.uninstrument() }); + gate.open(); + report({ answer: await reading }); + }); + break; + } + case "uninstrument-reuse": { + // One Agent instance: run while instrumented, then again (both ways) + // after uninstrument(). + await failproofai.session({ sessionId: "req-1" }, async () => { + const agent = weatherAgent(); + await agent.generate(question); + report({ removed: failproofai.uninstrument() }); + const again = await agent.generate(question); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + report({ answers: [again.text, text] }); + }); + break; + } + case "reinstrument": { + report({ removed: failproofai.uninstrument() }); + report({ instrumented: await failproofai.instrument("mastra") }); + const out = await weatherAgent().generate(question); + report({ answer: out.text }); + break; + } + case "reinstrument-reuse": { + // The same Agent instance across an uninstrument()/instrument() cycle. + const agent = weatherAgent(); + await agent.generate(question); + report({ removed: failproofai.uninstrument() }); + report({ instrumented: await failproofai.instrument("mastra") }); + const out = await agent.generate(question); + report({ answer: out.text }); + break; + } + case "mastra-instance": { + // Registered on a Mastra instance and fetched back through it — the + // shape `mastra dev` and every deployer use. + const mastra = new Mastra({ + agents: { weatherAgent: weatherAgent() }, + workflows: { weatherFlow: buildWorkflow() }, + logger: false, + }); + const out = await mastra.getAgent("weatherAgent").generate(question); + const run = await mastra.getWorkflow("weatherFlow").createRun(); + const flow = await run.start({ inputData: { question } }); + report({ answer: out.text, status: flow.status }); + break; + } + case "concurrent": { + // Ten runs of ONE Agent instance at once, in one session. + const agent = routedAgent("weather-agent"); + await failproofai.session({ sessionId: "req-1" }, async () => { + const outs = await Promise.all(CITIES.map((city) => agent.generate(`What is the weather in ${city}?`))); + report({ answers: outs.map((out) => out.text) }); + }); + break; + } + case "concurrent-sessions": { + // The same, with no scope: every run is its own session, so a leaked + // event would land in the wrong one where the test can see it. + const agent = routedAgent("weather-agent"); + const outs = await Promise.all( + CITIES.map((city, i) => + i % 2 === 0 + ? agent.generate(`What is the weather in ${city}?`).then((out) => out.text) + : agent.stream(`What is the weather in ${city}?`).then(async (out) => { + let text = ""; + for await (const chunk of out.textStream) text += chunk; + return text; + }), + ), + ); + report({ answers: outs }); + break; + } + case "memory": { + // Two turns of one conversation (thread) through @mastra/memory. + const agent = routedAgent("weather-agent", { weather }, { + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + const memory = { thread: "thread-42", resource: "user-7" }; + const first = await agent.generate("What is the weather in Paris?", { memory }); + const second = await agent.stream("And what is the weather in Rome?", { memory }); + let text = ""; + for await (const chunk of second.textStream) text += chunk; + report({ answers: [first.text, text] }); + break; + } + case "memory-scoped": { + // An enclosing session scope wins over the thread. + const agent = routedAgent("weather-agent", { weather }, { + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + await failproofai.session({ sessionId: "req-1" }, () => + agent.generate(question, { memory: { thread: "thread-42", resource: "user-7" } }), + ); + break; + } + case "wf-branch": + case "wf-parallel": + case "wf-loop": + case "wf-nested": + case "wf-agent-step": { + const name = scenario.slice(3); + const run = await workflowCase(name).createRun(); + const out = await run.start({ inputData: (name === "loop" ? { n: 0 } : { question }) as never }); + report({ status: out.status, result: out.status === "success" ? out.result : undefined }); + break; + } + case "wf-stream": { + // A workflow run streamed rather than awaited. + const run = await workflowCase("agent-step").createRun(); + const streamed = await run.stream({ inputData: { question } }); + let chunks = 0; + for await (const _ of streamed.fullStream) chunks += 1; + report({ chunks, status: await streamed.status }); + break; + } + case "wf-suspend": { + // A resume reads the suspended run's snapshot back from storage. + const mastra = new Mastra({ workflows: { approval: workflowCase("suspend") as never }, storage: new InMemoryStore(), logger: false }); + const run = await (mastra.getWorkflow("approval") as unknown as ReturnType).createRun(); + const first = await run.start({ inputData: { question } }); + report({ status: first.status }); + const second = await run.resume({ step: "approve", resumeData: { approved: true } }); + report({ status: second.status, result: second.status === "success" ? second.result : undefined }); + break; + } + case "processors": { + // An input processor that rewrites the prompt and an output processor + // that rewrites the answer: neither may add or lose an event. + const agent = routedAgent("weather-agent", { weather }, { + inputProcessors: [ + { + id: "tag-input", + processInput: ({ messages }: { messages: unknown[] }) => messages, + }, + ], + outputProcessors: [ + { + id: "shout", + processOutputResult: ({ messages }: { messages: unknown[] }) => messages, + }, + ], + }); + const out = await agent.generate(question); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + report({ answers: [out.text, text] }); + break; + } + case "tripwire": { + // A guardrail that blocks the prompt: no model call, the run rejected. + const agent = routedAgent("weather-agent", { weather }, { + inputProcessors: [ + { + id: "block-paris", + processInput: ({ messages, abort }: { messages: unknown[]; abort: (reason: string) => never }) => + JSON.stringify(messages).includes("Paris") ? abort("Paris is blocked") : messages, + }, + ], + }); + const out = await agent.generate(question); + const streamed = await agent.stream(question); + for await (const _ of streamed.fullStream) void _; + report({ tripwire: Boolean((out as { tripwire?: unknown }).tripwire), text: out.text }); + break; + } + case "tripwire-output": { + // A guardrail on the OUTPUT stream: the model answers, the processor + // blocks the answer mid-stream. + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", () => ({ text: "It is sunny in Paris.", usage: [9, 4] })) as never, + outputProcessors: [ + { + id: "block-answer", + processOutputStream: ({ part, abort }: { part: { type: string }; abort: (reason: string) => never }) => + part.type === "text-delta" ? abort("answer blocked") : part, + }, + ], + } as never); + const streamed = await agent.stream(question); + let chunks = 0; + for await (const _ of streamed.fullStream) chunks += 1; + report({ chunks, tripwire: Boolean((streamed as { tripwire?: unknown }).tripwire) }); + break; + } + case "usage-openai-compatible": { + // Mastra's own model for an OpenAI-compatible endpoint (`{ id, url }`), + // against a loopback server that — like OpenAI — sends a streamed + // step's usage only when the request asks for it, or always when + // USAGE_ALWAYS=1 (as some compatible servers do). + const requests: Array<{ stream?: boolean; streamOptions?: unknown }> = []; + const usage = { prompt_tokens: 17, completion_tokens: 5, total_tokens: 22 }; + const server = createServer((req, res) => { + let raw = ""; + req.on("data", (chunk: Buffer) => (raw += chunk.toString())); + req.on("end", () => { + const body = JSON.parse(raw || "{}") as { stream?: boolean; stream_options?: { include_usage?: boolean } }; + requests.push({ stream: body.stream, streamOptions: body.stream_options }); + const base = { id: "c1", created: 0, model: "compat-model" }; + if (!body.stream) { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ ...base, object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "Sunny." }, finish_reason: "stop" }], usage })); + return; + } + res.setHeader("content-type", "text/event-stream"); + const send = (chunk: unknown) => res.write(`data: ${JSON.stringify(chunk)}\n\n`); + send({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "Sunny." }, finish_reason: null }] }); + send({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }); + if (process.env.USAGE_ALWAYS === "1" || body.stream_options?.include_usage) { + send({ ...base, object: "chat.completion.chunk", choices: [], usage }); + } + res.end("data: [DONE]\n\n"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`; + const agent = new Agent({ id: "compat-agent", name: "compat-agent", instructions: "Be brief.", model: { id: "custom/compat-model", url, apiKey: "test-key" } as never }); + const streamed = await agent.stream(question); + let text = ""; + for await (const chunk of streamed.textStream) text += chunk; + const streamedUsage = await streamed.usage; + const generated = await agent.generate(question); + report({ text, mastraStreamTokens: streamedUsage.inputTokens ?? 0, mastraGenerateTokens: generated.usage.inputTokens ?? 0, requests }); + } finally { + // Deno's node:http keeps idle keep-alive sockets open through close(), + // so the process would outlive the test; Node closes idle ones itself. + server.closeAllConnections(); + server.close(); + } + break; + } + case "mcp": + case "mcp-toolsets": { + const mcp = new MCPClient({ id: `mcp-${process.pid}`, servers: { weatherServer: mcpServer() } }); + try { + if (scenario === "mcp") { + const tools = await mcp.listTools(); + const agent = routedAgent("weather-agent", tools); + const out = await agent.generate(question); + report({ tools: Object.keys(tools), answer: out.text }); + } else { + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer("forecast")) as never, + }); + const out = await agent.generate(question, { toolsets: await mcp.listToolsets() }); + report({ answer: out.text }); + } + } finally { + await mcp.disconnect(); + } + break; + } + case "structured": { + const agent = new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", (options) => ({ + text: JSON.stringify({ city: cityOf(lastUserText(options.prompt)), forecast: "sunny" }), + usage: [15, 6], + })) as never, + }); + const schema = z.object({ city: z.string(), forecast: z.string() }); + const out = await agent.generate(question, { structuredOutput: { schema } }); + const streamed = await agent.stream(question, { structuredOutput: { schema } }); + report({ object: out.object, streamed: await streamed.object }); + break; + } + case "structured-model": { + // Structuring by a SECOND model: Mastra runs it through an agent of + // its own, after the main loop. + const agent = routedAgent("weather-agent"); + const structurer = decidingModel("structuring-model", () => ({ text: JSON.stringify({ city: "Paris", forecast: "sunny" }), usage: [30, 5] })); + const out = await agent.generate(question, { + structuredOutput: { schema: z.object({ city: z.string(), forecast: z.string() }), model: structurer as never }, + }); + report({ object: out.object }); + break; + } + case "maxsteps-tool-error": { + // One step only, and the tool it calls throws. + const agent = routedAgent("weather-agent", { weather: brokenWeather }); + const out = await agent.generate(question, { maxSteps: 1 }); + report({ finishReason: out.finishReason, text: out.text }); + break; + } + case "network": { + // An agent network: the planner's model routes to `helper` once, the + // helper answers with its tool, the planner's model judges it complete. + const helper = new Agent({ + id: "helper", + name: "helper", + description: "Looks up the weather.", + instructions: "Answer weather questions.", + model: decidingModel("routed-model", toolThenAnswer("weather")) as never, + tools: { weather }, + }); + const planner = new Agent({ + id: "planner", + name: "planner", + instructions: "Coordinate the specialists.", + model: decidingModel("router-model", (options) => { + // Tell the calls apart by the schema they ask for, falling back to + // the prompt when a release injects the schema as text instead. The + // completion check comes first: its prompt quotes the routing answer. + const fields = options.responseFormat?.schema?.properties ?? {}; + const asked = lastUserText(options.prompt); + if ("isComplete" in fields || (asked.includes("evaluate") && asked.includes("complete"))) { + return { text: JSON.stringify({ isComplete: true, completionReason: "answered", finalResult: "It is sunny in Paris." }), usage: [40, 6] }; + } + if ("primitiveId" in fields || promptText(options.prompt).includes("primitiveId")) { + return { + text: JSON.stringify({ primitiveId: "helper", primitiveType: "agent", prompt: question, selectionReason: "weather" }), + usage: [50, 10], + }; + } + return { text: "Weather in Paris", usage: [5, 3] }; + }) as never, + agents: { helper }, + memory: new Memory({ storage: new InMemoryStore(), options: { lastMessages: 10 } }), + }); + const stream = await planner.network(question, { memory: { thread: "thread-net", resource: "user-7" } }); + for await (const _ of stream) void _; + report({ status: await stream.status }); + break; + } + case "agent-in-tool": { + // Delegation by hand: a tool whose body runs another agent. + const helper = routedAgent("helper"); + const ask = createTool({ + id: "ask-helper", + description: "Ask the helper", + inputSchema: z.object({ city: z.string() }), + execute: async (input: { city: string }) => ({ answer: (await helper.generate(`What is the weather in ${input.city}?`)).text }), + }); + const boss = routedAgent("boss", { "ask-helper": ask }); + const out = await boss.generate(question); + report({ answer: out.text }); + break; + } + default: + throw new Error(`unknown scenario ${scenario}`); + } + await failproofai.flush(); +} + +main(process.argv[2] ?? "generate").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/mastra-1/mcp-server.mjs b/sdk/typescript/integration/fixtures/mastra-1/mcp-server.mjs new file mode 100644 index 000000000..95632abb3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-1/mcp-server.mjs @@ -0,0 +1,66 @@ +// A minimal MCP server over stdio, for the `mcp` case of agent.ts. +// +// Hand-written JSON-RPC rather than the MCP SDK's server so the fixture does +// not depend on which SDK major `@mastra/mcp` happens to pull in: newline- +// delimited JSON on stdin/stdout, which is the whole stdio transport. It +// serves one tool, `forecast`, and nothing reaches a network. +import { createInterface } from "node:readline"; + +const TOOLS = [ + { + name: "forecast", + description: "Forecast for a city", + inputSchema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, +]; + +const send = (message) => process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", ...message })}\n`); + +function handle(request) { + switch (request.method) { + case "initialize": + return { + // Echo the client's version: this server speaks the subset every + // version shares. + protocolVersion: request.params?.protocolVersion ?? "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "weather-mcp", version: "1.0.0" }, + }; + case "ping": + return {}; + case "tools/list": + return { tools: TOOLS }; + case "tools/call": { + const city = request.params?.arguments?.city; + return { + content: [{ type: "text", text: `Forecast for ${String(city)}: sunny` }], + isError: false, + }; + } + case "resources/list": + return { resources: [] }; + case "prompts/list": + return { prompts: [] }; + default: + return undefined; + } +} + +createInterface({ input: process.stdin }).on("line", (line) => { + if (!line.trim()) return; + let request; + try { + request = JSON.parse(line); + } catch { + return; + } + // A notification (no id) needs no answer. + if (request.id === undefined || request.id === null) return; + const result = handle(request); + if (result === undefined) send({ id: request.id, error: { code: -32601, message: `unknown method ${request.method}` } }); + else send({ id: request.id, result }); +}); diff --git a/sdk/typescript/integration/fixtures/mastra-1/package-lock.json b/sdk/typescript/integration/fixtures/mastra-1/package-lock.json new file mode 100644 index 000000000..432442d22 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-1/package-lock.json @@ -0,0 +1,2540 @@ +{ + "name": "failproofai-it-mastra-1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-mastra-1", + "dependencies": { + "@mastra/core": "1.68.0", + "@mastra/mcp": "2.0.0", + "@mastra/memory": "1.31.0", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@a2a-js/sdk-v0_3": { + "name": "@a2a-js/sdk", + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@a2a-js/sdk/-/sdk-0.3.14.tgz", + "integrity": "sha512-F6Ew1AtPzCLhTn8h9yiqTe7DiDf6XVrSnq9V1YqSl9eWqPm6anMveTiKdCSb/76cW0YiJc24rNaUrVezFFHbqQ==", + "license": "Apache-2.0", + "dependencies": { + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.11.0", + "express": "^4.21.2 || ^5.1.0" + }, + "peerDependenciesMeta": { + "@bufbuild/protobuf": { + "optional": true + }, + "@grpc/grpc-js": { + "optional": true + }, + "express": { + "optional": true + } + } + }, + "node_modules/@a2a-js/sdk-v1": { + "name": "@a2a-js/sdk", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@a2a-js/sdk/-/sdk-1.0.1.tgz", + "integrity": "sha512-CJQdh3Wzwo8qIx5UUkSJ7+7BEI16PB+MXMHHNSmx8JQsQed2HlQgvx1ENOiKUfYA3PlcEvxIwv14dBblhDuPmw==", + "license": "Apache-2.0", + "dependencies": { + "jose": "^6.2.3", + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.11.0", + "express": "^4.21.2 || ^5.1.0" + }, + "peerDependenciesMeta": { + "@bufbuild/protobuf": { + "optional": true + }, + "@grpc/grpc-js": { + "optional": true + }, + "express": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.14.tgz", + "integrity": "sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils-v6": { + "name": "@ai-sdk/provider-utils", + "version": "4.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.40.tgz", + "integrity": "sha512-OL5IrpUm9Y8Dwy+w/vvFwPotS6m52O9W0op2oXgXdCROMJIBalBI0oro6OIBYkPxvm5Xg02GSkoQN25RlR0bnw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.14", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v7": { + "name": "@ai-sdk/provider-utils", + "version": "5.0.13", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.13.tgz", + "integrity": "sha512-fScDJMDnTbx32kLDQqp0MvPjvwkgiwvlBxlmIg7XW5PbS91LG6JjH3PQG+34oMFglqfpQA355e24OdGj5PPoDw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.4", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v7/node_modules/@ai-sdk/provider": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.4.tgz", + "integrity": "sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-v5": { + "name": "@ai-sdk/provider", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.3.tgz", + "integrity": "sha512-h88OPkavHTiN9tMn2l5awAznGB0lXzjcLhgR1/rvjB2zlLprsNxbM2tt6OJsHUxduLC3klq0/eqaSf6fX5XVww==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-v6": { + "name": "@ai-sdk/provider", + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.14.tgz", + "integrity": "sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-v7": { + "name": "@ai-sdk/provider", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.4.tgz", + "integrity": "sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@hono/node-server": { + "version": "1.19.17", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.17.tgz", + "integrity": "sha512-dSneS5qhiauZWGDCeK4o695Xd9nUNjviSZCMQrj10eetr8Uln1ucn6bbphOM6UynAMMtNIzZNSpL9vnASJwrPQ==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@hono/standard-validator": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@hono/standard-validator/-/standard-validator-0.4.0.tgz", + "integrity": "sha512-MxOm1asDh9j7c1D0KiWwppSbFgAQxTRO4YEhjMrJn3lF1BIcXenPdgjSGKDRfQmZdaTaMA8slQLETgVOKyJxFw==", + "license": "MIT", + "peerDependencies": { + "@standard-schema/spec": "^1.0.0", + "hono": ">=4.11.2" + } + }, + "node_modules/@isaacs/ttlcache": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-2.1.5.tgz", + "integrity": "sha512-VwGZqqjAWPICTmxUZnbpEfO60LhPWzquik+bmyXGY7pYRn6diEvCI5i6Ca+J6o2y4vS73HrpuMTo2dOvUevH8w==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=12" + } + }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@lukeed/uuid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.1.tgz", + "integrity": "sha512-qC72D4+CDdjGqJvkFMMEAtancHUQ7/d/tAiHf64z8MopFDmcrtbcJuerDtFceuAfQJ2pDSfCKCtbqoGBNnwg0w==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@mastra/core": { + "version": "1.68.0", + "resolved": "https://registry.npmjs.org/@mastra/core/-/core-1.68.0.tgz", + "integrity": "sha512-NdhQpjlOgaCzx/UePKkTfvnZk6/O26HSm46qNyY/XBgkTe1Qas5ks4IOYUTmhQsEkK1vi+fz8cNj6XAdLrZcpQ==", + "license": "Apache-2.0", + "dependencies": { + "@a2a-js/sdk-v0_3": "npm:@a2a-js/sdk@~0.3.14", + "@a2a-js/sdk-v1": "npm:@a2a-js/sdk@~1.0.1", + "@ai-sdk/provider-utils-v6": "npm:@ai-sdk/provider-utils@4.0.40", + "@ai-sdk/provider-utils-v7": "npm:@ai-sdk/provider-utils@5.0.13", + "@ai-sdk/provider-v5": "npm:@ai-sdk/provider@2.0.3", + "@ai-sdk/provider-v6": "npm:@ai-sdk/provider@3.0.14", + "@ai-sdk/provider-v7": "npm:@ai-sdk/provider@4.0.4", + "@isaacs/ttlcache": "^2.1.5", + "@lukeed/uuid": "^2.0.1", + "@mastra/schema-compat": "1.3.11", + "@standard-schema/spec": "^1.1.0", + "chat": "^4.37.0", + "croner": "^10.0.1", + "dotenv": "^17.3.1", + "execa": "^9.6.1", + "fastq": "^1.20.1", + "gray-matter": "^4.0.3", + "hono": "^4.13.7", + "hono-openapi": "^1.3.1", + "ignore": "^7.0.5", + "jpeg-js": "^0.4.4", + "json-schema": "^0.4.0", + "lru-cache": "^11.2.7", + "p-map": "^7.0.4", + "p-retry": "^7.1.1", + "picomatch": "^4.0.3", + "posthog-node": "^5.46.1", + "ws": "^8.21.3", + "xxhash-wasm": "^1.1.0" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/mcp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@mastra/mcp/-/mcp-2.0.0.tgz", + "integrity": "sha512-UygCyxyosUvtbt2hoHXVW3tfdycoTQ7QfTZhmgMCQvyZto1gvbYCAPJwMoIsMN1kH81BLleOXUWBiNdwKirDlw==", + "license": "Apache-2.0", + "dependencies": { + "@modelcontextprotocol/client": "2.0.0", + "@modelcontextprotocol/core": "2.0.0", + "@modelcontextprotocol/ext-apps": "^2.0.0", + "@modelcontextprotocol/node": "2.0.0", + "@modelcontextprotocol/server": "2.0.0", + "exit-hook": "^5.1.0", + "fast-deep-equal": "^3.1.3" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "@mastra/core": ">=1.68.0-0 <2.0.0-0" + } + }, + "node_modules/@mastra/memory": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/@mastra/memory/-/memory-1.31.0.tgz", + "integrity": "sha512-qt+J++XfotafhF19SaA+gQL/XoP3ACISfEOjhcevgcO3AjLKsbIuBFsGN4jJMmPtYJK0xsV47O0PD2ZlE7h+og==", + "license": "Apache-2.0", + "dependencies": { + "@mastra/schema-compat": "1.3.11", + "async-mutex": "^0.5.0", + "diff": "^8.0.3", + "json-schema": "^0.4.0", + "lru-cache": "^11.2.7", + "probe-image-size": "^7.2.3", + "xxhash-wasm": "^1.1.0", + "zod": "^4.6.4" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "@mastra/core": ">=1.4.1-0 <2.0.0-0" + } + }, + "node_modules/@mastra/schema-compat": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@mastra/schema-compat/-/schema-compat-1.3.11.tgz", + "integrity": "sha512-NvR+wYk4/d0uqzk702Aehd8Vbop+844c1Dp7TMkZVis71LzPORWJbFPQswrom9o+v5ollUfif+28lZzutGMq+A==", + "license": "Apache-2.0", + "dependencies": { + "json-schema-to-zod": "^2.7.0", + "zod-from-json-schema": "^0.5.2" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@modelcontextprotocol/client": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.0.0.tgz", + "integrity": "sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "jose": "^6.1.3", + "pkce-challenge": "^5.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.0.0.tgz", + "integrity": "sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==", + "license": "MIT", + "dependencies": { + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/ext-apps": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/ext-apps/-/ext-apps-2.0.0.tgz", + "integrity": "sha512-a6tXzFcIbIIdnqumQ7W8Oxd8W/KAPkAKYpoxpD9nDgxZ24ywgxG5pK/8G9W5pOFUygS7gWHQhPv8cwRB44/8yg==", + "license": "MIT", + "workspaces": [ + "examples/*" + ], + "dependencies": { + "@standard-schema/spec": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@modelcontextprotocol/client": "^2.0.0", + "@modelcontextprotocol/core": "^2.0.0", + "@modelcontextprotocol/server": "^2.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0", + "zod": "^4.2.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/server": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/node": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/node/-/node-2.0.0.tgz", + "integrity": "sha512-Y4hAC2XdGDUdDOCbLDOCA4+aL3NUldjsOWlDL/YwpAxrPhRm1xHd7lZ+mLacvZ9t3PaH28wgNoaLQGrIk1P2pg==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@modelcontextprotocol/server": "^2.0.0", + "hono": "^4.11.4" + }, + "peerDependenciesMeta": { + "hono": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/server": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/server/-/server-2.0.0.tgz", + "integrity": "sha512-YhHWdHfpFMQfd0prsEnxKeS3Qz3ytIGmsS0sth4KDjnacIT7hxk6hXHkJ9KysxlkvTM+WZAtQbbcUhdoP4Hvtw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@posthog/core": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.55.1.tgz", + "integrity": "sha512-S76bSbCHGVC8oQa1zyue6A6WzkNf1Ue4oRvfvBNLRnFM3UoWnpWnQ4T/4IzrIEtVCQYv1OI7gk8uL8/U6Tuu7w==", + "license": "MIT", + "dependencies": { + "@posthog/types": "^1.412.3" + } + }, + "node_modules/@posthog/types": { + "version": "1.412.4", + "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.412.4.tgz", + "integrity": "sha512-Q7lV9O9TbLngjOYw1ucm3bS3tn48xb1B4ZQdDy8gv7yfKe99hCTeWYG4xZ36nJM80VxpTF9TriiJt5hngDOkBg==", + "license": "MIT" + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@standard-community/standard-json": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@standard-community/standard-json/-/standard-json-0.3.6.tgz", + "integrity": "sha512-tuXKz1Ps4al0W4xhfwii7v4lskKtOv3QE7mcouRVThrBdOR6JzY4Vea3nsxcEnte84bOJvMvXGmOMWE3IBbHyA==", + "license": "MIT", + "dependencies": { + "quansync": "^0.2.11" + }, + "peerDependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/json-schema": "^7.0.15", + "@valibot/to-json-schema": "^1.3.0", + "arktype": "^2.1.20", + "effect": "^3.20.0", + "sury": "^10.0.0", + "typebox": "^1.0.17", + "valibot": "^1.4.2", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.24.5" + }, + "peerDependenciesMeta": { + "@valibot/to-json-schema": { + "optional": true + }, + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "sury": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-to-json-schema": { + "optional": true + } + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chat": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/chat/-/chat-4.41.0.tgz", + "integrity": "sha512-vMeODhulMdi7od1c6e4B/ZfIp/zq0f5KP3WKNCAAdNrTOeDgjRNZQZahaIax/t/gu+LE5vwJSjFU46Ro9Kx9tA==", + "license": "MIT", + "dependencies": { + "@workflow/serde": "4.1.0-beta.2", + "mdast-util-to-string": "^4.0.0", + "remark-gfm": "^4.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "remend": "^1.2.1", + "unified": "^11.0.5" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "ai": "^6.0.182 || ^7.0.0", + "workflow": "^5.0.0-beta.35", + "zod": "^3.0.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "ai": { + "optional": true + }, + "workflow": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/chat/node_modules/@workflow/serde": { + "version": "4.1.0-beta.2", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0-beta.2.tgz", + "integrity": "sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==", + "license": "Apache-2.0" + }, + "node_modules/croner": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/croner/-/croner-10.0.1.tgz", + "integrity": "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==", + "funding": [ + { + "type": "other", + "url": "https://paypal.me/hexagonpp" + }, + { + "type": "github", + "url": "https://github.com/sponsors/hexagon" + } + ], + "license": "MIT", + "engines": { + "node": ">=18.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/execa": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-9.6.1.tgz", + "integrity": "sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "cross-spawn": "^7.0.6", + "figures": "^6.1.0", + "get-stream": "^9.0.0", + "human-signals": "^8.0.1", + "is-plain-obj": "^4.1.0", + "is-stream": "^4.0.1", + "npm-run-path": "^6.0.0", + "pretty-ms": "^9.2.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^4.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": "^18.19.0 || >=20.5.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit-hook": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-5.1.0.tgz", + "integrity": "sha512-INjr2xyxHo7bhAqf5ong++GZPPnpcuBcaXUKt03yf7Fie9yWD7FapL4teOU0+awQazGs5ucBh7xWs/AD+6nhog==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", + "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/hono-openapi": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/hono-openapi/-/hono-openapi-1.3.3.tgz", + "integrity": "sha512-+LVFPQEc4eDrXr/Rd0QQkRh/DbEZRXnm1RZU4bVaoqmWiWgspT4nNHIQMgi44X+D9/8k1eGzHvR0KLNPJ/0xaQ==", + "license": "MIT", + "dependencies": { + "@hono/standard-validator": "^0.4.0", + "@standard-community/standard-json": "^0.3.5", + "@standard-community/standard-openapi": "^0.2.9", + "@standard-schema/spec": "^1.0.0", + "@types/json-schema": "^7.0.15", + "openapi-types": "^12.1.3" + }, + "peerDependencies": { + "hono": "^4.11.2" + } + }, + "node_modules/hono-openapi/node_modules/@standard-community/standard-openapi": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@standard-community/standard-openapi/-/standard-openapi-0.2.10.tgz", + "integrity": "sha512-whaCN9eu5zv5gu6Kyeh5am/S3AyS1nqo57/wAxs/KPS7+94UhBUZmkIetmXzm22qIV0zMrHt3y+2VfjkbkYYTA==", + "license": "MIT", + "peerDependencies": { + "@standard-community/standard-json": "^0.3.5", + "@standard-schema/spec": "^1.0.0", + "arktype": "^2.1.20", + "effect": "^3.20.0", + "openapi-types": "^12.1.3", + "sury": "^10.0.0", + "typebox": "^1.0.0", + "valibot": "^1.4.2", + "zod": "^3.25.0 || ^4.0.0", + "zod-openapi": "^4" + }, + "peerDependenciesMeta": { + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "sury": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-openapi": { + "optional": true + } + } + }, + "node_modules/human-signals": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.1.tgz", + "integrity": "sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "7.0.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.10.tgz", + "integrity": "sha512-HpbUakT7xp5miBUywCHf36ZEuAJNklBJDDsGpUIjMzOSmM8ELSfA9Sa/QDPeNeqeoN31u+UTCkL4klCOVvRm4Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause" + }, + "node_modules/js-yaml": { + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/json-schema-to-zod": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/json-schema-to-zod/-/json-schema-to-zod-2.8.1.tgz", + "integrity": "sha512-fRr1mHgZ7hboLKBUdR428gd9dIHUFGivUqOeiDcSmyXkNZCtB1uGaZLvsjZ4GaN5pwBIs+TGIOf6s+Rp5/R/zA==", + "license": "ISC", + "bin": { + "json-schema-to-zod": "dist/cjs/cli.js" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.2.tgz", + "integrity": "sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/needle": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.9.1.tgz", + "integrity": "sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==", + "license": "MIT", + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/needle/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/npm-run-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", + "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0", + "unicorn-magic": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT" + }, + "node_modules/p-map": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.8.tgz", + "integrity": "sha512-MitaVsCuCFIvOLLPIU7NnfrZvS9H9h7kwMUkDo+T2pEISaJD48IV9S8iIdXB7PsvvdxyYcsSTTrr90XKsbulNw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-ms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", + "integrity": "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/posthog-node": { + "version": "5.52.5", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.52.5.tgz", + "integrity": "sha512-r4KRXh2MvcHYh1s3NUca022GqujkJjpmD4qbRzWOLVTu8BfN3fqZj+MyKIxvHPXsS8sKsNVX+MlKk2qzW0Y4GQ==", + "license": "MIT", + "dependencies": { + "@posthog/core": "^1.55.1" + }, + "engines": { + "node": "^20.20.0 || >=22.22.0" + }, + "peerDependencies": { + "rxjs": "^7.0.0" + }, + "peerDependenciesMeta": { + "rxjs": { + "optional": true + } + } + }, + "node_modules/pretty-ms": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.1.tgz", + "integrity": "sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==", + "license": "MIT", + "dependencies": { + "parse-ms": "^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/probe-image-size": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/probe-image-size/-/probe-image-size-7.4.0.tgz", + "integrity": "sha512-cdEprVtZxV+awMde9X+4jILBFYh4CARxVrQaMl4wY4YcPWbul9jntXrIW95NInBDyJwcVUP3U0T6yukN8rMBaQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "lodash.merge": "^4.6.2", + "needle": "^2.5.2", + "stream-parser": "~0.3.1" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", + "integrity": "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remend": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/remend/-/remend-1.3.1.tgz", + "integrity": "sha512-N3DiY5qbRPoa5vkxn1oDLMyXOVTeo6Hp+XOj6SIqJAYUgLS0Q587gILPMom/qm86AQ/ZrcOdwEIzCz8V3J0nxQ==", + "license": "Apache-2.0" + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-parser": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/stream-parser/-/stream-parser-0.3.1.tgz", + "integrity": "sha512-bJ/HgKq41nlKvlhccD5kaCr/P+Hu0wPNKPJOH7en+YrJu/9EgqUF+88w5Jb6KNcjOFMhfX4B2asfeAtIGuHObQ==", + "license": "MIT", + "dependencies": { + "debug": "2" + } + }, + "node_modules/stream-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/stream-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", + "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", + "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/uuid": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xxhash-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", + "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", + "license": "MIT" + }, + "node_modules/yoctocolors": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.2.0.tgz", + "integrity": "sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-from-json-schema": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/zod-from-json-schema/-/zod-from-json-schema-0.5.6.tgz", + "integrity": "sha512-U33AJ7ZWS6y9XNSzMWcdy8hRAvZmWhTtpYJu0SXPT5AArbc9nq2ur7Magzmn5RF9KBV4b3FP0nCpmqqlfXlR9w==", + "license": "MIT", + "dependencies": { + "zod": "^4.0.17" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/mastra-1/package.json b/sdk/typescript/integration/fixtures/mastra-1/package.json new file mode 100644 index 000000000..fdce4e9ed --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-1/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-mastra-1", + "private": true, + "type": "module", + "description": "Integration fixture: Mastra 1.x (@mastra/core) against the packed @failproofai/sdk.", + "dependencies": { + "@mastra/core": "1.68.0", + "@mastra/mcp": "2.0.0", + "@mastra/memory": "1.31.0", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/mastra-1/tsconfig.json b/sdk/typescript/integration/fixtures/mastra-1/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/mastra-1/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/actions.ts b/sdk/typescript/integration/fixtures/nextjs/app/actions.ts new file mode 100644 index 000000000..b2d9a9831 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/actions.ts @@ -0,0 +1,18 @@ +"use server"; + +import { telemetry } from "@failproofai/sdk/ai"; +import { generateText } from "ai"; + +import { loop, scripted, tools } from "../lib/ai"; + +/** A server action running an agent (twin: ai-7 `generate`). */ +export async function askWeather(): Promise { + const { text } = await generateText({ + model: scripted(), + prompt: "Weather in Paris?", + tools: tools(), + ...loop, + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + }); + return text; +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/ai/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/ai/route.ts new file mode 100644 index 000000000..6933f0f8e --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/ai/route.ts @@ -0,0 +1,42 @@ +import { telemetry } from "@failproofai/sdk/ai"; +import { generateText, streamText } from "ai"; + +import { loop, scripted, tools } from "../../../lib/ai"; + +export const dynamic = "force-dynamic"; + +/** + * `?mode=telemetry` generateText, call-site `telemetry()` (twin: ai-7 `generate`) + * `?mode=instrument` generateText, `instrument()` + isEnabled (twin: ai-7 `instrument`) + * `?mode=stream` streamText → toUIMessageStreamResponse(), + * call-site `telemetry()` (twin: ai-7 `stream`) + * `?mode=instrument-stream` the same through `instrument()` (twin: ai-7 `instrument-stream`) + * + * The streaming modes return the stream itself, as a chat route does: the + * model is still producing when the handler returns. + */ +export async function GET(request: Request): Promise { + const mode = new URL(request.url).searchParams.get("mode") ?? "telemetry"; + const viaInstrument = mode.startsWith("instrument"); + const settings = viaInstrument + ? { isEnabled: true as const, functionId: "weather-agent" } + : telemetry({ functionId: "weather-agent" }); + if (mode.endsWith("stream")) { + const result = streamText({ + model: scripted(), + prompt: "Weather in Rome?", + tools: tools(), + ...loop, + experimental_telemetry: settings, + }); + return result.toUIMessageStreamResponse(); + } + const { text } = await generateText({ + model: scripted(), + prompt: "Weather in Paris?", + tools: tools(), + ...loop, + experimental_telemetry: settings, + }); + return Response.json({ text }); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/edge/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/edge/route.ts new file mode 100644 index 000000000..67a4b804d --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/edge/route.ts @@ -0,0 +1,29 @@ +import * as failproofai from "@failproofai/sdk"; + +export const runtime = "edge"; +export const dynamic = "force-dynamic"; + +/** + * The SDK in the Edge runtime: no filesystem, so no spool. Importing it must + * not take the route down; what each call does instead is what this reports. + */ +export async function GET(): Promise { + const outcome: Record = { imported: typeof failproofai.event === "object" }; + try { + failproofai.session({ sessionId: "edge-session" }, () => + failproofai.agent("edge-agent", () => { + failproofai.event.modelRequest({ model: "mock-model", requestId: "r-1" }); + }), + ); + outcome.emitted = true; + } catch (error) { + outcome.emitted = String(error); + } + try { + await failproofai.flush(); + outcome.flushed = true; + } catch (error) { + outcome.flushed = String(error); + } + return Response.json(outcome); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/langgraph/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/langgraph/route.ts new file mode 100644 index 000000000..31c92b2c5 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/langgraph/route.ts @@ -0,0 +1,21 @@ +import { langchainHandler } from "@failproofai/sdk/langchain"; + +import { ScriptedModel, buildGraph, question } from "../../../lib/langgraph"; + +export const dynamic = "force-dynamic"; + +/** + * `?mode=instrument` — relies on `instrument()` from instrumentation.ts + * (twin: langchain-1 `graph`). + * `?mode=handler` — passes the call-site handler, no patching + * (twin: langchain-1 `handler`). + */ +export async function GET(request: Request): Promise { + const mode = new URL(request.url).searchParams.get("mode") ?? "instrument"; + const graph = buildGraph(new ScriptedModel()); + const out = + mode === "handler" + ? await graph.invoke(question(), { callbacks: [langchainHandler() as never] }) + : await graph.invoke(question()); + return Response.json({ answer: out.messages.at(-1)?.content }); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/llamaindex/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/llamaindex/route.ts new file mode 100644 index 000000000..094f9f661 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/llamaindex/route.ts @@ -0,0 +1,11 @@ +import { agent } from "@llamaindex/workflow"; + +import { QUESTION, ScriptedLLM, getWeather } from "../../../lib/llamaindex"; + +export const dynamic = "force-dynamic"; + +/** A LlamaIndex agent workflow through `instrument()` (twin: llamaindex-0.12 `workflow`). */ +export async function GET(): Promise { + const out = await agent({ llm: new ScriptedLLM(), tools: [getWeather] }).run(QUESTION); + return Response.json({ answer: out.data.result }); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/mastra/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/mastra/route.ts new file mode 100644 index 000000000..f5dcdf9b1 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/mastra/route.ts @@ -0,0 +1,19 @@ +import { wrapTool } from "@failproofai/sdk/mastra"; + +import { question, weather, weatherAgent } from "../../../lib/mastra"; + +export const dynamic = "force-dynamic"; + +/** + * `?mode=instrument` — Agent.generate() through `instrument()` (twin: mastra-1 `generate`). + * `?mode=wraptool` — the call-site `wrapTool()` (twin: mastra-1 `wraptool`). + */ +export async function GET(request: Request): Promise { + const mode = new URL(request.url).searchParams.get("mode") ?? "instrument"; + if (mode === "wraptool") { + const tool = wrapTool(weather); + return Response.json({ out: await tool.execute!({ city: "Rome" }, {} as never) }); + } + const out = await weatherAgent().generate(question); + return Response.json({ answer: out.text }); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/api/status/route.ts b/sdk/typescript/integration/fixtures/nextjs/app/api/status/route.ts new file mode 100644 index 000000000..85017e3c6 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/api/status/route.ts @@ -0,0 +1,7 @@ +export const dynamic = "force-dynamic"; + +/** What instrumentation.ts's `instrument()` returned, and whether it ran at all. */ +export function GET(): Response { + const instrumented = (globalThis as { __failproofaiInstrumented?: string[] }).__failproofaiInstrumented ?? null; + return Response.json({ instrumented, runtime: process.env.NEXT_RUNTIME ?? null }); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/layout.tsx b/sdk/typescript/integration/fixtures/nextjs/app/layout.tsx new file mode 100644 index 000000000..e53180eeb --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/layout.tsx @@ -0,0 +1,9 @@ +import type { ReactNode } from "react"; + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + {children} + + ); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/app/page.tsx b/sdk/typescript/integration/fixtures/nextjs/app/page.tsx new file mode 100644 index 000000000..10497fd86 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/app/page.tsx @@ -0,0 +1,15 @@ +import { askWeather } from "./actions"; + +export const dynamic = "force-dynamic"; + +export default function Page() { + async function submit(): Promise { + "use server"; + await askWeather(); + } + return ( +
+ +
+ ); +} diff --git a/sdk/typescript/integration/fixtures/nextjs/instrumentation.ts b/sdk/typescript/integration/fixtures/nextjs/instrumentation.ts new file mode 100644 index 000000000..5356dde6e --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/instrumentation.ts @@ -0,0 +1,15 @@ +/** + * Next.js's documented startup hook: `register()` runs once per server process, + * before any request is handled. The place the README should tell Next users + * to call `instrument()`. + * + * `NEXT_RUNTIME` guards the Node-only work: Next also evaluates this file for + * the Edge runtime, where the SDK has no filesystem to spool to. + */ +export async function register(): Promise { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + if (process.env.FAILPROOFAI_IT_INSTRUMENT === "0") return; + const failproofai = await import("@failproofai/sdk"); + const instrumented = await failproofai.instrument(); + (globalThis as { __failproofaiInstrumented?: string[] }).__failproofaiInstrumented = instrumented; +} diff --git a/sdk/typescript/integration/fixtures/nextjs/lib/ai.ts b/sdk/typescript/integration/fixtures/nextjs/lib/ai.ts new file mode 100644 index 000000000..7a766ed2d --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/lib/ai.ts @@ -0,0 +1,61 @@ +// The ai-7 fixture's scripted model and tool, verbatim, so a route's trace can +// be compared with that fixture's trace under plain Node. +import { simulateReadableStream, stepCountIs, tool } from "ai"; +import { MockLanguageModelV4 } from "ai/test"; +import { z } from "zod"; + +type StreamPart = Awaited>["stream"] extends ReadableStream ? P : never; + +const usage = (input: number, output: number) => ({ + inputTokens: { total: input, noCache: input, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: output, text: output, reasoning: undefined }, +}); +const finish = (reason: "stop" | "tool-calls") => ({ unified: reason, raw: reason }); + +export function scripted() { + let generated = 0; + let streamed = 0; + return new MockLanguageModelV4({ + provider: "mock-provider", + modelId: "mock-model", + doGenerate: async () => { + generated += 1; + if (generated === 1) { + return { + content: [{ type: "tool-call", toolCallId: "call-1", toolName: "weather", input: '{"city":"Paris"}' }], + finishReason: finish("tool-calls"), + usage: usage(11, 7), + warnings: [], + }; + } + return { content: [{ type: "text", text: "It is 20C in Paris." }], finishReason: finish("stop"), usage: usage(23, 9), warnings: [] }; + }, + doStream: async () => { + streamed += 1; + const text = (id: string, ...deltas: string[]): StreamPart[] => [ + { type: "text-start", id }, + ...deltas.map((delta): StreamPart => ({ type: "text-delta", id, delta })), + { type: "text-end", id }, + ]; + const chunks: StreamPart[] = + streamed === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "tool-call", toolCallId: "call-s1", toolName: "weather", input: '{"city":"Rome"}' }, + { type: "finish", finishReason: finish("tool-calls"), usage: usage(13, 4) }, + ] + : [{ type: "stream-start", warnings: [] }, ...text("t", "Rome is ", "25C."), { type: "finish", finishReason: finish("stop"), usage: usage(30, 6) }]; + return { stream: simulateReadableStream({ chunks }) }; + }, + }); +} + +export const tools = () => ({ + weather: tool({ + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => ({ city, celsius: city === "Paris" ? 20 : 25 }), + }), +}); + +export const loop = { stopWhen: stepCountIs(4) }; diff --git a/sdk/typescript/integration/fixtures/nextjs/lib/langgraph.ts b/sdk/typescript/integration/fixtures/nextjs/lib/langgraph.ts new file mode 100644 index 000000000..f5cff65a3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/lib/langgraph.ts @@ -0,0 +1,68 @@ +// The langchain-1 fixture's graph and scripted model, verbatim, so a route's +// trace can be compared with that fixture's trace under plain Node. +import { BaseChatModel, type BaseChatModelParams } from "@langchain/core/language_models/chat_models"; +import { AIMessage, HumanMessage, type BaseMessage } from "@langchain/core/messages"; +import type { ChatResult } from "@langchain/core/outputs"; +import { tool } from "@langchain/core/tools"; +import { END, MessagesAnnotation, START, StateGraph } from "@langchain/langgraph"; +import { ToolNode } from "@langchain/langgraph/prebuilt"; +import { z } from "zod"; + +export class ScriptedModel extends BaseChatModel { + calls = 0; + + constructor(fields: BaseChatModelParams = {}) { + super(fields); + } + + _llmType(): string { + return "scripted"; + } + + override bindTools(): this { + return this; + } + + async _generate(messages: BaseMessage[]): Promise { + this.calls += 1; + const answered = messages.some((m) => m.getType() === "tool"); + const message = answered + ? new AIMessage({ + content: "It is sunny in Paris.", + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + }) + : new AIMessage({ + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + response_metadata: { finish_reason: "tool_calls" }, + }); + return { generations: [{ text: typeof message.content === "string" ? message.content : "", message }] }; + } +} + +const getWeather = tool(async ({ city }: { city: string }) => `sunny in ${city}`, { + name: "get_weather", + description: "Current weather for a city", + schema: z.object({ city: z.string() }), +}); + +export function buildGraph(model: ScriptedModel) { + const callModel = async (state: typeof MessagesAnnotation.State) => ({ + messages: [await model.invoke(state.messages)], + }); + const route = (state: typeof MessagesAnnotation.State) => { + const last = state.messages[state.messages.length - 1] as AIMessage; + return last.tool_calls?.length ? "tools" : END; + }; + return new StateGraph(MessagesAnnotation) + .addNode("agent", callModel) + .addNode("tools", new ToolNode([getWeather])) + .addEdge(START, "agent") + .addConditionalEdges("agent", route, ["tools", END]) + .addEdge("tools", "agent") + .compile({ name: "weather_graph" }); +} + +export const question = () => ({ messages: [new HumanMessage("weather?")] }); diff --git a/sdk/typescript/integration/fixtures/nextjs/lib/llamaindex.ts b/sdk/typescript/integration/fixtures/nextjs/lib/llamaindex.ts new file mode 100644 index 000000000..c4fdf3928 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/lib/llamaindex.ts @@ -0,0 +1,98 @@ +// The llamaindex-0.12 fixture's scripted LLM and tool, so a route's trace can +// be compared with that fixture's `workflow` trace under plain Node. +// +// One difference, in spelling only: the fixture decorates `chat` with +// `@wrapEventCaller @wrapLLMEvent`, and Next's compiler does not take standard +// decorators. They are applied here by hand — the same two functions, in the +// same order, with the context object a decorator receives. +import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator"; +import { + ToolCallLLM, + type ChatMessage, + type ChatResponse, + type ChatResponseChunk, + type LLMChatParamsNonStreaming, + type LLMChatParamsStreaming, + type LLMMetadata, + type ToolCallLLMMessageOptions, +} from "@llamaindex/core/llms"; +import { tool } from "llamaindex"; +import { z } from "zod"; + +type Options = ToolCallLLMMessageOptions; + +interface Turn { + tool?: { name: string; input: Record; id: string }; + text?: string; + usage: { prompt_tokens: number; completion_tokens: number }; +} + +const weatherTurns = (): Turn[] => [ + { tool: { name: "get_weather", input: { city: "Paris" }, id: "call_1" }, usage: { prompt_tokens: 12, completion_tokens: 5 } }, + { text: "It is sunny in Paris.", usage: { prompt_tokens: 30, completion_tokens: 7 } }, +]; + +const initializers: Array<(this: unknown) => void> = []; + +export class ScriptedLLM extends ToolCallLLM { + supportToolCall = true; + metadata: LLMMetadata = { + model: "scripted-1", + temperature: 0, + topP: 1, + contextWindow: 4096, + tokenizer: undefined, + structuredOutput: false, + }; + private readonly turns: Turn[]; + + constructor(turns: Turn[] = weatherTurns()) { + super(); + this.turns = turns; + for (const init of initializers) init.call(this); + } + + chat(params: LLMChatParamsStreaming): Promise>>; + chat(params: LLMChatParamsNonStreaming): Promise>; + async chat( + params: LLMChatParamsStreaming | LLMChatParamsNonStreaming, + ): Promise> | ChatResponse> { + const turn = this.turns.shift() ?? { text: "done", usage: { prompt_tokens: 1, completion_tokens: 1 } }; + const options: Options = turn.tool ? { toolCall: [turn.tool] } : {}; + const message: ChatMessage = { role: "assistant", content: turn.text ?? "", options }; + if (params.stream) { + return (async function* (): AsyncGenerator> { + yield { delta: turn.text ?? "", raw: { choices: [{ delta: {} }] }, options }; + yield { delta: "", raw: { choices: [], usage: turn.usage }, options: {} }; + })(); + } + return { message, raw: { choices: [{ finish_reason: turn.tool ? "tool_calls" : "stop" }], usage: turn.usage } }; + } +} + +{ + const context = { + kind: "method", + name: "chat", + static: false, + private: false, + access: { has: () => true, get: () => undefined }, + metadata: {}, + addInitializer: (init: (this: unknown) => void) => void initializers.push(init), + } as never; + const proto = ScriptedLLM.prototype as unknown as { chat: (...args: unknown[]) => unknown }; + // `@wrapEventCaller @wrapLLMEvent` applies bottom-up: wrapLLMEvent first. + proto.chat = (wrapEventCaller as (m: unknown, c: never) => typeof proto.chat)( + (wrapLLMEvent as (m: unknown, c: never) => typeof proto.chat)(proto.chat, context), + context, + ); +} + +export const getWeather = tool({ + name: "get_weather", + description: "Current weather for a city", + parameters: z.object({ city: z.string() }), + execute: ({ city }) => `sunny in ${city}`, +}); + +export const QUESTION = "weather in Paris?"; diff --git a/sdk/typescript/integration/fixtures/nextjs/lib/mastra.ts b/sdk/typescript/integration/fixtures/nextjs/lib/mastra.ts new file mode 100644 index 000000000..3895bf0ca --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/lib/mastra.ts @@ -0,0 +1,90 @@ +// The mastra-1 fixture's scripted model, tool and agent, verbatim (minus the +// gate/failure options no route uses), so a route's trace can be compared with +// that fixture's trace under plain Node. +import { Agent } from "@mastra/core/agent"; +import { createTool } from "@mastra/core/tools"; +import { z } from "zod"; + +type Turn = { tool: string; input: Record; usage: [number, number] } | { text: string; usage: [number, number] }; + +function scriptedModel(turns: Turn[]) { + let calls = 0; + const next = (): Turn => turns[calls++ % turns.length]!; + const usage = ([input, output]: [number, number]) => ({ + inputTokens: input, + outputTokens: output, + totalTokens: input + output, + }); + return { + specificationVersion: "v2" as const, + provider: "scripted", + modelId: "scripted-model", + supportedUrls: {}, + async doGenerate() { + const turn = next(); + return "tool" in turn + ? { + content: [{ type: "tool-call" as const, toolCallId: `call_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }], + finishReason: "tool-calls" as const, + usage: usage(turn.usage), + warnings: [], + } + : { + content: [{ type: "text" as const, text: turn.text }], + finishReason: "stop" as const, + usage: usage(turn.usage), + warnings: [], + }; + }, + async doStream() { + const turn = next(); + const parts: unknown[] = [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: `resp_${calls}`, modelId: "scripted-model", timestamp: new Date(0) }, + ]; + if ("tool" in turn) { + parts.push({ type: "tool-call", toolCallId: `call_${calls}`, toolName: turn.tool, input: JSON.stringify(turn.input) }); + parts.push({ type: "finish", finishReason: "tool-calls", usage: usage(turn.usage) }); + } else { + parts.push({ type: "text-start", id: "t" }); + const cut = turn.text.indexOf(" ", turn.text.length / 2) + 1; + for (const delta of [turn.text.slice(0, cut), turn.text.slice(cut)]) { + parts.push({ type: "text-delta", id: "t", delta }); + } + parts.push({ type: "text-end", id: "t" }); + parts.push({ type: "finish", finishReason: "stop", usage: usage(turn.usage) }); + } + return { + stream: new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }), + }; + }, + }; +} + +const WEATHER_TURNS: Turn[] = [ + { tool: "weather", input: { city: "Paris" }, usage: [11, 7] }, + { text: "It is sunny in Paris.", usage: [23, 9] }, +]; + +export const weather = createTool({ + id: "weather", + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async (input: { city: string }) => ({ city: input.city, forecast: "sunny" }), +}); + +export const weatherAgent = () => + new Agent({ + id: "weather-agent", + name: "weather-agent", + instructions: "Answer weather questions.", + model: scriptedModel(WEATHER_TURNS) as never, + tools: { weather }, + }); + +export const question = "What is the weather in Paris?"; diff --git a/sdk/typescript/integration/fixtures/nextjs/next.config.ts b/sdk/typescript/integration/fixtures/nextjs/next.config.ts new file mode 100644 index 000000000..7df843aff --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/next.config.ts @@ -0,0 +1,52 @@ +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { withFailproofai } from "@failproofai/sdk/next"; +import type { NextConfig } from "next"; + +const root = dirname(fileURLToPath(import.meta.url)); + +/** + * One app, built several ways by `nextjs.test.ts`, each into its own distDir: + * + * FAILPROOFAI_IT_NEXT_EXTERNAL=1 the frameworks (and the SDK) are listed in + * `serverExternalPackages`, so the server + * loads them from node_modules at run time + * (with `import()`, under both bundlers). Unset: Next's DEFAULT, which bundles + * all four — none of them is on Next's + * built-in external list. + * + * Every build includes `app/api/edge/route.ts`, an Edge-runtime route that + * imports the SDK: an SDK whose import broke the Edge runtime fails the build. + */ +const external = process.env.FAILPROOFAI_IT_NEXT_EXTERNAL === "1"; +/** + * FAILPROOFAI_IT_NEXT_WRAP=1 the documented setup: the app lists nothing + * itself and exports `withFailproofai(config)`. + */ +const wrap = process.env.FAILPROOFAI_IT_NEXT_WRAP === "1"; + +const config: NextConfig = { + distDir: process.env.FAILPROOFAI_IT_NEXT_DIST ?? ".next", + serverExternalPackages: external + ? [ + "@failproofai/sdk", + "@langchain/core", + "@langchain/langgraph", + "ai", + "@mastra/core", + "llamaindex", + "@llamaindex/core", + "@llamaindex/workflow", + ] + : [], + // The build is about what runs, not about the fixtures' types (which the + // framework fixtures typecheck on their own). + typescript: { ignoreBuildErrors: true }, + // This fixture sits inside a repo with other lockfiles; without these, Next + // guesses the workspace root from them and traces files from the wrong one. + outputFileTracingRoot: root, + turbopack: { root }, +}; + +export default wrap ? withFailproofai(config) : config; diff --git a/sdk/typescript/integration/fixtures/nextjs/package-lock.json b/sdk/typescript/integration/fixtures/nextjs/package-lock.json new file mode 100644 index 000000000..b7393e142 --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/package-lock.json @@ -0,0 +1,4343 @@ +{ + "name": "failproofai-it-nextjs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-nextjs", + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "@llamaindex/core": "0.6.22", + "@llamaindex/workflow": "1.1.24", + "@mastra/core": "1.68.0", + "ai": "7.0.111", + "llamaindex": "0.12.1", + "next": "16.3.6", + "react": "19.3.0", + "react-dom": "19.3.0", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4", + "@types/react": "19.3.0", + "typescript": "5.9.3" + } + }, + "node_modules/@a2a-js/sdk-v0_3": { + "name": "@a2a-js/sdk", + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@a2a-js/sdk/-/sdk-0.3.14.tgz", + "integrity": "sha512-F6Ew1AtPzCLhTn8h9yiqTe7DiDf6XVrSnq9V1YqSl9eWqPm6anMveTiKdCSb/76cW0YiJc24rNaUrVezFFHbqQ==", + "license": "Apache-2.0", + "dependencies": { + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.11.0", + "express": "^4.21.2 || ^5.1.0" + }, + "peerDependenciesMeta": { + "@bufbuild/protobuf": { + "optional": true + }, + "@grpc/grpc-js": { + "optional": true + }, + "express": { + "optional": true + } + } + }, + "node_modules/@a2a-js/sdk-v1": { + "name": "@a2a-js/sdk", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@a2a-js/sdk/-/sdk-1.0.1.tgz", + "integrity": "sha512-CJQdh3Wzwo8qIx5UUkSJ7+7BEI16PB+MXMHHNSmx8JQsQed2HlQgvx1ENOiKUfYA3PlcEvxIwv14dBblhDuPmw==", + "license": "Apache-2.0", + "dependencies": { + "jose": "^6.2.3", + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.11.0", + "express": "^4.21.2 || ^5.1.0" + }, + "peerDependenciesMeta": { + "@bufbuild/protobuf": { + "optional": true + }, + "@grpc/grpc-js": { + "optional": true + }, + "express": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.89", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.89.tgz", + "integrity": "sha512-n0Q88UUASYQBGK3Ogs9pCe8h3ZXeTHFSuGOLBJAyH73MVGNKUv9w8EJX2f340WUF3eDWWOV305gO7YBpSCPblA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/gateway/node_modules/@ai-sdk/provider": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz", + "integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.14.tgz", + "integrity": "sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.45", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.45.tgz", + "integrity": "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v6": { + "name": "@ai-sdk/provider-utils", + "version": "4.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.40.tgz", + "integrity": "sha512-OL5IrpUm9Y8Dwy+w/vvFwPotS6m52O9W0op2oXgXdCROMJIBalBI0oro6OIBYkPxvm5Xg02GSkoQN25RlR0bnw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.14", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v7": { + "name": "@ai-sdk/provider-utils", + "version": "5.0.13", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.13.tgz", + "integrity": "sha512-fScDJMDnTbx32kLDQqp0MvPjvwkgiwvlBxlmIg7XW5PbS91LG6JjH3PQG+34oMFglqfpQA355e24OdGj5PPoDw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.4", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider-utils-v7/node_modules/@ai-sdk/provider": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.4.tgz", + "integrity": "sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz", + "integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-v5": { + "name": "@ai-sdk/provider", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.3.tgz", + "integrity": "sha512-h88OPkavHTiN9tMn2l5awAznGB0lXzjcLhgR1/rvjB2zlLprsNxbM2tt6OJsHUxduLC3klq0/eqaSf6fX5XVww==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-v6": { + "name": "@ai-sdk/provider", + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.14.tgz", + "integrity": "sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-v7": { + "name": "@ai-sdk/provider", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.4.tgz", + "integrity": "sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz", + "integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@finom/zod-to-json-schema": { + "version": "3.24.11", + "resolved": "https://registry.npmjs.org/@finom/zod-to-json-schema/-/zod-to-json-schema-3.24.11.tgz", + "integrity": "sha512-fL656yBPiWebtfGItvtXLWrFNGlF1NcDFS0WdMQXMs9LluVg0CfT5E2oXYp0pidl0vVG53XkW55ysijNkU5/hA==", + "deprecated": "Use https://www.npmjs.com/package/zod-v3-to-json-schema instead. See issue comment for details: https://github.com/StefanTerdell/zod-to-json-schema/issues/178#issuecomment-3533122539", + "license": "ISC", + "peerDependencies": { + "zod": "^4.0.14" + } + }, + "node_modules/@hono/standard-validator": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@hono/standard-validator/-/standard-validator-0.4.0.tgz", + "integrity": "sha512-MxOm1asDh9j7c1D0KiWwppSbFgAQxTRO4YEhjMrJn3lF1BIcXenPdgjSGKDRfQmZdaTaMA8slQLETgVOKyJxFw==", + "license": "MIT", + "peerDependencies": { + "@standard-schema/spec": "^1.0.0", + "hono": ">=4.11.2" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", + "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", + "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", + "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", + "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", + "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", + "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", + "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", + "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", + "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", + "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", + "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", + "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", + "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", + "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", + "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", + "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", + "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", + "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", + "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", + "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", + "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", + "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", + "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", + "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", + "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", + "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@isaacs/ttlcache": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-2.1.5.tgz", + "integrity": "sha512-VwGZqqjAWPICTmxUZnbpEfO60LhPWzquik+bmyXGY7pYRn6diEvCI5i6Ca+J6o2y4vS73HrpuMTo2dOvUevH8w==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=12" + } + }, + "node_modules/@langchain/core": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.12.tgz", + "integrity": "sha512-DvNsrN5Gz+bgEy/B/kK1AvxxcrwdrflI8hZJxqo/DtA17FMUeWM5k0o1c6G8ChH8+sIakV2UsDudFmNZJp1ugQ==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.17", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.17.tgz", + "integrity": "sha512-gkd34M42D5SdRaaHS6NSD+zTuZBgmEqecld3BenVlql11WshvarKH/2ZQbNmQQ0knbHgtaQEg3VeH0bUFt2B1Q==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.11.2", + "@langchain/protocol": "^0.0.19", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.11.2.tgz", + "integrity": "sha512-b2s4qdFKePgZudPJZfDCtQOmVfHoiKCTgSSdOxXtMYZzg9A+g2kI/ll1ZPMYIydGE3QIgGLIhyFCz5ULedi5Nw==", + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.19", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.2.tgz", + "integrity": "sha512-prbX4Z3YszrFNgH+MW5Zoeq3baXrMtP/MQnFeET90UB/GtGcGDQ5Usg9OCy6ETjTTntOw1SL2z9fMPUppN3Guw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/protocol": { + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", + "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", + "license": "MIT" + }, + "node_modules/@llamaindex/core": { + "version": "0.6.22", + "resolved": "https://registry.npmjs.org/@llamaindex/core/-/core-0.6.22.tgz", + "integrity": "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg==", + "dependencies": { + "@finom/zod-to-json-schema": "3.24.11", + "@llamaindex/env": "0.1.30", + "@types/node": "^24.0.13", + "magic-bytes.js": "^1.10.0", + "zod": "^4.1.5" + } + }, + "node_modules/@llamaindex/core/node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@llamaindex/core/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/@llamaindex/env": { + "version": "0.1.30", + "resolved": "https://registry.npmjs.org/@llamaindex/env/-/env-0.1.30.tgz", + "integrity": "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "js-tiktoken": "^1.0.12", + "pathe": "^1.1.2" + }, + "peerDependencies": { + "@huggingface/transformers": "^3.5.0", + "gpt-tokenizer": "^2.5.0" + }, + "peerDependenciesMeta": { + "@huggingface/transformers": { + "optional": true + }, + "gpt-tokenizer": { + "optional": true + } + } + }, + "node_modules/@llamaindex/node-parser": { + "version": "2.0.22", + "resolved": "https://registry.npmjs.org/@llamaindex/node-parser/-/node-parser-2.0.22.tgz", + "integrity": "sha512-uj5O89WShAAyiSZ8f8tU7hnLJ6pSmlY2a6hkAOs8odkUgT87dEqaPHpsK7w0iJdEFiob7GoLeRhv2K624FooXg==", + "dependencies": { + "html-to-text": "^9.0.5" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30", + "tree-sitter": "^0.22.0", + "web-tree-sitter": "^0.24.3" + } + }, + "node_modules/@llamaindex/workflow": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@llamaindex/workflow/-/workflow-1.1.24.tgz", + "integrity": "sha512-VyKsbRkFlnT5dRNKbgLXQV+ZpQ+CAFgmC9LaZv6hD/fIKo6wq1wQW/ZqLZgZt569xeHgxmrXPB6KHdqn/AhPbQ==", + "dependencies": { + "@llamaindex/workflow-core": "^1.3.2" + }, + "peerDependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@llamaindex/workflow-core": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@llamaindex/workflow-core/-/workflow-core-1.3.3.tgz", + "integrity": "sha512-WJIcD4K2suGbNkwU5CC70jKKrA5tARba42nMs8Pou1RGzmoxqg+K+b7vyLBmiDtImR8P40YLmkayCIRVQPBmsg==", + "license": "MIT", + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.7.0", + "hono": "^4.7.4", + "next": "^15.2.2", + "p-retry": "^6.2.1", + "rxjs": "^7.8.2", + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + }, + "hono": { + "optional": true + }, + "next": { + "optional": true + }, + "p-retry": { + "optional": true + }, + "rxjs": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/env": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.26.tgz", + "integrity": "sha512-NJBz9q10LU9h3KjHLEbdgWIV+ow/x+MYzKBRfqhm9/QmML3tPMhYmXF/UIV9SDVCNtOqFNc5oX7kZqeiigMCEA==", + "extraneous": true, + "license": "MIT" + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-darwin-arm64": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.26.tgz", + "integrity": "sha512-So8eoJxIcXw/TexNUvvh3uY72J9nDo5BpJsAwUKx+FK57CrWXg6RqVufV7U9OT3BO+siMzJ2FuAwBhaHoPlLGg==", + "cpu": [ + "arm64" + ], + "extraneous": true, + "license": "MIT", + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-darwin-x64": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.26.tgz", + "integrity": "sha512-jImzLUTClVWKhP91e5sgDumjxCLhaFSt7DuN5cnRYw99Dppxxhhq5jKRyDa2aTv3JE7dQmTSTsY3EFkSOp9Pog==", + "cpu": [ + "x64" + ], + "extraneous": true, + "license": "MIT", + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-linux-arm64-gnu": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.26.tgz", + "integrity": "sha512-CaWd+T/Lud2BmZbrsa1CzCnIOdU3YX9Nuk89virZaSB1O+C+8Yrrevgmnl68u4dvZIfOAzQ77S+Njrq7v1XwSA==", + "cpu": [ + "arm64" + ], + "extraneous": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-linux-arm64-musl": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.26.tgz", + "integrity": "sha512-97AyKI34yjpaudlkWHswAf7c1PjWQAC7lLyrw3R5K+bq834EOA+9IG68rIVy0VqrqGjtPSMjJmMmgeJ3wHR5og==", + "cpu": [ + "arm64" + ], + "extraneous": true, + "libc": [ + "musl" + ], + "license": "MIT", + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-linux-x64-gnu": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.26.tgz", + "integrity": "sha512-eVtuOCew1sBPV7BEgxy7qxuVyqoU3tJIS/xDZwa1/NiQ4Q0LM2JMH7rny+/uVIN6hsQ3PTb0hTQWypA0L2QxtA==", + "cpu": [ + "x64" + ], + "extraneous": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-linux-x64-musl": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.26.tgz", + "integrity": "sha512-EiUXADp+Z+OdQnSqbX10YgOSUrs0CXVODoTfySfsP2jdhngn9bq5RJd376FJzqMPe/XX25FMr1aXtYUVPA0qDw==", + "cpu": [ + "x64" + ], + "extraneous": true, + "libc": [ + "musl" + ], + "license": "MIT", + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-win32-arm64-msvc": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.26.tgz", + "integrity": "sha512-HPl41fgkC4kdM5CCIoqNW6KlKEj1N+xS6bjNFFxDNKooUNUu09frgD948zo95TPw/C3XINZpkdkBGNU2RhjEnw==", + "cpu": [ + "arm64" + ], + "extraneous": true, + "license": "MIT", + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@next/swc-win32-x64-msvc": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.26.tgz", + "integrity": "sha512-TgmJ5ginKr34RPsz01/swpYtFBxh51d66jM26aytpE6NIypU5KtBVI8C2hJjUNpWr7v6sWj8a6+og2MyntcnpA==", + "cpu": [ + "x64" + ], + "extraneous": true, + "license": "MIT", + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@llamaindex/workflow/node_modules/@swc/helpers": { + "version": "0.5.15", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", + "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "extraneous": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@llamaindex/workflow/node_modules/next": { + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.26.tgz", + "integrity": "sha512-EVCqhvq8Hs+nX9udH2VzE/iXAg9QodZBZnwVJTuAMl386GIYvlJtYhFytV9nSlDYxKw3kEyv8I2dCQs0+on0sQ==", + "extraneous": true, + "license": "MIT", + "dependencies": { + "@next/env": "15.5.26", + "@swc/helpers": "0.5.15", + "caniuse-lite": "^1.0.30001579", + "postcss": "8.4.31", + "styled-jsx": "5.1.6" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" + }, + "optionalDependencies": { + "@next/swc-darwin-arm64": "15.5.26", + "@next/swc-darwin-x64": "15.5.26", + "@next/swc-linux-arm64-gnu": "15.5.26", + "@next/swc-linux-arm64-musl": "15.5.26", + "@next/swc-linux-x64-gnu": "15.5.26", + "@next/swc-linux-x64-musl": "15.5.26", + "@next/swc-win32-arm64-msvc": "15.5.26", + "@next/swc-win32-x64-msvc": "15.5.26", + "sharp": "^0.34.3 || ^0.35.4" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.51.1", + "babel-plugin-react-compiler": "*", + "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@playwright/test": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/@llamaindex/workflow/node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "extraneous": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@llamaindex/workflow/node_modules/postcss": { + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "extraneous": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@lukeed/uuid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.1.tgz", + "integrity": "sha512-qC72D4+CDdjGqJvkFMMEAtancHUQ7/d/tAiHf64z8MopFDmcrtbcJuerDtFceuAfQJ2pDSfCKCtbqoGBNnwg0w==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@mastra/core": { + "version": "1.68.0", + "resolved": "https://registry.npmjs.org/@mastra/core/-/core-1.68.0.tgz", + "integrity": "sha512-NdhQpjlOgaCzx/UePKkTfvnZk6/O26HSm46qNyY/XBgkTe1Qas5ks4IOYUTmhQsEkK1vi+fz8cNj6XAdLrZcpQ==", + "license": "Apache-2.0", + "dependencies": { + "@a2a-js/sdk-v0_3": "npm:@a2a-js/sdk@~0.3.14", + "@a2a-js/sdk-v1": "npm:@a2a-js/sdk@~1.0.1", + "@ai-sdk/provider-utils-v6": "npm:@ai-sdk/provider-utils@4.0.40", + "@ai-sdk/provider-utils-v7": "npm:@ai-sdk/provider-utils@5.0.13", + "@ai-sdk/provider-v5": "npm:@ai-sdk/provider@2.0.3", + "@ai-sdk/provider-v6": "npm:@ai-sdk/provider@3.0.14", + "@ai-sdk/provider-v7": "npm:@ai-sdk/provider@4.0.4", + "@isaacs/ttlcache": "^2.1.5", + "@lukeed/uuid": "^2.0.1", + "@mastra/schema-compat": "1.3.11", + "@standard-schema/spec": "^1.1.0", + "chat": "^4.37.0", + "croner": "^10.0.1", + "dotenv": "^17.3.1", + "execa": "^9.6.1", + "fastq": "^1.20.1", + "gray-matter": "^4.0.3", + "hono": "^4.13.7", + "hono-openapi": "^1.3.1", + "ignore": "^7.0.5", + "jpeg-js": "^0.4.4", + "json-schema": "^0.4.0", + "lru-cache": "^11.2.7", + "p-map": "^7.0.4", + "p-retry": "^7.1.1", + "picomatch": "^4.0.3", + "posthog-node": "^5.46.1", + "ws": "^8.21.3", + "xxhash-wasm": "^1.1.0" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@mastra/schema-compat": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@mastra/schema-compat/-/schema-compat-1.3.11.tgz", + "integrity": "sha512-NvR+wYk4/d0uqzk702Aehd8Vbop+844c1Dp7TMkZVis71LzPORWJbFPQswrom9o+v5ollUfif+28lZzutGMq+A==", + "license": "Apache-2.0", + "dependencies": { + "json-schema-to-zod": "^2.7.0", + "zod-from-json-schema": "^0.5.2" + }, + "engines": { + "node": ">=22.13.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@next/env": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.6.tgz", + "integrity": "sha512-x9Vblze1EbtltQYnNH38xCPWU3TVfBd1eXqA3+w9+BTpedkkdNpAaltXlGQ/nsc1+E0mVTNrtcbX3GoO09zeLQ==", + "license": "MIT" + }, + "node_modules/@next/swc-darwin-arm64": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.6.tgz", + "integrity": "sha512-E/7GEqaUkt8mk/T8v9lAnrhzR06kdq1ZBkC12F8tAMkdIadwNp3H1KqHynDHrpcTlGCUdq/qu6vUL2aYVyYBdw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.6.tgz", + "integrity": "sha512-yBE893/nDWTlaiBD1p+qgt7NUen4U5R6FXyH0s67Npq1S3E0cVSef1WIXC2xBRgQvwAvJq6DnS6Y6PrY0cy4Ew==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.6.tgz", + "integrity": "sha512-KJDpjBqBPYlvkivmyrp+Qys6k/7ksbqGQvRVc6ZEGfR+cjQxx+nUkJaWmNZJsmoOrqYNbaXByF8wa0lBwDhB3Q==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.6.tgz", + "integrity": "sha512-mqNg2K+hvWskSRb/QM+Ix412DvBsuSF0XV+frTSw5vmoucNnIlynFwKYew8D01bfATErMOM7Bujrf0BA5DRKFA==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.6.tgz", + "integrity": "sha512-nFncBNGAYouRHjRVaITs9beZRfhX4ssVwpnvPIAbkZVH6LtGoAVlH4bJ8Cnf9SOo9bsXgPFer/GdHtEE3JNOkw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.6.tgz", + "integrity": "sha512-5Mf3cHDGR/Iz0ng2Bj3zUR3p5QS9YK3Hn2QiAfavFmyF48zwThAjpFoiTKNIcOHLYS4zEk+gzyJ/9deQ2ZB8yQ==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.6.tgz", + "integrity": "sha512-0jkJy0C2kbrJWTk4YLa3xk80pVBpx8FCHJym7CnUfDAXe/FWv5qT7SQJbR0KuemyxaEDlEx5WT4VQJoTW+/9Qw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.6.tgz", + "integrity": "sha512-/YXjI1e5OXcZ7YpxRwgP/1jAV/SBKTzeVKqN2mk7mLpcICsyn3Gl5+dIfDTJp70M0ccMhyMMRso4v6mPDCGepg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@posthog/core": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.55.1.tgz", + "integrity": "sha512-S76bSbCHGVC8oQa1zyue6A6WzkNf1Ue4oRvfvBNLRnFM3UoWnpWnQ4T/4IzrIEtVCQYv1OI7gk8uL8/U6Tuu7w==", + "license": "MIT", + "dependencies": { + "@posthog/types": "^1.412.3" + } + }, + "node_modules/@posthog/types": { + "version": "1.412.4", + "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.412.4.tgz", + "integrity": "sha512-Q7lV9O9TbLngjOYw1ucm3bS3tn48xb1B4ZQdDy8gv7yfKe99hCTeWYG4xZ36nJM80VxpTF9TriiJt5hngDOkBg==", + "license": "MIT" + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@standard-community/standard-json": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@standard-community/standard-json/-/standard-json-0.3.6.tgz", + "integrity": "sha512-tuXKz1Ps4al0W4xhfwii7v4lskKtOv3QE7mcouRVThrBdOR6JzY4Vea3nsxcEnte84bOJvMvXGmOMWE3IBbHyA==", + "license": "MIT", + "dependencies": { + "quansync": "^0.2.11" + }, + "peerDependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/json-schema": "^7.0.15", + "@valibot/to-json-schema": "^1.3.0", + "arktype": "^2.1.20", + "effect": "^3.20.0", + "sury": "^10.0.0", + "typebox": "^1.0.17", + "valibot": "^1.4.2", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.24.5" + }, + "peerDependenciesMeta": { + "@valibot/to-json-schema": { + "optional": true + }, + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "sury": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-to-json-schema": { + "optional": true + } + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@swc/helpers": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", + "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/lodash": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.25.tgz", + "integrity": "sha512-+K1NIO8I+F9/wNulfVvu23QYd0Pe9/OCqRrim4NoYIf1VoEDL90Ve4ClzpyqBLc7NpGGWRvYNCKZ1BE/Jpf8dQ==", + "license": "MIT" + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "extraneous": true, + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.111", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.111.tgz", + "integrity": "sha512-jd35WisL5FT7lFB6wIsZ38G0EaL4p1o+bUuRfkZGTXqO21U3SWyEZ+muV7N191QQfNeExgoyaeFoa8Jx6Uo+0A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.89", + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ai/node_modules/@ai-sdk/provider": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz", + "integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.25", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.25.tgz", + "integrity": "sha512-gMmEShwwq7FJqMwvfRwvCl00v4kN+KOfJqXn+f4nrufak5gNHJOksd/60Dvjuz7sI8Y5WiSFBa8FEYr+zoyqCw==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chat": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/chat/-/chat-4.41.0.tgz", + "integrity": "sha512-vMeODhulMdi7od1c6e4B/ZfIp/zq0f5KP3WKNCAAdNrTOeDgjRNZQZahaIax/t/gu+LE5vwJSjFU46Ro9Kx9tA==", + "license": "MIT", + "dependencies": { + "@workflow/serde": "4.1.0-beta.2", + "mdast-util-to-string": "^4.0.0", + "remark-gfm": "^4.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "remend": "^1.2.1", + "unified": "^11.0.5" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "ai": "^6.0.182 || ^7.0.0", + "workflow": "^5.0.0-beta.35", + "zod": "^3.0.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "ai": { + "optional": true + }, + "workflow": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/chat/node_modules/@workflow/serde": { + "version": "4.1.0-beta.2", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0-beta.2.tgz", + "integrity": "sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==", + "license": "Apache-2.0" + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", + "license": "MIT" + }, + "node_modules/croner": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/croner/-/croner-10.0.1.tgz", + "integrity": "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==", + "funding": [ + { + "type": "other", + "url": "https://paypal.me/hexagonpp" + }, + { + "type": "github", + "url": "https://github.com/sponsors/hexagon" + } + ], + "license": "MIT", + "engines": { + "node": ">=18.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/execa": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-9.6.1.tgz", + "integrity": "sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "cross-spawn": "^7.0.6", + "figures": "^6.1.0", + "get-stream": "^9.0.0", + "human-signals": "^8.0.1", + "is-plain-obj": "^4.1.0", + "is-stream": "^4.0.1", + "npm-run-path": "^6.0.0", + "pretty-ms": "^9.2.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^4.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": "^18.19.0 || >=20.5.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", + "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/hono-openapi": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/hono-openapi/-/hono-openapi-1.3.3.tgz", + "integrity": "sha512-+LVFPQEc4eDrXr/Rd0QQkRh/DbEZRXnm1RZU4bVaoqmWiWgspT4nNHIQMgi44X+D9/8k1eGzHvR0KLNPJ/0xaQ==", + "license": "MIT", + "dependencies": { + "@hono/standard-validator": "^0.4.0", + "@standard-community/standard-json": "^0.3.5", + "@standard-community/standard-openapi": "^0.2.9", + "@standard-schema/spec": "^1.0.0", + "@types/json-schema": "^7.0.15", + "openapi-types": "^12.1.3" + }, + "peerDependencies": { + "hono": "^4.11.2" + } + }, + "node_modules/hono-openapi/node_modules/@standard-community/standard-openapi": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@standard-community/standard-openapi/-/standard-openapi-0.2.10.tgz", + "integrity": "sha512-whaCN9eu5zv5gu6Kyeh5am/S3AyS1nqo57/wAxs/KPS7+94UhBUZmkIetmXzm22qIV0zMrHt3y+2VfjkbkYYTA==", + "license": "MIT", + "peerDependencies": { + "@standard-community/standard-json": "^0.3.5", + "@standard-schema/spec": "^1.0.0", + "arktype": "^2.1.20", + "effect": "^3.20.0", + "openapi-types": "^12.1.3", + "sury": "^10.0.0", + "typebox": "^1.0.0", + "valibot": "^1.4.2", + "zod": "^3.25.0 || ^4.0.0", + "zod-openapi": "^4" + }, + "peerDependenciesMeta": { + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "sury": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-openapi": { + "optional": true + } + } + }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/human-signals": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.1.tgz", + "integrity": "sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/ignore": { + "version": "7.0.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.10.tgz", + "integrity": "sha512-HpbUakT7xp5miBUywCHf36ZEuAJNklBJDDsGpUIjMzOSmM8ELSfA9Sa/QDPeNeqeoN31u+UTCkL4klCOVvRm4Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause" + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-yaml": { + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/json-schema-to-zod": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/json-schema-to-zod/-/json-schema-to-zod-2.8.1.tgz", + "integrity": "sha512-fRr1mHgZ7hboLKBUdR428gd9dIHUFGivUqOeiDcSmyXkNZCtB1uGaZLvsjZ4GaN5pwBIs+TGIOf6s+Rp5/R/zA==", + "license": "ISC", + "bin": { + "json-schema-to-zod": "dist/cjs/cli.js" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/langsmith": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.10.5.tgz", + "integrity": "sha512-VUh6LQYGb+wjmMz5Ulazqx1rUHLbWi2XZvVkSHfxDjz6V7u9TJOB6jW5CC84xcpffu7Jr1MZpBbEz3gg972HbA==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/llamaindex": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/llamaindex/-/llamaindex-0.12.1.tgz", + "integrity": "sha512-/tXXITk/iVGBycOFaDhev6dgTBIr6Ycu4FoPIt6A5JcEAiB6ujONjiV36flVXUR8JdqwMtS767XMjV+36nV4yQ==", + "license": "MIT", + "dependencies": { + "@llamaindex/core": "0.6.22", + "@llamaindex/env": "0.1.30", + "@llamaindex/node-parser": "2.0.22", + "@llamaindex/workflow": "1.1.24", + "@types/lodash": "^4.17.7", + "@types/node": "^24.0.13", + "lodash": "^4.17.21", + "magic-bytes.js": "^1.10.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/llamaindex/node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/llamaindex/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/magic-bytes.js": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/magic-bytes.js/-/magic-bytes.js-1.13.1.tgz", + "integrity": "sha512-x5sn4UX2k5gCWlcfmoFwG4TPie8+dctESyqOBdhB5p6MsgWXdBKGmt9nXPObj/JI50TTL928lc5Yt1WntMn1bw==", + "license": "MIT" + }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.2.tgz", + "integrity": "sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/next": { + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/next/-/next-16.3.6.tgz", + "integrity": "sha512-L+otWM/aQbYTx98aZhgEoMb4bZAXx1YVW4UMA/vuCyCoWG5HJyZUili8QAkqzrcC+5///tsz3s0M+SlyB5bLMw==", + "license": "MIT", + "dependencies": { + "@next/env": "16.3.6", + "@swc/helpers": "0.5.23", + "baseline-browser-mapping": "^2.9.19", + "caniuse-lite": "^1.0.30001579", + "postcss": "8.5.23", + "styled-jsx": "5.1.6" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": ">=20.9.0" + }, + "optionalDependencies": { + "@next/swc-darwin-arm64": "16.3.6", + "@next/swc-darwin-x64": "16.3.6", + "@next/swc-linux-arm64-gnu": "16.3.6", + "@next/swc-linux-arm64-musl": "16.3.6", + "@next/swc-linux-x64-gnu": "16.3.6", + "@next/swc-linux-x64-musl": "16.3.6", + "@next/swc-win32-arm64-msvc": "16.3.6", + "@next/swc-win32-x64-msvc": "16.3.6", + "sharp": "^0.35.4" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.51.1", + "babel-plugin-react-compiler": "*", + "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@playwright/test": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "license": "MIT", + "peer": true, + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "peer": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/npm-run-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", + "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0", + "unicorn-magic": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT" + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-map": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.8.tgz", + "integrity": "sha512-MitaVsCuCFIvOLLPIU7NnfrZvS9H9h7kwMUkDo+T2pEISaJD48IV9S8iIdXB7PsvvdxyYcsSTTrr90XKsbulNw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/parse-ms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", + "integrity": "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "license": "MIT", + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "license": "MIT" + }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/posthog-node": { + "version": "5.52.5", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.52.5.tgz", + "integrity": "sha512-r4KRXh2MvcHYh1s3NUca022GqujkJjpmD4qbRzWOLVTu8BfN3fqZj+MyKIxvHPXsS8sKsNVX+MlKk2qzW0Y4GQ==", + "license": "MIT", + "dependencies": { + "@posthog/core": "^1.55.1" + }, + "engines": { + "node": "^20.20.0 || >=22.22.0" + }, + "peerDependencies": { + "rxjs": "^7.0.0" + }, + "peerDependenciesMeta": { + "rxjs": { + "optional": true + } + } + }, + "node_modules/pretty-ms": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.1.tgz", + "integrity": "sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==", + "license": "MIT", + "dependencies": { + "parse-ms": "^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", + "integrity": "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remend": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/remend/-/remend-1.3.1.tgz", + "integrity": "sha512-N3DiY5qbRPoa5vkxn1oDLMyXOVTeo6Hp+XOj6SIqJAYUgLS0Q587gILPMom/qm86AQ/ZrcOdwEIzCz8V3J0nxQ==", + "license": "Apache-2.0" + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "extraneous": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "license": "MIT" + }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "license": "MIT", + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", + "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.4", + "@img/sharp-darwin-x64": "0.35.4", + "@img/sharp-freebsd-wasm32": "0.35.4", + "@img/sharp-libvips-darwin-arm64": "1.3.3", + "@img/sharp-libvips-darwin-x64": "1.3.3", + "@img/sharp-libvips-linux-arm": "1.3.3", + "@img/sharp-libvips-linux-arm64": "1.3.3", + "@img/sharp-libvips-linux-ppc64": "1.3.3", + "@img/sharp-libvips-linux-riscv64": "1.3.3", + "@img/sharp-libvips-linux-s390x": "1.3.3", + "@img/sharp-libvips-linux-x64": "1.3.3", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", + "@img/sharp-libvips-linuxmusl-x64": "1.3.3", + "@img/sharp-linux-arm": "0.35.4", + "@img/sharp-linux-arm64": "0.35.4", + "@img/sharp-linux-ppc64": "0.35.4", + "@img/sharp-linux-riscv64": "0.35.4", + "@img/sharp-linux-s390x": "0.35.4", + "@img/sharp-linux-x64": "0.35.4", + "@img/sharp-linuxmusl-arm64": "0.35.4", + "@img/sharp-linuxmusl-x64": "0.35.4", + "@img/sharp-webcontainers-wasm32": "0.35.4", + "@img/sharp-win32-arm64": "0.35.4", + "@img/sharp-win32-ia32": "0.35.4", + "@img/sharp-win32-x64": "0.35.4" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", + "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/styled-jsx": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", + "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==", + "license": "MIT", + "dependencies": { + "client-only": "0.0.1" + }, + "engines": { + "node": ">= 12.0.0" + }, + "peerDependencies": { + "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/tree-sitter": { + "version": "0.22.4", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.22.4.tgz", + "integrity": "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg==", + "hasInstallScript": true, + "license": "MIT", + "peer": true, + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", + "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/uuid": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/web-tree-sitter": { + "version": "0.24.7", + "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.24.7.tgz", + "integrity": "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ==", + "license": "MIT", + "peer": true + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xxhash-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", + "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", + "license": "MIT" + }, + "node_modules/yoctocolors": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.2.0.tgz", + "integrity": "sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-from-json-schema": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/zod-from-json-schema/-/zod-from-json-schema-0.5.6.tgz", + "integrity": "sha512-U33AJ7ZWS6y9XNSzMWcdy8hRAvZmWhTtpYJu0SXPT5AArbc9nq2ur7Magzmn5RF9KBV4b3FP0nCpmqqlfXlR9w==", + "license": "MIT", + "dependencies": { + "zod": "^4.0.17" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/nextjs/package.json b/sdk/typescript/integration/fixtures/nextjs/package.json new file mode 100644 index 000000000..87e8283ef --- /dev/null +++ b/sdk/typescript/integration/fixtures/nextjs/package.json @@ -0,0 +1,28 @@ +{ + "name": "failproofai-it-nextjs", + "private": true, + "type": "module", + "description": "Integration fixture: a Next.js App Router app with one route handler per framework and a server action, built with `next build` and served with `next start` against the packed @failproofai/sdk. Framework pins match the langchain-1, ai-7, mastra-1 and llamaindex-0.12 fixtures, so each route's trace is compared with the same program's trace under plain Node.", + "scripts": { + "build": "next build", + "start": "next start" + }, + "dependencies": { + "@langchain/core": "1.2.12", + "@langchain/langgraph": "1.4.17", + "@llamaindex/core": "0.6.22", + "@llamaindex/workflow": "1.1.24", + "@mastra/core": "1.68.0", + "ai": "7.0.111", + "llamaindex": "0.12.1", + "next": "16.3.6", + "react": "19.3.0", + "react-dom": "19.3.0", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4", + "@types/react": "19.3.0", + "typescript": "5.9.3" + } +} diff --git a/sdk/typescript/integration/fixtures/runtimes/agent.ts b/sdk/typescript/integration/fixtures/runtimes/agent.ts new file mode 100644 index 000000000..8005ee9a8 --- /dev/null +++ b/sdk/typescript/integration/fixtures/runtimes/agent.ts @@ -0,0 +1,51 @@ +// The core SDK with no framework: scopes, all 15 `event.*` methods, flush. +// Run under Node, Bun and Deno by `runtimes.core.test.ts`: ` agent.{mjs,cjs} `. +import * as failproofai from "@failproofai/sdk"; + +const { agent, event, session, toolCall } = failproofai; +const report = (value: unknown) => console.log(JSON.stringify(value)); + +/** Every public surface once, in one session. */ +async function core(): Promise { + await session({ sessionId: "core-session" }, async () => { + await agent("planner", { goal: "smoke every surface" }, async () => { + event.modelRequest({ model: "mock-model", messages: [{ role: "user", content: "hi" }], requestId: "r-1" }); + event.modelResponse({ model: "mock-model", stopReason: "stop", inputTokens: 3, outputTokens: 2, content: "hello", requestId: "r-1" }); + await toolCall("lookup", { toolCallId: "call-1", input: { q: "x" } }, async (call) => { + call.output = { found: true }; + }); + event.toolUse({ toolName: "manual", toolCallId: "call-2", input: { a: 1 } }); + event.toolResult({ toolName: "manual", toolCallId: "call-2", output: "ok" }); + event.hookTriggered({ hookName: "guard", hookId: "h-1", triggerEvent: "pre_tool" }); + event.hookCompleted({ hookName: "guard", hookId: "h-1", outcome: "allowed" }); + event.agentPause({ pauseId: "p-1", reason: "approval" }); + event.humanWait({ inputId: "i-1", prompt: "approve?" }); + event.humanInput({ inputId: "i-1", response: "yes" }); + event.agentResume({ pauseId: "p-1" }); + event.humanPause({ reason: "coffee" }); + event.humanInterrupt({ reason: "stop that", atStep: "2" }); + event.error({ errorType: "Recoverable", message: "retrying" }); + await agent("helper", async () => { + event.modelRequest({ model: "mock-model", requestId: "r-2" }); + event.modelResponse({ model: "mock-model", stopReason: "stop", inputTokens: 1, outputTokens: 1, requestId: "r-2" }); + }); + }); + }); + await failproofai.flush(); + report({ core: "done" }); +} + +async function main(scenario: string): Promise { + switch (scenario) { + case "core": + await core(); + break; + default: + throw new Error(`unknown scenario ${scenario}`); + } +} + +main(process.argv[2] ?? "core").catch((error: unknown) => { + console.error("FATAL", error); + process.exit(1); +}); diff --git a/sdk/typescript/integration/fixtures/runtimes/deno-npm.ts b/sdk/typescript/integration/fixtures/runtimes/deno-npm.ts new file mode 100644 index 000000000..b2817635b --- /dev/null +++ b/sdk/typescript/integration/fixtures/runtimes/deno-npm.ts @@ -0,0 +1,76 @@ +// Deno, the way a Deno app is written: every package through an `npm:` +// specifier, TypeScript run as-is. `deno run --allow-all deno-npm.ts `. +// +// With a package.json beside it Deno resolves `npm:` specifiers against this +// fixture's node_modules — the packed SDK and the pinned `ai` — so nothing is +// fetched at run time. +import * as failproofai from "npm:@failproofai/sdk"; +import { telemetry, wrapModel } from "npm:@failproofai/sdk/ai"; +import { generateText, stepCountIs, tool } from "npm:ai@6.0.288"; +import { MockLanguageModelV3 } from "npm:ai@6.0.288/test"; +import { z } from "npm:zod@4.6.5"; + +const usage = (input: number, output: number) => ({ + inputTokens: { total: input, noCache: input, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: output, text: output, reasoning: undefined }, +}); +const finish = (reason: "stop" | "tool-calls") => ({ unified: reason, raw: reason }); + +/** Asks for `weather` once, then answers. */ +function scripted() { + let calls = 0; + return new MockLanguageModelV3({ + provider: "mock-provider", + modelId: "mock-model", + doGenerate: async () => { + calls += 1; + if (calls === 1) { + return { + content: [{ type: "tool-call", toolCallId: "call-1", toolName: "weather", input: '{"city":"Paris"}' }], + finishReason: finish("tool-calls"), + usage: usage(11, 7), + warnings: [], + }; + } + return { content: [{ type: "text", text: "It is 20C in Paris." }], finishReason: finish("stop"), usage: usage(23, 9), warnings: [] }; + }, + }); +} + +const tools = { + weather: tool({ + description: "Current weather for a city", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }: { city: string }) => ({ city, celsius: 20 }), + }), +}; + +async function main(scenario: string): Promise { + let text: string; + if (scenario === "telemetry") { + ({ text } = await generateText({ + model: scripted(), + prompt: "Weather in Paris?", + tools, + stopWhen: stepCountIs(4), + experimental_telemetry: telemetry({ functionId: "weather-agent" }), + })); + } else if (scenario === "wrap") { + ({ text } = await failproofai.agent("weather-agent", () => + wrapModel(scripted()).then((model) => + generateText({ model, prompt: "Weather in Paris?", tools, stopWhen: stepCountIs(4) }), + ), + )); + } else { + throw new Error(`unknown scenario ${scenario}`); + } + console.log(JSON.stringify({ text })); + await failproofai.flush(); +} + +main(Deno.args[0] ?? "telemetry").catch((error: unknown) => { + console.error("FATAL", error); + Deno.exit(1); +}); + +declare const Deno: { args: string[]; exit(code: number): never }; diff --git a/sdk/typescript/integration/fixtures/runtimes/package-lock.json b/sdk/typescript/integration/fixtures/runtimes/package-lock.json new file mode 100644 index 000000000..7f78911e3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/runtimes/package-lock.json @@ -0,0 +1,484 @@ +{ + "name": "failproofai-it-runtimes", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-runtimes", + "dependencies": { + "ai": "6.0.288", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4", + "bun": "1.4.2", + "deno": "2.9.6" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.198", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.198.tgz", + "integrity": "sha512-Ji11GLswEsPkDZHMrG1s/Yq7NhmSUZGHM0IJB3lfpjE0MnGrYk1egmWzPL3GmDWJCw2wab8BgerSi68JiabaXQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.52", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.16.tgz", + "integrity": "sha512-9Av6kg0t/IN/dcYAAmEJ4B9OPhcEJqwSR+GfHEA8olRkindXpUHadc3p3cyvgFUQFTVm1G5thtsmgZ9yVb2w3A==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.52", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.52.tgz", + "integrity": "sha512-cUimyIz1jjwYoF9n6HO7sU+UBW7Nu3T4lW7562yRvEhM9/8JqlW4cecxSpkiafjRzAphZG2NfJ3/Ch5cHoCIhg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" + }, + "engines": { + "node": ">=18.17" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@deno/darwin-arm64": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/darwin-arm64/-/darwin-arm64-2.9.6.tgz", + "integrity": "sha512-V8uO1Aolrl/yvdMb5lOtgtigs8YuZBjrOrgviVMiYkQ0swymFfzkae1wZak4Xi24t7vf7d/eiMb/7ryjjVm+tg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@deno/darwin-x64": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/darwin-x64/-/darwin-x64-2.9.6.tgz", + "integrity": "sha512-tPi3CWPRdjW1MaWlvWjORE7Ass5SADdWdj1BF1Q1PEdzi+5u/mC5YhmZzniMiPr81PpaFeqjXaLAsfb8lFsZ0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@deno/linux-arm64-glibc": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/linux-arm64-glibc/-/linux-arm64-glibc-2.9.6.tgz", + "integrity": "sha512-poqhoH9B3nYloigEMPT5t8HY+wBeiuea1An5Q7fqgJzP0SYmWsoS/oUnWO/pEiA4ofzIBy1Elt/in58xKtxEoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@deno/linux-x64-glibc": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/linux-x64-glibc/-/linux-x64-glibc-2.9.6.tgz", + "integrity": "sha512-3md4TKLCuzsLDmfOoC5KN7NAZaxSVkMgm64vg6foD00JBADCpYS2/5QqXfzgnP5ZQz3iLP9vBQJtEgc116OAoQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@deno/win32-arm64": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/win32-arm64/-/win32-arm64-2.9.6.tgz", + "integrity": "sha512-iHSHs89g2Uy6k0bGHmXkgzTIW8WXImYPpvqRRO/zNsK1/N79PcjvAth1TuQatuIALcLdocstPs/PAYtndbfr0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@deno/win32-x64": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@deno/win32-x64/-/win32-x64-2.9.6.tgz", + "integrity": "sha512-qRGnmVz/Ea6UWPht/S3xFRK17UZ/ls38DSVfgGB17fpASLxPkksqzWUI/BzL9YYoOknYzfc94mySGyInZu/JXw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@oven/bun-darwin-aarch64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-darwin-aarch64/-/bun-darwin-aarch64-1.4.2.tgz", + "integrity": "sha512-MXdZkP1featqxZ+/VTXWG1BVjM4OGBehVY2Q88EeUj/7L0UMeCGItmyPYTN+wxvlGJ6F66JEtzsw+GvQWewnag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oven/bun-darwin-x64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-darwin-x64/-/bun-darwin-x64-1.4.2.tgz", + "integrity": "sha512-gZTxZuLjkUhAWjTETu3tw0WhsEdNkJ64daj60ybhPf835a2yollV3yTkK9JozvzKPx4TRFzLSl8C+U525pxVbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oven/bun-freebsd-aarch64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-freebsd-aarch64/-/bun-freebsd-aarch64-1.4.2.tgz", + "integrity": "sha512-SMNItMw1Z8QeeQVKnw8jA7xQNkeXdP+OPgin4Wi/QTx/B8RHHLnuZfqmFy7NtVeT2NF0kKYppW4WWd2CCYZjhQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@oven/bun-freebsd-x64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-freebsd-x64/-/bun-freebsd-x64-1.4.2.tgz", + "integrity": "sha512-THbPKXhO54N0DpFRKZNDZpQ7dpbX0bWASuARckAUS9wRtFIHsiY+uULXJvxJGo2YD1YewvXQ4G8Fj7XT5oBCiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@oven/bun-linux-aarch64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64/-/bun-linux-aarch64-1.4.2.tgz", + "integrity": "sha512-3BBP9ovJ2RGHFH6Ae1CAtxNtG1+YY6GD6rmYbsUosoAk9+OEl6zeDQ/k4fBkc6dYOJCtWnx8hUxzNzQATSmvYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oven/bun-linux-aarch64-android": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-android/-/bun-linux-aarch64-android-1.4.2.tgz", + "integrity": "sha512-3mZKO2rhsNgbAUtAHC1UKUlF2zTxFraDZT/Elv8wzyH0fJL9h+Iv3TgB9lO63w89PRn3eFe+NRA1bhVgikKNPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@oven/bun-linux-aarch64-musl": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-musl/-/bun-linux-aarch64-musl-1.4.2.tgz", + "integrity": "sha512-+Sm6y+lSiSFBOtXmnekp5Q6n1tUKlyv71FCPWBc61Cgb14T5eBs8SN/nh4MUCOKzONkI3O+as3MGUgikS4aCBQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oven/bun-linux-x64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-x64/-/bun-linux-x64-1.4.2.tgz", + "integrity": "sha512-9/E/UXOTpSo3YsV5g+FhtTd/qTpiWoKuxS12cqtuYA1ssu9fRAoPQnipFgGyck3tWO63iUdxBiygq+kELFawng==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oven/bun-linux-x64-android": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-android/-/bun-linux-x64-android-1.4.2.tgz", + "integrity": "sha512-6HC5tzcC79113n2IHCTJMWv+HsQImv4ZFEK2XpYLxY6HbT8tM4cUM2Zv1bHZBQsS3jv/zYBamDJ1UX7If0d5tw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@oven/bun-linux-x64-musl": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-musl/-/bun-linux-x64-musl-1.4.2.tgz", + "integrity": "sha512-vVTKUg1bnPhRP/Hp73jIVoFh2vPFNYEqYX0ERKfZBOQEEHitNAeukZzzuUDZS0SoDCIpuWUGSpd/CDMbjdR+Uw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oven/bun-windows-aarch64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-windows-aarch64/-/bun-windows-aarch64-1.4.2.tgz", + "integrity": "sha512-8EJ1ST7339WJE3poPW5nBgVW/lWf9HBz4W27ZUNhburKmcBLOByPyE6DP9fHD8FQGm5c+ilUN2hX1mrW0jxq9Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@oven/bun-windows-x64": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@oven/bun-windows-x64/-/bun-windows-x64-1.4.2.tgz", + "integrity": "sha512-+bN6OuVld/9diT/RLSXSW7JE6CvNE3gL9XsAEjULi1nUsXd6DNO6GuA9jNdNb3r8PdJFnYHr5aypNV1Oj3Rd9g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/ai": { + "version": "6.0.288", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.288.tgz", + "integrity": "sha512-p7N1TMAPkTWlapALDVQQY3+QbUKP1nKatz/yg9SArWWjUdUyzahyCN43WG+cTQBlN5mj5sysRTjDvZWRezpZeA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.198", + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.52", + "@opentelemetry/api": "^1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/bun": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/bun/-/bun-1.4.2.tgz", + "integrity": "sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA==", + "cpu": [ + "arm64", + "x64" + ], + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "os": [ + "darwin", + "linux", + "android", + "freebsd", + "win32" + ], + "bin": { + "bun": "bin/bun.exe", + "bunx": "bin/bunx.exe" + }, + "optionalDependencies": { + "@oven/bun-darwin-aarch64": "1.4.2", + "@oven/bun-darwin-x64": "1.4.2", + "@oven/bun-freebsd-aarch64": "1.4.2", + "@oven/bun-freebsd-x64": "1.4.2", + "@oven/bun-linux-aarch64": "1.4.2", + "@oven/bun-linux-aarch64-android": "1.4.2", + "@oven/bun-linux-aarch64-musl": "1.4.2", + "@oven/bun-linux-x64": "1.4.2", + "@oven/bun-linux-x64-android": "1.4.2", + "@oven/bun-linux-x64-musl": "1.4.2", + "@oven/bun-windows-aarch64": "1.4.2", + "@oven/bun-windows-x64": "1.4.2" + } + }, + "node_modules/deno": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/deno/-/deno-2.9.6.tgz", + "integrity": "sha512-cr16GxYiUeH1juNvFTMOxt99V5QRVmANfy5UsZvE5hjC6P/o4Op/MBbRhLeBOL8WJpV6tr9hb+Kjb1MW0nEoVg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "deno": "bin.cjs" + }, + "optionalDependencies": { + "@deno/darwin-arm64": "2.9.6", + "@deno/darwin-x64": "2.9.6", + "@deno/linux-arm64-glibc": "2.9.6", + "@deno/linux-x64-glibc": "2.9.6", + "@deno/win32-arm64": "2.9.6", + "@deno/win32-x64": "2.9.6" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/undici": { + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/runtimes/package.json b/sdk/typescript/integration/fixtures/runtimes/package.json new file mode 100644 index 000000000..ba88a5346 --- /dev/null +++ b/sdk/typescript/integration/fixtures/runtimes/package.json @@ -0,0 +1,15 @@ +{ + "name": "failproofai-it-runtimes", + "private": true, + "type": "module", + "description": "Integration fixture: the pinned non-Node runtimes (Bun, Deno) that the core smoke (agent.ts), the short-lived-handler cases and every framework fixture's agent run under. Installed with --ignore-scripts, so each runtime is its platform package's own binary, which the harness locates. `ai` and `zod` are for deno-npm.ts, which imports them through `npm:` specifiers.", + "dependencies": { + "ai": "6.0.288", + "zod": "4.6.5" + }, + "devDependencies": { + "@types/node": "22.20.4", + "bun": "1.4.2", + "deno": "2.9.6" + } +} diff --git a/sdk/typescript/integration/fixtures/types/agent.ts b/sdk/typescript/integration/fixtures/types/agent.ts new file mode 100644 index 000000000..cc18f41c3 --- /dev/null +++ b/sdk/typescript/integration/fixtures/types/agent.ts @@ -0,0 +1,79 @@ +/** + * A consumer of every public entry point — TYPECHECKED ONLY, never run. + * + * `types.test.ts` compiles this file under every module/moduleResolution pair a + * customer's tsconfig can plausibly carry, as `.ts`, `.mts` and `.cts`. Each + * import must resolve to real declarations of the right module format, and the + * `@ts-expect-error` lines prove the names arrived TYPED: an entry that silently + * resolved to `any` would leave them unused, which is itself an error. + */ +import { + agent, + configure, + flush, + instrument, + session, + version, + type AgentScope, + type FrameworkName, + type SessionScope, +} from "@failproofai/sdk"; +import { adapter as aiAdapter, telemetry, wrapTool } from "@failproofai/sdk/ai"; +import { adapter as mastraAdapter, workflow } from "@failproofai/sdk/mastra"; +import { adapter as langchainAdapter, langchainHandler } from "@failproofai/sdk/langchain"; +import { adapter as llamaindexAdapter } from "@failproofai/sdk/llamaindex"; +import { EvalResult, Evaluator, Score } from "@failproofai/sdk/evaluator"; +import { withFailproofai } from "@failproofai/sdk/next"; +import * as sandboxWorker from "@failproofai/sdk/sandbox-worker"; + +configure({}); +// @ts-expect-error configure takes an options object, not a number. +configure(42); + +const span: AgentScope = agent.open("planner", { goal: "typecheck" }); +const scope: SessionScope = session.open({ sessionId: "s-1" }); +const disposers: Array<() => void> = [span.dispose.bind(span), scope.dispose.bind(scope)]; +const answer: number = agent("planner", (identity) => identity.depth); +// @ts-expect-error agent() returns what its body returns. +const wrong: string = agent("planner", () => 1); + +const frameworks: Promise = instrument(); +const flushed: Promise = flush(); +const sdkVersion: string = version; + +const adapterNames: string[] = [aiAdapter, mastraAdapter, langchainAdapter, llamaindexAdapter].map( + (adapter) => adapter.name, +); +const handler: Record = langchainHandler(); +const settings = telemetry(); +const tool = wrapTool("get_weather", { execute: (input: { city: string }) => `sunny in ${input.city}` }); +const workflowResult: number = workflow("pipeline", () => 7); + +const evaluator = new Evaluator({ name: "quality", version: "1" }); +evaluator.eval("score", { version: "1" }, () => new EvalResult({ score: new Score(0.5) })); +// @ts-expect-error a Score's value is a number. +new Score("high"); + +const nextConfig = withFailproofai({ reactStrictMode: true }); +const externals: string[] = nextConfig.serverExternalPackages; +const strict: boolean = nextConfig.reactStrictMode; +// @ts-expect-error the wrapped config keeps its own field types. +const notStrict: string = nextConfig.reactStrictMode; + +export type SandboxWorker = typeof sandboxWorker; +export { + adapterNames, + answer, + disposers, + externals, + notStrict, + strict, + flushed, + frameworks, + handler, + sdkVersion, + settings, + tool, + workflowResult, + wrong, +}; diff --git a/sdk/typescript/integration/fixtures/types/package-lock.json b/sdk/typescript/integration/fixtures/types/package-lock.json new file mode 100644 index 000000000..18edbb29c --- /dev/null +++ b/sdk/typescript/integration/fixtures/types/package-lock.json @@ -0,0 +1,740 @@ +{ + "name": "failproofai-it-types", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-types", + "devDependencies": { + "@arethetypeswrong/cli": "0.18.5", + "@types/node": "22.20.4", + "typescript": "5.9.3", + "typescript-min": "npm:typescript@5.4.5" + } + }, + "node_modules/@andrewbranch/untar.js": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@andrewbranch/untar.js/-/untar.js-1.0.4.tgz", + "integrity": "sha512-pVXSwPsLuw8IGLo2Di0EaOfsk+ntVvpkk942J/sHYIkwvtKUakEcPh7HBgZ6tuimgzKSEHgCvO4XgQ05DEbwDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@arethetypeswrong/cli": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/@arethetypeswrong/cli/-/cli-0.18.5.tgz", + "integrity": "sha512-gM+8vRsQOD/Uc7EnBedUhkG5OCsDWE4uoak5QvomGpMpaky0Eh41p04nIMgrWb8EOmqZUJGc6zz9hsP6E56R7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@arethetypeswrong/core": "0.18.5", + "chalk": "^4.1.2", + "cli-table3": "^0.6.3", + "commander": "^10.0.1", + "marked": "^9.1.2", + "marked-terminal": "^7.1.0", + "semver": "^7.5.4" + }, + "bin": { + "attw": "dist/index.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@arethetypeswrong/core": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/@arethetypeswrong/core/-/core-0.18.5.tgz", + "integrity": "sha512-9ytjzGwxjm9Uz7I9avfbt5vlQt6uk9uRRESzJjqrznl6WKvI6dwYTo+vJ3U02Wrq/mR3iql/PzhvHhKdJIAjDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@andrewbranch/untar.js": "^1.0.3", + "@loaderkit/resolve": "^1.0.2", + "cjs-module-lexer": "^1.2.3", + "fflate": "^0.8.3", + "lru-cache": "^11.0.1", + "semver": "^7.5.4", + "typescript": "5.6.1-rc", + "validate-npm-package-name": "^5.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@arethetypeswrong/core/node_modules/typescript": { + "version": "5.6.1-rc", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.1-rc.tgz", + "integrity": "sha512-E3b2+1zEFu84jB0YQi9BORDjz9+jGbwwy1Zi3G0LUNw7a7cePUrHMRNy8aPh53nXpkFGVHSxIZo5vKTfYaFiBQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/@braidai/lang": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@braidai/lang/-/lang-1.1.2.tgz", + "integrity": "sha512-qBcknbBufNHlui137Hft8xauQMTZDKdophmLFv05r2eNmdIv/MlPuP4TdUknHG68UdWLgVZwgxVe735HzJNIwA==", + "dev": true, + "license": "ISC" + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@loaderkit/resolve": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@loaderkit/resolve/-/resolve-1.0.6.tgz", + "integrity": "sha512-G8FdIoF5CypfwmD9rl8BXod5HDn8JqB0CCNBXDTaRZ+yRYhARrrSToX1zg1zy9jX3zLqigsELwhT4gNtkdQAUg==", + "dev": true, + "license": "ISC", + "dependencies": { + "@braidai/lang": "^1.0.0" + } + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true, + "license": "MIT" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-highlight": { + "version": "2.1.11", + "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", + "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", + "dev": true, + "license": "ISC", + "dependencies": { + "chalk": "^4.0.0", + "highlight.js": "^10.7.1", + "mz": "^2.4.0", + "parse5": "^5.1.1", + "parse5-htmlparser2-tree-adapter": "^6.0.0", + "yargs": "^16.0.0" + }, + "bin": { + "highlight": "bin/highlight" + }, + "engines": { + "node": ">=8.0.0", + "npm": ">=5.0.0" + } + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/emojilib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/emojilib/-/emojilib-2.4.0.tgz", + "integrity": "sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "dev": true, + "license": "MIT" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/marked": { + "version": "9.1.6", + "resolved": "https://registry.npmjs.org/marked/-/marked-9.1.6.tgz", + "integrity": "sha512-jcByLnIFkd5gSXZmjNvS1TlmRhCXZjIzHYlaGkPlLIekG55JDR2Z4va9tZwCiP+/RDERiNhMOFu01xd6O5ct1Q==", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/marked-terminal": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/marked-terminal/-/marked-terminal-7.3.0.tgz", + "integrity": "sha512-t4rBvPsHc57uE/2nJOLmMbZCQ4tgAccAED3ngXQqW6g+TxA488JzJ+FK3lQkzBQOI1mRV/r/Kq+1ZlJ4D0owQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "ansi-regex": "^6.1.0", + "chalk": "^5.4.1", + "cli-highlight": "^2.1.11", + "cli-table3": "^0.6.5", + "node-emoji": "^2.2.0", + "supports-hyperlinks": "^3.1.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "marked": ">=1 <16" + } + }, + "node_modules/marked-terminal/node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/node-emoji": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-2.2.0.tgz", + "integrity": "sha512-Z3lTE9pLaJF47NyMhd4ww1yFTAP8YhYI8SleJiHzM46Fgpm5cnNzSl9XfzFNqbaz+VlJrIj3fXQ4DeN1Rjm6cw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.6.0", + "char-regex": "^1.0.2", + "emojilib": "^2.4.0", + "skin-tone": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse5": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", + "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==", + "dev": true, + "license": "MIT" + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz", + "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "parse5": "^6.0.1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/skin-tone": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/skin-tone/-/skin-tone-2.0.0.tgz", + "integrity": "sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "unicode-emoji-modifier-base": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-hyperlinks": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.2.0.tgz", + "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">=14.18" + }, + "funding": { + "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-min": { + "name": "typescript", + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unicode-emoji-modifier-base": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unicode-emoji-modifier-base/-/unicode-emoji-modifier-base-1.0.0.tgz", + "integrity": "sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/validate-npm-package-name": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-5.0.1.tgz", + "integrity": "sha512-OljLrQ9SQdOUqTaQxqL5dEfZWrXExyyWsozYlAWFawPVNuD83igl7uJD2RTkNMbniIYgt8l81eCJGIdQF7avLQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "16.2.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.2.tgz", + "integrity": "sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + } + } +} diff --git a/sdk/typescript/integration/fixtures/types/package.json b/sdk/typescript/integration/fixtures/types/package.json new file mode 100644 index 000000000..06d5d6f98 --- /dev/null +++ b/sdk/typescript/integration/fixtures/types/package.json @@ -0,0 +1,11 @@ +{ + "name": "failproofai-it-types", + "private": true, + "description": "Integration fixture: the packed @failproofai/sdk's type declarations under every consumer tsconfig, on the newest and the oldest supported TypeScript. No \"type\" field on purpose: a .ts file here is CommonJS, as in a default `npm init` project.", + "devDependencies": { + "@arethetypeswrong/cli": "0.18.5", + "@types/node": "22.20.4", + "typescript": "5.9.3", + "typescript-min": "npm:typescript@5.4.5" + } +} diff --git a/sdk/typescript/integration/fixtures/vanilla/agent.ts b/sdk/typescript/integration/fixtures/vanilla/agent.ts new file mode 100644 index 000000000..6b01130e1 --- /dev/null +++ b/sdk/typescript/integration/fixtures/vanilla/agent.ts @@ -0,0 +1,197 @@ +/** + * No framework — a real agent loop, hand-instrumented. + * + * npm install @failproofai/sdk openai + * OPENAI_API_KEY=… npx tsx examples/research-agent.ts + * + * A working tool-calling loop against the OpenAI API with no agent framework at + * all, instrumented by hand. The reference for "my agent is bespoke" — and the + * TypeScript twin of the Python SDK's `docs/manual/examples/research_agent.py`. + * + * Every hand-built agent already has three places this touches, whatever its + * functions are called: + * + * 1. where ONE RUN starts and ends → failproofai.agent() agent_start / agent_end + * 2. the ONE FUNCTION that calls a model → event.modelRequest/Response one pair per model turn + * 3. the ONE FUNCTION that runs a tool → failproofai.toolCall() tool_use / tool_result + * + * Identity is ambient: everything inside `agent()` lands on its session, so + * nothing else in the program changes. The one rule: emit the pairs — a + * `model_request` with no `model_response` is a span the dashboard shows as + * running forever. + * + * Environment: OPENAI_API_KEY; optionally OPENAI_BASE_URL (any OpenAI-compatible + * endpoint) and MODEL (default gpt-4o-mini). + */ +import { randomUUID } from "node:crypto"; + +import * as failproofai from "@failproofai/sdk"; +import OpenAI from "openai"; +import type { ChatCompletionMessageParam, ChatCompletionTool } from "openai/resources/chat/completions"; + +failproofai.configure({ environment: "examples" }); + +const client = new OpenAI(); // reads OPENAI_API_KEY / OPENAI_BASE_URL +const MODEL = process.env.MODEL ?? "gpt-4o-mini"; + +// ---------------------------------------------------------------- the tools + +const PRICE: Record = { widget: 42.0, gadget: 17.5 }; +const STOCK: Record = { widget: 120, gadget: 0 }; + +const TOOLS: ChatCompletionTool[] = [ + { + type: "function", + function: { + name: "price_of", + description: "Unit price of an item. Valid: widget, gadget.", + parameters: { type: "object", properties: { item: { type: "string" } }, required: ["item"] }, + }, + }, + { + type: "function", + function: { + name: "stock_of", + description: "Units in stock. Valid: widget, gadget.", + parameters: { type: "object", properties: { item: { type: "string" } }, required: ["item"] }, + }, + }, +]; + +function runTool(name: string, args: { item?: string }): string { + const item = String(args.item ?? "").toLowerCase().trim(); + const table = name === "price_of" ? PRICE : name === "stock_of" ? STOCK : null; + if (table === null) throw new Error(`unknown tool ${name}`); + if (!(item in table)) throw new Error(`unknown item ${JSON.stringify(item)}`); + return String(table[item]); +} + +// ---------------------------------------------------------- edit site 2 of 3 +// The one function that calls the model. `requestId` pairs the two halves even +// when calls overlap; `duration_ms` is yours to set — model events are not +// timed for you. A failed call still closes its pair, with the error, before +// rethrowing: the enclosing agent() then ends "failed". Only the provider call +// sits in the `try`, so nothing but a failed call can reach the error path. + +async function callModel(messages: ChatCompletionMessageParam[]) { + const requestId = randomUUID(); + const started = Date.now(); + failproofai.event.modelRequest({ + model: MODEL, + requestId, + // Role and content, plus the ids that link a tool result to the call that + // asked for it: what a reader of the trace needs, not the provider's full + // message objects. + messages: messages.map((m) => ({ + role: m.role, + content: typeof m.content === "string" ? m.content : m.content == null ? "" : JSON.stringify(m.content), + ...(m.role === "tool" ? { tool_call_id: m.tool_call_id } : {}), + ...(m.role === "assistant" && m.tool_calls + ? { tool_calls: m.tool_calls.map((c) => ({ id: c.id, name: c.type === "function" ? c.function.name : c.type })) } + : {}), + })), + tools: TOOLS.flatMap((t) => (t.type === "function" ? [{ name: t.function.name, description: t.function.description ?? "" }] : [])), + }); + let reply: OpenAI.Chat.Completions.ChatCompletion; + try { + reply = await client.chat.completions.create({ model: MODEL, messages, tools: TOOLS }); + } catch (error) { + failproofai.event.modelResponse({ + model: MODEL, + requestId, + stopReason: "error", + error: error instanceof Error ? `${error.constructor.name}: ${error.message}` : String(error), + duration_ms: Date.now() - started, + }); + throw error; + } + const choice = reply.choices[0]!; + const calls = (choice.message.tool_calls ?? []).filter((c) => c.type === "function"); + failproofai.event.modelResponse({ + model: reply.model, + requestId, + role: choice.message.role, + content: choice.message.content ?? "", + stopReason: choice.finish_reason, + inputTokens: reply.usage?.prompt_tokens ?? null, + outputTokens: reply.usage?.completion_tokens ?? null, + duration_ms: Date.now() - started, + // What the model asked for, so a turn that is only tool calls is not blank — + // the field and shape the framework adapters write. + fw_tool_calls: calls.map((c) => ({ toolCallId: c.id, toolName: c.function.name, input: c.function.arguments })), + }); + return choice.message; +} + +// ---------------------------------------------------------- edit site 3 of 3 +// The one function that runs tools. Reuse the model's own tool-call id, so a +// tool_use lines up with the tool_calls[] entry that asked for it. toolCall() +// times it and records a throw as tool_result.error — then rethrows, and here +// the loop turns that into a tool message so the model can recover. + +async function dispatch(call: { id: string; function: { name: string; arguments: string } }): Promise { + // Malformed arguments from the model are still a tool call: recorded with the + // raw text as input and failed inside toolCall(), so the trace shows it and + // the model gets an error it can recover from — not a crashed run, and not a + // call that silently never appears. + let args: { item?: string } = {}; + let malformed: unknown; + try { + const parsed: unknown = JSON.parse(call.function.arguments || "{}"); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new TypeError("tool arguments must be a JSON object"); + } + args = parsed as { item?: string }; + } catch (error) { + malformed = error; + } + try { + const input = malformed === undefined ? args : { arguments: call.function.arguments }; + return await failproofai.toolCall(call.function.name, { toolCallId: call.id, input }, async () => { + if (malformed !== undefined) throw malformed; + return runTool(call.function.name, args); + }); + } catch (error) { + return `error: ${error instanceof Error ? error.message : String(error)}`; + } +} + +// ---------------------------------------------------------- edit site 1 of 3 +// Where one run starts and ends. In a service, pass your own request or job id +// as `sessionId`, so a session on the dashboard and a record in your own +// database are the same string. + +async function main(): Promise { + const question = process.argv[2] || "Price and stock for widget and gadget?"; + const messages: ChatCompletionMessageParam[] = [ + { role: "system", content: "Use the tools for every number. Be terse." }, + { role: "user", content: question }, + ]; + + let sessionId = ""; + const answer = await failproofai.agent("inventory", { goal: question }, async (identity) => { + sessionId = identity.sessionId ?? ""; + for (let turn = 0; turn < 6; turn++) { + // bounded: an unbounded agent loop is its own bug + const message = await callModel(messages); + const calls = (message.tool_calls ?? []).filter((c) => c.type === "function"); + if (calls.length === 0) return message.content ?? ""; + messages.push(message); + for (const call of calls) { + messages.push({ role: "tool", tool_call_id: call.id, content: await dispatch(call) }); + } + } + return "(gave up after 6 turns)"; + }); + + console.log(answer); + console.log(`session ${sessionId}`); +} + +main() + .catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }) + // A short script must flush before it exits; a server flushes on its own. + .finally(() => failproofai.flush()); diff --git a/sdk/typescript/integration/fixtures/vanilla/package-lock.json b/sdk/typescript/integration/fixtures/vanilla/package-lock.json new file mode 100644 index 000000000..e3129b4ba --- /dev/null +++ b/sdk/typescript/integration/fixtures/vanilla/package-lock.json @@ -0,0 +1,70 @@ +{ + "name": "failproofai-it-vanilla", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "failproofai-it-vanilla", + "dependencies": { + "openai": "7.23.0" + }, + "devDependencies": { + "@types/node": "22.20.4" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/openai": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-7.23.0.tgz", + "integrity": "sha512-0ecOXnFSMNWZq6cUBcTV6Lf93y+fm8BH/+QzFvpoP1UGNOE5pnytRa6HAPGCw+3IS9SIAW7rc1M8yyzN1PiBAQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", + "undici": ">=5 <9", + "ws": "^8.21.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "undici": { + "optional": true + }, + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/sdk/typescript/integration/fixtures/vanilla/package.json b/sdk/typescript/integration/fixtures/vanilla/package.json new file mode 100644 index 000000000..433789289 --- /dev/null +++ b/sdk/typescript/integration/fixtures/vanilla/package.json @@ -0,0 +1,12 @@ +{ + "name": "failproofai-it-vanilla", + "private": true, + "type": "module", + "description": "Integration fixture: a hand-instrumented agent with no framework (examples/research-agent.ts), on the real openai client, against the packed @failproofai/sdk.", + "dependencies": { + "openai": "7.23.0" + }, + "devDependencies": { + "@types/node": "22.20.4" + } +} diff --git a/sdk/typescript/integration/fixtures/vanilla/tsconfig.json b/sdk/typescript/integration/fixtures/vanilla/tsconfig.json new file mode 100644 index 000000000..3665b2029 --- /dev/null +++ b/sdk/typescript/integration/fixtures/vanilla/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "files": ["agent.ts"] +} diff --git a/sdk/typescript/integration/global-setup.ts b/sdk/typescript/integration/global-setup.ts new file mode 100644 index 000000000..cc1fa13dd --- /dev/null +++ b/sdk/typescript/integration/global-setup.ts @@ -0,0 +1,29 @@ +import type { TestProject } from "vitest/node"; + +import { fixtures, installFixture, packedTarball } from "./harness.js"; + +declare module "vitest" { + export interface ProvidedContext { + tarball: string; + } +} + +/** + * Pack the SDK once and install every fixture before any test runs. + * + * A fixture that fails to install FAILS the run. There is no skip: the Python + * SDK's integration job learned the hard way that "4 skipped" reads as green + * while testing nothing (`AGENTEYE_TESTS_REQUIRE_FRAMEWORKS`), and a harness + * whose only job is to prove the frameworks work cannot be allowed to pass + * without them. + */ +export default async function setup(project: TestProject): Promise { + const tarball = packedTarball(); + project.provide("tarball", tarball); + const only = process.env.FAILPROOFAI_IT_FIXTURES?.split(",").filter(Boolean); + await Promise.all( + fixtures() + .filter((name) => !only || only.includes(name)) + .map((name) => installFixture(name, tarball)), + ); +} diff --git a/sdk/typescript/integration/harness.ts b/sdk/typescript/integration/harness.ts new file mode 100644 index 000000000..e1b2407be --- /dev/null +++ b/sdk/typescript/integration/harness.ts @@ -0,0 +1,451 @@ +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; + +/** + * The real-framework harness. + * + * Everything under `test/` runs against `src/` with no framework installed, so + * it can prove an adapter's logic and nothing about whether that logic ever + * reaches the framework. The failures this directory exists for are invisible + * from there and total from outside it: + * + * * an adapter that patches the CommonJS copy of a dual-published framework + * while an ES-module app runs the ESM copy — `instrument()` reports + * success, every event is lost, nothing warns; + * * a framework major that moved the extension point — the adapter still + * installs, the callbacks never fire; + * * type declarations that resolve in the SDK's own `nodenext` tsconfig and + * in no CommonJS project's. + * + * So each fixture is a real consumer project: its own `package.json`, its own + * lockfile pinning a real framework release, the PACKED tarball extracted into + * its `node_modules` exactly as `npm install @failproofai/sdk` would lay it out, + * and one `agent.ts` run twice — once transpiled to an ES module and once to + * CommonJS — because the two module systems load different copies of the + * framework and both are what customers run. + */ + +export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const FIXTURES = join(ROOT, "integration", "fixtures"); + +export type Format = "esm" | "cjs" | "bun-esm" | "bun-cjs" | "deno-esm" | "deno-cjs"; +/** The two module systems, under Node. What every framework file runs. */ +export const FORMATS: readonly Format[] = ["esm", "cjs"]; +/** + * The same two transpiled agents, under Bun. A framework test file opts in with + * `describe.each([...FORMATS, ...BUN_FORMATS])`; `runtimes.bun.test.ts` runs + * every scenario of every fixture this way against its Node twin. + */ +export const BUN_FORMATS: readonly Format[] = ["bun-esm", "bun-cjs"]; +/** The same two transpiled agents, under Deno (`deno run -A`). */ +export const DENO_FORMATS: readonly Format[] = ["deno-esm", "deno-cjs"]; + +export type Runtime = "node" | "bun" | "deno"; + +/** Which runtime a format runs under, and which transpiled agent it runs. */ +export function splitFormat(format: Format): { runtime: Runtime; module: "esm" | "cjs" } { + const [head, tail] = format.split("-") as [string, string | undefined]; + if (tail === undefined) return { runtime: "node", module: head as "esm" | "cjs" }; + return { runtime: head as Runtime, module: tail as "esm" | "cjs" }; +} + +/** + * The command line that starts `runtime`, before the script and its arguments. + * + * Bun and Deno come from the `runtimes` fixture — pinned in its lockfile and + * installed with `--ignore-scripts`, so each is its platform package's own + * binary rather than whatever a developer or a CI image happens to have on + * PATH. `FAILPROOFAI_IT_BUN` / `FAILPROOFAI_IT_DENO` override that with an + * explicit binary. There is no fall-back to PATH and no skip: a runtime that + * cannot be found fails the test that needed it. + */ +export function runtimeCommand(runtime: Runtime): string[] { + if (runtime === "node") return [process.execPath]; + const override = process.env[runtime === "bun" ? "FAILPROOFAI_IT_BUN" : "FAILPROOFAI_IT_DENO"]; + const binary = override || runtimeBinary(runtime); + return runtime === "deno" ? [binary, "run", "--allow-all", "--no-lock"] : [binary]; +} + +function runtimeBinary(runtime: "bun" | "deno"): string { + const modules = join(FIXTURES, "runtimes", "node_modules"); + const scope = join(modules, runtime === "bun" ? "@oven" : "@deno"); + const candidates = existsSync(scope) + ? readdirSync(scope).map((dir) => + runtime === "bun" ? join(scope, dir, "bin", "bun") : join(scope, dir, "deno"), + ) + : []; + // npm installs every platform package whose os/cpu match; on a glibc Linux + // that can include a musl build that will not start. Take the first that runs. + for (const candidate of candidates) { + if (!existsSync(candidate)) continue; + const probe = spawnSync(candidate, ["--version"], { encoding: "utf8" }); + if (probe.status === 0) return candidate; + } + throw new Error( + `no runnable ${runtime} binary under ${scope}. The runtimes fixture installs it; ` + + `run the suite with FAILPROOFAI_IT_FIXTURES including "runtimes", or set ` + + `FAILPROOFAI_IT_${runtime.toUpperCase()} to a binary.`, + ); +} + +/** + * Every scenario a fixture's agent accepts: the `case "…":` labels of its + * dispatch `switch`. Lets a runtime-parity test cover a fixture completely + * without restating what each framework test already asserts. + */ +export function scenarios(fixture: string): string[] { + const source = readFileSync(join(FIXTURES, fixture, "agent.ts"), "utf8"); + return [...new Set([...source.matchAll(/^\s*case "([a-z0-9-]+)":/gm)].map((match) => match[1]!))]; +} + +export type Event = Record & { + type: string; + session_id: string; + agent_id: string; +}; + +export interface RunResult { + events: Event[]; + stdout: string; + stderr: string; + status: number | null; +} + +/** Pack `dist/` exactly as `npm publish` would, once per process. */ +let tarball: string | null = null; +export function packedTarball(): string { + if (tarball !== null) return tarball; + if (!existsSync(join(ROOT, "dist", "esm", "index.js"))) { + throw new Error("dist/ is not built. Run `npm run build` first (`npm run test:integration` does)."); + } + const out = mkdtempSync(join(tmpdir(), "failproofai-sdk-pack-")); + const name = execFileSync("npm", ["pack", "--silent", "--ignore-scripts", "--pack-destination", out], { + cwd: ROOT, + encoding: "utf8", + }) + .trim() + .split("\n") + .pop()!; + tarball = join(out, name); + return tarball; +} + +/** + * Install one fixture: `npm ci` against its lockfile, then the packed SDK. + * + * The SDK is extracted rather than `npm install`ed. It has no dependencies to + * resolve, and an install would rewrite the fixture's lockfile — which is the + * one file that pins the framework release this fixture exists to test. + * + * `npm ci` is skipped when `node_modules` was already built from this exact + * lockfile, so a local re-run costs seconds rather than minutes. + */ +export async function installFixture(fixture: string, pack: string): Promise { + const dir = join(FIXTURES, fixture); + const lock = join(dir, "package-lock.json"); + if (!existsSync(lock)) throw new Error(`${fixture} has no package-lock.json`); + const stamp = join(dir, "node_modules", ".failproofai-lock"); + const lockText = readFileSync(lock, "utf8"); + if (!existsSync(stamp) || readFileSync(stamp, "utf8") !== lockText) { + await run("npm", ["ci", "--no-audit", "--no-fund", "--ignore-scripts"], dir, `npm ci in ${fixture}`); + writeFileSync(stamp, lockText); + } + const target = join(dir, "node_modules", "@failproofai", "sdk"); + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + execFileSync("tar", ["-xzf", pack, "-C", target, "--strip-components=1"]); + // Runtime fixtures (`nextjs`, `runtimes`, `workers`) are not one-agent + // projects; they build or run their own sources. + if (existsSync(join(dir, "agent.ts"))) transpile(fixture); +} + +function run(command: string, args: string[], cwd: string, label: string): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { cwd, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + child.stdout.on("data", (chunk: Buffer) => (output += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (output += chunk.toString())); + child.on("error", reject); + child.on("close", (code) => + code === 0 ? resolvePromise() : reject(new Error(`${label} failed (exit ${String(code)}):\n${output}`)), + ); + }); +} + +/** Every fixture directory, i.e. every framework release under test. */ +export function fixtures(): string[] { + return readdirSync(FIXTURES, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && existsSync(join(FIXTURES, entry.name, "package.json"))) + .map((entry) => entry.name) + .sort(); +} + +/** + * Transpile `agent.ts` into both module systems, beside the fixture's + * `node_modules` so resolution is exactly the fixture's. + * + * `transpileModule` rather than `tsc`: this step is about RUNTIME resolution and + * must not fail because of a type error — type correctness is checked + * separately, by `typecheck()`, where a failure names itself. + */ +export function transpile(fixture: string): void { + const dir = join(FIXTURES, fixture); + const out = join(dir, ".run"); + mkdirSync(out, { recursive: true }); + // `agent.ts`, plus any other top-level `agent-*.ts` program a fixture carries + // for APIs only its own framework release has, and the named EXTRA_PROGRAMS + // (the ai fixtures' `surfaces.ts`) — all run through `runAgent`'s `program`. + // Each is standalone: programs never import one another. + const programs = readdirSync(dir).filter( + (name) => /^agent(-[\w-]+)?\.ts$/.test(name) || EXTRA_PROGRAMS.some((extra) => name === `${extra}.ts`), + ); + for (const name of programs) { + const base = name.slice(0, -".ts".length); + const source = readFileSync(join(dir, name), "utf8"); + const emit = (module: ts.ModuleKind, file: string): void => { + const { outputText } = ts.transpileModule(source, { + compilerOptions: { module, target: ts.ScriptTarget.ES2022, esModuleInterop: true }, + fileName: join(dir, name), + }); + writeFileSync(join(out, file), outputText); + }; + emit(ts.ModuleKind.ESNext, `${base}.mjs`); + emit(ts.ModuleKind.CommonJS, `${base}.cjs`); + } +} + +/** Entry programs besides `agent.ts` that a fixture may ship, run with `runAgent(..., program)`. */ +const EXTRA_PROGRAMS = ["surfaces"]; + +/** Run one case of a fixture's agent in one module system. */ +export function runAgent( + fixture: string, + format: Format, + scenario: string, + env: Record = {}, + /** Which `agent*.ts` program of the fixture to run; `agent` unless it carries more. */ + program = "agent", +): RunResult { + const dir = join(FIXTURES, fixture); + const home = mkdtempSync(join(tmpdir(), `failproofai-it-${fixture}-`)); + try { + const { runtime, module } = splitFormat(format); + const entry = join(dir, ".run", `${program}.${module === "esm" ? "mjs" : "cjs"}`); + const [command, ...prefix] = runtimeCommand(runtime); + const result = spawnSync(command!, [...prefix, entry, scenario], { + cwd: dir, + encoding: "utf8", + timeout: 90_000, + env: { + ...process.env, + // Never the developer's real spool: a running daemon would collect it. + FAILPROOFAI_HOME: home, + FAILPROOFAI_SDK_STRICT: "1", + NODE_OPTIONS: "", + ...env, + }, + }); + return { + events: readSpool(join(home, "custom-agents", "events")), + stdout: result.stdout, + stderr: result.stderr, + status: result.status, + }; + } finally { + rmSync(home, { recursive: true, force: true }); + } +} + +/** + * `runAgent`, without blocking the test worker — so a file that runs hundreds + * of cases (the runtime-parity suites) can run them `concurrent`ly. + */ +export async function runAgentAsync( + fixture: string, + format: Format, + scenario: string, + env: Record = {}, +): Promise { + const dir = join(FIXTURES, fixture); + const { runtime, module } = splitFormat(format); + const entry = join(dir, ".run", module === "esm" ? "agent.mjs" : "agent.cjs"); + const [command, ...prefix] = runtimeCommand(runtime); + return await runProcess([command!, ...prefix, entry, scenario], { cwd: dir, env, label: fixture }); +} + +/** + * Run any command against a fresh scratch spool and collect what it wrote. + * The building block of `runAgentAsync`, and of the runtime suites whose + * programs are not a fixture's `agent.ts`. + */ +export async function runProcess( + argv: string[], + options: { cwd: string; env?: Record; label?: string; timeout?: number; home?: string }, +): Promise { + const home = options.home ?? mkdtempSync(join(tmpdir(), `failproofai-it-${options.label ?? "run"}-`)); + try { + const { stdout, stderr, status } = await new Promise<{ stdout: string; stderr: string; status: number | null }>( + (resolvePromise) => { + const child = spawn(argv[0]!, argv.slice(1), { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + FAILPROOFAI_HOME: home, + FAILPROOFAI_SDK_STRICT: "1", + NODE_OPTIONS: "", + ...options.env, + }, + }); + let out = ""; + let err = ""; + child.stdout.on("data", (chunk: Buffer) => (out += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (err += chunk.toString())); + const timer = setTimeout(() => child.kill("SIGKILL"), options.timeout ?? 90_000); + child.on("error", (error) => { + clearTimeout(timer); + resolvePromise({ stdout: out, stderr: `${err}\n${String(error)}`, status: null }); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolvePromise({ stdout: out, stderr: err, status: code }); + }); + }, + ); + return { events: readSpool(join(home, "custom-agents", "events")), stdout, stderr, status }; + } finally { + if (options.home === undefined) rmSync(home, { recursive: true, force: true }); + } +} + +/** Every event in a spool directory, in the order the run produced them. */ +export function readSpool(dir: string): Event[] { + if (!existsSync(dir)) return []; + const events: Event[] = []; + for (const name of readdirSync(dir).filter((n) => n.endsWith(".jsonl")).sort()) { + for (const line of readFileSync(join(dir, name), "utf8").split("\n")) { + if (line.trim()) events.push(JSON.parse(line) as Event); + } + } + // Batches are named by time, then pid, then sequence; order within the run + // is the order the timestamps say, with the file order breaking ties. + return events + .map((event, index) => ({ event, index })) + .sort((a, b) => String(a.event.timestamp).localeCompare(String(b.event.timestamp)) || a.index - b.index) + .map(({ event }) => event); +} + +/** `tsc --noEmit` over a fixture with a given tsconfig; returns diagnostics. */ +export function typecheck(fixture: string, tsconfig = "tsconfig.json"): string { + const dir = join(FIXTURES, fixture); + const tsc = join(ROOT, "node_modules", "typescript", "bin", "tsc"); + const result = spawnSync(process.execPath, [tsc, "-p", join(dir, tsconfig), "--noEmit"], { + cwd: dir, + encoding: "utf8", + }); + return (result.stdout + result.stderr).trim(); +} + +export const count = (events: Event[], type: string): number => + events.filter((event) => event.type === type).length; + +export const ofType = (events: Event[], type: string): Event[] => + events.filter((event) => event.type === type); + +/** + * The four checks every adapter must pass, from the Python SDK's + * `skill/references/frameworks.md` ("Verifying an adapter"), plus the two that + * make them meaningful: something was recorded, and nothing was recorded twice. + * Returns the list of violations so a failure names every problem at once. + */ +export function traceViolations(events: Event[]): string[] { + const problems: string[] = []; + if (events.length === 0) return ["no events were recorded"]; + + const bySession = new Map(); + for (const event of events) { + const list = bySession.get(event.session_id) ?? []; + list.push(event); + bySession.set(event.session_id, list); + } + const UUID = /^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$/i; + + for (const [session, list] of bySession) { + // 1. The first event of each session is the root agent_start. + if (list[0]!.type !== "agent_start") { + problems.push(`session ${session} starts with ${list[0]!.type}, not agent_start`); + } + // 2. agent_id values are names, not UUIDs. + for (const id of new Set(list.map((e) => e.agent_id))) { + if (UUID.test(id)) problems.push(`session ${session} has a UUID agent_id ${id}`); + } + // 3. model_request/model_response pair on request_id, response has duration. + const requests = new Set(ofType(list, "model_request").map((e) => e.request_id)); + for (const response of ofType(list, "model_response")) { + if (!requests.has(response.request_id)) { + problems.push(`model_response ${String(response.request_id)} has no matching model_request`); + } + if (typeof response.duration_ms !== "number") { + problems.push(`model_response ${String(response.request_id)} has no duration_ms`); + } + } + if (count(list, "model_request") !== count(list, "model_response")) { + problems.push( + `session ${session}: ${count(list, "model_request")} model_request vs ` + + `${count(list, "model_response")} model_response`, + ); + } + // 4. Nothing left open. + const pairs: Array<[string, string, string]> = [ + ["tool_use", "tool_result", "tool_call_id"], + ["hook_triggered", "hook_completed", "hook_id"], + ]; + for (const [open, close, key] of pairs) { + const opened = ofType(list, open).map((e) => String(e[key])); + const closed = new Set(ofType(list, close).map((e) => String(e[key]))); + for (const id of opened) if (!closed.has(id)) problems.push(`${open} ${id} never closed`); + const dupes = opened.filter((id, i) => opened.indexOf(id) !== i); + for (const id of new Set(dupes)) problems.push(`${open} ${id} emitted more than once`); + } + const depth = new Map(); + for (const event of list) { + if (event.type === "agent_start") depth.set(event.agent_id, (depth.get(event.agent_id) ?? 0) + 1); + if (event.type === "agent_end") depth.set(event.agent_id, (depth.get(event.agent_id) ?? 0) - 1); + } + for (const [agent, open] of depth) { + if (open !== 0) problems.push(`session ${session}: agent ${agent} start/end imbalance ${open}`); + } + } + return problems; +} + +/** Shorthand for a failure message that shows the trace it is about. */ +export function describeTrace(result: RunResult): string { + const lines = result.events.map( + (e) => + ` ${e.session_id.slice(0, 12)} ${e.agent_id} ${e.type}` + + (e.tool_name ? ` tool=${JSON.stringify(e.tool_name)}` : "") + + (e.hook_name ? ` hook=${JSON.stringify(e.hook_name)}` : "") + + (e.input_tokens !== undefined ? ` tokens=${JSON.stringify([e.input_tokens, e.output_tokens])}` : ""), + ); + return [ + `exit=${String(result.status)}`, + `trace (${result.events.length}):`, + ...lines, + `stdout: ${result.stdout.slice(-1500)}`, + `stderr: ${result.stderr.slice(-3000)}`, + ].join("\n"); +} diff --git a/sdk/typescript/integration/langchain.test.ts b/sdk/typescript/integration/langchain.test.ts new file mode 100644 index 000000000..f45359645 --- /dev/null +++ b/sdk/typescript/integration/langchain.test.ts @@ -0,0 +1,682 @@ +import { describe, expect, it } from "vitest"; + +import { + FORMATS, + count, + describeTrace, + ofType, + runAgent, + traceViolations, + typecheck, + type Event, +} from "./harness.js"; + +/** + * LangChain.js / LangGraph.js, against real releases. + * + * The expected traces are the Python SDK's, captured from its LangChain adapter + * running the same graph with the same scripted model (langchain-core 1.6.3, + * langgraph 1.2.11). The two SDKs write into one pipe and the dashboard cannot + * tell which language wrote what — so for the same program they must draw the + * same tree. In particular (`sdk/python/skill/references/frameworks.md`): + * + * * the root run is the agent, named after the graph; + * * a LangGraph node is a HOOK (`hook_triggered`/`hook_completed`, + * `trigger_event="graph_node"`), never a nested agent; + * * `__start__`, routers, `RunnableSequence` steps and every other piece of + * machinery emit nothing; + * * a failure is carried by the events it happened in — no stack of `error` + * events, one per layer the exception unwound through. + */ + +/** + * Both ends of the declared peer range (`@langchain/core >=0.3.0 <2`): the 0.3 + * line with LangGraph.js 0.4, and the 1.x line with LangGraph.js 1.x. They + * differ in exactly the places an adapter breaks on — 0.3 passes no + * `toolCallId` to `handleToolStart` and LangGraph 0.4 has no graph lifecycle + * callbacks — so one set of expectations over both is the proof that the + * adapter covers the range it declares, not just the release it was written on. + */ +const FIXTURES = ["langchain-0.3", "langchain-1"] as const; + +/** One line per event: `agent_id type [hook|tool]`. */ +const shape = (events: Event[]): string[] => + events.map((e) => + [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim(), + ); + +const GRAPH = [ + "weather_graph agent_start", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph hook_triggered tools", + "weather_graph tool_use get_weather", + "weather_graph tool_result get_weather", + "weather_graph hook_completed tools", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph agent_end", +]; + +/** + * The expected trace of each plain-LangChain surface, per session. + * + * Captured from the Python adapter running the equivalent program + * (langchain-core 1.6.3) with `GenericFakeChatModel`; the rule on both sides is + * the table in `frameworks.md`: the ROOT run is the agent, named by the + * framework's own run name; an LCEL step, a prompt, a parser, a lambda and a + * `RunnableParallel` branch emit NOTHING (no `includeChains`); a chat model is + * its request/response pair; a retriever is a `retriever:` tool pair. + * + * Where the agent_id differs from Python's it is the FRAMEWORK's run name that + * differs, not the mapping: + * * `RunnableParallel.from({...})` runs as `RunnableMap` in LangChain.js + * (Python: `RunnableParallel`); + * * a `RunnableLambda` is named `RunnableLambda` whatever its function is + * called in JS (Python names it after the function), so the case sets the + * run name to Python's (`shout`) instead; + * * `withStructuredOutput` names its sequence `StructuredOutput` in JS + * (Python: `RunnableSequence`). + * `.batch()` of three inputs is three roots — three sessions, exactly as + * Python's `.batch()` is. + */ +const SURFACES: Record = { + lcel: { trace: modelRun("RunnableSequence") }, + sequence: { trace: ["RunnableSequence agent_start", "RunnableSequence agent_end"] }, + parallel: { trace: ["RunnableMap agent_start", "RunnableMap agent_end"] }, + lambda: { trace: ["shout agent_start", "shout agent_end"] }, + retriever: { trace: retrieverRun("Docs", "Docs") }, + vectorstore: { trace: retrieverRun("VectorStoreRetriever", "VectorStoreRetriever") }, + rag: { + trace: [ + "RunnableSequence agent_start", + "RunnableSequence tool_use retriever:Docs", + "RunnableSequence tool_result retriever:Docs", + "RunnableSequence model_request", + "RunnableSequence model_response", + "RunnableSequence agent_end", + ], + }, + batch3: { trace: modelRun("RunnableSequence"), sessions: 3 }, + // `streamEvents` v2 attaches LangChain's own event-stream handler beside + // ours; the trace is the `.invoke()` one, with the stream folded in. + "stream-events": { trace: modelRun("RunnableSequence") }, + "lcel-stream": { trace: modelRun("RunnableSequence") }, + structured: { trace: modelRun("StructuredOutput") }, + // `bindTools` returns a binding, which is not a run of its own: the model is + // the root, exactly as Python's `bind_tools(...).invoke()`. + "bind-tools": { trace: modelRun("ToolCallingModel") }, +}; + +function modelRun(agent: string): string[] { + return [`${agent} agent_start`, `${agent} model_request`, `${agent} model_response`, `${agent} agent_end`]; +} + +function retrieverRun(agent: string, retriever: string): string[] { + return [ + `${agent} agent_start`, + `${agent} tool_use retriever:${retriever}`, + `${agent} tool_result retriever:${retriever}`, + `${agent} agent_end`, + ]; +} + +/** + * The v1 `langchain` package's `createAgent` (langchain 1.5.12), which only the + * 1.x line has — so it lives in the langchain-1 fixture's own `agent-v1.ts`. + * + * Python golden: `langchain.agents.create_agent` 1.4.2 with the same scripted + * model, `name="weather_agent"`. Identical but for one name: LangChain.js calls + * the model node `model_request` where Python calls it `model` — the graph's + * own node name, which both adapters report verbatim. + */ +const CREATE_AGENT = [ + "weather_agent agent_start", + "weather_agent hook_triggered model_request", + "weather_agent model_request", + "weather_agent model_response", + "weather_agent hook_completed model_request", + "weather_agent hook_triggered tools", + "weather_agent tool_use get_weather", + "weather_agent tool_result get_weather", + "weather_agent hook_completed tools", + "weather_agent hook_triggered model_request", + "weather_agent model_request", + "weather_agent model_response", + "weather_agent hook_completed model_request", + "weather_agent agent_end", +]; + +describe.each(FIXTURES)("%s", (fixture) => { + it("typechecks as a customer's nodenext ES-module project", () => { + expect(typecheck(fixture)).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string) => { + const result = runAgent(fixture, format, scenario); + expect(result.status, describeTrace(result)).toBe(0); + return result; + }; + + it("records a graph run as one agent with its nodes as hooks", () => { + const result = run("graph"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + for (const hook of ofType(result.events, "hook_triggered")) { + expect(hook.trigger_event).toBe("graph_node"); + } + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + // The model's own tool call id, so a tool_use joins to the tool_calls[] + // entry that asked for it. + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + for (const event of result.events) { + expect(event.framework).toBe("langchain"); + expect(typeof event.framework_version).toBe("string"); + } + expect(result.stdout).toContain("It is sunny in Paris."); + }); + + it("records .stream() exactly like .invoke()", () => { + const result = run("stream"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + }); + + it("records a prebuilt ReAct agent under its own name", () => { + const result = run("react"); + expect(shape(result.events), describeTrace(result)).toEqual( + GRAPH.map((line) => line.replace("weather_graph", "react_bot")), + ); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a failure once, where it happened", () => { + const result = run("error"); + expect(result.stdout).toContain("model exploded"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather_graph agent_start", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph agent_end", + ]); + expect(ofType(result.events, "model_response")[0]!.error).toMatch(/model exploded/); + expect(ofType(result.events, "hook_completed")[0]!.outcome).toBe("failed"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(count(result.events, "error")).toBe(0); + }); + + it("records a bare model call as its own run", () => { + const result = run("model"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "ScriptedModel agent_start", + "ScriptedModel model_request", + "ScriptedModel model_response", + "ScriptedModel agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a bare tool call as its own run", () => { + const result = run("tool"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "get_weather agent_start", + "get_weather tool_use get_weather", + "get_weather tool_result get_weather", + "get_weather agent_end", + ]); + }); + + it("nests the graph under an enclosing agent() scope", () => { + const result = run("scope"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + ...GRAPH, + "planner agent_end", + ]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("planner"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records through an explicit langchainHandler() without instrument()", () => { + const result = run("handler"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + }); + + it("does not double-record when the handler and instrument() are both active", () => { + const result = run("handler-and-instrument"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + }); + + it("records nothing after uninstrument()", () => { + const result = run("uninstrument"); + expect(result.events, describeTrace(result)).toEqual([]); + expect(result.stdout).toContain('"removed":["langchain"]'); + }); + + it("records a tool that failed on its tool_result, and the run carries on", () => { + const result = run("tool-error"); + // `ToolNode` turns the throw into an error ToolMessage, so the graph goes + // on to answer: the same shape as a clean run, with the failure on the + // tool span and nowhere else. + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(ofType(result.events, "tool_result")[0]!.error).toBe("Error: no weather for Paris"); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(count(result.events, "error")).toBe(0); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records an interrupt() and its Command resume as one paused agent", () => { + const result = run("hitl"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "hitl_graph agent_start", + "hitl_graph hook_triggered plan", + "hitl_graph hook_completed plan", + "hitl_graph hook_triggered approve", + // The node did not fail, it stopped to ask a human. + "hitl_graph hook_completed approve", + "hitl_graph human_wait", + "hitl_graph agent_pause", + // The second `.invoke()` is the SAME agent: no second agent_start. + "hitl_graph agent_resume", + "hitl_graph human_input", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph hook_triggered act", + "hitl_graph hook_completed act", + "hitl_graph agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["t-1"])); + expect(ofType(result.events, "hook_completed")[1]!.outcome).toBe("paused"); + const wait = ofType(result.events, "human_wait")[0]!; + expect(wait.prompt).toBe("ship it?"); + expect(wait.options).toEqual(["yes", "no"]); + expect(wait.reason).toBe("langgraph_interrupt"); + const pauseId = wait.input_id; + expect(typeof pauseId).toBe("string"); + expect(ofType(result.events, "agent_pause")[0]!.pause_id).toBe(pauseId); + expect(ofType(result.events, "agent_resume")[0]!.pause_id).toBe(pauseId); + const answer = ofType(result.events, "human_input")[0]!; + expect(answer.input_id).toBe(pauseId); + expect(answer.response).toBe("yes"); + // Control flow, not failure: nothing red anywhere. + expect(count(result.events, "error")).toBe(0); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + }); + + it("closes a paused agent as cancelled at uninstrument()", () => { + const result = run("hitl-uninstrument"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "hitl_graph agent_start", + "hitl_graph hook_triggered plan", + "hitl_graph hook_completed plan", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph human_wait", + "hitl_graph agent_pause", + "hitl_graph agent_end", + ]); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(result.stdout).toContain('"removed":["langchain"]'); + }); + + it("closes a pause opened by another process when this one takes the answer", () => { + // Process A interrupts and exits; process B shares only the checkpointer. + // B has no memory of the pause, so the id is rebuilt from the interrupted + // task's checkpoint namespace — the same derivation `interrupt()` used. + const result = run("remote-resume"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "hitl_graph agent_start", + "hitl_graph hook_triggered plan", + "hitl_graph hook_completed plan", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph human_wait", + "hitl_graph agent_pause", + // Process B: its own root, then the pause closed where the answered + // node finished. + "hitl_graph agent_start", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph agent_resume", + "hitl_graph human_input", + "hitl_graph hook_triggered act", + "hitl_graph hook_completed act", + "hitl_graph agent_end", + ]); + const opened = ofType(result.events, "human_wait")[0]!.input_id; + expect(ofType(result.events, "agent_resume")[0]!.pause_id).toBe(opened); + const answer = ofType(result.events, "human_input")[0]!; + expect(answer.input_id).toBe(opened); + expect(answer.response).toBe("yes"); + expect(answer.fw_resumed_elsewhere).toBe(true); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["t-1"])); + expect(count(result.events, "error")).toBe(0); + }); + + it("closes an aborted run as cancelled, not failed", () => { + const result = run("abort"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "abort_graph agent_start", + "abort_graph hook_triggered slow", + "abort_graph hook_completed slow", + "abort_graph agent_end", + ]); + // A caller that gave up is not a crash: nothing red, and the session + // closed rather than left running forever. + expect(ofType(result.events, "hook_completed")[0]!.outcome).toBe("cancelled"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(count(result.events, "error")).toBe(0); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a compiled subgraph as a nested agent under its host node", () => { + const result = run("subgraph"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "parent_graph agent_start", + "parent_graph hook_triggered pre", + "parent_graph hook_completed pre", + "parent_graph hook_triggered child", + "parent_graph/child agent_start", + "parent_graph/child hook_triggered inner", + "parent_graph/child hook_completed inner", + "parent_graph/child agent_end", + "parent_graph hook_completed child", + "parent_graph agent_end", + ]); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("parent_graph"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records each input of a .batch() as its own root", () => { + const result = run("batch"); + const sessions = [...new Set(result.events.map((e) => e.session_id))]; + expect(sessions).toHaveLength(2); + for (const session of sessions) { + expect(shape(result.events.filter((e) => e.session_id === session))).toEqual([ + "ScriptedModel agent_start", + "ScriptedModel model_request", + "ScriptedModel model_response", + "ScriptedModel agent_end", + ]); + } + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("folds a streamed model call into one model_response", () => { + const result = run("stream-model"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "StreamingModel agent_start", + "StreamingModel model_request", + "StreamingModel model_response", + "StreamingModel agent_end", + ]); + const response = ofType(result.events, "model_response")[0]!; + expect(response.fw_streamed).toBe(true); + expect(response.fw_chunks).toBe(5); + expect(typeof response.fw_ttft_ms).toBe("number"); + expect(response.content).toBe("hello there friend"); + }); + + it("records an intermediate chain only when includeChains names it", () => { + const result = run("include-chains"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "pipeline agent_start", + "pipeline hook_triggered summarise", + "pipeline hook_completed summarise", + "pipeline agent_end", + ]); + expect(ofType(result.events, "hook_triggered")[0]!.trigger_event).toBe("pipeline"); + }); + + it("pins every run to the sessionId option", () => { + const result = run("session-option"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["fixed-session"])); + }); + + it("takes the session from the documented metadata key", () => { + const result = run("metadata-session"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["meta-sid"])); + }); + + it("falls back to the LangGraph thread_id for the session", () => { + const result = run("thread-session"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["thread-9"])); + }); + + it("drops prompts, messages and outputs under captureContent: false", () => { + const result = run("capture-off"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + for (const event of result.events) { + for (const key of ["goal", "input", "output", "messages", "content"]) { + expect(event[key], `${event.type}.${key}`).toBeUndefined(); + } + } + // Structure, durations and token counts survive. + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + }); + + // -- plain LangChain: every surface, instrumented and by explicit handler -- + + describe.each(["instrument", "handler"] as const)("through %s", (mode) => { + const runSurface = (surface: string) => run(mode === "handler" ? `handler:${surface}` : surface); + + it.each(Object.keys(SURFACES))("records %s as one root agent per input", (surface) => { + const expected = SURFACES[surface]!; + const result = runSurface(surface); + const sessions = [...new Set(result.events.map((e) => e.session_id))]; + expect(sessions, describeTrace(result)).toHaveLength(expected.sessions ?? 1); + for (const session of sessions) { + expect(shape(result.events.filter((e) => e.session_id === session)), describeTrace(result)).toEqual( + expected.trace, + ); + } + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + if (mode === "handler") expect(result.stdout).not.toContain("instrumented"); + }); + + it("summarises a retriever's documents, never their text", () => { + for (const surface of ["retriever", "rag"]) { + const result = runSurface(surface); + const use = ofType(result.events, "tool_use")[0]!; + const done = ofType(result.events, "tool_result")[0]!; + expect(use.tool_call_id, describeTrace(result)).toBe(done.tool_call_id); + expect(use.input).toEqual({ query: surface === "rag" ? "weather?" : "weather in Paris" }); + expect(done.output).toEqual({ n: 2, sources: ["wx.txt", "wx2.txt"] }); + // Not in the retrieval's own events. (In `rag` the documents DO reach + // the model_request — the chain put them into the prompt.) + expect(JSON.stringify([use, done])).not.toContain("Rome is rainy"); + } + const store = runSurface("vectorstore"); + expect(ofType(store.events, "tool_result")[0]!.output, describeTrace(store)).toEqual({ n: 1, sources: ["wx.txt"] }); + }); + + it("folds a streamed chain's tokens into one model_response, usage included", () => { + for (const surface of ["lcel-stream", "stream-events"]) { + const result = runSurface(surface); + const response = ofType(result.events, "model_response")[0]!; + expect(response.fw_streamed, describeTrace(result)).toBe(true); + expect(response.fw_chunks).toBe(5); + expect(typeof response.fw_ttft_ms).toBe("number"); + expect(response.content).toBe("It is sunny in Paris."); + // Usage arrives on the LAST chunk only; it survives the aggregation. + expect([response.input_tokens, response.output_tokens]).toEqual([30, 7]); + } + }); + + it("records the tools a model was bound to, for bindTools and withStructuredOutput", () => { + const bound = runSurface("bind-tools"); + const request = ofType(bound.events, "model_request")[0]!; + expect(request.model, describeTrace(bound)).toBe("tool-calling-1"); + expect((request.tools as Array<{ function: { name: string } }>).map((t) => t.function.name)).toEqual([ + "get_weather", + ]); + expect([ofType(bound.events, "model_response")[0]!.input_tokens]).toEqual([12]); + const structured = runSurface("structured"); + const asked = ofType(structured.events, "model_request")[0]!; + expect((asked.tools as Array<{ function: { name: string } }>).map((t) => t.function.name)).toEqual(["Weather"]); + expect(structured.stdout).toContain('{"out":{"city":"Paris","sky":"sunny"}}'); + }); + }); + }); +}); + +describe("langchain-1: createAgent", () => { + it("typechecks as a customer's nodenext ES-module project", () => { + // `tsconfig.json` lists both programs, so this covers agent-v1.ts too. + expect(typecheck("langchain-1")).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string) => { + const result = runAgent("langchain-1", format, scenario, {}, "agent-v1"); + expect(result.status, describeTrace(result)).toBe(0); + return result; + }; + + it.each(["create-agent", "handler:create-agent", "create-agent-stream"])( + "records %s as one agent with its nodes as hooks", + (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual(CREATE_AGENT); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + for (const hook of ofType(result.events, "hook_triggered")) expect(hook.trigger_event).toBe("graph_node"); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + }, + ); + + it("records middleware: a node-shaped hook is a hook, a wrapping hook adds nothing", () => { + // Python golden (`AgentMiddleware` with `before_model` + `wrap_model_call`): + // `Audit.before_model` is a node before each model turn; `wrap_model_call` + // runs INSIDE the model node and emits nothing of its own. + const result = run("create-agent-mw"); + const before = ["weather_agent hook_triggered Audit.before_model", "weather_agent hook_completed Audit.before_model"]; + expect(shape(result.events), describeTrace(result)).toEqual([ + CREATE_AGENT[0], + ...before, + ...CREATE_AGENT.slice(1, 9), + ...before, + ...CREATE_AGENT.slice(9), + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("renders the model node's Command output as messages, not LangChain's serialization envelope", () => { + // The model node returns `{ output: [Command] }`; its messages used to be + // captured as `{lc: 1, type: "constructor", id: [...], kwargs}`. + const result = run("create-agent"); + const done = ofType(result.events, "hook_completed").find((e) => e.hook_name === "model_request")!; + const text = JSON.stringify(done.output); + expect(text, describeTrace(result)).not.toContain('"lc":1'); + expect(text).toContain('"type":"ai"'); + expect(text).toContain('"lg_name":"Command"'); + }); + }); +}); + +/** + * Two copies of `@langchain/core` in one process: the app's 1.2.12 and 0.3.80 + * nested under a provider that pinned it (see the fixture's `agent.ts`). + * + * The nested copy's runs INSIDE an app run were always recorded — the child is + * handed the parent's manager, handler and all. Its ROOT runs were recorded by + * nothing under `instrument()` until the adapter learned to find and patch the + * nested copy too. Python has no equivalent (one interpreter imports one + * `langchain_core`), so the expectation is the mapping rule itself: identical + * to the same run through the app's own copy. + */ +describe("langchain-dup-core: a provider nested on its own @langchain/core", () => { + const DUP = "langchain-dup-core"; + + it("typechecks as a customer's nodenext ES-module project", () => { + expect(typecheck(DUP)).toBe(""); + }); + + const NESTED_ROOTS: Record = { + "nested-model": modelRun("ChatWeather"), + "nested-tool": [ + "get_weather agent_start", + "get_weather tool_use get_weather", + "get_weather tool_result get_weather", + "get_weather agent_end", + ], + "nested-retriever": retrieverRun("WeatherRetriever", "WeatherRetriever"), + "nested-runnable": ["forecast agent_start", "forecast agent_end"], + }; + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string) => { + const result = runAgent(DUP, format, scenario); + expect(result.status, describeTrace(result)).toBe(0); + return result; + }; + + it.each(Object.keys(NESTED_ROOTS))("records a root started by the nested copy (%s)", (scenario) => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual(NESTED_ROOTS[scenario]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it.each(Object.keys(NESTED_ROOTS))("records it through an explicit handler too (%s)", (scenario) => { + const result = run(`handler:${scenario}`); + expect(shape(result.events), describeTrace(result)).toEqual(NESTED_ROOTS[scenario]); + }); + + it("does not double-record a nested root when the handler and instrument() are both active", () => { + const result = run("both:nested-model"); + expect(shape(result.events), describeTrace(result)).toEqual(NESTED_ROOTS["nested-model"]); + }); + + it("records nothing through the nested copy after uninstrument()", () => { + const result = run("uninstrument:nested-model"); + expect(result.events, describeTrace(result)).toEqual([]); + }); + + it("records the nested copy's model inside the app's chain", () => { + const result = run("app-chain"); + expect(shape(result.events), describeTrace(result)).toEqual(modelRun("RunnableSequence")); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records the nested copy's model inside the app's lambda, parented through the shared async context", () => { + const result = run("app-lambda"); + expect(shape(result.events), describeTrace(result)).toEqual(modelRun("outer")); + }); + + it("records the nested copy's model inside the app's LangGraph graph", () => { + const result = run("app-graph"); + expect(shape(result.events), describeTrace(result)).toEqual(GRAPH); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + }); + }); +}); diff --git a/sdk/typescript/integration/llamaindex.test.ts b/sdk/typescript/integration/llamaindex.test.ts new file mode 100644 index 000000000..c1ef8d864 --- /dev/null +++ b/sdk/typescript/integration/llamaindex.test.ts @@ -0,0 +1,709 @@ +import { describe, expect, it } from "vitest"; + +import { + FORMATS, + count, + describeTrace, + ofType, + runAgent, + traceViolations, + typecheck, + type Event, +} from "./harness.js"; + +/** + * LlamaIndex.TS, against real releases. + * + * The expected traces follow the Python SDK's LlamaIndex adapter, captured from + * llama-index-core 0.14.24 running the equivalent `FunctionAgent` program with a + * scripted model (same tool, same tool call id, same two turns). The two SDKs + * write into one pipe, so for the same program they must draw the same tree + * (`sdk/python/skill/references/frameworks.md`): + * + * * a workflow run is the agent — session + `agent_start`/`agent_end` — named + * after the agent (`FunctionAgent.name`, `"Agent"` by default), never an id; + * * each workflow step is a HOOK (`trigger_event="workflow_step"`); + * * a multi-agent workflow is `AgentWorkflow` with one nested agent per agent + * holding the turn; + * * a bare model call outside any run is its own run, named after the class; + * * a tool failure lives on its `tool_result`, not in an `error` event. + * + * Where TS genuinely differs, it is the framework and not the adapter: the step + * NAMES are LlamaIndex.TS's own handler names (`runAgentStep`, where Python has + * `run_agent_step`), TS executes every tool call of a turn in ONE + * `executeToolCalls` step (Python runs a `call_tool` step per call), and + * `parseAgentOutput` starts INSIDE `runAgentStep` because the TS runtime + * dispatches the next step synchronously from `sendEvent`. + */ + +/** + * The supported floor and the latest release. 0.11.4 is the first `llamaindex` + * on `@llamaindex/workflow` 1.1 (then still `@llama-flow/core` underneath, with + * step handlers called as `handler(event)` and no bus event for a tool call); + * 0.9–0.11.3 ship workflow 1.0, a different runtime whose `AgentWorkflow` has no + * `runStream` at all. + */ +const FIXTURES = ["llamaindex-0.11", "llamaindex-0.12"] as const; + +/** One line per event: `agent_id type [hook|tool]`. */ +const shape = (events: Event[]): string[] => + events.map((e) => [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim()); + +const hook = (agentId: string, name: string, body: string[] = []): string[] => [ + `${agentId} hook_triggered ${name}`, + ...body, + `${agentId} hook_completed ${name}`, +]; + +const MODEL = (agentId: string) => [`${agentId} model_request`, `${agentId} model_response`]; + +/** One `FunctionAgent` turn that asks for `tool`, then the turn that answers. */ +const agentLoop = (agentId: string, tool: string): string[] => [ + ...hook(agentId, "setupAgent"), + `${agentId} hook_triggered runAgentStep`, + ...MODEL(agentId), + ...hook(agentId, "parseAgentOutput"), + `${agentId} hook_completed runAgentStep`, + ...hook(agentId, "executeToolCalls", [`${agentId} tool_use ${tool}`, `${agentId} tool_result ${tool}`]), + ...hook(agentId, "processToolResults"), + ...hook(agentId, "setupAgent"), + `${agentId} hook_triggered runAgentStep`, + ...MODEL(agentId), + `${agentId} hook_triggered parseAgentOutput`, + `${agentId} hook_completed runAgentStep`, + `${agentId} hook_completed parseAgentOutput`, +]; + +const WORKFLOW = (agentId = "Agent", tool = "get_weather"): string[] => [ + `${agentId} agent_start`, + ...hook(agentId, "handleInputStep"), + ...agentLoop(agentId, tool), + `${agentId} agent_end`, +]; + +describe.each(FIXTURES)("%s", (fixture) => { + it("typechecks as a customer's nodenext ES-module project", () => { + expect(typecheck(fixture)).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string, env: Record = {}) => { + const result = runAgent(fixture, format, scenario, env); + expect(result.status, describeTrace(result)).toBe(0); + // Instrumenting must never load a second copy of the framework: LlamaIndex + // detects it and prints this to the customer's terminal. + expect(result.stderr).not.toContain("already imported"); + expect(result.stdout).toContain('"instrumented":["llamaindex"]'); + return result; + }; + + it("records a workflow agent run as one agent with its steps as hooks", () => { + const result = run("workflow"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW()); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + for (const event of ofType(result.events, "hook_triggered")) { + expect(event.trigger_event).toBe("workflow_step"); + } + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + for (const event of [...ofType(result.events, "model_request"), ...responses]) { + expect(event.model).toBe("scripted-1"); + } + // The model's own tool call id, so a tool_use joins to the tool call that + // asked for it. + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(result.events, "tool_result")[0]!.output).toBe("sunny in Paris"); + const end = ofType(result.events, "agent_end")[0]!; + expect(end.outcome).toBe("success"); + expect(end.summary).toBe("It is sunny in Paris."); + expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?"); + for (const event of result.events) { + expect(event.framework).toBe("llamaindex"); + expect(typeof event.framework_version).toBe("string"); + } + expect(result.stdout).toContain("It is sunny in Paris."); + }); + + it("names the run after the agent", () => { + const result = run("named"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("weather_bot")); + }); + + it("records an agent built before instrument()", () => { + const result = run("early"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW()); + }); + + it("records runStream() exactly like run()", () => { + const result = run("stream"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW()); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a multi-agent handoff as nested agents under the workflow", () => { + const result = run("handoff"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "AgentWorkflow agent_start", + ...hook("AgentWorkflow", "handleInputStep"), + "triage agent_start", + ...hook("triage", "setupAgent"), + "triage hook_triggered runAgentStep", + ...MODEL("triage"), + ...hook("triage", "parseAgentOutput"), + "triage hook_completed runAgentStep", + ...hook("triage", "executeToolCalls", ["triage tool_use handOff", "triage tool_result handOff"]), + ...hook("triage", "processToolResults"), + "triage agent_end", + "forecaster agent_start", + ...agentLoop("forecaster", "get_weather"), + "forecaster agent_end", + "AgentWorkflow agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + for (const start of ofType(result.events, "agent_start").slice(1)) { + expect(start.parent_id).toBe("AgentWorkflow"); + } + }); + + it("records a legacy LLMAgent as one agent across all its steps", () => { + const result = run("legacy"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "LLMAgent agent_start", + ...MODEL("LLMAgent"), + "LLMAgent tool_use get_weather", + "LLMAgent tool_result get_weather", + ...MODEL("LLMAgent"), + "LLMAgent agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + expect(ofType(result.events, "model_request")[0]!.model).toBe("scripted-1"); + }); + + it("records a bare model call as its own run", () => { + const result = run("model"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "ScriptedLLM agent_start", + ...MODEL("ScriptedLLM"), + "ScriptedLLM agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([3, 1]); + expect(response.model).toBe("scripted-1"); + }); + + it("nests the run under an enclosing agent() scope", () => { + const result = run("scope"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + ...WORKFLOW(), + "planner agent_end", + ]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("planner"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a failing tool on its tool_result, once", () => { + const result = run("tool-error"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("Agent", "broken")); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(ofType(result.events, "tool_result")[0]!.error).toMatch(/tool exploded/); + expect(ofType(result.events, "hook_completed").every((e) => e.outcome === "success")).toBe(true); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(count(result.events, "error")).toBe(0); + }); + + /** + * Two requests in flight on ONE object built at startup (before + * instrument()), Paris slow and Rome fast, so Rome starts and ends inside + * Paris. Each must be its own session with its own run, and every event in + * a session must belong to that session's request. + */ + const concurrent = (scenario: string, expected: string[], agentId: string) => { + const result = run(scenario); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const sessions = new Map(); + for (const e of result.events) sessions.set(e.session_id, [...(sessions.get(e.session_id) ?? []), e]); + expect(sessions.size, describeTrace(result)).toBe(2); + const byCity = new Map(); + for (const list of sessions.values()) { + expect(shape(list), describeTrace(result)).toEqual(expected); + const start = list[0]!; + expect(start.agent_id).toBe(agentId); + expect(start.parent_id ?? null).toBeNull(); + const city = String(start.goal).includes("Rome") ? "Rome" : "Paris"; + byCity.set(city, list); + } + expect([...byCity.keys()].sort()).toEqual(["Paris", "Rome"]); + for (const [city, list] of byCity) { + const other = city === "Paris" ? "Rome" : "Paris"; + // Nothing of the other request in this session. + expect(JSON.stringify(list), describeTrace(result)).not.toContain(other); + for (const response of ofType(list, "model_response")) { + expect([response.input_tokens, response.output_tokens]).toEqual(city === "Paris" ? [11, 2] : [21, 3]); + expect(response.model).toBe("echo-1"); + } + const end = ofType(list, "agent_end")[0]!; + expect(end.outcome).toBe("success"); + expect(end.summary).toBe(`It is sunny in ${city}.`); + } + // They really did overlap: Rome started after Paris and ended before it. + const index = (city: string, type: string) => result.events.indexOf(ofType(byCity.get(city)!, type)[0]!); + expect(index("Rome", "agent_start")).toBeGreaterThan(index("Paris", "agent_start")); + expect(index("Rome", "agent_end")).toBeLessThan(index("Paris", "agent_end")); + return byCity; + }; + + it("keeps concurrent queries on one shared query engine in separate sessions", () => { + const byCity = concurrent( + "shared-query", + [ + "RetrieverQueryEngine agent_start", + "RetrieverQueryEngine tool_use NotesRetriever", + "RetrieverQueryEngine tool_result NotesRetriever", + ...MODEL("RetrieverQueryEngine"), + "RetrieverQueryEngine agent_end", + ], + "RetrieverQueryEngine", + ); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` }); + } + }); + + it("keeps concurrent chats on one shared legacy LLMAgent in separate sessions", () => { + const byCity = concurrent( + "shared-legacy", + [ + "LLMAgent agent_start", + ...MODEL("LLMAgent"), + "LLMAgent tool_use get_weather", + "LLMAgent tool_result get_weather", + ...MODEL("LLMAgent"), + "LLMAgent agent_end", + ], + "LLMAgent", + ); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`); + expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`); + } + }); + + it("keeps concurrent runs of one shared workflow agent in separate sessions", () => { + const byCity = concurrent("shared-workflow", WORKFLOW(), "Agent"); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`); + expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`); + } + }); + + it("records nothing after uninstrument()", () => { + const result = run("uninstrument"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW()); + expect(result.stdout).toContain('"removed":["llamaindex"]'); + }); + + // -- the coverage sweep ------------------------------------------------------ + // + // Every commonly used surface, each against the Python adapter's golden + // trace for the same program (llama-index-core 0.14.24, mock LLM and + // embeddings): a top-level chat engine, query engine or retriever call is + // the session's root agent named after its class, and a retrieval is a + // `tool_use`/`tool_result` named after the retriever's class, its output + // summarised. (Python's MockLLM records each call twice, chat inside + // complete; that is the mock, not the mapping.) + + /** The fewest checks every trace must pass: sound, one session, the framework's name. */ + const sound = (result: ReturnType, sessions = 1, scopes: string[] = []) => { + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size, describeTrace(result)).toBe(sessions); + // The application's own agent() scope is not the framework's. + for (const event of result.events) { + if (!scopes.includes(event.agent_id)) expect(event.framework).toBe("llamaindex"); + } + }; + const RETRIEVAL = (agentId: string, retriever = "VectorIndexRetriever") => [ + `${agentId} tool_use ${retriever}`, + `${agentId} tool_result ${retriever}`, + ]; + /** The retrieval found the Paris note, summarised: count, score, text prefix. */ + const retrievedParis = (event: Event) => { + const output = event.output as { num_nodes: number; top: Array<{ text: string; score: number }> }; + expect(output.num_nodes).toBe(1); + expect(output.top[0]!.text).toBe("Paris is sunny."); + expect(typeof output.top[0]!.score).toBe("number"); + }; + + describe.each(["chat-simple", "chat-simple-stream"])("SimpleChatEngine (%s)", (scenario) => { + it("is one run named after the engine, with the chat history in the model call", () => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual([ + "SimpleChatEngine agent_start", + ...MODEL("SimpleChatEngine"), + "SimpleChatEngine agent_end", + ]); + sound(result); + const request = ofType(result.events, "model_request")[0]!; + expect(request.model).toBe("scripted-1"); + expect(request.messages).toEqual([ + { role: "user", content: "hi" }, + { role: "assistant", content: "hello" }, + { role: "user", content: "weather in Paris?" }, + ]); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + }); + }); + + describe.each(["chat-context", "chat-context-stream"])("ContextChatEngine (%s)", (scenario) => { + it("is one run: the retrieval, then the model call with the history and the context", () => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual([ + "ContextChatEngine agent_start", + ...RETRIEVAL("ContextChatEngine"), + ...MODEL("ContextChatEngine"), + "ContextChatEngine agent_end", + ]); + sound(result); + expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "weather in Paris?" }); + retrievedParis(ofType(result.events, "tool_result")[0]!); + const messages = JSON.stringify(ofType(result.events, "model_request")[0]!.messages); + expect(messages).toContain("Paris is sunny."); + expect(messages).toContain("hello"); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + }); + }); + + describe.each(["chat-condense", "chat-condense-stream"])("CondenseQuestionChatEngine (%s)", (scenario) => { + it("is one run: the condensing model call, the query's retrieval and its answer", () => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual([ + "CondenseQuestionChatEngine agent_start", + ...MODEL("CondenseQuestionChatEngine"), + ...RETRIEVAL("CondenseQuestionChatEngine"), + ...MODEL("CondenseQuestionChatEngine"), + "CondenseQuestionChatEngine agent_end", + ]); + sound(result); + // The history reaches the model through the condense prompt. + expect(JSON.stringify(ofType(result.events, "model_request")[0]!.messages)).toContain("hello"); + expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "What is the weather in Paris?" }); + retrievedParis(ofType(result.events, "tool_result")[0]!); + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [6, 3], + [9, 4], + ]); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + }); + }); + + it("records a retriever used directly as its own run", () => { + const result = run("retriever"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "VectorIndexRetriever agent_start", + ...RETRIEVAL("VectorIndexRetriever"), + "VectorIndexRetriever agent_end", + ]); + sound(result); + expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "weather in Paris?" }); + retrievedParis(ofType(result.events, "tool_result")[0]!); + }); + + describe.each(["index-query", "index-query-stream"])("VectorStoreIndex.asQueryEngine() (%s)", (scenario) => { + it("is one run named after the engine, with its retrieval and its answer", () => { + const result = run(scenario); + expect(shape(result.events), describeTrace(result)).toEqual([ + "RetrieverQueryEngine agent_start", + ...RETRIEVAL("RetrieverQueryEngine"), + ...MODEL("RetrieverQueryEngine"), + "RetrieverQueryEngine agent_end", + ]); + sound(result); + expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?"); + retrievedParis(ofType(result.events, "tool_result")[0]!); + const response = ofType(result.events, "model_response")[0]!; + expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + }); + }); + + it("records nothing for building an index, with or without embeddings: true", () => { + // LlamaIndex.TS dispatches no embedding event on its bus, so there is + // nothing for `embeddings: true` to record; the option is accepted and + // changes nothing. (Python records embedding calls under it.) + expect(run("index-build").events).toEqual([]); + expect(run("index-build", { FAILPROOFAI_IT_OPTIONS: JSON.stringify({ embeddings: true }) }).events).toEqual([]); + }); + + /** workflow-core ≥1.1 exposes the hook plain workflows are recorded through; the floor's runtime does not. */ + const plainWorkflows = fixture !== "llamaindex-0.11"; + + it( + plainWorkflows + ? "records a createWorkflow() workflow as a run with its steps as hooks" + : "leaves a createWorkflow() workflow's model call a run of its own on the floor's runtime", + () => { + const result = run("custom-workflow"); + sound(result); + if (plainWorkflows) { + expect(shape(result.events), describeTrace(result)).toEqual([ + "Workflow agent_start", + ...hook("Workflow", "research"), + ...hook("Workflow", "answer", MODEL("Workflow")), + "Workflow agent_end", + ]); + expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?"); + for (const event of ofType(result.events, "hook_triggered")) expect(event.trigger_event).toBe("workflow_step"); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + } else { + // @llama-flow/core binds its handler context in a closure: there is + // no run boundary and no step to observe, only the model call. + expect(shape(result.events), describeTrace(result)).toEqual([ + "ScriptedLLM agent_start", + ...MODEL("ScriptedLLM"), + "ScriptedLLM agent_end", + ]); + } + }, + ); + + it("nests a createWorkflow() workflow under an enclosing agent() scope", () => { + const result = run("custom-workflow-scoped"); + sound(result, 1, ["forecast_flow"]); + const inner = plainWorkflows + ? ["Workflow agent_start", ...hook("Workflow", "research"), ...hook("Workflow", "answer", MODEL("Workflow")), "Workflow agent_end"] + : ["ScriptedLLM agent_start", ...MODEL("ScriptedLLM"), "ScriptedLLM agent_end"]; + expect(shape(result.events), describeTrace(result)).toEqual(["forecast_flow agent_start", ...inner, "forecast_flow agent_end"]); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("forecast_flow"); + }); + + it("records a three-agent handoff chain as nested agents under the workflow", () => { + const result = run("handoff3"); + const handingOff = (agentId: string) => [ + `${agentId} agent_start`, + ...hook(agentId, "setupAgent"), + `${agentId} hook_triggered runAgentStep`, + ...MODEL(agentId), + ...hook(agentId, "parseAgentOutput"), + `${agentId} hook_completed runAgentStep`, + ...hook(agentId, "executeToolCalls", [`${agentId} tool_use handOff`, `${agentId} tool_result handOff`]), + ...hook(agentId, "processToolResults"), + `${agentId} agent_end`, + ]; + expect(shape(result.events), describeTrace(result)).toEqual([ + "AgentWorkflow agent_start", + ...hook("AgentWorkflow", "handleInputStep"), + ...handingOff("triage"), + ...handingOff("researcher"), + "forecaster agent_start", + ...agentLoop("forecaster", "get_weather"), + "forecaster agent_end", + "AgentWorkflow agent_end", + ]); + sound(result); + for (const start of ofType(result.events, "agent_start").slice(1)) expect(start.parent_id).toBe("AgentWorkflow"); + expect(ofType(result.events, "tool_use").map((e) => e.tool_call_id)).toEqual(["call_h1", "call_h2", "call_1"]); + expect(ofType(result.events, "agent_end").at(-1)!.summary).toBe("It is sunny in Paris."); + }); + + // `responseFormat` arrived after the floor (@llamaindex/workflow 1.1.5 ignores it). + it.skipIf(fixture === "llamaindex-0.11")("records agent() with responseFormat: the structured-output call and tool", () => { + const result = run("structured"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "Agent agent_start", + ...hook("Agent", "handleInputStep"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + "Agent hook_triggered parseAgentOutput", + "Agent hook_completed runAgentStep", + ...MODEL("Agent"), + "Agent tool_use format_output", + "Agent tool_result format_output", + "Agent hook_completed parseAgentOutput", + "Agent agent_end", + ]); + sound(result); + expect(ofType(result.events, "tool_result")[0]!.output).toEqual({ city: "Paris", sky: "sunny" }); + expect(result.stdout).toContain('"object":{"city":"Paris","sky":"sunny"}'); + }); + + it("records a FunctionTool.from() tool like a tool() one", () => { + const result = run("function-tool"); + expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("Agent", "lookup_weather")); + sound(result); + expect(ofType(result.events, "tool_use")[0]).toMatchObject({ tool_call_id: "call_1", input: { city: "Paris" } }); + expect(ofType(result.events, "tool_result")[0]!.output).toBe("sunny in Paris"); + }); + + it("records a QueryEngineTool with the query's retrieval and model call inside it", () => { + const result = run("query-engine-tool"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "Agent agent_start", + ...hook("Agent", "handleInputStep"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + ...hook("Agent", "parseAgentOutput"), + "Agent hook_completed runAgentStep", + ...hook("Agent", "executeToolCalls", [ + "Agent tool_use city_notes", + ...RETRIEVAL("Agent"), + ...MODEL("Agent"), + "Agent tool_result city_notes", + ]), + ...hook("Agent", "processToolResults"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + "Agent hook_triggered parseAgentOutput", + "Agent hook_completed runAgentStep", + "Agent hook_completed parseAgentOutput", + "Agent agent_end", + ]); + sound(result); + expect(ofType(result.events, "tool_result").at(-1)!.output).toEqual({ content: "Paris is sunny." }); + expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [12, 5], + [7, 3], + [30, 7], + ]); + }); + + it("records parallel tool calls from one model turn, each on its own id", () => { + const result = run("parallel-tools"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "Agent agent_start", + ...hook("Agent", "handleInputStep"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + ...hook("Agent", "parseAgentOutput"), + "Agent hook_completed runAgentStep", + ...hook("Agent", "executeToolCalls", [ + "Agent tool_use get_weather", + "Agent tool_result get_weather", + "Agent tool_use get_time", + "Agent tool_result get_time", + ]), + ...hook("Agent", "processToolResults"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + "Agent hook_triggered parseAgentOutput", + "Agent hook_completed runAgentStep", + "Agent hook_completed parseAgentOutput", + "Agent agent_end", + ]); + sound(result); + expect(ofType(result.events, "tool_result").map((e) => [e.tool_call_id, e.output])).toEqual([ + ["call_1", "sunny in Paris"], + ["call_2", "noon in Paris"], + ]); + }); + + it("closes the run as failed when the provider fails mid-stream (an unhandled rejection in LlamaIndex)", () => { + // The workflow runtime lets this error escape as an unhandled rejection + // and `run()` never settles — LlamaIndex's behaviour, with or without + // the SDK. The failed step is the signal: the run still closes, failed. + const result = run("stream-error"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "Agent agent_start", + ...hook("Agent", "handleInputStep"), + ...hook("Agent", "setupAgent"), + "Agent hook_triggered runAgentStep", + ...MODEL("Agent"), + "Agent hook_completed runAgentStep", + "Agent agent_end", + ]); + sound(result); + expect(result.stdout).toContain('"settled":"pending"'); + expect(result.stdout).toContain('"unhandled":["Error: provider dropped the stream"]'); + expect(ofType(result.events, "model_response")[0]!.error).toMatch(/provider dropped the stream/); + expect(ofType(result.events, "hook_completed").at(-1)).toMatchObject({ outcome: "failed" }); + expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "failed" }); + expect(count(result.events, "error")).toBe(0); + }); + + /** + * Ten requests in flight at once on ONE object: each its own session, and + * nothing of any other request in it. CityLLM's usage is per city + * (`[100 + n, n]`), so every model response says which request it answered. + */ + const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Delhi", "Tokyo", "Quito", "Accra", "Hanoi"]; + const tenWay = (scenario: string, expected: (agentId: string) => string[], agentId: string) => { + const result = run(scenario); + sound(result, CITIES.length); + const byCity = new Map(); + for (const e of result.events) { + const list = [...result.events.filter((x) => x.session_id === e.session_id)]; + const text = JSON.stringify(list); + const city = CITIES.find((c) => text.includes(`weather in ${c}?`)); + expect(city, describeTrace(result)).toBeDefined(); + byCity.set(city!, list); + } + expect([...byCity.keys()].sort()).toEqual([...CITIES].sort()); + for (const [city, list] of byCity) { + expect(shape(list), describeTrace(result)).toEqual(expected(agentId)); + expect(list[0]!.agent_id).toBe(agentId); + expect(list[0]!.parent_id ?? null).toBeNull(); + const text = JSON.stringify(list); + for (const other of CITIES) if (other !== city) expect(text, `${city}'s session mentions ${other}`).not.toContain(other); + const n = CITIES.indexOf(city); + for (const response of ofType(list, "model_response")) { + expect([response.input_tokens, response.output_tokens]).toEqual([100 + n, n]); + expect(response.model).toBe("city-1"); + } + expect(ofType(list, "agent_end")[0]).toMatchObject({ outcome: "success", summary: `It is sunny in ${city}.` }); + } + return byCity; + }; + + it("keeps 10 concurrent run()s of one agent object apart", () => { + const byCity = tenWay("concurrent-agents", (id) => WORKFLOW(id), "Agent"); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`); + expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`); + } + }); + + it("keeps 10 concurrent queries on one shared query engine apart", () => { + const byCity = tenWay( + "concurrent-queries", + (id) => [`${id} agent_start`, ...RETRIEVAL(id), ...MODEL(id), `${id} agent_end`], + "RetrieverQueryEngine", + ); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` }); + expect((ofType(list, "tool_result")[0]!.output as { top: Array<{ text: string }> }).top[0]!.text).toBe( + `${city} is sunny.`, + ); + } + }); + + it("keeps 10 concurrent chats on one shared chat engine apart", () => { + const byCity = tenWay( + "concurrent-chats", + (id) => [`${id} agent_start`, ...RETRIEVAL(id), ...MODEL(id), `${id} agent_end`], + "ContextChatEngine", + ); + for (const [city, list] of byCity) { + expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` }); + } + }); + }); +}); diff --git a/sdk/typescript/integration/mastra-coverage.test.ts b/sdk/typescript/integration/mastra-coverage.test.ts new file mode 100644 index 000000000..671c54ebe --- /dev/null +++ b/sdk/typescript/integration/mastra-coverage.test.ts @@ -0,0 +1,386 @@ +import { describe, expect, it } from "vitest"; + +import { FORMATS, count, describeTrace, ofType, runAgent, traceViolations, type Event } from "./harness.js"; + +/** + * Mastra, every commonly used surface beyond a bare `new Agent().generate()`, + * against real 0.x and 1.x releases in both module systems. The mapping each + * case expects is the one `src/integrations/mastra.ts` documents; the cases in + * `mastra.test.ts` cover the core loop, errors and the instrument lifecycle. + * + * Every model here is a mock that answers from what it is ASKED (the city in + * the question, whether the last message is a tool result, which schema the + * call requests), so concurrent runs, memory, networks and structured output + * can share one without a script going out of step. Nothing reaches a + * network; the MCP server is a local stdio child process. + */ + +const FIXTURES = ["mastra-1", "mastra-0"] as const; + +const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Tokyo", "Quito", "Dakar", "Hanoi", "Perth"]; + +/** One line per event: `agent_id type [hook|tool]`. */ +const shape = (events: Event[]): string[] => + events.map((e) => [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim()); + +const loop = (agent: string, tool = "weather"): string[] => [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} tool_use ${tool}`, + `${agent} tool_result ${tool}`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, +]; + +const sessions = (events: Event[]): string[] => [...new Set(events.map((e) => e.session_id))]; + +const bySession = (events: Event[]): Event[][] => sessions(events).map((id) => events.filter((e) => e.session_id === id)); + +const hooks = (flow: string, ...steps: string[]): string[] => + steps.flatMap((step) => [`${flow} hook_triggered ${step}`, `${flow} hook_completed ${step}`]); + +describe.each(FIXTURES)("%s", (fixture) => { + const zero = fixture === "mastra-0"; + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string) => { + const result = runAgent(fixture, format, scenario); + expect(result.status, describeTrace(result)).toBe(0); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + return result; + }; + + it("records an agent and a workflow fetched back from a Mastra instance", () => { + const result = run("mastra-instance"); + expect(shape(result.events), describeTrace(result)).toEqual([ + ...loop("weather-agent"), + "weather-flow agent_start", + ...hooks("weather-flow", "fetch-city"), + "weather-flow hook_triggered ask-agent", + ...loop("weather-agent"), + "weather-flow hook_completed ask-agent", + "weather-flow agent_end", + ]); + expect(sessions(result.events)).toHaveLength(2); + }); + + it("keeps 10 concurrent runs of one Agent apart in one session", () => { + const result = run("concurrent"); + expect(sessions(result.events)).toEqual(["req-1"]); + expect(count(result.events, "agent_start")).toBe(10); + expect(count(result.events, "agent_end")).toBe(10); + expect(count(result.events, "model_request")).toBe(20); + expect(count(result.events, "model_response")).toBe(20); + const uses = ofType(result.events, "tool_use"); + expect(uses.map((e) => (e.input as { city: string }).city).sort()).toEqual([...CITIES].sort()); + expect(new Set(uses.map((e) => e.tool_call_id)).size).toBe(10); + for (const end of ofType(result.events, "agent_end")) expect(end.outcome).toBe("success"); + expect(new Set(ofType(result.events, "agent_start").map((e) => e.goal)).size).toBe(10); + }); + + it("keeps 10 concurrent runs of one Agent (generate and stream) apart across sessions", () => { + const result = run("concurrent-sessions"); + const perSession = bySession(result.events); + expect(perSession).toHaveLength(10); + const seen: string[] = []; + for (const events of perSession) { + expect(shape(events), describeTrace(result)).toEqual(loop("weather-agent")); + // Every event of the run is about the run's own city: nothing crossed. + const city = /in ([A-Z][a-z]+)/.exec(String(events[0]!.goal))![1]!; + seen.push(city); + expect((ofType(events, "tool_use")[0]!.input as { city: string }).city).toBe(city); + expect(ofType(events, "tool_result")[0]!.output).toEqual({ city, forecast: "sunny" }); + expect(ofType(events, "model_response")[1]!.content).toBe(`It is sunny in ${city}.`); + for (const request of ofType(events, "model_request")) { + expect(JSON.stringify(request.messages)).toContain(city); + for (const other of CITIES.filter((c) => c !== city)) { + expect(JSON.stringify(request.messages)).not.toContain(other); + } + } + } + expect(seen.sort()).toEqual([...CITIES].sort()); + }); + + it("makes a memory thread the session of every run on it", () => { + const result = run("memory"); + expect(sessions(result.events)).toEqual(["thread-42"]); + const starts = ofType(result.events, "agent_start"); + expect(starts.map((e) => e.goal)).toEqual(["What is the weather in Paris?", "And what is the weather in Rome?"]); + for (const start of starts) expect(start).toMatchObject({ fw_thread_id: "thread-42", fw_resource_id: "user-7" }); + expect(count(result.events, "agent_end")).toBe(2); + expect(ofType(result.events, "tool_use").map((e) => (e.input as { city: string }).city)).toEqual(["Paris", "Rome"]); + // 0.x titles a new thread with the agent's own model: a real call, + // recorded on the run that made it. + expect(count(result.events, "model_request")).toBe(zero ? 5 : 4); + // Memory is real: the second run's first request carries the first run. + const second = result.events.findIndex((e) => e.type === "agent_start" && e.goal !== starts[0]!.goal); + const recalled = result.events.slice(second).find((e) => e.type === "model_request")!; + expect(JSON.stringify(recalled.messages)).toContain("What is the weather in Paris?"); + }); + + it("lets an enclosing session scope win over the memory thread", () => { + const result = run("memory-scoped"); + expect(sessions(result.events)).toEqual(["req-1"]); + expect(ofType(result.events, "agent_start")[0]).toMatchObject({ fw_thread_id: "thread-42" }); + }); + + it("records a branch as the branch taken", () => { + const result = run("wf-branch"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "branch-flow agent_start", + ...hooks("branch-flow", "city", "sunny"), + "branch-flow agent_end", + ]); + }); + + it("records parallel steps each as a hook", () => { + const result = run("wf-parallel"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "parallel-flow agent_start", + ...hooks("parallel-flow", "city"), + "parallel-flow hook_triggered high", + "parallel-flow hook_triggered low", + "parallel-flow hook_completed high", + "parallel-flow hook_completed low", + "parallel-flow agent_end", + ]); + }); + + it("records every iteration of a dowhile and a foreach", () => { + const result = run("wf-loop"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "loop-flow agent_start", + ...hooks("loop-flow", "count", "count", "count", "cities", "answer", "answer"), + "loop-flow agent_end", + ]); + expect(new Set(ofType(result.events, "hook_triggered").map((e) => e.hook_id)).size).toBe(6); + }); + + it("nests a nested workflow under the step that runs it", () => { + const result = run("wf-nested"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "outer-flow agent_start", + "outer-flow hook_triggered inner-flow", + "inner-flow agent_start", + ...hooks("inner-flow", "pick-city", "answer"), + "inner-flow agent_end", + "outer-flow hook_completed inner-flow", + "outer-flow agent_end", + ]); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("outer-flow"); + }); + + it("nests an agent used as a step (createStep(agent)) under the workflow", () => { + const result = run("wf-agent-step"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "agent-step-flow agent_start", + ...hooks("agent-step-flow", "to-prompt"), + "agent-step-flow hook_triggered weather-agent", + ...loop("weather-agent"), + "agent-step-flow hook_completed weather-agent", + "agent-step-flow agent_end", + ]); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("agent-step-flow"); + }); + + it("records a streamed workflow run (run.stream()) exactly like an awaited one", () => { + const result = run("wf-stream"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "agent-step-flow agent_start", + ...hooks("agent-step-flow", "to-prompt"), + "agent-step-flow hook_triggered weather-agent", + ...loop("weather-agent"), + "agent-step-flow hook_completed weather-agent", + "agent-step-flow agent_end", + ]); + expect(ofType(result.events, "agent_end").at(-1)!.outcome).toBe("success"); + }); + + it("records suspend / resume as a human wait on ONE workflow span", () => { + const result = run("wf-suspend"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "approval-flow agent_start", + ...hooks("approval-flow", "city", "approve"), + "approval-flow human_wait", + "approval-flow agent_pause", + "approval-flow agent_resume", + "approval-flow human_input", + ...hooks("approval-flow", "approve", "done"), + "approval-flow agent_end", + ]); + const runId = String(ofType(result.events, "agent_start")[0]!.fw_workflow_run_id); + expect(sessions(result.events)).toEqual([runId]); + const pauseId = `${runId}:approve`; + expect(ofType(result.events, "human_wait")[0]).toMatchObject({ input_id: pauseId, prompt: "Look up Paris?" }); + expect(ofType(result.events, "agent_pause")[0]).toMatchObject({ pause_id: pauseId }); + expect(ofType(result.events, "agent_resume")[0]).toMatchObject({ pause_id: pauseId }); + expect(ofType(result.events, "human_input")[0]).toMatchObject({ input_id: pauseId, response: '{"approved":true}' }); + expect(ofType(result.events, "hook_completed")[1]!.outcome).toBe("suspended"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(result.stdout).toContain('"status":"suspended"'); + expect(result.stdout).toContain('"answer":"sunny in Paris"'); + }); + + it("records a run through input and output processors exactly as one without", () => { + const result = run("processors"); + expect(shape(result.events), describeTrace(result)).toEqual([...loop("weather-agent"), ...loop("weather-agent")]); + expect(sessions(result.events)).toHaveLength(2); + expect(result.stdout).toContain('"answers":["It is sunny in Paris.","It is sunny in Paris."]'); + }); + + it("closes a run an input processor blocks, rejected — generate and stream", () => { + const result = run("tripwire"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent agent_end", + "weather-agent agent_start", + "weather-agent agent_end", + ]); + expect(ofType(result.events, "agent_end").map((e) => e.outcome)).toEqual(["rejected", "rejected"]); + }); + + // 0.x's output offers no end signal for a stream an OUTPUT processor + // blocks (no callback fires, no `_waitUntilFinished`), so there its agent + // stays open until uninstrument() — a known limitation. + it.skipIf(zero)("closes a stream an output processor blocks, rejected, with its model step", () => { + const result = run("tripwire-output"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + expect(ofType(result.events, "model_response")[0]).toMatchObject({ input_tokens: 9, output_tokens: 4 }); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("rejected"); + }); + + // The manual finding "Mastra streams from OpenAI-compatible providers carry + // no tokens": Mastra's own model never asks for streamed usage, so there is + // nothing to record — and when the stream does carry it, it is recorded. + it("records a streamed OpenAI-compatible step's tokens only when the stream carries them", () => { + const result = run("usage-openai-compatible"); + const report = JSON.parse(result.stdout.trim().split("\n").at(-1)!) as { + mastraStreamTokens: number; + mastraGenerateTokens: number; + requests: Array<{ stream?: boolean; streamOptions?: unknown }>; + }; + // The streamed request did not ask (no `stream_options`), and Mastra + // itself reports zero; the non-streamed call reports the real count. + expect(report.requests[0]).toEqual({ stream: true }); + expect(report.mastraStreamTokens).toBe(0); + expect(report.mastraGenerateTokens).toBe(17); + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [undefined, undefined], + [17, 5], + ]); + + const always = runAgent(fixture, format, "usage-openai-compatible", { USAGE_ALWAYS: "1" }); + expect(always.status, describeTrace(always)).toBe(0); + expect(ofType(always.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [17, 5], + [17, 5], + ]); + }); + + it("records an MCP tool (listTools) under its namespaced MCP name", () => { + const result = run("mcp"); + expect(shape(result.events), describeTrace(result)).toEqual(loop("weather-agent", "weatherServer_forecast")); + expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ city: "Paris" }); + expect(JSON.stringify(ofType(result.events, "tool_result")[0]!.output)).toContain("Forecast for Paris: sunny"); + expect(ofType(result.events, "tool_result")[0]!.error).toBeUndefined(); + }); + + it("records an MCP tool passed as a toolset under the name the model called", () => { + const result = run("mcp-toolsets"); + expect(shape(result.events), describeTrace(result)).toEqual(loop("weather-agent", "forecast")); + const call = (ofType(result.events, "model_response")[0]!.fw_tool_calls as Array<{ name: string }>)[0]!; + expect(call.name).toBe("forecast"); + expect(JSON.stringify(ofType(result.events, "tool_result")[0]!.output)).toContain("Forecast for Paris: sunny"); + }); + + it("records structured output (generate and stream) as one model step each", () => { + const result = run("structured"); + const one = [ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]; + expect(shape(result.events), describeTrace(result)).toEqual([...one, ...one]); + for (const response of ofType(result.events, "model_response")) { + expect(response).toMatchObject({ input_tokens: 15, output_tokens: 6 }); + expect(JSON.parse(String(response.content))).toEqual({ city: "Paris", forecast: "sunny" }); + } + expect(result.stdout).toContain('"object":{"city":"Paris","forecast":"sunny"}'); + }); + + it("nests structuring by a second model as its own agent, inside the run", () => { + const result = run("structured-model"); + expect(shape(result.events), describeTrace(result)).toEqual([ + ...loop("weather-agent").slice(0, -1), + "structured-output-structurer agent_start", + "structured-output-structurer model_request", + "structured-output-structurer model_response", + "structured-output-structurer agent_end", + "weather-agent agent_end", + ]); + const structurer = ofType(result.events, "agent_start")[1]!; + expect(structurer.parent_id).toBe("weather-agent"); + expect(ofType(result.events, "model_response")[2]).toMatchObject({ model: "structuring-model", input_tokens: 30 }); + }); + + it("records a one-step run whose tool fails: one model step, the failure on the tool", () => { + const result = run("maxsteps-tool-error"); + expect(shape(result.events), describeTrace(result)).toEqual(loop("weather-agent").filter((_, i) => i !== 5 && i !== 6)); + expect(String(ofType(result.events, "tool_result")[0]!.error)).toMatch(/tool exploded/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(count(result.events, "error")).toBe(0); + }); + + it("nests an agent a tool runs by hand under that tool call", () => { + const result = run("agent-in-tool"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "boss agent_start", + "boss model_request", + "boss model_response", + "boss tool_use ask-helper", + ...loop("helper"), + "boss tool_result ask-helper", + "boss model_request", + "boss model_response", + "boss agent_end", + ]); + expect(sessions(result.events)).toHaveLength(1); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("boss"); + }); + + it("records an agent network as one agent: its router's steps, its delegate nested", () => { + const result = run("network"); + // 0.x titles the network's new memory thread with the router's model. + const title = zero ? ["planner model_request", "planner model_response"] : []; + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + ...title, + "planner model_request", + "planner model_response", + ...loop("helper"), + "planner model_request", + "planner model_response", + "planner agent_end", + ]); + expect(sessions(result.events)).toEqual(["thread-net"]); + const [planner, helper] = ofType(result.events, "agent_start"); + expect(planner).toMatchObject({ fw_method: "network", fw_thread_id: "thread-net" }); + expect(helper!.parent_id).toBe("planner"); + const tokens = ofType(result.events, "model_response") + .filter((e) => e.agent_id === "planner") + .map((e) => [e.input_tokens, e.output_tokens]); + expect(tokens).toEqual([...(zero ? [[5, 3]] : []), [50, 10], [40, 6]]); + expect(ofType(result.events, "agent_end").at(-1)!.outcome).toBe("success"); + expect(result.stdout).toContain('"status":"success"'); + }); + }); +}); diff --git a/sdk/typescript/integration/mastra.test.ts b/sdk/typescript/integration/mastra.test.ts new file mode 100644 index 000000000..3ca46e00a --- /dev/null +++ b/sdk/typescript/integration/mastra.test.ts @@ -0,0 +1,311 @@ +import { describe, expect, it } from "vitest"; + +import { + FORMATS, + count, + describeTrace, + ofType, + runAgent, + traceViolations, + typecheck, + type Event, +} from "./harness.js"; + +/** + * Mastra (`@mastra/core`), against real releases. + * + * Mastra has no Python counterpart, so the expected traces are derived from the + * rule every adapter follows (`sdk/python/skill/references/frameworks.md`, "The + * rule the mappings follow"), using CrewAI and LlamaIndex as the analogues: + * + * * an `Agent.generate()` / `.stream()` call owns an LLM decision loop, so it + * is an AGENT, named after the Mastra agent — never its id if that is a + * UUID, never a constant; + * * a Mastra agent another agent calls (Mastra's `agents: {…}` delegation) is + * a nested agent under the caller, reached through the caller's tool call; + * * every LLM step is its own `model_request`/`model_response` pair — a + * two-step tool loop is two pairs, not one — carrying the model id, integer + * tokens and a string stop reason; + * * a tool is a `tool_use`/`tool_result` pair carrying the MODEL's tool call + * id, attributed to the agent that called it; + * * a workflow run is an agent named after the workflow, and its steps are + * HOOKS (`trigger_event="workflow_step"`), exactly as LlamaIndex workflow + * steps are; + * * Mastra's own machinery — the agentic loop is itself built from internal + * workflows — emits nothing; + * * a failure is carried by the events it happened in: no `error` events. + */ + +const FIXTURES = ["mastra-1", "mastra-0"] as const; + +/** One line per event: `agent_id type [hook|tool]`. */ +const shape = (events: Event[]): string[] => + events.map((e) => + [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim(), + ); + +const loop = (agent: string): string[] => [ + `${agent} agent_start`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} tool_use weather`, + `${agent} tool_result weather`, + `${agent} model_request`, + `${agent} model_response`, + `${agent} agent_end`, +]; + +const LOOP = loop("weather-agent"); + +describe.each(FIXTURES)("%s", (fixture) => { + it("typechecks as a customer's nodenext ES-module project", () => { + expect(typecheck(fixture)).toBe(""); + }); + + describe.each(FORMATS)("as %s", (format) => { + const run = (scenario: string) => { + const result = runAgent(fixture, format, scenario); + expect(result.status, describeTrace(result)).toBe(0); + return result; + }; + + it("records generate() as one agent with one model pair per LLM step", () => { + const result = run("generate"); + expect(shape(result.events), describeTrace(result)).toEqual(LOOP); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [11, 7], + [23, 9], + ]); + expect(responses.map((e) => e.stop_reason)).toEqual(["tool-calls", "stop"]); + for (const event of [...ofType(result.events, "model_request"), ...responses]) { + expect(event.model).toBe("scripted-model"); + } + expect(responses[1]!.content).toBe("It is sunny in Paris."); + const request = ofType(result.events, "model_request")[0]!; + expect(JSON.stringify(request.messages)).toContain("What is the weather in Paris?"); + + // The model's own tool call id, so a tool_use joins to the tool call in + // the model_response that asked for it. + const use = ofType(result.events, "tool_use")[0]!; + expect(use.tool_call_id).toBe("call_1"); + expect(use.input).toEqual({ city: "Paris" }); + const toolResult = ofType(result.events, "tool_result")[0]!; + expect(toolResult.tool_call_id).toBe("call_1"); + expect(toolResult.output).toEqual({ city: "Paris", forecast: "sunny" }); + expect(typeof toolResult.duration_ms).toBe("number"); + + const end = ofType(result.events, "agent_end")[0]!; + expect(end.outcome).toBe("success"); + expect(typeof end.duration_ms).toBe("number"); + for (const event of result.events) { + expect(event.framework).toBe("mastra"); + expect(typeof event.framework_version).toBe("string"); + } + expect(result.stdout).toContain("It is sunny in Paris."); + }); + + it("records a fully consumed stream() exactly like generate()", () => { + const result = run("stream"); + expect(shape(result.events), describeTrace(result)).toEqual(LOOP); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [11, 7], + [23, 9], + ]); + expect(responses.map((e) => e.stop_reason)).toEqual(["tool-calls", "stop"]); + expect(responses[1]!.content).toBe("It is sunny in Paris."); + expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(result.stdout).toContain("It is sunny in Paris."); + }); + + it("nests a sub-agent under the agent whose tool call delegated to it", () => { + const result = run("subagent"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "boss agent_start", + "boss model_request", + "boss model_response", + "boss tool_use agent-helper", + ...loop("helper"), + "boss tool_result agent-helper", + "boss model_request", + "boss model_response", + "boss agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + const starts = ofType(result.events, "agent_start"); + expect(starts[0]!.parent_id).toBeUndefined(); + expect(starts[1]!.parent_id).toBe("boss"); + expect( + ofType(result.events, "model_response") + .filter((e) => e.agent_id === "boss") + .map((e) => [e.input_tokens, e.output_tokens]), + ).toEqual([ + [40, 12], + [60, 8], + ]); + }); + + it("records a workflow run as one agent with its steps as hooks", () => { + const result = run("workflow"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-flow agent_start", + "weather-flow hook_triggered fetch-city", + "weather-flow hook_completed fetch-city", + "weather-flow hook_triggered ask-agent", + ...LOOP, + "weather-flow hook_completed ask-agent", + "weather-flow agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1); + for (const hook of ofType(result.events, "hook_triggered")) { + expect(hook.trigger_event).toBe("workflow_step"); + } + for (const hook of ofType(result.events, "hook_completed")) { + expect(hook.outcome).toBe("success"); + } + // The agent a step calls is nested under the workflow, and the tool that + // agent calls is ITS tool, not the workflow's. + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("weather-flow"); + expect(ofType(result.events, "tool_use")[0]!.agent_id).toBe("weather-agent"); + expect(ofType(result.events, "agent_end").at(-1)!.outcome).toBe("success"); + expect(result.stdout).toContain('"status":"success"'); + }); + + it("records a failed workflow step once, on the step", () => { + const result = run("workflow-error"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-flow agent_start", + "weather-flow hook_triggered fetch-city", + "weather-flow hook_completed fetch-city", + "weather-flow hook_triggered ask-agent", + "weather-flow hook_completed ask-agent", + "weather-flow agent_end", + ]); + const failed = ofType(result.events, "hook_completed")[1]!; + expect(failed.outcome).toBe("failed"); + expect(String(failed.error)).toMatch(/step exploded/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(count(result.events, "error")).toBe(0); + }); + + it("records a bare wrapTool() call as its own run", () => { + const result = run("wraptool"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather agent_start", + "weather tool_use weather", + "weather tool_result weather", + "weather agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ city: "Rome" }); + expect(ofType(result.events, "tool_result")[0]!.output).toEqual({ city: "Rome", forecast: "sunny" }); + }); + + it("records a tool failure on the tool, and the agent carries on", () => { + const result = run("tool-error"); + expect(shape(result.events), describeTrace(result)).toEqual(LOOP); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect(String(ofType(result.events, "tool_result")[0]!.error)).toMatch(/tool exploded/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(count(result.events, "error")).toBe(0); + }); + + for (const scenario of ["model-error", "stream-model-error"]) { + it(`records a model failure once, where it happened (${scenario})`, () => { + const result = run(scenario); + expect(result.stdout).toContain("model exploded"); + // Once per provider call. Mastra 0.x retries a failed model call + // itself (`maxRetries: 2` by default); each retry is a real request — + // it costs latency and, against a real provider, tokens — so each is + // its own pair, each carrying the error. 1.x does not retry. + const attempts = fixture === "mastra-0" ? 3 : 1; + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + ...Array.from({ length: attempts }, () => [ + "weather-agent model_request", + "weather-agent model_response", + ]).flat(), + "weather-agent agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + for (const response of ofType(result.events, "model_response")) { + expect(String(response.error)).toMatch(/model exploded/); + } + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(count(result.events, "error")).toBe(0); + }); + } + + it("nests the agent under an enclosing agent() scope", () => { + const result = run("scope"); + expect(shape(result.events), describeTrace(result)).toEqual([ + "planner agent_start", + ...LOOP, + "planner agent_end", + ]); + expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("planner"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records nothing after uninstrument()", () => { + const result = run("uninstrument"); + expect(result.events, describeTrace(result)).toEqual([]); + expect(result.stdout).toContain('"removed":["mastra"]'); + }); + + // uninstrument() does not reach what an instrumented run already built — + // a model behind a proxy, tools behind wrappers, a stream the caller is + // still reading — so each of these runs inside a session scope, where a + // stray event would have somewhere to land instead of being dropped. + + it("stops a stream in flight at uninstrument(), closing its agent cancelled", () => { + const result = run("uninstrument-midstream"); + expect(result.stdout).toContain('"removed":["mastra"]'); + // The stream is the caller's: it runs to the end, tool call and all. + expect(result.stdout).toContain("It is sunny in Paris."); + expect(shape(result.events), describeTrace(result)).toEqual([ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const response = ofType(result.events, "model_response")[0]!; + expect(response.stop_reason).toBe("incomplete"); + expect(response.fw_incomplete).toBe(true); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("cancelled"); + }); + + it("records nothing from an Agent reused after uninstrument()", () => { + const result = run("uninstrument-reuse"); + expect(result.stdout).toContain('"answers":["It is sunny in Paris.","It is sunny in Paris."]'); + expect(shape(result.events), describeTrace(result)).toEqual(LOOP); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a run once after instrument() → uninstrument() → instrument()", () => { + const result = run("reinstrument"); + expect(result.stdout).toContain('"removed":["mastra"]'); + expect(shape(result.events), describeTrace(result)).toEqual(LOOP); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records an Agent reused across uninstrument() → instrument() once per run", () => { + const result = run("reinstrument-reuse"); + expect(shape(result.events), describeTrace(result)).toEqual([...LOOP, ...LOOP]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + const sessions = result.events.map((e) => e.session_id); + expect(new Set(sessions.slice(0, LOOP.length)).size).toBe(1); + expect(new Set(sessions.slice(LOOP.length)).size).toBe(1); + }); + }); +}); diff --git a/sdk/typescript/integration/nextjs.test.ts b/sdk/typescript/integration/nextjs.test.ts new file mode 100644 index 000000000..98ce00593 --- /dev/null +++ b/sdk/typescript/integration/nextjs.test.ts @@ -0,0 +1,341 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { FIXTURES, describeTrace, runAgentAsync, traceViolations, type Event, type RunResult } from "./harness.js"; +import { digest } from "./runtime-parity.js"; + +/** + * Next.js (App Router), the way it is deployed next to failproofaid: `next + * build`, then `next start` as a long-running server, `instrument()` called + * from `instrumentation.ts` — Next's documented startup hook. + * + * Each route runs the same program as a framework fixture's scenario (its + * "twin"), with the same scripted model; the trace the route leaves in the + * spool must equal the twin's trace under plain Node. + * + * ## Bundled or external — the question this file answers per framework + * + * Next 16's built-in `serverExternalPackages` list includes none of the four + * frameworks, so by DEFAULT every one of them is bundled into the server + * chunks: a copy that no `require`/`import` of `node_modules` can reach. + * + * * `ai` records through `instrument()` either way — ai 7 reads its + * telemetry integrations from a global, which the bundled copy reads too. + * * LangChain, Mastra and LlamaIndex record through `instrument()` ONLY when + * listed in `serverExternalPackages`. Bundled, `instrument()` returns their + * names and records NOTHING: it patched the `node_modules` copy, the routes + * run the bundled one. Asserted below as empty, so a change shows up. + * * The call-site helpers (`langchainHandler()`, `telemetry()`, Mastra + * `wrapTool()`) work in every configuration: they travel with the call. + * + * External packages are loaded by Next with `import()` (Turbopack and webpack + * alike) from a CommonJS launcher; before `appImportsReachCommonJs` the SDK + * patched the CommonJS copies there and even the external configuration + * recorded nothing for those three. + * + * Twin fixtures (langchain-1, ai-7, mastra-1, llamaindex-0.12) must be + * installed alongside `nextjs`: `FAILPROOFAI_IT_FIXTURES=nextjs,langchain-1,ai-7,mastra-1,llamaindex-0.12`. + */ + +const APP = join(FIXTURES, "nextjs"); +const NEXT = join(APP, "node_modules", "next", "dist", "bin", "next"); + +interface Variant { + name: string; + dist: string; + bundler: "turbopack" | "webpack"; + external: boolean; + /** Built with `withFailproofai(config)` instead of a hand-written list. */ + wrapped?: boolean; +} + +const VARIANTS: Variant[] = [ + { name: "turbopack, default config (frameworks bundled)", dist: ".next-it-turbopack", bundler: "turbopack", external: false }, + { name: "turbopack, frameworks in serverExternalPackages", dist: ".next-it-turbopack-external", bundler: "turbopack", external: true }, + { name: "webpack, default config (frameworks bundled)", dist: ".next-it-webpack", bundler: "webpack", external: false }, + { name: "webpack, frameworks in serverExternalPackages", dist: ".next-it-webpack-external", bundler: "webpack", external: true }, + { name: "turbopack, withFailproofai(nextConfig)", dist: ".next-it-turbopack-wrapped", bundler: "turbopack", external: true, wrapped: true }, +]; + +const nextEnv = (variant: Variant): Record => ({ + NEXT_TELEMETRY_DISABLED: "1", + FAILPROOFAI_IT_NEXT_DIST: variant.dist, + FAILPROOFAI_IT_NEXT_EXTERNAL: variant.external && !variant.wrapped ? "1" : "", + FAILPROOFAI_IT_NEXT_WRAP: variant.wrapped ? "1" : "", + // A hand-written list carries no marker, so the documented override tells + // `instrument()` the app is configured. The wrapper needs none: it records + // what it externalized when Next evaluates the config. + FAILPROOFAI_NEXT_EXTERNALS: variant.external && !variant.wrapped ? "1" : "", +}); + +async function freePort(): Promise { + return await new Promise((resolvePort, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close(() => resolvePort(typeof address === "object" && address !== null ? address.port : 0)); + }); + }); +} + +const sleep = (ms: number) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); + +/** Reads only the spool files that appeared since the last call. */ +class Spool { + private seen = new Set(); + constructor(readonly dir: string) {} + + private fresh(): string[] { + if (!existsSync(this.dir)) return []; + return readdirSync(this.dir) + .filter((name) => name.endsWith(".jsonl") && !this.seen.has(name)) + .sort(); + } + + /** Wait for the request's events to land and stop arriving, then take them. */ + async take(options: { expectNone?: boolean } = {}): Promise { + const deadline = Date.now() + 20_000; + let lastCount = -1; + let stableSince = Date.now(); + // Four flush intervals with nothing new is "done"; for a case expected to + // record nothing, that same quiet period is the whole wait. + const quiet = 2_000; + for (;;) { + const count = this.fresh().length; + if (count !== lastCount) { + lastCount = count; + stableSince = Date.now(); + } + if ((count > 0 || options.expectNone) && Date.now() - stableSince >= quiet) break; + if (Date.now() > deadline) break; + await sleep(200); + } + const events: Event[] = []; + for (const name of this.fresh()) { + this.seen.add(name); + for (const line of readFileSync(join(this.dir, name), "utf8").split("\n")) { + if (line.trim()) events.push(JSON.parse(line) as Event); + } + } + return events.sort((a, b) => String(a.timestamp).localeCompare(String(b.timestamp))); + } +} + +/** Twins run once per file: the same scenario always yields the same trace. */ +const twins = new Map>(); +function twin(fixture: string, scenario: string): Promise { + const key = `${fixture}:${scenario}`; + if (!twins.has(key)) { + if (!existsSync(join(FIXTURES, fixture, ".run", "agent.mjs"))) { + throw new Error(`twin fixture ${fixture} is not installed; add it to FAILPROOFAI_IT_FIXTURES`); + } + twins.set(key, runAgentAsync(fixture, "esm", scenario)); + } + return twins.get(key)!; +} + +const asTrace = (events: Event[], stderr = ""): RunResult => ({ events, stdout: "", stderr, status: 0 }); + +describe.each(VARIANTS)("Next.js 16, $name", (variant) => { + let server: ChildProcess | null = null; + let base = ""; + let home = ""; + let spool: Spool; + let stderr = ""; + + beforeAll(async () => { + // Build from scratch. The harness extracts the freshly PACKED SDK into + // node_modules, and `npm pack` stamps every file with the same fixed mtime — + // so webpack's persistent cache (in the distDir) saw an unchanged SDK and + // kept compiling the previous run's copy into the bundle. Turbopack hashes + // content and was not fooled; webpack silently tested stale SDK code. + rmSync(join(APP, variant.dist), { recursive: true, force: true }); + const args = [NEXT, "build", ...(variant.bundler === "webpack" ? ["--webpack"] : [])]; + const build = spawnSync(process.execPath, args, { + cwd: APP, + encoding: "utf8", + env: { ...process.env, ...nextEnv(variant), NODE_OPTIONS: "" }, + timeout: 600_000, + }); + if (build.status !== 0) { + throw new Error(`next build failed (${variant.name}):\n${build.stdout}\n${build.stderr}`); + } + + home = mkdtempSync(join(tmpdir(), "failproofai-it-nextjs-")); + spool = new Spool(join(home, "custom-agents", "events")); + const port = await freePort(); + base = `http://127.0.0.1:${port}`; + server = spawn(process.execPath, [NEXT, "start", "-p", String(port), "-H", "127.0.0.1"], { + cwd: APP, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + ...nextEnv(variant), + NODE_OPTIONS: "", + FAILPROOFAI_HOME: home, + FAILPROOFAI_SDK_STRICT: "1", + }, + }); + server.stdout!.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + server.stderr!.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + for (let i = 0; i < 150; i += 1) { + try { + if ((await fetch(`${base}/api/status`)).ok) return; + } catch { + /* not up yet */ + } + await sleep(200); + } + throw new Error(`next start never answered:\n${stderr}`); + }); + + afterAll(async () => { + if (server !== null) { + server.kill("SIGTERM"); + await new Promise((resolveExit) => server!.once("exit", resolveExit)); + } + if (home) rmSync(home, { recursive: true, force: true }); + }); + + /** GET a route (reading the whole body, as a client does) and take its trace. */ + const call = async (path: string, options: { expectNone?: boolean } = {}) => { + const response = await fetch(`${base}${path}`); + const body = await response.text(); + expect(response.status, body).toBe(200); + return { body, events: await spool.take(options) }; + }; + + const expectTwin = async (events: Event[], fixture: string, scenario: string) => { + const expected = await twin(fixture, scenario); + expect(expected.status, describeTrace(expected)).toBe(0); + const context = `NEXT (${variant.name})\n${describeTrace(asTrace(events, stderr))}\n\nTWIN ${fixture} ${scenario}\n${describeTrace(expected)}`; + let got = events; + let want = expected.events; + if (fixture.startsWith("langchain")) { + // KNOWN ADAPTER ISSUE, pinned so its fix shows up: for a chat model with + // no model-name parameter the LangChain adapter falls back to the model's + // CLASS name, and Next minifies the app's classes — `ScriptedModel` + // arrives as `d` or `v5`. The rest of the trace must still match. + const models = (list: Event[]) => + [...new Set(list.filter((e) => e.type.startsWith("model_")).map((e) => e.model))]; + expect(models(want)).toEqual(["ScriptedModel"]); + expect(models(got), context).not.toEqual(["ScriptedModel"]); + const mask = (list: Event[]) => list.map((e) => (e.type.startsWith("model_") ? { ...e, model: "" } : e)); + got = mask(got); + want = mask(want); + } + expect(digest(got), context).toEqual(digest(want)); + expect(traceViolations(events), context).toEqual([]); + }; + + it("runs instrument() from instrumentation.ts before the first request", async () => { + const status = (await (await fetch(`${base}/api/status`)).json()) as { instrumented: string[] | null }; + expect(status.instrumented).toEqual(["ai", "langchain", "llamaindex", "mastra"]); + }); + + describe("call-site helpers work whether the framework is bundled or not", () => { + it("LangGraph with langchainHandler()", async () => { + const { body, events } = await call("/api/langgraph?mode=handler"); + expect(body).toContain("It is sunny in Paris."); + await expectTwin(events, "langchain-1", variant.external ? "handler-and-instrument" : "handler"); + }); + + it("ai generateText with telemetry()", async () => { + const { body, events } = await call("/api/ai?mode=telemetry"); + expect(body).toContain("It is 20C in Paris."); + await expectTwin(events, "ai-7", "generate"); + }); + + it("ai streamText with telemetry(), returned as toUIMessageStreamResponse()", async () => { + const { body, events } = await call("/api/ai?mode=stream"); + expect(body).toContain('"type":"finish"'); + await expectTwin(events, "ai-7", "stream"); + }); + + it("Mastra wrapTool()", async () => { + const { body, events } = await call("/api/mastra?mode=wraptool"); + expect(body).toContain("sunny"); + await expectTwin(events, "mastra-1", "wraptool"); + }); + + it("a server action running an agent", async () => { + const manifest = JSON.parse( + readFileSync(join(APP, variant.dist, "server", "server-reference-manifest.json"), "utf8"), + ) as { node: Record }; + const id = Object.entries(manifest.node).find(([, entry]) => entry.exportedName === "askWeather")?.[0]; + expect(id, "askWeather is in the server-reference manifest").toBeDefined(); + const response = await fetch(`${base}/`, { + method: "POST", + headers: { "Next-Action": id!, "Content-Type": "text/plain;charset=UTF-8", Accept: "text/x-component" }, + body: "[]", + }); + expect(await response.text()).toContain("It is 20C in Paris."); + await expectTwin(await spool.take(), "ai-7", "generate"); + }); + }); + + describe("instrument()", () => { + it("records the Vercel AI SDK, bundled or not", async () => { + await expectTwin((await call("/api/ai?mode=instrument")).events, "ai-7", "instrument"); + await expectTwin((await call("/api/ai?mode=instrument-stream")).events, "ai-7", "instrument-stream"); + }); + + const patched: Array<[string, string, string, string]> = [ + ["LangGraph", "/api/langgraph?mode=instrument", "langchain-1", "graph"], + ["a Mastra agent", "/api/mastra?mode=instrument", "mastra-1", "generate"], + ["a LlamaIndex agent workflow", "/api/llamaindex", "llamaindex-0.12", "workflow"], + ]; + if (variant.external) { + it.each(patched)("records %s when it is in serverExternalPackages", async (_label, path, fixture, scenario) => { + await expectTwin((await call(path)).events, fixture, scenario); + }); + } else { + it.each(patched)( + "records NOTHING for %s when Next bundles it — the known limitation", + async (_label, path) => { + const { body, events } = await call(path, { expectNone: true }); + expect(body).toContain("sunny"); + expect(events, describeTrace(asTrace(events, stderr))).toEqual([]); + }, + ); + } + }); + + it("serves an Edge-runtime route that imports the SDK: its no-op build, announced once, nothing recorded", async () => { + const { body, events } = await call("/api/edge", { expectNone: true }); + expect(JSON.parse(body)).toEqual({ imported: true, emitted: true, flushed: true }); + expect(events).toEqual([]); + expect(stderr.match(/loaded its no-op build/g) ?? []).toHaveLength(1); + }); + + it("warns for each adapter Next bundles, and otherwise prints nothing from the SDK", () => { + const lines = stderr + .split("\n") + .filter((line) => line.includes("[failproofai-sdk]") && !line.includes("loaded its no-op build")); + const warned = lines + .map((line) => /instrument\("([a-z]+)"\) is running under Next\.js/.exec(line)?.[1]) + .filter((name): name is string => name !== undefined) + .sort(); + // Bundled: exactly the three adapters that cannot reach a bundled copy, + // each once, naming the wrapper. Never the Vercel AI SDK, which records + // bundled or not. Configured (wrapper or hand-written list + override): + // silence. + expect(warned, stderr).toEqual(variant.external ? [] : ["langchain", "llamaindex", "mastra"]); + if (!variant.external) { + for (const line of lines) expect(line).toContain("withFailproofai"); + } + expect(lines.length, stderr).toBe(warned.length); + // Loading a second copy of LlamaIndex beside the one the app runs makes + // LlamaIndex itself warn. External: the SDK patches the app's own copy, so + // there is no second one. Bundled: `instrument()` loads the node_modules + // copy the bundle never uses — LlamaIndex notices, and says so. + const doubled = stderr.includes("llamaindex was already imported"); + expect(doubled, stderr).toBe(!variant.external); + }); +}); diff --git a/sdk/typescript/integration/runtime-parity.ts b/sdk/typescript/integration/runtime-parity.ts new file mode 100644 index 000000000..7e3f18b7a --- /dev/null +++ b/sdk/typescript/integration/runtime-parity.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from "vitest"; + +import { + describeTrace, + runAgentAsync, + scenarios, + traceViolations, + type Event, + type Format, + type RunResult, +} from "./harness.js"; + +/** + * Runtime parity: every scenario of every framework fixture, run under another + * runtime and under Node, must record the same trace. + * + * The framework files (`ai.test.ts`, …) say what a correct trace IS; this says + * only that Bun or Deno produce the one Node does. That keeps a runtime suite + * from restating — and drifting from — a hundred and eighty expectations, and + * it still catches every way a runtime differs that matters here: the adapter + * patched a copy the app does not run (no events), a hook never fired (a + * missing pair), the exit flush never ran (a short trace), a warning Node does + * not print. + */ + +export const FRAMEWORK_FIXTURES = [ + "ai-4", + "ai-5", + "ai-6", + "ai-7", + "langchain-0.3", + "langchain-1", + "mastra-0", + "mastra-1", + "llamaindex-0.11", + "llamaindex-0.12", +] as const; + +/** + * The fields of an event that are a property of the RUN, not of the moment: + * no timestamps, durations, generated ids or session ids. + */ +const STABLE = [ + "type", + "agent_id", + "parent_id", + "tool_name", + "hook_name", + "trigger_event", + "model", + "input_tokens", + "output_tokens", + "stop_reason", + "tool_call_id", + "outcome", + "framework", +] as const; + +/** A generated id (a tool call with no model-assigned id gets a UUID). */ +const GENERATED = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Each session's events in order, sessions sorted. Two sessions run + * concurrently (`shared-query`) interleave by scheduling, which is a property + * of the event loop and not of the SDK; the order WITHIN a session is not. + */ +export const digest = (events: Event[], unordered = false): string[][] => { + const sessions = new Map(); + for (const event of events) sessions.set(event.session_id, [...(sessions.get(event.session_id) ?? []), event]); + return [...sessions.values()] + .map((session) => (unordered ? digestSession(session).sort() : digestSession(session))) + .sort((a, b) => a.join().localeCompare(b.join())); +}; + +/** + * Scenarios that run several agents at once INSIDE ONE session (ten + * `generate`s on one agent, say). Their events interleave by scheduling within + * the session too, so for these the comparison is the same MULTISET of events + * per session rather than the same sequence — still exact about what was + * recorded, silent only about an order no runtime (Node included) repeats. + */ +const UNORDERED = /^concurrent/; + +/** + * Scenarios not compared at all, with the reason. Each is one its own + * framework suite already skips as a documented limit, where the trace is not + * a stable property of the run on any runtime. + */ +const SKIPPED: Record = { + "mastra-0:tripwire-output": + "Mastra 0.24 gives no end signal for an output-processor tripwire on a stream, so the run is left open (skipped in mastra-coverage.test.ts)", +}; + +const digestSession = (events: Event[]): string[] => + events.map((event) => + JSON.stringify( + Object.fromEntries( + STABLE.filter((key) => key in event).map((key) => { + const value = event[key]; + return [key, typeof value === "string" && GENERATED.test(value) ? "" : value]; + }), + ), + ), + ); + +/** The SDK's own diagnostics, without the parts that name paths or numbers. */ +export const sdkLines = (stderr: string): string[] => + stderr + .split("\n") + .filter((line) => line.includes("[failproofai-sdk]")) + .map((line) => line.replace(/\/[^\s"')]+/g, "").replace(/\d+/g, "N")); + +/** + * Labels in an agent's `switch` that are not scenarios: each is one half of a + * scenario that spawns it with arguments of its own, and is exercised through + * that scenario — under the runtime being tested, since the parent re-executes + * `process.execPath`. Run bare, they only exercise argument handling (and Deno, + * whose `writeFileSync(undefined)` does not throw, writes a file named + * `undefined` into the fixture). + */ +const HELPERS: Record = { + "remote-resume-pause": "the pausing process of remote-resume, which passes it a checkpoint path", +}; + +export interface Divergence { + /** Why this scenario is allowed to differ, stated as the observed behaviour. */ + reason: string; +} + +/** + * One `describe` per fixture and module system; one concurrent case per + * scenario. `known` lists scenarios whose difference is understood and is NOT + * the SDK's (each is asserted to still differ, so a fix upstream shows up), + * keyed `fixture:format:scenario` with `*` for either of the first two. + */ +export function parity( + label: string, + pairs: ReadonlyArray, + known: Record = {}, +): void { + describe.each(FRAMEWORK_FIXTURES)(`%s under ${label}`, (fixture) => { + describe.each(pairs)("%s vs %s", (nodeFormat, otherFormat) => { + it.concurrent.each( + scenarios(fixture).filter((name) => !(name in HELPERS) && !(`${fixture}:${name}` in SKIPPED)), + )("%s", async (scenario) => { + const [node, other] = await Promise.all([ + runAgentAsync(fixture, nodeFormat, scenario), + runAgentAsync(fixture, otherFormat, scenario), + ]); + const key = `${fixture}:${otherFormat}:${scenario}`; + const unordered = UNORDERED.test(scenario); + const matches = same(node, other, unordered); + const divergence = + known[key] ?? + known[`${fixture}:*:${scenario}`] ?? + known[`*:${otherFormat}:${scenario}`] ?? + known[`*:*:${scenario}`]; + if (divergence !== undefined) { + expect(matches, `${key} is listed as a known divergence (${divergence.reason}) but now matches Node`).toBe( + false, + ); + return; + } + const context = `NODE ${nodeFormat}\n${describeTrace(node)}\n\n${otherFormat.toUpperCase()}\n${describeTrace(other)}`; + expect(other.status, context).toBe(node.status); + expect(digest(other.events, unordered), context).toEqual(digest(node.events, unordered)); + expect(traceViolations(other.events), context).toEqual(traceViolations(node.events)); + expect(sdkLines(other.stderr), context).toEqual(sdkLines(node.stderr)); + }); + }); + }); +} + +function same(a: RunResult, b: RunResult, unordered: boolean): boolean { + return ( + a.status === b.status && + JSON.stringify(digest(a.events, unordered)) === JSON.stringify(digest(b.events, unordered)) && + JSON.stringify(sdkLines(a.stderr)) === JSON.stringify(sdkLines(b.stderr)) + ); +} diff --git a/sdk/typescript/integration/runtimes.bun.test.ts b/sdk/typescript/integration/runtimes.bun.test.ts new file mode 100644 index 000000000..f735c172d --- /dev/null +++ b/sdk/typescript/integration/runtimes.bun.test.ts @@ -0,0 +1,15 @@ +import { parity } from "./runtime-parity.js"; + +/** + * Bun: every scenario of every framework fixture, as an ES module and as + * CommonJS, against Node's trace for the same agent. + * + * Bun implements `node:module`'s `createRequire`, `require.cache`, + * `require.main`, `AsyncLocalStorage` and `process.on("exit")`, which is + * everything the SDK's copy resolution and exit flush rely on — this proves it + * on real frameworks instead of assuming it. + */ +parity("bun", [ + ["esm", "bun-esm"], + ["cjs", "bun-cjs"], +]); diff --git a/sdk/typescript/integration/runtimes.core.test.ts b/sdk/typescript/integration/runtimes.core.test.ts new file mode 100644 index 000000000..7a373c1c6 --- /dev/null +++ b/sdk/typescript/integration/runtimes.core.test.ts @@ -0,0 +1,113 @@ +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + BUN_FORMATS, + DENO_FORMATS, + FIXTURES, + FORMATS, + count, + describeTrace, + runAgentAsync, + runProcess, + runtimeCommand, + traceViolations, + type Event, +} from "./harness.js"; + +/** + * The SDK with no framework, under every runtime it can be imported in with a + * filesystem — Node, Bun, Deno — as an ES module and as CommonJS. + * + * Agent: `fixtures/runtimes/agent.ts`. + */ + +const ALL = [...FORMATS, ...BUN_FORMATS, ...DENO_FORMATS]; + +/** All 15 event types, each exactly as often as `core()` emits it. */ +const CORE_COUNTS: Record = { + agent_start: 2, + agent_end: 2, + model_request: 2, + model_response: 2, + tool_use: 2, + tool_result: 2, + hook_triggered: 1, + hook_completed: 1, + agent_pause: 1, + agent_resume: 1, + human_wait: 1, + human_input: 1, + human_pause: 1, + human_interrupt: 1, + error: 1, +}; + +const shape = (events: Event[]) => events.map((e) => `${e.agent_id} ${e.type}`); + +/** + * `traceViolations`, less the one rule this program breaks by design: a + * hand-emitted `model_response` carries no `duration_ms` (the SDK measures only + * the pairs it can see open — tools, hooks, pauses, human input — exactly like + * the Python SDK). The adapters are what supply model timings. + */ +const violations = (events: Event[]) => + traceViolations(events).filter((problem) => !problem.endsWith("has no duration_ms")); + +describe.each(ALL)("as %s", (format) => { + const run = (scenario: string) => runAgentAsync("runtimes", format, scenario); + + it("records every scope and all 15 event methods, and flush() writes them", async () => { + const result = await run("core"); + expect(result.status, describeTrace(result)).toBe(0); + expect(violations(result.events), describeTrace(result)).toEqual([]); + const counts = Object.fromEntries(Object.keys(CORE_COUNTS).map((type) => [type, count(result.events, type)])); + expect(counts, describeTrace(result)).toEqual(CORE_COUNTS); + expect(result.events).toHaveLength(21); + for (const event of result.events) expect(event.session_id).toBe("core-session"); + const helper = result.events.find((e) => e.type === "agent_start" && e.agent_id === "helper")!; + expect(helper.parent_id).toBe("planner"); + expect(result.stderr, describeTrace(result)).not.toContain("[failproofai-sdk]"); + }); +}); + +describe("Deno with npm: specifiers", () => { + const GEN = [ + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_use", + "weather-agent tool_result", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]; + const run = (scenario: string) => + runProcess([...runtimeCommand("deno"), "deno-npm.ts", scenario], { + cwd: join(FIXTURES, "runtimes"), + label: "deno-npm", + }); + + // wrapModel() sees model calls only — the tools run outside the model — so + // its trace is GEN without the tool pair, exactly as `ai.test.ts` expects. + const WRAPPED = GEN.filter((line) => !line.includes(" tool_")); + + it.each([ + ["telemetry", GEN], + ["wrap", WRAPPED], + ])("records an ai tool loop through %s()", async (scenario, expected) => { + const result = await run(scenario); + expect(result.status, describeTrace(result)).toBe(0); + expect(result.stdout).toContain("It is 20C in Paris."); + expect(shape(result.events), describeTrace(result)).toEqual(expected); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + expect( + result.events.filter((e) => e.type === "model_response").map((e) => [e.input_tokens, e.output_tokens]), + ).toEqual([ + [11, 7], + [23, 9], + ]); + expect(result.stderr, describeTrace(result)).not.toContain("[failproofai-sdk]"); + }); +}); diff --git a/sdk/typescript/integration/runtimes.deno.test.ts b/sdk/typescript/integration/runtimes.deno.test.ts new file mode 100644 index 000000000..7285b5c6b --- /dev/null +++ b/sdk/typescript/integration/runtimes.deno.test.ts @@ -0,0 +1,19 @@ +import { parity } from "./runtime-parity.js"; + +/** + * Deno (`deno run --allow-all`), running each fixture's transpiled agent + * straight out of the fixture's `node_modules` — Deno 2's npm compatibility, + * with the frameworks resolved exactly as Node resolves them. + * + * This file found the SDK's one Deno bug: Deno answers `require.main === null` + * (not `undefined`) for an ES-module entry, which the SDK read as "CommonJS", + * so `instrument()` patched the CommonJS copy of LangChain, Mastra and + * LlamaIndex while the app ran the ES-module copy — success reported, nothing + * recorded. See `isCommonJsMain` in `src/node-require.ts`. + * + * `npm:` specifiers and the core SDK under Deno are in `runtimes.core.test.ts`. + */ +parity("deno", [ + ["esm", "deno-esm"], + ["cjs", "deno-cjs"], +]); diff --git a/sdk/typescript/integration/types.test.ts b/sdk/typescript/integration/types.test.ts new file mode 100644 index 000000000..a18a608ce --- /dev/null +++ b/sdk/typescript/integration/types.test.ts @@ -0,0 +1,141 @@ +import { spawnSync } from "node:child_process"; +import { closeSync, copyFileSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { describe, expect, inject, it } from "vitest"; + +import { FIXTURES } from "./harness.js"; + +/** + * The packed tarball's type declarations, as every kind of TypeScript project + * sees them. + * + * The SDK's own tsconfig is `nodenext` ESM, the one setup in which a dual + * package whose `exports` hand ESM declarations to `require` looks fine. Every + * other setup a customer can carry was broken by exactly that, and silently + * from here: + * + * * a CommonJS project on `module: node16` (or `nodenext` before TS 5.8) got + * TS1479 — "the referenced file is an ECMAScript module" — on every import; + * * `moduleResolution: node` (node10, still what `module: commonjs` implies, + * e.g. every NestJS project) resolved the root and no subpath at all. + * + * So `agent.ts` — importing the root and every public subpath, using the names + * in ways a wrong or `any` type would fail — is compiled under each consumer + * mode, on the newest TypeScript and on the oldest one this package supports, + * with `skipLibCheck: false` so the declarations themselves are checked too. + * `@arethetypeswrong/cli` then runs over the same tarball: it checks JavaScript + * and declarations AGREE on module format, which no compile of ours can see. + * + * Minimum TypeScript: 5.4. It is the oldest release in which every mode below + * exists (`module: preserve` arrived in 5.4). The declarations themselves need + * only a type for `Symbol.dispose` (the `using` scopes), which `@types/node` + * supplies, as does `lib: esnext.disposable` from TS 5.2 — measured: 5.2 + * passes every mode it has; 5.0 passes them only with `skipLibCheck`, because + * `@types/node` 22 itself no longer compiles there. + */ + +const FIXTURE = "types"; +const DIR = join(FIXTURES, FIXTURE); + +interface Mode { + /** The consumer file's extension, which fixes its module format under node16/nodenext. */ + ext: "ts" | "mts" | "cts"; + module: string; + moduleResolution: string; + /** Written as the mode directory's package.json `type`; absent = CommonJS. */ + type?: "module"; +} + +const MODES: Record = { + "ESM nodenext": { ext: "ts", module: "nodenext", moduleResolution: "nodenext", type: "module" }, + "CJS .cts nodenext": { ext: "cts", module: "nodenext", moduleResolution: "nodenext" }, + "CJS node16": { ext: "ts", module: "node16", moduleResolution: "node16" }, + "CJS commonjs + node10": { ext: "ts", module: "commonjs", moduleResolution: "node" }, + "esnext + bundler": { ext: "ts", module: "esnext", moduleResolution: "bundler" }, + "preserve + bundler": { ext: "ts", module: "preserve", moduleResolution: "bundler" }, +}; + +/** The installed compiler packages, and the release each must be. */ +const COMPILERS = { typescript: "5.9.3", "typescript-min": "5.4.5" } as const; + +const ENTRYPOINTS = [".", "./ai", "./mastra", "./langchain", "./llamaindex", "./next", "./evaluator", "./sandbox-worker"]; + +function writeMode(name: string, mode: Mode): string { + const dir = join(DIR, ".run", "types", name.replace(/[^a-z0-9]+/gi, "-").toLowerCase()); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + const file = `agent.${mode.ext}`; + copyFileSync(join(DIR, "agent.ts"), join(dir, file)); + // Every mode gets a package.json, so the fixture's own cannot decide the + // format of a mode that expects the other one. + writeFileSync(join(dir, "package.json"), JSON.stringify(mode.type ? { type: mode.type } : {})); + writeFileSync( + join(dir, "tsconfig.json"), + JSON.stringify({ + compilerOptions: { + target: "ES2022", + module: mode.module, + moduleResolution: mode.moduleResolution, + strict: true, + noEmit: true, + skipLibCheck: false, + types: ["node"], + }, + files: [file], + }), + ); + return dir; +} + +function tsc(compiler: string, project: string): string { + const bin = join(DIR, "node_modules", compiler, "bin", "tsc"); + const result = spawnSync(process.execPath, [bin, "-p", project], { cwd: project, encoding: "utf8" }); + return (result.stdout + result.stderr).trim(); +} + +describe.each(Object.entries(COMPILERS))("TypeScript %s", (compiler, release) => { + it(`is ${release}`, () => { + const manifest = JSON.parse(readFileSync(join(DIR, "node_modules", compiler, "package.json"), "utf8")); + expect(manifest.version).toBe(release); + }); + + it.each(Object.entries(MODES))("typechecks every entry point as a %s project", (name, mode) => { + expect(tsc(compiler, writeMode(`${compiler}-${name}`, mode))).toBe(""); + }); +}); + +describe("@arethetypeswrong/cli", () => { + it("finds no problem in any entry point under any resolution mode", () => { + const attw = join(DIR, "node_modules", "@arethetypeswrong", "cli", "dist", "index.js"); + // Into a file, not a pipe: attw calls process.exit() straight after + // writing, which truncates a large report written to a pipe. + const out = join(DIR, ".run", "attw.json"); + mkdirSync(dirname(out), { recursive: true }); + const fd = openSync(out, "w"); + let result; + try { + result = spawnSync(process.execPath, [attw, inject("tarball"), "--format", "json"], { + cwd: DIR, + encoding: "utf8", + stdio: ["ignore", fd, "pipe"], + }); + } finally { + closeSync(fd); + } + const report = JSON.parse(readFileSync(out, "utf8")) as { + analysis: { + entrypoints: Record }>; + problems: unknown[]; + }; + }; + expect(Object.keys(report.analysis.entrypoints).filter((e) => e !== "./package.json").sort()).toEqual( + [...ENTRYPOINTS].sort(), + ); + for (const [entry, { resolutions }] of Object.entries(report.analysis.entrypoints)) { + expect(Object.keys(resolutions).sort(), entry).toEqual(["bundler", "node10", "node16-cjs", "node16-esm"]); + } + expect(report.analysis.problems).toEqual([]); + expect(result.status, result.stderr).toBe(0); + }); +}); diff --git a/sdk/typescript/integration/vanilla.test.ts b/sdk/typescript/integration/vanilla.test.ts new file mode 100644 index 000000000..556c13be2 --- /dev/null +++ b/sdk/typescript/integration/vanilla.test.ts @@ -0,0 +1,255 @@ +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { FIXTURES, FORMATS, ROOT, describeTrace, ofType, runAgentAsync, traceViolations, typecheck, type Event } from "./harness.js"; + +/** + * An agent with NO framework — the documented `examples/research-agent.ts`, + * run as a customer would: the real `openai` client, the packed SDK, as an ES + * module and as CommonJS. + * + * The fixture's `agent.ts` IS the example, byte for byte (the first test holds + * that), so what the docs tell a customer to copy is what this proves records a + * full trace: agent_start/agent_end around the run, a model_request/ + * model_response pair per turn with tokens and duration, a tool_use/tool_result + * pair per tool carrying the model's own tool-call id — the same shape the + * framework adapters produce. + * + * The model is a local OpenAI-compatible server playing a fixed conversation: + * two tools in parallel, then one that fails (an unknown item), then the answer. + */ + +const FIXTURE = "vanilla"; + +type Mode = "conversation" | "model-fails" | "bad-arguments"; + +interface ChatRequest { + messages: Array<{ role: string; content?: unknown }>; +} + +const usage = (input: number, output: number) => ({ + prompt_tokens: input, + completion_tokens: output, + total_tokens: input + output, +}); + +const toolCall = (id: string, name: string, item: string) => ({ + id, + type: "function", + function: { name, arguments: JSON.stringify({ item }) }, +}); + +/** The reply for this point of the conversation, decided by how many tool results it carries. */ +function reply(body: ChatRequest): Record { + const results = body.messages.filter((m) => m.role === "tool").length; + const message = + results === 0 + ? { role: "assistant", content: null, tool_calls: [toolCall("call_1", "price_of", "widget"), toolCall("call_2", "stock_of", "gadget")] } + : results === 2 + ? { role: "assistant", content: null, tool_calls: [toolCall("call_3", "stock_of", "doohickey")] } + : { role: "assistant", content: "widget 42 (120 in stock); gadget 17.5, out of stock." }; + return { + id: `chatcmpl-${results}`, + object: "chat.completion", + created: 0, + model: "fake-model", + choices: [{ index: 0, message, finish_reason: message.content ? "stop" : "tool_calls" }], + usage: usage(20 + results * 10, 7 + results), + }; +} + +let server: Server | null = null; + +async function startModel(mode: Mode): Promise { + server = createServer((req: IncomingMessage, res) => { + let raw = ""; + req.on("data", (chunk: Buffer) => (raw += chunk.toString())); + req.on("end", () => { + res.setHeader("content-type", "application/json"); + if (mode === "model-fails") { + // 400: not retried by the openai client, so the failure is immediate. + res.statusCode = 400; + res.end(JSON.stringify({ error: { message: "model gpt-imaginary does not exist", type: "invalid_request_error" } })); + return; + } + const body = JSON.parse(raw || "{}") as ChatRequest; + if (mode === "bad-arguments") { + // First turn: a tool call whose arguments are not JSON. Second: answer. + const answered = body.messages.some((m) => m.role === "tool"); + const message = answered + ? { role: "assistant", content: "could not look it up" } + : { + role: "assistant", + content: null, + tool_calls: [{ id: "call_9", type: "function", function: { name: "price_of", arguments: "{not json" } }], + }; + res.end( + JSON.stringify({ + id: "chatcmpl-bad", + object: "chat.completion", + created: 0, + model: "fake-model", + choices: [{ index: 0, message, finish_reason: answered ? "stop" : "tool_calls" }], + usage: usage(10, 5), + }), + ); + return; + } + res.end(JSON.stringify(reply(body))); + }); + }); + await new Promise((resolveListen) => server!.listen(0, "127.0.0.1", resolveListen)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`; +} + +afterEach(async () => { + server?.closeAllConnections(); + await new Promise((resolveClose) => (server ? server.close(() => resolveClose()) : resolveClose())); + server = null; +}); + +const shape = (events: Event[]): string[] => + events.map((e) => [e.agent_id, e.type, (e.tool_name ?? "") as string].join(" ").trim()); + +describe("an agent with no framework (examples/research-agent.ts)", () => { + it("is the documented example, byte for byte", () => { + const example = readFileSync(join(ROOT, "examples", "research-agent.ts"), "utf8"); + const fixture = readFileSync(join(FIXTURES, FIXTURE, "agent.ts"), "utf8"); + expect(fixture, "copy examples/research-agent.ts over integration/fixtures/vanilla/agent.ts").toBe(example); + }); + + it("typechecks against the real openai types", () => { + expect(typecheck(FIXTURE)).toBe(""); + }); + + it("the skill's no-framework snippet typechecks against the real openai types too", () => { + // Users copy this block, not the example file. It failed `tsc --strict` once + // openai's tool and tool-call types became unions, and nothing caught it. + const page = readFileSync(join(ROOT, "..", "python", "skill", "references", "typescript.md"), "utf8"); + const blocks = [...page.matchAll(/```ts\n([\s\S]*?)```/g)].map((m) => m[1]!); + const snippet = blocks.find((block) => block.includes("// 1. the run")); + expect(snippet, "the no-framework block in typescript.md").toBeDefined(); + const dir = join(FIXTURES, FIXTURE); + // What the snippet leaves to the reader's own program. + const context = [ + "declare const MODEL: string;", + "declare const client: OpenAI;", + "declare const TOOLS: OpenAI.Chat.Completions.ChatCompletionTool[];", + "declare const question: string;", + "declare const messages: ChatCompletionMessageParam[];", + "declare function runTool(name: string, input: Record): string;", + "export {};", + ].join("\n"); + writeFileSync(join(dir, "skill-snippet.ts"), `${snippet}\n${context}\n`); + writeFileSync( + join(dir, "tsconfig.skill.json"), + JSON.stringify({ extends: "./tsconfig.json", files: ["skill-snippet.ts"] }), + ); + try { + expect(typecheck(FIXTURE, "tsconfig.skill.json")).toBe(""); + } finally { + rmSync(join(dir, "skill-snippet.ts"), { force: true }); + rmSync(join(dir, "tsconfig.skill.json"), { force: true }); + } + }); + + describe.each(FORMATS)("as %s", (format) => { + it("records the full trace: the run, every model turn, every tool", async () => { + const baseUrl = await startModel("conversation"); + const result = await runAgentAsync(FIXTURE, format, "Price and stock for widget and gadget?", { + OPENAI_API_KEY: "test-key", + OPENAI_BASE_URL: baseUrl, + MODEL: "fake-model", + }); + expect(result.status, describeTrace(result)).toBe(0); + expect(result.stdout).toContain("gadget 17.5, out of stock"); + + expect(shape(result.events), describeTrace(result)).toEqual([ + "inventory agent_start", + "inventory model_request", + "inventory model_response", + "inventory tool_use price_of", + "inventory tool_result price_of", + "inventory tool_use stock_of", + "inventory tool_result stock_of", + "inventory model_request", + "inventory model_response", + "inventory tool_use stock_of", + "inventory tool_result stock_of", + "inventory model_request", + "inventory model_response", + "inventory agent_end", + ]); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + + const responses = ofType(result.events, "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([ + [20, 7], + [40, 9], + [50, 10], + ]); + expect(responses.map((e) => e.stop_reason)).toEqual(["tool_calls", "tool_calls", "stop"]); + // A turn that is only tool calls still says what the model asked for. + expect(responses.map((e) => (e.fw_tool_calls as Array<{ toolCallId: string }>).map((c) => c.toolCallId))).toEqual([ + ["call_1", "call_2"], + ["call_3"], + [], + ]); + for (const e of responses) expect(typeof e.duration_ms).toBe("number"); + + // The model's own tool-call ids, and the failure on its tool_result only. + expect(ofType(result.events, "tool_use").map((e) => e.tool_call_id)).toEqual(["call_1", "call_2", "call_3"]); + const results = ofType(result.events, "tool_result"); + expect(results.map((e) => e.output ?? null)).toEqual(["42", "0", null]); + expect(results[2]!.error).toMatch(/unknown item "doohickey"/); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(ofType(result.events, "agent_start")[0]!.goal).toBe("Price and stock for widget and gadget?"); + }); + + it("records a tool call with malformed arguments, and the run recovers", async () => { + const baseUrl = await startModel("bad-arguments"); + const result = await runAgentAsync(FIXTURE, format, "Price of widget?", { + OPENAI_API_KEY: "test-key", + OPENAI_BASE_URL: baseUrl, + MODEL: "fake-model", + }); + expect(result.status, describeTrace(result)).toBe(0); + const use = ofType(result.events, "tool_use")[0]!; + expect(use.tool_call_id).toBe("call_9"); + expect(use.input).toEqual({ arguments: "{not json" }); + const toolResult = ofType(result.events, "tool_result")[0]!; + expect(toolResult.tool_call_id).toBe("call_9"); + expect(toolResult.error).toMatch(/^SyntaxError: /); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + + it("records a failed model call once, and ends the run failed", async () => { + const baseUrl = await startModel("model-fails"); + const result = await runAgentAsync(FIXTURE, format, "Price of widget?", { + OPENAI_API_KEY: "test-key", + OPENAI_BASE_URL: baseUrl, + MODEL: "gpt-imaginary", + }); + expect(result.status, describeTrace(result)).toBe(1); + expect(shape(result.events), describeTrace(result)).toEqual([ + "inventory agent_start", + "inventory model_request", + "inventory model_response", + "inventory error", + "inventory agent_end", + ]); + const response = ofType(result.events, "model_response")[0]!; + expect(response.stop_reason).toBe("error"); + expect(response.error).toMatch(/does not exist/); + // The class, not the `name` openai's errors leave as "Error". + expect(response.error).toMatch(/^BadRequestError: /); + expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("failed"); + expect(traceViolations(result.events), describeTrace(result)).toEqual([]); + }); + }); +}); diff --git a/sdk/typescript/package-lock.json b/sdk/typescript/package-lock.json new file mode 100644 index 000000000..931b15380 --- /dev/null +++ b/sdk/typescript/package-lock.json @@ -0,0 +1,2640 @@ +{ + "name": "@failproofai/sdk", + "version": "0.0.1-beta.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@failproofai/sdk", + "version": "0.0.1-beta.0", + "license": "SEE LICENSE IN LICENSE", + "bin": { + "failproofai-evaluator": "dist/esm/evaluator/cli.js" + }, + "devDependencies": { + "@eslint/js": "^9.38.0", + "@types/node": "^22.10.2", + "eslint": "^9.38.0", + "typescript": "^5.9.2", + "typescript-eslint": "^8.46.0", + "vitest": "^5.0.0" + }, + "engines": { + "node": ">=20.9.0" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.0 <2", + "@mastra/core": ">=0.20.0 <2", + "ai": ">=4.0.0 <8", + "llamaindex": ">=0.11.4 <1" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "@mastra/core": { + "optional": true + }, + "ai": { + "optional": true + }, + "llamaindex": { + "optional": true + } + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.7.tgz", + "integrity": "sha512-F42g89Qd5oAWtp0k0nnSrjziAKza7w8SVT4mStc18LZMaRb4J1HQAHLCalEtDCxrTuksx7NU9qsmeLwpOfPqWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.3.2", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.5.tgz", + "integrity": "sha512-QywQuszQh77pIXCsq998c8hbhSTI/azTty1Z6N53dmAudKHhy573j3yvRLsX2BSp8YpLtoCEG8E9DJe+8zUh4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.150.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", + "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", + "dev": true, + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", + "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", + "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", + "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", + "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", + "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", + "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", + "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", + "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", + "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", + "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", + "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", + "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", + "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", + "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", + "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.70.1.tgz", + "integrity": "sha512-nDNrUQ/4ruSNYbu749TRY7cfrzPtoLHEXSNBI8aaNY32LlZCajixqRf3FqcKC4p5Cam4VOHYx/t+i5+nKXvrqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.70.1", + "@typescript-eslint/type-utils": "8.70.1", + "@typescript-eslint/utils": "8.70.1", + "@typescript-eslint/visitor-keys": "8.70.1", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.70.1", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.10.tgz", + "integrity": "sha512-HpbUakT7xp5miBUywCHf36ZEuAJNklBJDDsGpUIjMzOSmM8ELSfA9Sa/QDPeNeqeoN31u+UTCkL4klCOVvRm4Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.70.1.tgz", + "integrity": "sha512-nO974WLllwhSFWQXnMLj6nDGa8f0khKEz1JzpPJ1u7Vm/4X1X6ZHajpoknU4bb41vJyMB0HHVyS2GqdhWfIXZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.70.1", + "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/typescript-estree": "8.70.1", + "@typescript-eslint/visitor-keys": "8.70.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.70.1.tgz", + "integrity": "sha512-62xOgboPfwc3/IgPSX/W6oQR3ZbF04194FPGUGH8HL8iLFHbt/456/8Ph1wLNUgVF+s94FlHoipBsz+v7+LMnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.70.1", + "@typescript-eslint/types": "^8.70.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.70.1.tgz", + "integrity": "sha512-Pa0EeSeAusQc1WbjQMac+YfenewYTBu0KjgYvkUKwhXaHUKbFog23Dm/rp0DX/6tyYOQ3Xl1a+3EcFNZynGHCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/visitor-keys": "8.70.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.70.1.tgz", + "integrity": "sha512-jumze1fPI+sDOaM2TWGQdn39PDxTr7TZGeuyLkAbNyx2vtMT3uRnVKChN0hfht5V2TugphJzF6bYXvBcE09qqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.70.1.tgz", + "integrity": "sha512-7zKTnyvaVWqzLZHPFQtX1hVHqgkMC+WebPWakNCSyrQVbIP1AM0L0TlBZtACldIRb6PptI8Odk+jyZ5kP3B1VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/typescript-estree": "8.70.1", + "@typescript-eslint/utils": "8.70.1", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.70.1.tgz", + "integrity": "sha512-Dm1ypdhhrGCTyyehxElhgJ6kgk8MVCv5qXdoOVqPr1uqk42jX8KjrZqhROvdShczA8qrDoYiOWn1ykWlx2k81Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.70.1.tgz", + "integrity": "sha512-TU8PwyGN0PQJUcE96mw8eCQ44SmxGdQlJmlWakHaHQ15eIuuvye5yNtmh/i6oS88jzXVQB71xdNkbkB/fMwL0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.70.1", + "@typescript-eslint/tsconfig-utils": "8.70.1", + "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/visitor-keys": "8.70.1", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.70.1.tgz", + "integrity": "sha512-Esgul8MsnKnRLdYU2Eb2cRV9bS5HJYtKj1ByJnOzzG2M58DGdSUQ1jUuILxipqcpB2h9WLrbD5GijIWUjX/Tqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.70.1", + "@typescript-eslint/types": "8.70.1", + "@typescript-eslint/typescript-estree": "8.70.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.70.1.tgz", + "integrity": "sha512-Vwj9lUIW5Xq3wQ9w6gv3R86g1hMK8f2zNOdGTAgeXUMMXFK78G9ruCjjqutHMNJc0+CH7LYRnHeUB9IT8wFmcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.70.1", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@vitest/mocker": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.1.tgz", + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", + "@vitest/spy": "5.0.1", + "estree-walker": "^3.0.3", + "magic-string": "^1.2.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/spy": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.1.tgz", + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.5.tgz", + "integrity": "sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.6", + "@eslint/js": "9.39.5", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "peer": true, + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.1.tgz", + "integrity": "sha512-8lyCu36ErXR0J9uaGKlKQoiLZKmtI63YGLE8G2o9jyRPdr4X47LusSOwgOJOzcVtp81fTAAjxR7BwKz682Jhow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rolldown": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", + "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@oxc-project/types": "=0.150.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.9", + "@rolldown/binding-android-arm64": "1.2.9", + "@rolldown/binding-darwin-arm64": "1.2.9", + "@rolldown/binding-darwin-x64": "1.2.9", + "@rolldown/binding-freebsd-x64": "1.2.9", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", + "@rolldown/binding-linux-arm64-gnu": "1.2.9", + "@rolldown/binding-linux-arm64-musl": "1.2.9", + "@rolldown/binding-linux-ppc64-gnu": "1.2.9", + "@rolldown/binding-linux-s390x-gnu": "1.2.9", + "@rolldown/binding-linux-x64-gnu": "1.2.9", + "@rolldown/binding-linux-x64-musl": "1.2.9", + "@rolldown/binding-openharmony-arm64": "1.2.9", + "@rolldown/binding-win32-arm64-msvc": "1.2.9", + "@rolldown/binding-win32-x64-msvc": "1.2.9" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tinybench": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.1.4.tgz", + "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.70.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.70.1.tgz", + "integrity": "sha512-AcWG7KDjZ2THNXsgwttMaGmzVi0VFRlFYfqFHYQRbDpF3owuYbuiL8c7UUrd2k8s3PoSfIQrWfrGXfcElrWLYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.70.1", + "@typescript-eslint/parser": "8.70.1", + "@typescript-eslint/typescript-estree": "8.70.1", + "@typescript-eslint/utils": "8.70.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.1.tgz", + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/mocker": "5.0.1", + "chai": "^6.2.2", + "es-module-lexer": "^2.3.2", + "expect-type": "^1.4.0", + "magic-string": "^1.2.3", + "obug": "^2.1.4", + "picomatch": "^4.0.7", + "std-env": "^4.2.0", + "tinybench": "6.1.4", + "tinyexec": "1.3.0", + "tinyglobby": "^0.2.17", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^22.12.0 || ^24.0.0 || >=26.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "5.0.1", + "@vitest/browser-preview": "5.0.1", + "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", + "@vitest/coverage-istanbul": "5.0.1", + "@vitest/coverage-v8": "5.0.1", + "@vitest/ui": "5.0.1", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json new file mode 100644 index 000000000..ed6b5dfe0 --- /dev/null +++ b/sdk/typescript/package.json @@ -0,0 +1,401 @@ +{ + "name": "@failproofai/sdk", + "version": "0.0.1-beta.0", + "description": "Emit agent telemetry to FailproofAI Cloud \u2014 zero-dependency, Node stdlib only", + "keywords": [ + "agents", + "observability", + "ai", + "monitoring", + "failproofai", + "telemetry", + "sdk", + "langchain", + "langgraph", + "ai-sdk", + "mastra", + "llamaindex", + "evals" + ], + "homepage": "https://befailproof.ai", + "repository": { + "type": "git", + "url": "git+https://github.com/FailproofAI/failproofai.git", + "directory": "sdk/typescript" + }, + "bugs": { + "url": "https://github.com/FailproofAI/failproofai/issues" + }, + "author": "Failproof AI ", + "license": "SEE LICENSE IN LICENSE", + "type": "module", + "engines": { + "node": ">=20.9.0" + }, + "main": "./dist/cjs/index.js", + "module": "./dist/esm/index.js", + "types": "./dist/cjs/index.d.ts", + "typesVersions": { + "*": { + "ai": [ + "./dist/cjs/integrations/ai.d.ts" + ], + "mastra": [ + "./dist/cjs/integrations/mastra.d.ts" + ], + "langchain": [ + "./dist/cjs/integrations/langchain.d.ts" + ], + "llamaindex": [ + "./dist/cjs/integrations/llamaindex.d.ts" + ], + "evaluator": [ + "./dist/cjs/evaluator/index.d.ts" + ], + "sandbox-worker": [ + "./dist/cjs/evaluator/sandbox-worker.d.ts" + ], + "next": [ + "./dist/cjs/next.d.ts" + ] + } + }, + "exports": { + ".": { + "edge-light": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/edge/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/edge/index.js" + } + }, + "workerd": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/edge/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/edge/index.js" + } + }, + "worker": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/edge/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/edge/index.js" + } + }, + "browser": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/edge/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/edge/index.js" + } + }, + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "./ai": { + "edge-light": { + "import": { + "types": "./dist/esm/integrations/ai.d.ts", + "default": "./dist/esm/edge/ai.js" + }, + "require": { + "types": "./dist/cjs/integrations/ai.d.ts", + "default": "./dist/cjs/edge/ai.js" + } + }, + "workerd": { + "import": { + "types": "./dist/esm/integrations/ai.d.ts", + "default": "./dist/esm/edge/ai.js" + }, + "require": { + "types": "./dist/cjs/integrations/ai.d.ts", + "default": "./dist/cjs/edge/ai.js" + } + }, + "worker": { + "import": { + "types": "./dist/esm/integrations/ai.d.ts", + "default": "./dist/esm/edge/ai.js" + }, + "require": { + "types": "./dist/cjs/integrations/ai.d.ts", + "default": "./dist/cjs/edge/ai.js" + } + }, + "browser": { + "import": { + "types": "./dist/esm/integrations/ai.d.ts", + "default": "./dist/esm/edge/ai.js" + }, + "require": { + "types": "./dist/cjs/integrations/ai.d.ts", + "default": "./dist/cjs/edge/ai.js" + } + }, + "import": { + "types": "./dist/esm/integrations/ai.d.ts", + "default": "./dist/esm/integrations/ai.js" + }, + "require": { + "types": "./dist/cjs/integrations/ai.d.ts", + "default": "./dist/cjs/integrations/ai.js" + } + }, + "./mastra": { + "edge-light": { + "import": { + "types": "./dist/esm/integrations/mastra.d.ts", + "default": "./dist/esm/edge/mastra.js" + }, + "require": { + "types": "./dist/cjs/integrations/mastra.d.ts", + "default": "./dist/cjs/edge/mastra.js" + } + }, + "workerd": { + "import": { + "types": "./dist/esm/integrations/mastra.d.ts", + "default": "./dist/esm/edge/mastra.js" + }, + "require": { + "types": "./dist/cjs/integrations/mastra.d.ts", + "default": "./dist/cjs/edge/mastra.js" + } + }, + "worker": { + "import": { + "types": "./dist/esm/integrations/mastra.d.ts", + "default": "./dist/esm/edge/mastra.js" + }, + "require": { + "types": "./dist/cjs/integrations/mastra.d.ts", + "default": "./dist/cjs/edge/mastra.js" + } + }, + "browser": { + "import": { + "types": "./dist/esm/integrations/mastra.d.ts", + "default": "./dist/esm/edge/mastra.js" + }, + "require": { + "types": "./dist/cjs/integrations/mastra.d.ts", + "default": "./dist/cjs/edge/mastra.js" + } + }, + "import": { + "types": "./dist/esm/integrations/mastra.d.ts", + "default": "./dist/esm/integrations/mastra.js" + }, + "require": { + "types": "./dist/cjs/integrations/mastra.d.ts", + "default": "./dist/cjs/integrations/mastra.js" + } + }, + "./langchain": { + "edge-light": { + "import": { + "types": "./dist/esm/integrations/langchain.d.ts", + "default": "./dist/esm/edge/langchain.js" + }, + "require": { + "types": "./dist/cjs/integrations/langchain.d.ts", + "default": "./dist/cjs/edge/langchain.js" + } + }, + "workerd": { + "import": { + "types": "./dist/esm/integrations/langchain.d.ts", + "default": "./dist/esm/edge/langchain.js" + }, + "require": { + "types": "./dist/cjs/integrations/langchain.d.ts", + "default": "./dist/cjs/edge/langchain.js" + } + }, + "worker": { + "import": { + "types": "./dist/esm/integrations/langchain.d.ts", + "default": "./dist/esm/edge/langchain.js" + }, + "require": { + "types": "./dist/cjs/integrations/langchain.d.ts", + "default": "./dist/cjs/edge/langchain.js" + } + }, + "browser": { + "import": { + "types": "./dist/esm/integrations/langchain.d.ts", + "default": "./dist/esm/edge/langchain.js" + }, + "require": { + "types": "./dist/cjs/integrations/langchain.d.ts", + "default": "./dist/cjs/edge/langchain.js" + } + }, + "import": { + "types": "./dist/esm/integrations/langchain.d.ts", + "default": "./dist/esm/integrations/langchain.js" + }, + "require": { + "types": "./dist/cjs/integrations/langchain.d.ts", + "default": "./dist/cjs/integrations/langchain.js" + } + }, + "./llamaindex": { + "edge-light": { + "import": { + "types": "./dist/esm/integrations/llamaindex.d.ts", + "default": "./dist/esm/edge/llamaindex.js" + }, + "require": { + "types": "./dist/cjs/integrations/llamaindex.d.ts", + "default": "./dist/cjs/edge/llamaindex.js" + } + }, + "workerd": { + "import": { + "types": "./dist/esm/integrations/llamaindex.d.ts", + "default": "./dist/esm/edge/llamaindex.js" + }, + "require": { + "types": "./dist/cjs/integrations/llamaindex.d.ts", + "default": "./dist/cjs/edge/llamaindex.js" + } + }, + "worker": { + "import": { + "types": "./dist/esm/integrations/llamaindex.d.ts", + "default": "./dist/esm/edge/llamaindex.js" + }, + "require": { + "types": "./dist/cjs/integrations/llamaindex.d.ts", + "default": "./dist/cjs/edge/llamaindex.js" + } + }, + "browser": { + "import": { + "types": "./dist/esm/integrations/llamaindex.d.ts", + "default": "./dist/esm/edge/llamaindex.js" + }, + "require": { + "types": "./dist/cjs/integrations/llamaindex.d.ts", + "default": "./dist/cjs/edge/llamaindex.js" + } + }, + "import": { + "types": "./dist/esm/integrations/llamaindex.d.ts", + "default": "./dist/esm/integrations/llamaindex.js" + }, + "require": { + "types": "./dist/cjs/integrations/llamaindex.d.ts", + "default": "./dist/cjs/integrations/llamaindex.js" + } + }, + "./next": { + "import": { + "types": "./dist/esm/next.d.ts", + "default": "./dist/esm/next.js" + }, + "require": { + "types": "./dist/cjs/next.d.ts", + "default": "./dist/cjs/next.js" + } + }, + "./evaluator": { + "import": { + "types": "./dist/esm/evaluator/index.d.ts", + "default": "./dist/esm/evaluator/index.js" + }, + "require": { + "types": "./dist/cjs/evaluator/index.d.ts", + "default": "./dist/cjs/evaluator/index.js" + } + }, + "./sandbox-worker": { + "types": "./dist/cjs/evaluator/sandbox-worker.d.ts", + "require": "./dist/cjs/evaluator/sandbox-worker.js", + "default": "./dist/cjs/evaluator/sandbox-worker.js" + }, + "./package.json": "./package.json" + }, + "bin": { + "failproofai-evaluator": "./dist/esm/evaluator/cli.js" + }, + "files": [ + "dist/", + "README.md", + "CHANGELOG.md", + "LICENSE" + ], + "sideEffects": [ + "./dist/esm/writer.js", + "./dist/cjs/writer.js", + "./dist/esm/runtime.js", + "./dist/cjs/runtime.js", + "./dist/esm/evaluator/sandbox-worker.js", + "./dist/cjs/evaluator/sandbox-worker.js", + "./dist/esm/evaluator/cli.js", + "./dist/cjs/evaluator/cli.js" + ], + "scripts": { + "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", + "build": "npm run clean && tsc -p tsconfig.build.json && tsc -p tsconfig.cjs.json && node scripts/finalize-build.mjs", + "typecheck": "tsc -p tsconfig.json --noEmit", + "lint": "eslint . --config eslint.config.mjs", + "test": "npm run build && vitest run", + "test:integration": "npm run build && vitest run --config vitest.integration.config.ts", + "test:watch": "vitest", + "prepublishOnly": "npm run build" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.0 <2", + "@mastra/core": ">=0.20.0 <2", + "ai": ">=4.0.0 <8", + "llamaindex": ">=0.11.4 <1" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "@mastra/core": { + "optional": true + }, + "ai": { + "optional": true + }, + "llamaindex": { + "optional": true + } + }, + "devDependencies": { + "@eslint/js": "^9.38.0", + "@types/node": "^22.10.2", + "eslint": "^9.38.0", + "typescript": "^5.9.2", + "typescript-eslint": "^8.46.0", + "vitest": "^5.0.0" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/sdk/typescript/scripts/finalize-build.mjs b/sdk/typescript/scripts/finalize-build.mjs new file mode 100644 index 000000000..b9b328ded --- /dev/null +++ b/sdk/typescript/scripts/finalize-build.mjs @@ -0,0 +1,123 @@ +/** + * Post-process the two `tsc` outputs into something Node can actually load. + * + * Three things `tsc` will not do on its own: + * + * 1. **Tell Node which half is which.** The package is `"type": "module"`, so + * every `.js` under `dist/` is treated as ESM — including the CommonJS build, + * whose `require()` calls would then be a syntax error. A `package.json` + * carrying `"type": "commonjs"` inside `dist/cjs/` is the supported way to + * say otherwise, and it applies to that directory only. + * + * 2. **Make the CLI executable.** A `bin` entry that is not executable fails + * with a permission error on every platform where the installer links rather + * than copies, and the shebang in the source is not enough on its own. + * + * 3. **Check the promise this package makes.** Zero runtime dependencies is a + * contract, not an aspiration: this package installs into other people's + * agent processes, so any dependency we declare is a version constraint they + * inherit. Asserting it here means a `npm install --save` somebody forgot to + * revert fails the build rather than shipping. + */ + +import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); + +const problems = []; + +for (const field of ["dependencies", "optionalDependencies"]) { + const declared = Object.keys(manifest[field] ?? {}); + if (declared.length > 0) { + problems.push( + `package.json declares ${field}: ${declared.join(", ")}. ` + + "This package is contractually zero-dependency — see README.md.", + ); + } +} + +writeFileSync( + join(root, "dist/cjs/package.json"), + `${JSON.stringify({ type: "commonjs" }, null, 2)}\n`, +); +writeFileSync( + join(root, "dist/esm/package.json"), + `${JSON.stringify({ type: "module" }, null, 2)}\n`, +); + +const cli = join(root, "dist/esm/evaluator/cli.js"); +if (existsSync(cli)) { + chmodSync(cli, 0o755); +} else { + problems.push("dist/esm/evaluator/cli.js is missing; the `bin` entry would not resolve."); +} + +// Every path the `exports` map promises must exist, or the failure lands on a +// consumer's `import` rather than on this build. Conditions nest +// (`require.types`), so walk them. +// +// And every declaration must sit in the same half as the JavaScript it +// describes: `.d.ts` files take their module format from the nearest +// package.json exactly like `.js` files do, so ESM declarations under a +// `require` condition tell TypeScript a CommonJS file is an ES module. +const walkConditions = (path, value, visit) => { + if (typeof value === "string") { + visit(path, value); + return; + } + for (const [condition, next] of Object.entries(value)) walkConditions([...path, condition], next, visit); +}; +for (const [entry, conditions] of Object.entries(manifest.exports)) { + if (typeof conditions === "string") continue; + const targets = []; + walkConditions([], conditions, (path, target) => { + targets.push({ path, target }); + if (!existsSync(join(root, target))) { + problems.push(`exports["${entry}"].${path.join(".")} points at ${target}, which was not built.`); + } + }); + for (const half of ["import", "require"]) { + const dirs = new Set( + targets + .filter(({ path }) => path[0] === half) + .map(({ target }) => target.split("/").slice(0, 3).join("/")), + ); + if (dirs.size > 1) { + problems.push(`exports["${entry}"].${half} mixes ${[...dirs].join(" and ")}; types and JavaScript must match.`); + } + } +} + +// `moduleResolution: node` (node10) ignores `exports`, so each subpath reaches +// it only through `typesVersions` — and must land on the CommonJS declarations, +// which is what a node10 project compiles to. +for (const [range, mapping] of Object.entries(manifest.typesVersions ?? {})) { + for (const [subpath, targets] of Object.entries(mapping)) { + for (const target of targets) { + if (!existsSync(join(root, target))) { + problems.push(`typesVersions["${range}"]["${subpath}"] points at ${target}, which was not built.`); + } + if (!target.startsWith("./dist/cjs/")) { + problems.push(`typesVersions["${range}"]["${subpath}"] points at ${target}, not at dist/cjs.`); + } + } + } +} +for (const [field, prefix] of [ + ["types", "./dist/cjs/"], + ["main", "./dist/cjs/"], +]) { + if (!String(manifest[field]).startsWith(prefix) || !existsSync(join(root, manifest[field]))) { + problems.push(`package.json "${field}" is ${manifest[field]}; it must be a built file under ${prefix}.`); + } +} + +if (problems.length > 0) { + for (const problem of problems) process.stderr.write(`finalize-build: ${problem}\n`); + process.exit(1); +} + +process.stdout.write("finalize-build: dist/esm and dist/cjs are complete\n"); diff --git a/sdk/typescript/scripts/release.mjs b/sdk/typescript/scripts/release.mjs new file mode 100644 index 000000000..f7088f8b1 --- /dev/null +++ b/sdk/typescript/scripts/release.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** + * The release scheme for `@failproofai/sdk`, and the only place it is written + * down. + * + * The publish workflow calls this rather than restating the arithmetic, for the + * same reason `scripts/python-version.py` exists on the Python side: a release + * rule spelled out in YAML is a release rule nothing can test. + * + * ## The scheme + * + * | from | next | + * |-----------------------|-------------------------| + * | beta `X.Y.Z-beta.N` | `X.Y.Z-beta.(N+1)` | + * | stable `X.Y.Z` | `X.Y.(Z+1)-beta.0` | + * + * The same rule `publish.yml` applies to the `failproofai` npm package. Unlike + * PyPI, npm HAS dist-tags, so a pre-release publishes behind `beta` and a bare + * `npm install @failproofai/sdk` never resolves it — which is why a prerelease + * here is a cheaper mistake than one on PyPI, and why the guard rails below are + * about not BURNING a version rather than about not shipping one. + * + * ## Commands + * + * resolve print the version in src/version.ts, and what follows it + * write move src/version.ts and package.json to + * changelog print that version's CHANGELOG section + * + * Node's standard library only: this runs in the preflight job, which installs + * nothing precisely so that no third-party code decides whether a release may + * proceed. + */ + +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const VERSION_FILE = join(root, "src/version.ts"); +const MANIFEST_FILE = join(root, "package.json"); +const CHANGELOG_FILE = join(root, "CHANGELOG.md"); + +/** + * Only canonical spellings are accepted. + * + * `1.2.3-beta.01`, `v1.2.3` and `1.2.3-beta` are all things a human types and + * npm stores as something else or refuses outright. Every consumer in the + * pipeline compares version STRINGS — this file, the tarball name, the dist-tag + * and the "is this published" query — so a spelling that round-trips + * differently is a release that half-succeeds. + */ +const CANONICAL = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-beta\.(0|[1-9]\d*))?$/; + +function fail(message) { + process.stderr.write(`release: ${message}\n`); + process.exit(1); +} + +function readVersion() { + const source = readFileSync(VERSION_FILE, "utf8"); + const match = /export const VERSION = "([^"]+)";/.exec(source); + if (match === null) fail(`could not find VERSION in ${VERSION_FILE}`); + return match[1]; +} + +function parse(version) { + const match = CANONICAL.exec(version); + if (match === null) { + fail( + `${version} is not a canonical version. This package uses X.Y.Z or ` + + "X.Y.Z-beta.N with no leading zeroes and no other pre-release kinds — " + + "anything else stores on npm as a different string than the one in the " + + "source, and the pipeline compares strings.", + ); + } + return { + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + beta: match[5] === undefined ? null : Number(match[5]), + }; +} + +function next(version) { + const { major, minor, patch, beta } = parse(version); + return beta === null + ? `${major}.${minor}.${patch + 1}-beta.0` + : `${major}.${minor}.${patch}-beta.${beta + 1}`; +} + +/** + * The CHANGELOG section for `version`, which becomes the GitHub Release body. + * + * A release whose section is missing or empty is refused before anything is + * built. A published version nobody can read the changes for is a version that + * may as well not have shipped. + */ +function changelogSection(version) { + const lines = readFileSync(CHANGELOG_FILE, "utf8").split("\n"); + const heading = new RegExp(`^## ${version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`); + const start = lines.findIndex((line) => heading.test(line)); + if (start === -1) { + fail( + `CHANGELOG.md has no "## ${version} — " section. Add one before releasing; ` + + "it becomes the GitHub Release body.", + ); + } + const rest = lines.slice(start + 1); + const end = rest.findIndex((line) => line.startsWith("## ")); + const body = (end === -1 ? rest : rest.slice(0, end)).join("\n").trim(); + if (body === "") fail(`CHANGELOG.md's "## ${version}" section is empty.`); + return body; +} + +function write(version) { + parse(version); + const source = readFileSync(VERSION_FILE, "utf8"); + writeFileSync( + VERSION_FILE, + source.replace(/export const VERSION = "[^"]+";/, `export const VERSION = "${version}";`), + ); + const manifest = readFileSync(MANIFEST_FILE, "utf8"); + // A targeted replacement rather than a parse-and-re-serialise: rewriting the + // whole manifest would reformat every line and make the release commit + // unreadable in review. + const replaced = manifest.replace(/("version":\s*")[^"]+(")/, `$1${version}$2`); + if (replaced === manifest) fail("could not find a version field in package.json"); + writeFileSync(MANIFEST_FILE, replaced); +} + +function emit(name, value) { + const file = process.env.GITHUB_OUTPUT; + const multiline = value.includes("\n"); + const line = multiline + ? `${name}<<__RELEASE_EOF__\n${value}\n__RELEASE_EOF__\n` + : `${name}=${value}\n`; + if (file) writeFileSync(file, line, { flag: "a" }); + process.stdout.write(multiline ? `${name}:\n${value}\n` : `${name}=${value}\n`); +} + +const [command, argument] = process.argv.slice(2); + +switch (command) { + case "resolve": { + const version = readVersion(); + parse(version); + const manifest = JSON.parse(readFileSync(MANIFEST_FILE, "utf8")); + if (manifest.version !== version) { + fail( + `package.json says ${manifest.version} but src/version.ts says ${version}. ` + + "They are read by different consumers and must agree.", + ); + } + emit("version", version); + emit("next_version", next(version)); + emit("is_prerelease", String(parse(version).beta !== null)); + emit("dist_tag", parse(version).beta === null ? "latest" : "beta"); + emit("tag", `ts-sdk-v${version}`); + break; + } + case "write": { + if (!argument) fail("write needs a version"); + write(argument); + process.stdout.write(`release: moved to ${argument}\n`); + break; + } + case "changelog": { + emit("body", changelogSection(argument ?? readVersion())); + break; + } + case "next": { + process.stdout.write(`${next(argument ?? readVersion())}\n`); + break; + } + default: + fail("usage: release.mjs resolve | write | changelog [version] | next [version]"); +} diff --git a/sdk/typescript/src/clock.ts b/sdk/typescript/src/clock.ts new file mode 100644 index 000000000..b3f3bcaf7 --- /dev/null +++ b/sdk/typescript/src/clock.ts @@ -0,0 +1,58 @@ +/** + * Event timestamps: wall-clock milliseconds, with the microsecond digits used + * to keep emission order inside a millisecond. + * + * The wire format has six fractional digits (the Python SDK writes real + * microseconds), and the dashboard orders a session's events by that field. + * JavaScript's wall clock resolves to the millisecond, and a fast agent emits + * `model_response`, `tool_use`, `tool_result` and the next `model_request` + * inside one — so with the last three digits always `000` they tied, and the + * dashboard showed a `tool_result` before its own `tool_use`. + * + * So each stamp is `max(wall µs, previous + 1)`. The digits below the + * millisecond are a sequence, not a measurement — nothing here claims a + * precision the clock does not have — and the millisecond part is still the + * wall clock. The sequence is process-wide and shared through `globalThis`, so + * the ESM and CommonJS copies of this package loaded into one process (the + * dual-package case the adapters already handle) cannot interleave two + * sequences. + * + * A wall clock that steps BACK (NTP, a VM resume) is followed, not fought: if + * the sequence is more than a second ahead of the wall, it re-anchors. Holding + * `previous + 1` instead would freeze every timestamp at the old time for as + * long as the step was — an hour's step, an hour of events stamped with one + * instant. + */ + +const KEY = Symbol.for("@failproofai/sdk.clock"); +const MAX_LEAD_MICROS = 1_000_000; + +interface ClockState { + last: number; +} + +function state(): ClockState { + const holder = globalThis as unknown as Record; + let found = holder[KEY]; + if (found === undefined) { + found = { last: 0 }; + holder[KEY] = found; + } + return found; +} + +/** Microseconds since the epoch, strictly increasing within the process. */ +export function nowMicros(): number { + const clock = state(); + const wall = Date.now() * 1000; + const next = wall > clock.last || clock.last - wall > MAX_LEAD_MICROS ? wall : clock.last + 1; + clock.last = next; + return next; +} + +/** `2026-09-23T12:34:56.123456Z`: six fractional digits, as the ingest parser expects. */ +export function formatMicros(micros: number): string { + const ms = Math.floor(micros / 1000); + const sub = String(micros - ms * 1000).padStart(3, "0"); + return `${new Date(ms).toISOString().slice(0, -1)}${sub}Z`; +} diff --git a/sdk/typescript/src/context.ts b/sdk/typescript/src/context.ts new file mode 100644 index 000000000..afdbb24ca --- /dev/null +++ b/sdk/typescript/src/context.ts @@ -0,0 +1,214 @@ +/** + * Ambient run identity, carried on an `AsyncLocalStorage`. + * + * Without this the SDK would have no ambient session: every `event.*` call + * would take `sessionId` and `agentId` as required arguments and nothing would + * propagate them. Threading both through every function that might emit an + * event is what makes instrumentation sprawl into a diff nobody wants to + * review. `session()` / `agent()` bind them instead. + * + * ## Why `AsyncLocalStorage` and not a module-level variable + * + * An agent process runs many logical runs at once — a server handling + * concurrent requests, a supervisor fanning out to sub-agents. A plain variable + * is shared by all of them, so the last writer wins and every event after it is + * attributed to the wrong run. `AsyncLocalStorage` is Node's equivalent of + * Python's `contextvars`: the value follows the async call tree through + * `await`, `.then()`, timers and `EventEmitter` callbacks created inside the + * scope, and two concurrent branches see two different values. + * + * ## Why the stack is a frozen array + * + * `agentStack` is replaced, never mutated. A store object is shared BY + * REFERENCE with every async branch below it, so `push()` in one branch would + * mutate the value every sibling sees — which is exactly the cross-run event + * mixing this module exists to prevent, wearing an `AsyncLocalStorage` costume. + * It passes every single-branch test. + * + * There is deliberately no separate `agentId` slot: the top of the stack IS the + * current agent id, so the two cannot drift apart, and `parentId` is the entry + * beneath it. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; + +import { logger } from "./logger.js"; + +/** + * The agent id used when events are emitted with a session bound but no agent + * scope. "main" is the convention the skill and the reference integrations + * already teach, so an un-scoped event lands somewhere sensible rather than + * throwing. + */ +export const DEFAULT_AGENT_ID = "main"; + +/** The run identity in scope. Never null — check `sessionId === null` instead. */ +export interface Identity { + readonly sessionId: string | null; + readonly agentId: string | null; + readonly parentId: string | null; + readonly depth: number; +} + +export interface Store { + readonly sessionId: string | null; + readonly agentStack: readonly string[]; +} + +export const EMPTY_STORE: Store = Object.freeze({ + sessionId: null, + agentStack: Object.freeze([]), +}); + +const storage = new AsyncLocalStorage(); + +function read(): Store { + return storage.getStore() ?? EMPTY_STORE; +} + +/** + * The identity bound to the current context. + * + * `current().sessionId === null` means nothing is bound — either no scope was + * entered, or this callback escaped the async context that entered one (see + * `propagate`). + */ +export function current(): Identity { + const { sessionId: sid, agentStack: stack } = read(); + return { + sessionId: sid, + agentId: stack.length > 0 ? stack[stack.length - 1]! : null, + parentId: stack.length >= 2 ? stack[stack.length - 2]! : null, + depth: stack.length, + }; +} + +/** The bound session id, or null. Hot path — allocates no Identity. */ +export function sessionId(): string | null { + return read().sessionId; +} + +/** The current agent id, falling back to `DEFAULT_AGENT_ID`. */ +export function agentId(): string { + const { agentStack: stack } = read(); + return stack.length > 0 ? stack[stack.length - 1]! : DEFAULT_AGENT_ID; +} + +/** The enclosing agent id, or null at depth 0 or 1. */ +export function parentAgentId(): string | null { + const { agentStack: stack } = read(); + return stack.length >= 2 ? stack[stack.length - 2]! : null; +} + +export function snapshot(): Store { + return read(); +} + +export function withSessionBound(store: Store, sid: string): Store { + return { sessionId: sid, agentStack: store.agentStack }; +} + +export function withAgentPushed(store: Store, aid: string): Store { + return { sessionId: store.sessionId, agentStack: Object.freeze([...store.agentStack, aid]) }; +} + +/** Run `fn` with `store` bound. The clean form: nothing to unwind, ever. */ +export function runWith(store: Store, fn: () => T): T { + return storage.run(store, fn); +} + +/** + * Bind `store` for the REMAINDER of the current async context and return what + * was bound before. + * + * This is the `using`-statement half of the API, and it is strictly weaker than + * `runWith`: the binding escapes upward into the caller's context, so it has to + * be undone by hand. `scopes.ts` does that in a `[Symbol.dispose]`, which the + * runtime guarantees runs at the end of the block — including on `throw` and + * `return`. + * + * The hazard it cannot fix is a scope entered in one async context and disposed + * in another: `enterWith` in the disposing context cannot reach the context the + * value was bound in, so the frame stays bound there. `noteCrossContextExit` + * below is how that gets reported. + */ +export function enterWith(store: Store): Store { + const previous = read(); + storage.enterWith(store); + return previous; +} + +/** Deduplicates the cross-context warning — it fires from a scope exit. */ +let warnedCrossContext = false; + +export function resetCrossContextWarning(): void { + warnedCrossContext = false; +} + +/** + * Report, once, that a scope could not be unwound where it was entered. + * + * WARNING, not debug. The consequence is not cosmetic: the frame the scope + * pushed stays bound in whatever context it was set in, so identity is wrong + * for everything that follows there. The caller cannot discover that any other + * way — there is no exception, and the events look plausible. + */ +export function noteCrossContextExit(): void { + if (warnedCrossContext) return; + warnedCrossContext = true; + logger.warn( + "a scope was entered in one async context and exited in another, so its identity " + + "could not be unwound and later events in the entering context may be attributed " + + "to it. The usual cause is a `using` scope spanning a `yield` in an async " + + "generator; pass sessionId/agentId explicitly there, or use the callback form " + + "(`await agent('name', fn)`), which cannot reach this state.", + ); +} + +/** + * Remove ONE frame for `aid` from a stack, innermost first. + * + * The fallback for a scope that cannot be unwound by restoring the previous + * store. Removes a single occurrence rather than every match, because the same + * agent id may legitimately be on the stack twice (a recursive agent), and + * dropping both would corrupt the outer one to fix the inner. + */ +export function withAgentDiscarded(store: Store, aid: string): Store { + const stack = store.agentStack; + for (let i = stack.length - 1; i >= 0; i -= 1) { + if (stack[i] === aid) { + return { + sessionId: store.sessionId, + agentStack: Object.freeze([...stack.slice(0, i), ...stack.slice(i + 1)]), + }; + } + } + return store; +} + +/** + * Wrap `fn` so it runs with the identity bound *right now*. + * + * queue.push(propagate(work)); + * emitter.on("done", propagate(onDone)); + * new Worker(url).on("message", propagate(handle)); + * + * `AsyncLocalStorage` already follows `await`, `.then()`, `setTimeout` and any + * callback CREATED inside a scope, so most code needs nothing. This is for the + * cases where it genuinely cannot: a callback stored during one run and invoked + * during another (a module-level registry, a connection pool's handler), work + * handed to a `worker_threads` boundary, or anything re-entered from a native + * addon that does not carry async context. + * + * It snapshots VALUES rather than capturing the store object, so re-entering it + * twice — a retried job, a handler invoked per message — binds the same + * identity each time instead of inheriting whatever the previous call left. + */ +export function propagate( + fn: (...args: Args) => Result, +): (...args: Args) => Result { + const captured = snapshot(); + return function failproofaiPropagated(this: unknown, ...args: Args): Result { + return storage.run(captured, () => fn.apply(this, args)); + }; +} diff --git a/sdk/typescript/src/edge/adapter.ts b/sdk/typescript/src/edge/adapter.ts new file mode 100644 index 000000000..b97ac678e --- /dev/null +++ b/sdk/typescript/src/edge/adapter.ts @@ -0,0 +1,18 @@ +/** A framework adapter that installs nothing — see `src/edge/index.ts`. */ + +import { notice } from "./notice.js"; + +export function noopAdapter(name: string): { + name: string; + install(): Promise; + uninstall(): void; +} { + return { + name, + install() { + notice(); + return Promise.resolve(); + }, + uninstall() {}, + }; +} diff --git a/sdk/typescript/src/edge/ai.ts b/sdk/typescript/src/edge/ai.ts new file mode 100644 index 000000000..c8ab09cd3 --- /dev/null +++ b/sdk/typescript/src/edge/ai.ts @@ -0,0 +1,116 @@ +/** + * `@failproofai/sdk/ai` for runtimes with no filesystem — see `./index.ts`. + * + * Every helper hands the Vercel AI SDK something it accepts and that records + * nothing: `telemetry()` returns telemetry settings with `isEnabled: false`, + * the middleware passes calls straight through, the wrappers return what they + * were given. + */ + +import { noopAdapter } from "./adapter.js"; +import { notice } from "./notice.js"; + +type AttributeValue = string | number | boolean | Array; + +/** A span that records nothing, for a caller holding the tracer directly. */ +export class FailproofSpan { + readonly attributes: Record = {}; + spanContext() { + return { traceId: "0".repeat(32), spanId: "0".repeat(16), traceFlags: 0 }; + } + setAttribute(): this { + return this; + } + setAttributes(): this { + return this; + } + addEvent(): this { + return this; + } + addLink(): this { + return this; + } + addLinks(): this { + return this; + } + setStatus(): this { + return this; + } + updateName(): this { + return this; + } + recordException(): void {} + isRecording(): boolean { + return false; + } + end(): void {} +} + +export class FailproofTracer { + startSpan(): FailproofSpan { + return new FailproofSpan(); + } + + startActiveSpan(_name: string, ...rest: unknown[]): unknown { + const fn = rest[rest.length - 1] as (span: FailproofSpan) => unknown; + return fn(new FailproofSpan()); + } +} + +export function telemetry( + options: { functionId?: string; metadata?: Record } = {}, +): { isEnabled: false; functionId?: string; metadata?: Record } { + notice(); + return { + isEnabled: false, + ...(options.functionId === undefined ? {} : { functionId: options.functionId }), + ...(options.metadata === undefined ? {} : { metadata: options.metadata }), + }; +} + +export function tracer(): FailproofTracer { + notice(); + return new FailproofTracer(); +} + +const ignore = (): void => {}; + +/** The v7 telemetry integration, with every hook a no-op. */ +export const integration = Object.freeze({ + onStart: ignore, + onLanguageModelCallStart: ignore, + onLanguageModelCallEnd: ignore, + onObjectStepStart: ignore, + onObjectStepEnd: ignore, + onEmbedStart: ignore, + onEmbedEnd: ignore, + onToolExecutionStart: ignore, + onToolExecutionEnd: ignore, + onEnd: ignore, +}); + +export function middleware() { + notice(); + return { + specificationVersion: "v3" as const, + wrapGenerate: async ({ doGenerate }: { doGenerate: () => PromiseLike }): Promise => await doGenerate(), + wrapStream: async ({ doStream }: { doStream: () => PromiseLike }): Promise => await doStream(), + }; +} + +export function wrapModel(model: T): Promise { + notice(); + return Promise.resolve(model); +} + +export function wrapTool(_toolName: string, tool: T): T { + notice(); + return tool; +} + +export function wrapTools(tools: T): T { + notice(); + return tools; +} + +export const adapter = noopAdapter("ai"); diff --git a/sdk/typescript/src/edge/index.ts b/sdk/typescript/src/edge/index.ts new file mode 100644 index 000000000..e9ff394cd --- /dev/null +++ b/sdk/typescript/src/edge/index.ts @@ -0,0 +1,238 @@ +/** + * `@failproofai/sdk` for runtimes with no filesystem: the Edge runtime (Next.js + * `export const runtime = "edge"`, Vercel Edge Functions), Cloudflare Workers, + * and browser bundles. Selected by the package's `exports` conditions + * (`edge-light`, `workerd`, `worker`, `browser`), so an application's own + * `import "@failproofai/sdk"` lands here without a code change. + * + * ## Why a separate build + * + * The real entry statically imports `node:fs`, `node:os`, `node:module` and + * `node:crypto`. An ES module cannot catch a failed static import, so in the + * Edge runtime the IMPORT itself failed — and because Next evaluates a route + * module while building, `next build` failed ("Native module not found: + * node:fs"), taking the whole application down over a telemetry import. + * + * ## What it does instead + * + * The same public surface, recording nothing: scopes run their bodies and + * return their values, `event.*` accepts and drops, `flush()` resolves, + * `instrument()` returns `[]`. The first use logs ONE line to stderr saying + * so. These runtimes are not a deployment target — the SDK ships beside the + * `failproofaid` daemon on a machine with a filesystem — so this build exists + * only so that a shared module importing the SDK cannot break an Edge route. + * + * Type declarations stay the real ones (the `types` condition), so code that + * compiles against Node compiles here too. + */ + +import type { Identity } from "../context.js"; +import type { EventNamespace } from "../events.js"; +import type { + AgentFn, + AgentOptions, + SessionFn, + SessionOptions, + ToolCallFn, + ToolCallOptions, +} from "../scopes.js"; +import { VERSION } from "../version.js"; +import { notice } from "./notice.js"; + +export { VERSION as version }; + +export const DEFAULT_AGENT_ID = "main"; +/** The same registered symbol as the Node build's, so comparisons still hold. */ +export const AUTO: unique symbol = Symbol.for("failproofai.AUTO") as never; + +// `Symbol.dispose` is not in every Edge runtime. A missing one must not become +// the property key "undefined". +const DISPOSE: symbol = + (Symbol as unknown as { dispose?: symbol }).dispose ?? Symbol.for("nodejs.dispose"); + +let counter = 0; +function newId(): string { + try { + const uuid = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto?.randomUUID?.(); + if (typeof uuid === "string") return uuid.replace(/-/g, ""); + } catch { + /* fall through */ + } + counter += 1; + return `edge${Date.now().toString(16)}${counter.toString(16)}`; +} + +const identityOf = (sessionId: string | null, agentId: string | null): Identity => + Object.freeze({ sessionId, agentId, parentId: null, depth: agentId === null ? 0 : 1 }); + +export function current(): Identity { + return identityOf(null, null); +} + +export function propagate( + fn: (...args: Args) => Result, +): (...args: Args) => Result { + return fn; +} + +/** The handle a `toolCall` body receives. Set `.output`; read `.id`. */ +export class ToolCall { + readonly id: string; + output: unknown = undefined; + private assigned = false; + + constructor(toolCallId: string) { + this.id = toolCallId; + let stored: unknown; + Object.defineProperty(this, "output", { + get: () => stored, + set: (value: unknown) => { + stored = value; + this.assigned = true; + }, + enumerable: true, + configurable: true, + }); + } + + get outputAssigned(): boolean { + return this.assigned; + } +} + +class Handle { + dispose(): void {} + fail(): void {} + [DISPOSE](): void {} +} + +class SessionHandle extends Handle { + constructor(readonly id: string) { + super(); + } +} + +class AgentHandle extends Handle { + readonly identity: Identity; + constructor( + readonly agentId: string, + readonly sessionId: string, + ) { + super(); + this.identity = identityOf(sessionId, agentId); + } +} + +class ToolCallHandle extends Handle { + constructor(readonly call: ToolCall) { + super(); + } +} + +type Body = (arg: A) => T; + +function split(optionsOrBody: O | Body | undefined, body: Body | undefined) { + return typeof optionsOrBody === "function" + ? { options: {} as O, body: optionsOrBody as Body } + : { options: (optionsOrBody ?? {}) as O, body: body! }; +} + +function sessionImpl(optionsOrBody: SessionOptions | Body, maybeBody?: Body): T { + notice(); + const { options, body } = split(optionsOrBody, maybeBody); + return body(options.sessionId ?? newId()); +} +sessionImpl.open = (options: SessionOptions = {}) => { + notice(); + return new SessionHandle(options.sessionId ?? newId()); +}; + +function agentImpl( + agentId: string, + optionsOrBody: AgentOptions | Body, + maybeBody?: Body, +): T { + notice(); + const { options, body } = split(optionsOrBody, maybeBody); + return body(identityOf(options.sessionId ?? null, agentId)); +} +agentImpl.open = (agentId = DEFAULT_AGENT_ID, options: AgentOptions = {}) => { + notice(); + return new AgentHandle(agentId, options.sessionId ?? newId()); +}; + +function toolCallImpl( + toolName: string, + optionsOrBody: ToolCallOptions | Body, + maybeBody?: Body, +): T { + notice(); + void toolName; + const { options, body } = split(optionsOrBody, maybeBody); + return body(new ToolCall(options.toolCallId ?? newId())); +} +toolCallImpl.open = (toolName: string, options: ToolCallOptions = {}) => { + void toolName; + notice(); + return new ToolCallHandle(new ToolCall(options.toolCallId ?? newId())); +}; + +export const session = sessionImpl as unknown as SessionFn; +export const agent = agentImpl as unknown as AgentFn; +export const toolCall = toolCallImpl as unknown as ToolCallFn; + +const EVENT_METHODS = [ + "toolUse", + "toolResult", + "modelRequest", + "modelResponse", + "agentStart", + "agentEnd", + "agentPause", + "agentResume", + "hookTriggered", + "hookCompleted", + "error", + "humanWait", + "humanInput", + "humanPause", + "humanInterrupt", +] as const; + +/** The 15 event methods, each accepting its options and recording nothing. */ +export const event: EventNamespace = Object.freeze( + Object.fromEntries(EVENT_METHODS.map((name) => [name, () => notice()])), +) as unknown as EventNamespace; + +export function configure(): void { + notice(); +} + +export function flush(): Promise { + notice(); + return Promise.resolve(); +} + +export function flushSync(): void { + notice(); +} + +export function setLogger(): void {} +export function setLogLevel(): void {} + +export function instrument(): Promise { + notice(); + return Promise.resolve([]); +} + +export function uninstrument(): never[] { + return []; +} + +export function availableFrameworks(): never[] { + return []; +} + +export function activeFrameworks(): never[] { + return []; +} diff --git a/sdk/typescript/src/edge/langchain.ts b/sdk/typescript/src/edge/langchain.ts new file mode 100644 index 000000000..968789975 --- /dev/null +++ b/sdk/typescript/src/edge/langchain.ts @@ -0,0 +1,18 @@ +/** `@failproofai/sdk/langchain` for runtimes with no filesystem — see `./index.ts`. */ + +import { noopAdapter } from "./adapter.js"; +import { notice } from "./notice.js"; + +export const SESSION_METADATA_KEY = "failproofai_sdk_session_id"; + +/** + * A callback handler with no callbacks. LangChain accepts any object as a + * handler (`BaseCallbackHandler.fromMethods`), so passing this where the Node + * build's handler goes is valid and inert. + */ +export function langchainHandler(): Record { + notice(); + return { name: "failproofai_noop" }; +} + +export const adapter = noopAdapter("langchain"); diff --git a/sdk/typescript/src/edge/llamaindex.ts b/sdk/typescript/src/edge/llamaindex.ts new file mode 100644 index 000000000..b54121461 --- /dev/null +++ b/sdk/typescript/src/edge/llamaindex.ts @@ -0,0 +1,12 @@ +/** `@failproofai/sdk/llamaindex` for runtimes with no filesystem — see `./index.ts`. */ + +import { noopAdapter } from "./adapter.js"; +import { notice } from "./notice.js"; + +/** Attaching to a LlamaIndex object records nothing here; returns a detach. */ +export function attach(): () => void { + notice(); + return () => {}; +} + +export const adapter = noopAdapter("llamaindex"); diff --git a/sdk/typescript/src/edge/mastra.ts b/sdk/typescript/src/edge/mastra.ts new file mode 100644 index 000000000..5ae3bfb30 --- /dev/null +++ b/sdk/typescript/src/edge/mastra.ts @@ -0,0 +1,17 @@ +/** `@failproofai/sdk/mastra` for runtimes with no filesystem — see `./index.ts`. */ + +import { noopAdapter } from "./adapter.js"; +import { notice } from "./notice.js"; + +export function wrapTool(tool: T): T { + notice(); + return tool; +} + +export function workflow(workflowName: string, body: () => T): T { + void workflowName; + notice(); + return body(); +} + +export const adapter = noopAdapter("mastra"); diff --git a/sdk/typescript/src/edge/notice.ts b/sdk/typescript/src/edge/notice.ts new file mode 100644 index 000000000..7706d7388 --- /dev/null +++ b/sdk/typescript/src/edge/notice.ts @@ -0,0 +1,33 @@ +/** + * The one line an Edge / Worker build says, once per isolate. + * + * Deliberately self-contained: it imports nothing, not even `logger.ts`, + * because that module reads environment variables at import time and a Cloudflare + * Worker without `nodejs_compat` has no `process` at all. Everything under + * `src/edge/` holds to the same rule — the whole point of these modules is that + * importing them cannot fail anywhere JavaScript runs. + */ + +let said = false; + +export const EDGE_NOTICE = + "@failproofai/sdk loaded its no-op build: this runtime (an Edge / Worker runtime, " + + "or a browser bundle) has no filesystem for the local failproofaid spool, so NOTHING " + + "is recorded here; your code runs unchanged. Record from the Node.js runtime " + + '(Next.js: drop `export const runtime = "edge"` from the route).'; + +/** Say it once. Never throws: a diagnostic must not take the host down. */ +export function notice(): void { + if (said) return; + said = true; + try { + console.warn(`[failproofai-sdk] ${EDGE_NOTICE}`); + } catch { + /* empty */ + } +} + +/** Forget that the notice was given (tests). */ +export function resetNotice(): void { + said = false; +} diff --git a/sdk/typescript/src/environment.ts b/sdk/typescript/src/environment.ts new file mode 100644 index 000000000..adf591615 --- /dev/null +++ b/sdk/typescript/src/environment.ts @@ -0,0 +1,75 @@ +import { logger } from "./logger.js"; +import { shared } from "./shared.js"; + +const DEFAULT_ENVIRONMENT = "dev"; + + +/** + * Set once the comma warning below has been emitted. `getEnvironment()` runs + * from `build()`, i.e. once per event on the caller's own stack, and the + * warning had no once-flag at all in an early draft — so an + * `AGENTEYE_ENVIRONMENT` with a comma put one WARN line into the host + * application's log for every event emitted, for the life of the process. At + * the SDK's documented ceiling that is a logging-driven throughput collapse in + * a library whose first constraint is not to disrupt the host agent. + */ +let warnedComma = false; + +/** + * A comma in `environment` makes ingest skip EVERY event carrying it. + * + * The endpoint splits this field on commas to build its filter facets, so a + * line whose `environment` contains one is discarded — the whole line, not the + * field. It answers 200 with `{"accepted":0,"skipped":N}`, the daemon deletes + * the delivered batch, and the run that produced it is simply never in the + * dashboard: no exception here, nothing in the agent's output, and an empty + * session list that looks exactly like an agent nobody ran. + * + * `failproofaid` already refuses a comma in `collector.environment` for this + * reason (`crates/fpai-collect/src/config.rs`). The SDK is the other writer of + * the same field, so `AGENTEYE_ENVIRONMENT="prod,eu"` — a wholly reasonable + * thing to type — would silently throw away everything the process emitted. + */ +export function rejectComma(env: string, source: string): void { + if (env.includes(",")) { + throw new Error( + `environment must not contain a comma (got ${JSON.stringify(env)} from ${source}). ` + + "The ingest endpoint skips every event whose environment has one, so this would " + + "silently discard all telemetry from this process. Use a single label, e.g. 'prod-eu'.", + ); + } +} + +export function getEnvironment(): string { + const environment = shared().environment; + if (environment !== null) return environment; + + const raw = process.env.AGENTEYE_ENVIRONMENT; + if (!raw) return DEFAULT_ENVIRONMENT; + if (raw.includes(",")) { + // Throwing here would blow up inside `build()` on an arbitrary event, far + // from the thing that set it, and take the caller's agent down with it — a + // telemetry library must not do that. Warn once and fall back to a label + // ingest will actually accept, so the events land under a visibly-wrong + // environment instead of vanishing. + if (!warnedComma) { + warnedComma = true; + logger.warn( + `AGENTEYE_ENVIRONMENT=${JSON.stringify(raw)} contains a comma, which makes the ingest ` + + `endpoint skip every event carrying it. Falling back to ${JSON.stringify(DEFAULT_ENVIRONMENT)}. ` + + "Use a single label, e.g. 'prod-eu'.", + ); + } + return DEFAULT_ENVIRONMENT; + } + return raw; +} + +export function setEnvironment(env: string | null | undefined): void { + if (env) rejectComma(env, "configure({ environment })"); + shared().environment = env ? env : null; + // A new label means the env var may be worth complaining about again. + warnedComma = false; +} + +export { DEFAULT_ENVIRONMENT }; diff --git a/sdk/typescript/src/evaluator/authoring.ts b/sdk/typescript/src/evaluator/authoring.ts new file mode 100644 index 000000000..56942f225 --- /dev/null +++ b/sdk/typescript/src/evaluator/authoring.ts @@ -0,0 +1,480 @@ +/** + * Evaluator definition registry and typed author results. + * + * Every bound checked here is a bound the SERVER also checks, and rejects with + * a NON-RETRYABLE 422. Checking them at authoring time is what turns "a + * successful evaluation silently dead-lettered in production" into an error at + * the line that wrote it. + */ + +import { createHash } from "node:crypto"; + +import { + MAX_CATALOG_DEFINITIONS, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABEL_BYTES, + MAX_LABELS_PER_RESULT, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + ResultKind, + catalogDefinitionToWire, +} from "./protocol.js"; +import type { CatalogDefinition, ResultItem, SessionTranscript } from "./protocol.js"; + +const KEY_PATTERN = /^[a-z][a-z0-9_]*$/; + +export type EvalFunction = (session: SessionTranscript) => EvalResult | Promise; +export type ConditionFunction = ( + session: SessionTranscript, +) => boolean | ConditionResult | Promise; +export type CancellationFunction = (session: SessionTranscript) => unknown; + +/** + * Compiles server-authored (sandboxed) source into something callable with a + * session. Signature mirrors `source.compileEvaluator`, which is the default. + * + * A host that serves managed definitions whose source is NOT a restricted + * expression — a declarative judge document, say — installs its own compiler + * here. The SDK stays agnostic: it never inspects the source, it just hands it + * to whoever compiles. Returning an async function is supported and is the + * right shape for a compiler whose work is IO-bound. + */ +export type ManagedCompiler = ( + source: string, + options: { timeoutSeconds?: number | null; evalKey?: string | null }, +) => EvalFunction; + +function utf8Length(value: string): number { + return Buffer.byteLength(value, "utf8"); +} + +function bounded(value: unknown, fieldName: string, maximum: number): string { + if (typeof value !== "string") throw new TypeError(`${fieldName} must be a string`); + if (value === "") throw new Error(`${fieldName} must not be empty`); + const size = utf8Length(value); + if (size > maximum) throw new Error(`${fieldName} is ${size} bytes; maximum is ${maximum}`); + // Reject C0 control characters and DEL, matching the server's own check. + // Without this the SDK accepts a string — reasoning or a summary quoting + // transcript text that contains an ANSI escape or a NUL — that the server + // then rejects with a NON-RETRYABLE 422, so a successful evaluation is + // silently lost and its assignment dead-letters. TAB, LF and CR are kept + // because real multi-line reasoning uses them. + for (const char of value) { + const code = char.codePointAt(0)!; + if ((code < 0x20 && char !== "\t" && char !== "\n" && char !== "\r") || code === 0x7f) { + throw new Error( + `${fieldName} must not contain control characters (found U+${code + .toString(16) + .toUpperCase() + .padStart(4, "0")})`, + ); + } + } + return value; +} + +function finite(value: unknown, fieldName: string): number { + if (typeof value !== "number") throw new TypeError(`${fieldName} must be a number`); + if (!Number.isFinite(value)) throw new Error(`${fieldName} must be finite`); + return value; +} + +function normalizeLabels(values: readonly string[]): string[] { + if (values.length > MAX_LABELS_PER_RESULT) { + throw new Error(`at most ${MAX_LABELS_PER_RESULT} labels are allowed`); + } + const normalized = values.map((label) => bounded(label, "label", MAX_LABEL_BYTES)); + if (new Set(normalized).size !== normalized.length) throw new Error("labels must be unique"); + return [...normalized].sort(); +} + +function validateResultText( + unit: string, + displayValue: string | undefined, + description: string | undefined, +): void { + if (unit) bounded(unit, "unit", MAX_UNIT_BYTES); + if (displayValue !== undefined) bounded(displayValue, "display value", MAX_DISPLAY_VALUE_BYTES); + if (description !== undefined) bounded(description, "description", MAX_DESCRIPTION_BYTES); +} + +export function validateKey(value: string, fieldName = "eval_key"): string { + bounded(value, fieldName, MAX_EVAL_KEY_BYTES); + if (!KEY_PATTERN.test(value)) throw new Error(`${fieldName} must match ${KEY_PATTERN.source}`); + return value; +} + +export class Score { + readonly value: number; + readonly passed: boolean | undefined; + readonly unit: string; + readonly displayValue: string | undefined; + readonly description: string | undefined; + + constructor( + value: number, + options: { + passed?: boolean; + unit?: string; + displayValue?: string; + description?: string; + } = {}, + ) { + const numeric = finite(value, "score value"); + if (numeric < 0 || numeric > 1) throw new Error("score value must be between 0 and 1"); + if (options.passed !== undefined && typeof options.passed !== "boolean") { + throw new TypeError("score passed must be a boolean or undefined"); + } + this.value = numeric; + this.passed = options.passed; + this.unit = options.unit ?? "ratio"; + this.displayValue = options.displayValue; + this.description = options.description; + validateResultText(this.unit, this.displayValue, this.description); + Object.freeze(this); + } +} + +export class Metric { + readonly value: number; + readonly unit: string; + readonly displayValue: string | undefined; + readonly description: string | undefined; + + constructor( + value: number, + options: { unit?: string; displayValue?: string; description?: string } = {}, + ) { + this.value = finite(value, "metric value"); + this.unit = options.unit ?? ""; + this.displayValue = options.displayValue; + this.description = options.description; + validateResultText(this.unit, this.displayValue, this.description); + Object.freeze(this); + } +} + +export class Assertion { + readonly passed: boolean; + readonly description: string | undefined; + + constructor(passed: boolean, options: { description?: string } = {}) { + if (typeof passed !== "boolean") throw new TypeError("assertion passed must be a boolean"); + this.passed = passed; + this.description = options.description; + if (this.description !== undefined) { + bounded(this.description, "description", MAX_DESCRIPTION_BYTES); + } + Object.freeze(this); + } +} + +export class ConditionResult { + readonly applicable: boolean; + readonly reasonCode: string; + + constructor(applicable: boolean, reasonCode = "condition_false") { + if (typeof applicable !== "boolean") { + throw new TypeError("condition applicable must be a boolean"); + } + this.applicable = applicable; + this.reasonCode = reasonCode; + validateKey(reasonCode, "condition reason code"); + Object.freeze(this); + } +} + +export interface EvalResultOptions { + score?: Score; + metrics?: Record; + assertions?: Record; + reasoning?: string; + summary?: string; + labels?: readonly string[]; +} + +export class EvalResult { + readonly score: Score | undefined; + readonly metrics: Readonly>; + readonly assertions: Readonly>; + readonly reasoning: string | undefined; + readonly summary: string | undefined; + readonly labels: readonly string[]; + + constructor(options: EvalResultOptions = {}) { + this.score = options.score; + this.metrics = options.metrics ?? {}; + this.assertions = options.assertions ?? {}; + this.reasoning = options.reasoning; + this.summary = options.summary; + if (this.reasoning !== undefined) bounded(this.reasoning, "reasoning", MAX_REASONING_BYTES); + if (this.summary !== undefined) bounded(this.summary, "summary", MAX_SUMMARY_BYTES); + this.labels = normalizeLabels(options.labels ?? []); + Object.freeze(this); + } + + resultItems(evalKey: string): ResultItem[] { + const items: ResultItem[] = []; + if (this.score !== undefined) { + items.push({ + resultKey: evalKey, + resultKind: ResultKind.SCORE, + numericValue: this.score.value, + boolValue: this.score.passed ?? null, + unit: this.score.unit, + displayValue: this.score.displayValue ?? null, + description: this.score.description ?? null, + reasoning: this.reasoning ?? null, + labels: this.labels, + }); + } + for (const key of Object.keys(this.metrics).sort()) { + validateKey(key, "metric key"); + const raw = this.metrics[key]!; + const metric = raw instanceof Metric ? raw : new Metric(raw); + items.push({ + resultKey: key, + resultKind: ResultKind.METRIC, + numericValue: metric.value, + unit: metric.unit, + displayValue: metric.displayValue ?? null, + description: metric.description ?? null, + // A metric-kind eval's primary result IS the metric whose key equals + // `evalKey`; attach the eval's reasoning there so it is not silently + // dropped for non-score evals. + reasoning: key === evalKey ? (this.reasoning ?? null) : null, + labels: this.labels, + }); + } + for (const key of Object.keys(this.assertions).sort()) { + validateKey(key, "assertion key"); + const raw = this.assertions[key]!; + const assertion = raw instanceof Assertion ? raw : new Assertion(raw); + items.push({ + resultKey: key, + resultKind: ResultKind.ASSERTION, + boolValue: assertion.passed, + description: assertion.description ?? null, + reasoning: key === evalKey ? (this.reasoning ?? null) : null, + labels: this.labels, + }); + } + if (items.length === 0) { + throw new Error("an EvalResult must contain a score, metric, or assertion"); + } + if (items.length > MAX_RESULTS_PER_RUN) { + throw new Error(`an EvalResult may contain at most ${MAX_RESULTS_PER_RUN} results`); + } + const keys = items.map((item) => item.resultKey); + if (new Set(keys).size !== keys.length) { + throw new Error("result keys must be unique within one evaluation run"); + } + return items; + } +} + +export interface EvalDefinition { + evalKey: string; + displayName: string; + evalVersion: string; + resultKind: ResultKind; + labels: readonly string[]; + function: EvalFunction; + condition: ConditionFunction | null; + onCancel: CancellationFunction | null; + timeoutSeconds: number | null; +} + +export function catalogDefinitionOf(definition: EvalDefinition): CatalogDefinition { + return { + evalKey: definition.evalKey, + displayName: definition.displayName, + evalVersion: definition.evalVersion, + resultKind: definition.resultKind, + labels: definition.labels, + }; +} + +/** Deterministic JSON with sorted keys and no spaces, for the catalog hash. */ +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const entries = Object.entries(value as Record) + .filter(([, item]) => item !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`; +} + +export interface EvalOptions { + version: string; + displayName?: string; + resultKind?: ResultKind; + labels?: readonly string[]; + when?: ConditionFunction; + onCancel?: CancellationFunction; + timeoutSeconds?: number; +} + +/** + * Marks an `Evaluator` across the package's two builds. + * + * `failproofai-evaluator` is the ESM build, and a CommonJS evals file (plain + * `tsc` output, `require`) constructs its `Evaluator` from `dist/cjs` — a + * second, unrelated copy of the class. `instanceof` is false across the two, so + * the loader refused the commonest setup there is with "resolved to Evaluator, + * not an Evaluator". `Symbol.for` is one registry per process, so both copies + * stamp and read the same key. + */ +const EVALUATOR_BRAND = Symbol.for("@failproofai/sdk/evaluator.Evaluator"); + +/** + * True for an `Evaluator` from either build of this package. + * + * Safe to hand the result to `runFromEnv()`: that method belongs to the copy + * that built the object, and runs the runtime from that same copy, so the + * result classes it checks are the ones the evaluations construct. + * + * @internal + */ +export function isEvaluator(value: unknown): value is Evaluator { + return ( + typeof value === "object" && + value !== null && + (value as Record)[EVALUATOR_BRAND] === true && + typeof (value as { runFromEnv?: unknown }).runFromEnv === "function" + ); +} + +/** A process-local collection of explicitly versioned evaluations. */ +export class Evaluator { + declare readonly [EVALUATOR_BRAND]: true; + readonly name: string; + readonly version: string; + readonly managedCompiler: ManagedCompiler | null; + private readonly registry = new Map(); + + constructor(options: { name: string; version: string; managedCompiler?: ManagedCompiler }) { + Object.defineProperty(this, EVALUATOR_BRAND, { value: true }); + this.name = bounded(options.name, "name", MAX_DISPLAY_NAME_BYTES); + this.version = bounded(options.version, "version", MAX_VERSION_BYTES); + // Omitted keeps the default: server-authored source is compiled and + // sandboxed by `source.compileEvaluator`. Customer workers never set this — + // their definitions are LOCAL and never carry source at all. + if (options.managedCompiler !== undefined && typeof options.managedCompiler !== "function") { + throw new TypeError("managedCompiler must be a function"); + } + this.managedCompiler = options.managedCompiler ?? null; + } + + /** + * Register one evaluation. + * + * evaluator.eval("tool_success_rate", { version: "1" }, (session) => + * new EvalResult({ score: new Score(ratio) }), + * ); + */ + eval(evalKey: string, options: EvalOptions, fn: EvalFunction): EvalFunction { + const key = validateKey(evalKey); + const evalVersion = bounded(options.version, "eval version", MAX_VERSION_BYTES); + const display = bounded( + options.displayName ?? defaultDisplayName(evalKey), + "display name", + MAX_DISPLAY_NAME_BYTES, + ); + const kind = options.resultKind ?? ResultKind.SCORE; + if (!Object.values(ResultKind).includes(kind)) { + throw new Error(`resultKind must be one of ${Object.values(ResultKind).join(", ")}`); + } + const labels = normalizeLabels(options.labels ?? []); + let timeoutSeconds: number | null = null; + if (options.timeoutSeconds !== undefined) { + timeoutSeconds = finite(options.timeoutSeconds, "timeoutSeconds"); + if (timeoutSeconds <= 0) throw new Error("timeoutSeconds must be greater than zero"); + } + + if (this.registry.has(key)) throw new Error(`duplicate eval key: ${key}`); + if (this.registry.size >= MAX_CATALOG_DEFINITIONS) { + throw new Error( + `an evaluator may define at most ${MAX_CATALOG_DEFINITIONS} evaluations`, + ); + } + if (typeof fn !== "function") throw new TypeError("evaluation must be a function"); + if (options.when !== undefined && typeof options.when !== "function") { + throw new TypeError("when must be a function"); + } + if (options.onCancel !== undefined && typeof options.onCancel !== "function") { + throw new TypeError("onCancel must be a function"); + } + + this.registry.set(key, { + evalKey: key, + displayName: display, + evalVersion, + resultKind: kind, + labels, + function: fn, + condition: options.when ?? null, + onCancel: options.onCancel ?? null, + timeoutSeconds, + }); + return fn; + } + + get definitions(): readonly EvalDefinition[] { + return [...this.registry.keys()].sort().map((key) => this.registry.get(key)!); + } + + catalog(): CatalogDefinition[] { + return this.definitions.map(catalogDefinitionOf); + } + + get catalogRevision(): string { + const payload = this.catalog().map(catalogDefinitionToWire); + const canonical = Buffer.from(canonicalJson(payload), "utf8"); + return `sha256:${createHash("sha256").update(canonical).digest("hex")}`; + } + + definition(evalKey: string): EvalDefinition { + const found = this.registry.get(evalKey); + if (found === undefined) throw new Error(`unknown eval key: ${evalKey}`); + return found; + } + + /** Run this evaluator until the process receives a stop request. */ + async runFromEnv(): Promise { + const { WorkerRuntime, workerConfigFromEnv } = await import("./runtime.js"); + const runtime = new WorkerRuntime(this, workerConfigFromEnv()); + const stop = (): void => { + runtime.stop(); + }; + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, stop); + } + try { + await runtime.runForever(); + } finally { + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.removeListener(signal, stop); + } + } + } + + /** Call an evaluation or condition, awaiting it when it returns a promise. */ + static async call( + fn: EvalFunction | ConditionFunction, + session: SessionTranscript, + ): Promise { + return await (fn as (s: SessionTranscript) => unknown)(session); + } +} + +function defaultDisplayName(evalKey: string): string { + const spaced = evalKey.replaceAll("_", " "); + return spaced.charAt(0).toUpperCase() + spaced.slice(1); +} diff --git a/sdk/typescript/src/evaluator/cli.ts b/sdk/typescript/src/evaluator/cli.ts new file mode 100644 index 000000000..e2978d94d --- /dev/null +++ b/sdk/typescript/src/evaluator/cli.ts @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/** + * Run an evaluator declared as `module#export`. + * + * npx failproofai-evaluator ./my-evals.js + * npx failproofai-evaluator ./my-evals.js#app + * FAILPROOFAI_EVALUATOR_MODULE=./my-evals.js npx failproofai-evaluator + * + * `#` rather than `:` as the separator, because a module specifier on Windows + * routinely contains a colon (`C:\evals\index.js`) and a `file:` URL always + * does. Splitting on `:` would take `C` as the module. + */ + +import { pathToFileURL } from "node:url"; +import { isAbsolute, resolve } from "node:path"; + +import { importModule } from "../node-require.js"; +import { isEvaluator, type Evaluator } from "./authoring.js"; + +export async function loadEvaluator(spec: string): Promise { + const separator = spec.lastIndexOf("#"); + const moduleName = separator === -1 ? spec : spec.slice(0, separator); + const exportName = separator === -1 ? "app" : spec.slice(separator + 1); + if (!moduleName) throw new Error("evaluator module must not be empty"); + if (!exportName) throw new Error("evaluator export must not be empty"); + + // A relative specifier is relative to the USER'S working directory, not to + // this file inside `node_modules`. Bare specifiers (`my-evals`) are left + // alone so a package can be named. + const target = + moduleName.startsWith(".") || isAbsolute(moduleName) + ? pathToFileURL(resolve(process.cwd(), moduleName)).href + : moduleName; + + const module = (await importModule(target)) as Record; + const candidate = module[exportName] ?? (module.default as Record | undefined)?.[exportName]; + if (candidate === undefined) { + throw new Error(`${JSON.stringify(spec)} does not export ${JSON.stringify(exportName)}`); + } + if (!isEvaluator(candidate)) { + throw new TypeError( + `${JSON.stringify(spec)} resolved to ${ + (candidate as object)?.constructor?.name ?? typeof candidate + }, not an Evaluator`, + ); + } + return candidate; +} + +export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { + const args = argv.filter((arg) => arg !== "--"); + if (args.includes("--help") || args.includes("-h")) { + process.stdout.write( + "Usage: failproofai-evaluator [module[#export]]\n\n" + + " module Path or package to import (default: $FAILPROOFAI_EVALUATOR_MODULE)\n" + + " export Named export holding the Evaluator (default: app)\n\n" + + "Environment:\n" + + " FAILPROOFAI_EVALUATOR_URL required — the API origin\n" + + " FAILPROOFAI_EVALUATOR_TOKEN required — the worker credential\n" + + " FAILPROOFAI_EVALUATOR_WORKER_ID, _CONCURRENCY, _REQUEST_TIMEOUT_SECONDS,\n" + + " _DRAIN_TIMEOUT_SECONDS, _ALLOW_INSECURE_HTTP\n", + ); + return 0; + } + + const spec = args[0] ?? process.env.FAILPROOFAI_EVALUATOR_MODULE; + if (!spec) { + process.stderr.write( + "failproofai-evaluator: a module is required (or set FAILPROOFAI_EVALUATOR_MODULE)\n", + ); + return 2; + } + + const evaluator = await loadEvaluator(spec); + await evaluator.runFromEnv(); + return 0; +} + +// `process.argv[1]` is the script Node was started with. Comparing it to this +// module's own resolved path is the module-system-agnostic way to ask "was I +// run, or imported" — `import.meta.main` does not exist and `require.main` +// only answers for CommonJS. +const invokedPath = process.argv[1]; +if (invokedPath !== undefined && /failproofai-evaluator|evaluator[\\/]cli/.test(invokedPath)) { + main().then( + (code) => { + process.exitCode = code; + }, + (error: unknown) => { + process.stderr.write( + `failproofai-evaluator: ${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }, + ); +} diff --git a/sdk/typescript/src/evaluator/client.ts b/sdk/typescript/src/evaluator/client.ts new file mode 100644 index 000000000..01594a0b4 --- /dev/null +++ b/sdk/typescript/src/evaluator/client.ts @@ -0,0 +1,421 @@ +/** + * HTTP client for the Evaluator v2 worker protocol. + * + * Built on the global `fetch` (Node 18+), so it adds no dependency. Four things + * in here are security or correctness properties rather than style: + * + * * **Redirects are refused**, not followed. A redirect carries the + * `Authorization` header to wherever it points, so following one turns a + * compromised or misconfigured server into credential exfiltration. + * * **Every URL is pinned to the configured origin.** The server supplies + * `transcript_url` and `definitions_url`; a URL outside the origin we + * authenticated to is refused before a request is made. + * * **Responses are bounded while they are read**, not after. A limit checked + * on a fully-buffered body is a limit that has already been exceeded. + * * **`claim` is never retried at the transport layer.** A lost claim response + * may already have leased work; a blind retry would lease it twice. The + * runtime recalculates capacity and claims again on its own schedule. + */ + +import { + CLAIM_PATH, + DEFINITIONS_PATH, + HEARTBEAT_PATH, + LEASE_GENERATION_HEADER, + MAX_TRANSCRIPT_BYTES, + PLAN_PATH, + REGISTER_PATH, + RESULT_PATH, + WORKER_ID_HEADER, + claimRequestToWire, + claimResponseFromWire, + definitionsResponseFromWire, + errorResponseFromWire, + heartbeatRequestToWire, + heartbeatResponseFromWire, + planRequestToWire, + planResponseFromWire, + registerRequestToWire, + registerResponseFromWire, + resultRequestToWire, + resultResponseFromWire, + sessionTranscriptFromWire, +} from "./protocol.js"; +import type { + Assignment, + ClaimRequest, + ClaimResponse, + DefinitionsResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + WireObject, +} from "./protocol.js"; + +const DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024; +const RETRYABLE_HTTP_STATUSES = new Set([429, 502, 503, 504]); + +export class EvaluatorAPIError extends Error { + readonly status: number | null; + readonly code: string; + readonly retryable: boolean; + readonly requestId: string | null; + + constructor(options: { + status: number | null; + code: string; + message: string; + retryable: boolean; + requestId?: string | null; + }) { + super(`${options.code}: ${options.message}`); + this.name = "EvaluatorAPIError"; + this.status = options.status; + this.code = options.code; + this.retryable = options.retryable; + this.requestId = options.requestId ?? null; + } +} + +export interface EvaluatorClientOptions { + baseUrl: string; + credential: string; + timeoutSeconds?: number; + maxRetries?: number; + allowInsecureHttp?: boolean; + /** Injected by the tests; defaults to the global `fetch`. */ + fetchImpl?: typeof fetch; + /** Injected by the tests, so a retry schedule does not cost real seconds. */ + sleep?: (ms: number) => Promise; +} + +function isLoopback(hostname: string): boolean { + if (hostname === "localhost") return true; + const bare = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + if (bare === "::1") return true; + // 127.0.0.0/8 — the whole block, not just 127.0.0.1, because a dev server on + // 127.0.0.2 is as local as one on .1 and refusing it helps nobody. + const parts = bare.split("."); + if (parts.length !== 4 || parts.some((part) => !/^\d{1,3}$/.test(part))) return false; + const octets = parts.map(Number); + return octets[0] === 127 && octets.every((octet) => octet >= 0 && octet <= 255); +} + +async function defaultSleep(ms: number): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); +} + +/** + * Client for the public Evaluator v2 machine API. + * + * Hosted workers normally use the FailproofAI dashboard origin. Its `/v1` + * passthrough forwards this worker's bearer credential to the private server. + */ +export class EvaluatorClient { + private readonly baseUrl: string; + private readonly origin: string; + private readonly credential: string; + private readonly timeoutMs: number; + private readonly maxRetries: number; + private readonly fetchImpl: typeof fetch; + private readonly sleep: (ms: number) => Promise; + + constructor(options: EvaluatorClientOptions) { + let parsed: URL; + try { + parsed = new URL(options.baseUrl); + } catch { + throw new Error("baseUrl must be an absolute http(s) URL"); + } + if ((parsed.protocol !== "http:" && parsed.protocol !== "https:") || !parsed.host) { + throw new Error("baseUrl must be an absolute http(s) URL"); + } + if ( + parsed.protocol !== "https:" && + !isLoopback(parsed.hostname) && + options.allowInsecureHttp !== true + ) { + throw new Error("baseUrl must use https unless it targets loopback"); + } + if (!options.credential || options.credential.trim() === "") { + throw new Error("credential must not be empty"); + } + for (const char of options.credential) { + const code = char.codePointAt(0)!; + // A control character in a header value is a header-injection primitive + // and `fetch` rejects it with an opaque error; refusing here names the + // real problem. + if (code < 32 || code === 127) { + throw new Error("credential must not contain control characters"); + } + } + const timeoutSeconds = options.timeoutSeconds ?? 30; + if (!(timeoutSeconds > 0)) throw new Error("timeoutSeconds must be greater than zero"); + const maxRetries = options.maxRetries ?? 3; + if (maxRetries < 0) throw new Error("maxRetries must not be negative"); + + this.baseUrl = `${options.baseUrl.replace(/\/+$/, "")}/`; + this.origin = parsed.origin; + this.credential = options.credential; + this.timeoutMs = timeoutSeconds * 1000; + this.maxRetries = maxRetries; + this.fetchImpl = options.fetchImpl ?? globalThis.fetch; + this.sleep = options.sleep ?? defaultSleep; + if (typeof this.fetchImpl !== "function") { + throw new Error( + "no global fetch is available; pass fetchImpl, or run on Node 18 or newer", + ); + } + } + + async register(request: RegisterRequest): Promise { + return registerResponseFromWire( + await this.json("POST", REGISTER_PATH, registerRequestToWire(request), { retry: true }), + ); + } + + async claim(request: ClaimRequest): Promise { + return claimResponseFromWire( + await this.json("POST", CLAIM_PATH, claimRequestToWire(request), { retry: false }), + ); + } + + async transcript(assignment: Assignment, workerId: string): Promise { + return sessionTranscriptFromWire( + await this.json("GET", assignment.transcriptUrl, null, { + retry: true, + headers: this.leaseHeaders(assignment, workerId), + responseLimit: MAX_TRANSCRIPT_BYTES, + }), + ); + } + + async definitions(assignment: Assignment, workerId: string): Promise { + const path = + assignment.definitionsUrl || + DEFINITIONS_PATH.replace("{assignment_id}", encodeURIComponent(assignment.assignmentId)); + return definitionsResponseFromWire( + await this.json("GET", path, null, { + retry: true, + headers: this.leaseHeaders(assignment, workerId), + }), + ); + } + + async plan(assignmentId: string, request: PlanRequest): Promise { + return planResponseFromWire( + await this.json( + "POST", + PLAN_PATH.replace("{assignment_id}", encodeURIComponent(assignmentId)), + planRequestToWire(request), + { retry: true }, + ), + ); + } + + async heartbeat(request: HeartbeatRequest): Promise { + return heartbeatResponseFromWire( + await this.json("POST", HEARTBEAT_PATH, heartbeatRequestToWire(request), { retry: true }), + ); + } + + async submitResult(runId: string, request: ResultRequest): Promise { + return resultResponseFromWire( + await this.json( + "POST", + RESULT_PATH.replace("{evaluation_run_id}", encodeURIComponent(runId)), + resultRequestToWire(request), + { retry: true }, + ), + ); + } + + private leaseHeaders(assignment: Assignment, workerId: string): Record { + return { + [WORKER_ID_HEADER]: workerId, + [LEASE_GENERATION_HEADER]: String(assignment.leaseGeneration), + }; + } + + private url(path: string): string { + const resolved = new URL(path, this.baseUrl); + if (resolved.origin !== this.origin) { + throw new EvaluatorAPIError({ + status: null, + code: "invalid_transcript_url", + message: "server supplied a URL outside the configured API origin", + retryable: false, + }); + } + return resolved.toString(); + } + + private async json( + method: string, + path: string, + body: WireObject | null, + options: { retry: boolean; headers?: Record; responseLimit?: number }, + ): Promise { + const responseLimit = options.responseLimit ?? DEFAULT_RESPONSE_LIMIT; + const headers: Record = { + Accept: "application/json", + Authorization: `Bearer ${this.credential}`, + "User-Agent": "failproofai-sdk-evaluator/2", + ...options.headers, + }; + let encoded: string | undefined; + if (body !== null) { + encoded = JSON.stringify(body); + headers["Content-Type"] = "application/json"; + } + + const attempts = options.retry ? this.maxRetries + 1 : 1; + for (let attempt = 0; attempt < attempts; attempt += 1) { + const url = this.url(path); + let response: Response; + try { + response = await this.fetchImpl(url, { + method, + headers, + body: encoded, + // Never follow a redirect: it would carry the Authorization header to + // wherever the redirect points. + redirect: "error", + signal: AbortSignal.timeout(this.timeoutMs), + }); + } catch (error) { + if (attempt + 1 === attempts) { + throw new EvaluatorAPIError({ + status: null, + code: "transport_error", + message: error instanceof Error ? error.message : String(error), + retryable: true, + }); + } + await this.backoff(attempt); + continue; + } + + if (response.ok) return decode(await readLimited(response, responseLimit), responseLimit); + + const apiError = await this.httpError(response, responseLimit); + if (attempt + 1 === attempts || !apiError.retryable) throw apiError; + await this.backoff(attempt); + } + // Unreachable: the loop either returns or throws on its last attempt. + throw new Error("retry loop exhausted without returning or throwing"); + } + + private async backoff(attempt: number): Promise { + // Jitter is scheduling noise, not a security decision, so `Math.random` is + // the right tool — a CSPRNG here would buy nothing and cost entropy. + await this.sleep(Math.random() * Math.min(250 * 2 ** attempt, 2000)); + } + + private async httpError(response: Response, limit: number): Promise { + let raw: string; + try { + raw = await readLimited(response, limit); + } catch { + raw = ""; + } + try { + const parsed = errorResponseFromWire(decode(raw, limit)); + return new EvaluatorAPIError({ + status: response.status, + code: parsed.error.code, + message: parsed.error.message, + retryable: parsed.error.retryable, + requestId: parsed.error.requestId, + }); + } catch { + return new EvaluatorAPIError({ + status: response.status, + code: "http_error", + message: `server returned HTTP ${response.status}`, + retryable: RETRYABLE_HTTP_STATUSES.has(response.status), + }); + } + } +} + +/** + * Read a response body, stopping the moment it exceeds `limit`. + * + * Checking the size after buffering is checking it too late — the memory has + * already been allocated, which on a 25 MiB transcript ceiling and a hostile or + * broken server is the whole problem. `response.body` may be absent (a 204, or + * a mocked response in a test), in which case `text()` is bounded by whatever + * produced it. + */ +async function readLimited(response: Response, limit: number): Promise { + const body = response.body; + if (!body) return await response.text(); + + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value === undefined) continue; + total += value.byteLength; + if (total > limit) { + throw new EvaluatorAPIError({ + status: null, + code: "response_too_large", + message: `server response exceeds ${limit} bytes`, + retryable: false, + }); + } + chunks.push(value); + } + } finally { + // Releasing the lock lets the connection be reused; cancelling an + // over-limit body stops the server streaming the rest of it at us. + if (total > limit) await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8"); +} + +function decode(raw: string, limit: number): WireObject { + if (Buffer.byteLength(raw, "utf8") > limit) { + throw new EvaluatorAPIError({ + status: null, + code: "response_too_large", + message: `server response exceeds ${limit} bytes`, + retryable: false, + }); + } + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new EvaluatorAPIError({ + status: null, + code: "invalid_response", + message: "server response was not valid JSON", + retryable: false, + }); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new EvaluatorAPIError({ + status: null, + code: "invalid_response", + message: "server response must be a JSON object", + retryable: false, + }); + } + return value as WireObject; +} diff --git a/sdk/typescript/src/evaluator/expression.ts b/sdk/typescript/src/evaluator/expression.ts new file mode 100644 index 000000000..2bfab3759 --- /dev/null +++ b/sdk/typescript/src/evaluator/expression.ts @@ -0,0 +1,1292 @@ +/** + * A restricted deterministic expression language for server-authored + * evaluations, parsed and **interpreted** here — never `eval`'d, never handed + * to `node:vm`. + * + * ## Why an interpreter and not `vm` + * + * The Python SDK validates an AST against an allowlist and then calls `eval` + * with empty builtins. The same shape in JavaScript is not safe, because + * JavaScript has a reachable path from any value to arbitrary code: + * + * (() => {}).constructor("return process")() + * x["constructor"]["constructor"]("…")() + * + * A STATIC allowlist cannot close the second one: `x[k]` is a property read + * whose key is only known at runtime, so `x["constructor"]` passes any check + * made over the source text. `node:vm` does not close it either — a vm context + * has its own `Function`, and reaching it from inside is a well-known escape. + * + * Interpreting removes the question. Every property read goes through + * `readProperty` below, which is an allowlist evaluated against the ACTUAL key + * at the moment of the read, and there is no path from a value to a function + * constructor because the interpreter never constructs one. The + * `worker_threads` sandbox around this (see `source.ts`) is then a RESOURCE + * bound — CPU, heap, wall-clock — rather than the only thing standing between + * tenant source and the worker's credentials. + * + * ## The grammar + * + * One expression. Literals, arrays, objects, template strings, member access, + * calls, arrow functions (the analogue of Python's comprehensions — `.map` / + * `.filter` need them), the arithmetic, comparison and logical operators, and + * the conditional operator. No statements, no assignment, no `new`, no + * `function`, no `async`, no `await`, no regular-expression literals, no + * optional chaining into calls. + * + * DEFAULT-DENY throughout: anything the parser does not explicitly understand + * is a rejection, not a pass-through. A grammar gap is therefore a definition + * that will not run, never a definition that runs unchecked. + */ + +export class UnsafeEvaluatorSource extends Error { + constructor(message: string) { + super(message); + this.name = "UnsafeEvaluatorSource"; + } +} + +/** A sandboxed evaluation exceeded its step, depth or value budget. */ +export class EvaluationBudgetExceeded extends Error { + constructor(message: string) { + super(message); + this.name = "EvaluationBudgetExceeded"; + } +} + +export const MAX_AST_NODES = 5_000; +export const MAX_POW_EXPONENT = 64; +/** + * Interpreter step ceiling. There are no loop statements in this grammar, but + * `.map` over a large array and self-application (`(f => f(f))(f => f(f))`, the + * Y-combinator — reachable with nothing but arrow functions) both run forever. + * The worker's wall-clock kill is the outer bound; this one fails the + * evaluation with a message that says what happened instead of a silent + * timeout. + */ +export const MAX_STEPS = 2_000_000; +export const MAX_CALL_DEPTH = 64; +/** Cap on a single constructed string, so concatenation cannot build a bomb. */ +export const MAX_STRING_LENGTH = 1_000_000; +/** Cap on a single constructed array, for the same reason. */ +export const MAX_ARRAY_LENGTH = 100_000; + +// --------------------------------------------------------------------------- +// Allowlists +// --------------------------------------------------------------------------- + +/** + * Pure data methods. Allowed only at a CALL SITE — `_parse` requires each of + * these names to be the callee of a call, so a bare `payload.get` reference + * cannot be smuggled into a result field. + * + * Conspicuously absent: `constructor`, `apply`, `call`, `bind`, `toString`, + * `valueOf`, `__proto__`, `prototype`. Not as a denylist — they are absent + * because this is an allowlist, which is the whole point: a method added to + * `Array.prototype` by a future runtime is denied by default rather than + * needing to be discovered and listed. + */ +const METHOD_ATTRS = new Set([ + // Transcript methods. + "eventsOfType", + "count", + // Array data methods. + "map", + "filter", + "some", + "every", + "find", + "findIndex", + "includes", + "indexOf", + "lastIndexOf", + "slice", + "concat", + "join", + "flat", + "flatMap", + "reduce", + "at", + "reverse", + // String data methods. + "toLowerCase", + "toUpperCase", + "trim", + "trimStart", + "trimEnd", + "split", + "startsWith", + "endsWith", + "replace", + "replaceAll", + "padStart", + "padEnd", + "repeat", + "charAt", + "codePointAt", + "normalize", + "localeCompare", + // Number data methods. + "toFixed", +]); + +/** + * Property names that must never be readable, whatever the allowlist says and + * however the key is computed. `readProperty` checks this on the RUNTIME key, + * which is what closes `x[someExpressionThatYields("constructor")]`. + */ +const FORBIDDEN_KEYS = new Set([ + "constructor", + "prototype", + "__proto__", + "__defineGetter__", + "__defineSetter__", + "__lookupGetter__", + "__lookupSetter__", + "apply", + "call", + "bind", + "arguments", + "caller", + "toString", + "valueOf", + "toJSON", + "then", +]); + +/** Namespace globals, each with its own fixed member set. */ +const NAMESPACE_MEMBERS: Record> = { + Math: new Set([ + "abs", + "min", + "max", + "round", + "floor", + "ceil", + "trunc", + "sign", + "sqrt", + "log", + "log2", + "log10", + "exp", + "hypot", + "PI", + "E", + ]), + Object: new Set(["keys", "values", "entries", "fromEntries"]), + Array: new Set(["isArray", "from", "of"]), + Number: new Set(["isFinite", "isInteger", "isNaN", "parseFloat", "parseInt", "MAX_SAFE_INTEGER"]), + JSON: new Set(["stringify"]), +}; + +// --------------------------------------------------------------------------- +// Tokenizer +// --------------------------------------------------------------------------- + +type TokenType = "num" | "str" | "template" | "name" | "punct" | "eof"; + +interface TemplatePart { + readonly cooked: string; + readonly expression: string | null; +} + +interface Token { + readonly type: TokenType; + readonly value: string; + readonly start: number; + readonly numeric?: number; + readonly parts?: readonly TemplatePart[]; +} + +const PUNCTUATORS = [ + "===", + "!==", + "**", + "&&", + "||", + "??", + "==", + "!=", + "<=", + ">=", + "=>", + "(", + ")", + "[", + "]", + "{", + "}", + ",", + ".", + ":", + "?", + "+", + "-", + "*", + "/", + "%", + "<", + ">", + "!", +].sort((a, b) => b.length - a.length); + +const IDENT_START = /[A-Za-z_$]/; +const IDENT_PART = /[A-Za-z0-9_$]/; + +function fail(fieldName: string, detail: string): never { + throw new UnsafeEvaluatorSource(`${fieldName} ${detail}`); +} + +class Tokenizer { + private index = 0; + constructor( + private readonly text: string, + private readonly fieldName: string, + ) {} + + tokenize(): Token[] { + const tokens: Token[] = []; + for (;;) { + this.skipTrivia(); + if (this.index >= this.text.length) { + tokens.push({ type: "eof", value: "", start: this.index }); + return tokens; + } + tokens.push(this.next()); + } + } + + /** + * Whitespace only. Comments are NOT skipped — they are rejected by the + * punctuator table, because a language that accepts `//` and `/*` has to get + * their interaction with division exactly right to stay unambiguous, and an + * expression that needs a comment is one that should be simpler. + */ + private skipTrivia(): void { + while (this.index < this.text.length && /\s/.test(this.text[this.index]!)) this.index += 1; + } + + private next(): Token { + const start = this.index; + const char = this.text[start]!; + + if (char >= "0" && char <= "9") return this.number(start); + if (char === "." && /[0-9]/.test(this.text[start + 1] ?? "")) return this.number(start); + if (char === '"' || char === "'") return this.string(start, char); + if (char === "`") return this.template(start); + if (IDENT_START.test(char)) return this.name(start); + + for (const punctuator of PUNCTUATORS) { + if (this.text.startsWith(punctuator, start)) { + this.index = start + punctuator.length; + return { type: "punct", value: punctuator, start }; + } + } + fail(this.fieldName, `contains an unexpected character ${JSON.stringify(char)}`); + } + + private number(start: number): Token { + let end = start; + while (end < this.text.length && /[0-9]/.test(this.text[end]!)) end += 1; + if (this.text[end] === ".") { + end += 1; + while (end < this.text.length && /[0-9]/.test(this.text[end]!)) end += 1; + } + if (this.text[end] === "e" || this.text[end] === "E") { + let cursor = end + 1; + if (this.text[cursor] === "+" || this.text[cursor] === "-") cursor += 1; + if (/[0-9]/.test(this.text[cursor] ?? "")) { + cursor += 1; + while (cursor < this.text.length && /[0-9]/.test(this.text[cursor]!)) cursor += 1; + end = cursor; + } + } + // A trailing identifier character means `1n` (BigInt) or `0x…`; both are + // outside this grammar and must be rejected rather than silently truncated + // to the numeric prefix. + if (IDENT_PART.test(this.text[end] ?? "")) { + fail(this.fieldName, "contains an unsupported numeric literal"); + } + const raw = this.text.slice(start, end); + this.index = end; + return { type: "num", value: raw, start, numeric: Number(raw) }; + } + + private readEscape(): string { + // `this.index` points at the backslash. + this.index += 1; + const char = this.text[this.index]; + if (char === undefined) fail(this.fieldName, "ends inside a string escape"); + this.index += 1; + switch (char) { + case "n": + return "\n"; + case "t": + return "\t"; + case "r": + return "\r"; + case "b": + return "\b"; + case "f": + return "\f"; + case "v": + return "\v"; + case "0": + return "\0"; + case "u": { + if (this.text[this.index] === "{") { + const close = this.text.indexOf("}", this.index); + if (close === -1) fail(this.fieldName, "has an unterminated unicode escape"); + const hex = this.text.slice(this.index + 1, close); + if (!/^[0-9a-fA-F]{1,6}$/.test(hex)) fail(this.fieldName, "has a malformed unicode escape"); + this.index = close + 1; + return String.fromCodePoint(Number.parseInt(hex, 16)); + } + const hex = this.text.slice(this.index, this.index + 4); + if (!/^[0-9a-fA-F]{4}$/.test(hex)) fail(this.fieldName, "has a malformed unicode escape"); + this.index += 4; + return String.fromCharCode(Number.parseInt(hex, 16)); + } + case "x": { + const hex = this.text.slice(this.index, this.index + 2); + if (!/^[0-9a-fA-F]{2}$/.test(hex)) fail(this.fieldName, "has a malformed hex escape"); + this.index += 2; + return String.fromCharCode(Number.parseInt(hex, 16)); + } + default: + return char; + } + } + + private string(start: number, quote: string): Token { + this.index = start + 1; + let out = ""; + while (this.index < this.text.length) { + const char = this.text[this.index]!; + if (char === quote) { + this.index += 1; + return { type: "str", value: out, start }; + } + if (char === "\\") { + out += this.readEscape(); + continue; + } + if (char === "\n") fail(this.fieldName, "has an unterminated string literal"); + out += char; + this.index += 1; + } + fail(this.fieldName, "has an unterminated string literal"); + } + + private template(start: number): Token { + this.index = start + 1; + const parts: TemplatePart[] = []; + let cooked = ""; + while (this.index < this.text.length) { + const char = this.text[this.index]!; + if (char === "`") { + this.index += 1; + parts.push({ cooked, expression: null }); + return { type: "template", value: "", start, parts }; + } + if (char === "\\") { + cooked += this.readEscape(); + continue; + } + if (char === "$" && this.text[this.index + 1] === "{") { + const expression = this.readInterpolation(); + parts.push({ cooked, expression }); + cooked = ""; + continue; + } + cooked += char; + this.index += 1; + } + fail(this.fieldName, "has an unterminated template literal"); + } + + /** The source between `${` and its matching `}`, tracking nesting and strings. */ + private readInterpolation(): string { + this.index += 2; + const begin = this.index; + let depth = 1; + while (this.index < this.text.length) { + const char = this.text[this.index]!; + if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + const source = this.text.slice(begin, this.index); + this.index += 1; + return source; + } + } else if (char === '"' || char === "'" || char === "`") { + this.string(this.index, char); + continue; + } + this.index += 1; + } + fail(this.fieldName, "has an unterminated template interpolation"); + } + + private name(start: number): Token { + let end = start; + while (end < this.text.length && IDENT_PART.test(this.text[end]!)) end += 1; + this.index = end; + return { type: "name", value: this.text.slice(start, end), start }; + } +} + +// --------------------------------------------------------------------------- +// AST +// --------------------------------------------------------------------------- + +type Node = + | { kind: "literal"; value: unknown } + | { kind: "template"; parts: ReadonlyArray<{ cooked: string; expression: Node | null }> } + | { kind: "name"; name: string } + | { kind: "array"; elements: readonly Node[] } + | { kind: "object"; properties: ReadonlyArray<{ key: Node; value: Node }> } + | { kind: "member"; object: Node; property: Node; computed: boolean; optional: boolean } + | { kind: "call"; callee: Node; args: readonly Node[]; optional: boolean } + | { kind: "unary"; operator: string; argument: Node } + | { kind: "binary"; operator: string; left: Node; right: Node } + | { kind: "logical"; operator: string; left: Node; right: Node } + | { kind: "conditional"; test: Node; consequent: Node; alternate: Node } + | { kind: "arrow"; params: readonly string[]; body: Node }; + +const RESERVED_NAMES = new Set([ + "new", + "function", + "class", + "await", + "yield", + "delete", + "typeof", + "instanceof", + "void", + "this", + "super", + "import", + "export", + "var", + "let", + "const", + "return", + "if", + "else", + "for", + "while", + "do", + "switch", + "case", + "try", + "catch", + "finally", + "throw", + "in", + "of", + "with", + "debugger", +]); + +// --------------------------------------------------------------------------- +// Parser +// --------------------------------------------------------------------------- + +class Parser { + private position = 0; + private nodes = 0; + + constructor( + private readonly tokens: readonly Token[], + private readonly fieldName: string, + private readonly globals: ReadonlySet, + private scopes: readonly (readonly string[])[] = [], + ) {} + + static parse( + source: string, + fieldName: string, + globals: ReadonlySet, + scopes: readonly (readonly string[])[] = [], + ): Node { + const parser = new Parser(new Tokenizer(source, fieldName).tokenize(), fieldName, globals, scopes); + const node = parser.expression(); + parser.expect("eof"); + return node; + } + + private count(): void { + this.nodes += 1; + if (this.nodes > MAX_AST_NODES) { + fail(this.fieldName, `is too large (${MAX_AST_NODES}-node ceiling)`); + } + } + + private peek(offset = 0): Token { + return this.tokens[Math.min(this.position + offset, this.tokens.length - 1)]!; + } + + private at(value: string): boolean { + const token = this.peek(); + return token.type === "punct" && token.value === value; + } + + private eat(value: string): boolean { + if (!this.at(value)) return false; + this.position += 1; + return true; + } + + private expect(type: TokenType, value?: string): Token { + const token = this.peek(); + if (token.type !== type || (value !== undefined && token.value !== value)) { + fail( + this.fieldName, + `is not one valid expression (unexpected ${ + token.type === "eof" ? "end of input" : JSON.stringify(token.value) + })`, + ); + } + this.position += 1; + return token; + } + + private bound(name: string): boolean { + return this.scopes.some((scope) => scope.includes(name)); + } + + private withScope(params: readonly string[], body: () => T): T { + const previous = this.scopes; + this.scopes = [...previous, params]; + try { + return body(); + } finally { + this.scopes = previous; + } + } + + expression(): Node { + return this.conditional(); + } + + private conditional(): Node { + const test = this.nullish(); + if (!this.eat("?")) return test; + this.count(); + const consequent = this.expression(); + this.expect("punct", ":"); + const alternate = this.expression(); + return { kind: "conditional", test, consequent, alternate }; + } + + private nullish(): Node { + let left = this.logicalOr(); + while (this.at("??")) { + this.position += 1; + this.count(); + left = { kind: "logical", operator: "??", left, right: this.logicalOr() }; + } + return left; + } + + private logicalOr(): Node { + let left = this.logicalAnd(); + while (this.at("||")) { + this.position += 1; + this.count(); + left = { kind: "logical", operator: "||", left, right: this.logicalAnd() }; + } + return left; + } + + private logicalAnd(): Node { + let left = this.equality(); + while (this.at("&&")) { + this.position += 1; + this.count(); + left = { kind: "logical", operator: "&&", left, right: this.equality() }; + } + return left; + } + + private equality(): Node { + let left = this.relational(); + for (;;) { + const operator = ["===", "!==", "==", "!="].find((op) => this.at(op)); + if (operator === undefined) return left; + this.position += 1; + this.count(); + left = { kind: "binary", operator, left, right: this.relational() }; + } + } + + private relational(): Node { + let left = this.additive(); + for (;;) { + const operator = ["<=", ">=", "<", ">"].find((op) => this.at(op)); + if (operator === undefined) return left; + this.position += 1; + this.count(); + left = { kind: "binary", operator, left, right: this.additive() }; + } + } + + private additive(): Node { + let left = this.multiplicative(); + for (;;) { + const operator = ["+", "-"].find((op) => this.at(op)); + if (operator === undefined) return left; + this.position += 1; + this.count(); + left = { kind: "binary", operator, left, right: this.multiplicative() }; + } + } + + private multiplicative(): Node { + let left = this.exponent(); + for (;;) { + const operator = ["*", "/", "%"].find((op) => this.at(op)); + if (operator === undefined) return left; + this.position += 1; + this.count(); + left = { kind: "binary", operator, left, right: this.exponent() }; + } + } + + private exponent(): Node { + const left = this.unary(); + if (!this.at("**")) return left; + this.position += 1; + this.count(); + const right = this.exponent(); + // Defence in depth: a literal `10 ** 20` (or `2 ** (10 ** 8)`) builds a + // number — or, with strings, a memory bomb — at compile-time-visible size. + // Require the exponent to be a small non-negative integer literal. + // Runtime-sized bombs still exist and are caught by the step budget and the + // worker's limits, not here. + if ( + right.kind !== "literal" || + typeof right.value !== "number" || + !Number.isInteger(right.value) || + right.value < 0 || + right.value > MAX_POW_EXPONENT + ) { + fail(this.fieldName, `exponent must be an integer constant in 0..${MAX_POW_EXPONENT}`); + } + return { kind: "binary", operator: "**", left, right }; + } + + private unary(): Node { + const operator = ["!", "-", "+"].find((op) => this.at(op)); + if (operator === undefined) return this.postfix(); + this.position += 1; + this.count(); + return { kind: "unary", operator, argument: this.unary() }; + } + + private postfix(): Node { + let node = this.primary(); + for (;;) { + if (this.eat(".")) { + this.count(); + const name = this.expect("name"); + this.checkAttribute(name.value); + node = { + kind: "member", + object: node, + property: { kind: "literal", value: name.value }, + computed: false, + optional: false, + }; + // A method name must be the callee of a call. A bare reference is only + // ever useful for getting a function object somewhere it can be + // re-entered, and nothing a real evaluation does needs one. + if (METHOD_ATTRS.has(name.value) && !this.at("(")) { + fail( + this.fieldName, + `may reference method '${name.value}' only to call it`, + ); + } + continue; + } + if (this.eat("[")) { + this.count(); + const property = this.expression(); + this.expect("punct", "]"); + // A computed key that is a literal can be checked now, so + // `x["constructor"]` fails when the definition is published rather than + // on the first session it runs against. A computed key that is not a + // literal cannot be, which is why `readProperty` checks every key again + // at the moment of the read. + if (property.kind === "literal" && typeof property.value === "string") { + this.checkAttribute(property.value); + } + node = { kind: "member", object: node, property, computed: true, optional: false }; + continue; + } + if (this.at("(")) { + this.count(); + node = { kind: "call", callee: node, args: this.arguments(), optional: false }; + continue; + } + return node; + } + } + + /** + * What the PARSER can rule out, which is less than it looks. + * + * An event payload's keys are whatever the agent put in it — `error`, + * `tool_name`, `latency_ms`, anything — so a static allowlist of attribute + * names cannot be the boundary here without making the language unable to + * read the data it exists to read. And it would not be the boundary anyway: + * `x[k]` is a property read whose key is only known at runtime, so anything + * a static check rejects can be spelled computed. + * + * So `readProperty` is the boundary, checked against the ACTUAL key at the + * moment of the read, and this check rejects only the names that can never be + * legitimate — private/dunder, and the introspection surface — plus the rule + * that a method may be referenced only to call it. + */ + private checkAttribute(name: string): void { + if (name.startsWith("_")) { + fail(this.fieldName, "may not access private or dunder attributes"); + } + if (FORBIDDEN_KEYS.has(name)) { + fail(this.fieldName, `may not access attribute '${name}'`); + } + } + + private arguments(): Node[] { + this.expect("punct", "("); + const args: Node[] = []; + if (this.eat(")")) return args; + for (;;) { + args.push(this.expression()); + if (this.eat(",")) { + if (this.eat(")")) return args; // trailing comma + continue; + } + this.expect("punct", ")"); + return args; + } + } + + private primary(): Node { + this.count(); + const token = this.peek(); + + if (token.type === "num") { + this.position += 1; + return { kind: "literal", value: token.numeric }; + } + if (token.type === "str") { + this.position += 1; + return { kind: "literal", value: token.value }; + } + if (token.type === "template") { + this.position += 1; + return { + kind: "template", + parts: (token.parts ?? []).map((part) => ({ + cooked: part.cooked, + expression: + part.expression === null + ? null + : Parser.parse(part.expression, this.fieldName, this.globals, this.scopes), + })), + }; + } + if (token.type === "name") { + if (token.value === "true" || token.value === "false") { + this.position += 1; + return { kind: "literal", value: token.value === "true" }; + } + if (token.value === "null" || token.value === "undefined") { + this.position += 1; + return { kind: "literal", value: token.value === "null" ? null : undefined }; + } + // `x => …` + if (this.peek(1).type === "punct" && this.peek(1).value === "=>") { + return this.arrow([this.parameterName(token.value)]); + } + this.position += 1; + if (RESERVED_NAMES.has(token.value)) { + fail(this.fieldName, `may not use the reserved word '${token.value}'`); + } + if (token.value.startsWith("_")) fail(this.fieldName, "may not access private names"); + if (!this.bound(token.value) && !this.globals.has(token.value)) { + fail(this.fieldName, `may not reference '${token.value}'`); + } + return { kind: "name", name: token.value }; + } + if (this.at("[")) { + this.position += 1; + const elements: Node[] = []; + if (this.eat("]")) return { kind: "array", elements }; + for (;;) { + elements.push(this.expression()); + if (this.eat(",")) { + if (this.eat("]")) return { kind: "array", elements }; + continue; + } + this.expect("punct", "]"); + return { kind: "array", elements }; + } + } + if (this.at("{")) return this.objectLiteral(); + if (this.at("(")) { + const arrowParams = this.tryParenArrowParams(); + if (arrowParams !== null) return this.arrow(arrowParams); + this.position += 1; + const inner = this.expression(); + this.expect("punct", ")"); + return inner; + } + fail( + this.fieldName, + `is not one valid expression (unexpected ${ + token.type === "eof" ? "end of input" : JSON.stringify(token.value) + })`, + ); + } + + private parameterName(name: string): string { + if (RESERVED_NAMES.has(name)) fail(this.fieldName, `may not use '${name}' as a parameter`); + if (name.startsWith("_")) fail(this.fieldName, "may not declare private names"); + return name; + } + + /** + * `(a, b) =>` — decided by lookahead rather than by backtracking a parse. + * + * Returns the parameter names when the parenthesised group is a bare + * identifier list followed by `=>`, and null otherwise (in which case it is a + * parenthesised expression). Destructuring, defaults and rest are not part of + * the grammar, so anything else in the group means "not an arrow". + */ + private tryParenArrowParams(): string[] | null { + let offset = 1; + const params: string[] = []; + if (this.peek(offset).type === "punct" && this.peek(offset).value === ")") { + offset += 1; + } else { + for (;;) { + const token = this.peek(offset); + if (token.type !== "name") return null; + params.push(token.value); + offset += 1; + const next = this.peek(offset); + if (next.type !== "punct") return null; + if (next.value === ",") { + offset += 1; + continue; + } + if (next.value === ")") { + offset += 1; + break; + } + return null; + } + } + const arrow = this.peek(offset); + if (arrow.type !== "punct" || arrow.value !== "=>") return null; + this.position += offset; + return params.map((name) => this.parameterName(name)); + } + + private arrow(params: readonly string[]): Node { + // The caller has consumed everything up to but not including `=>` for the + // single-identifier form, and up to and including `)` for the parenthesised + // one; normalise by consuming the arrow here. + if (!this.at("=>")) { + this.position += 1; // the lone identifier + } + this.expect("punct", "=>"); + this.count(); + if (new Set(params).size !== params.length) { + fail(this.fieldName, "may not declare a parameter twice"); + } + if (this.at("{")) { + fail(this.fieldName, "arrow functions must have an expression body, not a block"); + } + const body = this.withScope(params, () => this.expression()); + return { kind: "arrow", params, body }; + } + + private objectLiteral(): Node { + this.expect("punct", "{"); + const properties: Array<{ key: Node; value: Node }> = []; + if (this.eat("}")) return { kind: "object", properties }; + for (;;) { + this.count(); + let key: Node; + const token = this.peek(); + if (token.type === "name") { + this.position += 1; + key = { kind: "literal", value: token.value }; + } else if (token.type === "str") { + this.position += 1; + key = { kind: "literal", value: token.value }; + } else if (token.type === "num") { + this.position += 1; + key = { kind: "literal", value: String(token.numeric) }; + } else if (this.eat("[")) { + key = this.expression(); + this.expect("punct", "]"); + } else { + fail(this.fieldName, "has an unsupported object key"); + } + this.expect("punct", ":"); + properties.push({ key, value: this.expression() }); + if (this.eat(",")) { + if (this.eat("}")) return { kind: "object", properties }; + continue; + } + this.expect("punct", "}"); + return { kind: "object", properties }; + } + } +} + +// --------------------------------------------------------------------------- +// Interpreter +// --------------------------------------------------------------------------- + +class Budget { + steps = 0; + depth = 0; + + step(): void { + this.steps += 1; + if (this.steps > MAX_STEPS) { + throw new EvaluationBudgetExceeded( + `evaluation exceeded ${MAX_STEPS} steps; it is looping or iterating unboundedly`, + ); + } + } +} + +type Scope = Map; + +/** + * Every property read in the language, checked against the RUNTIME key. + * + * This is the security boundary, not the parser: `x[k]` cannot be checked + * statically, and `x["constructor"]` is the whole escape. Own properties only, + * so nothing reaches a prototype; allowlisted names only, so nothing reaches an + * inherited method the allowlist has not vetted. + */ +function readProperty(target: unknown, key: unknown): unknown { + if (target === null || target === undefined) { + throw new TypeError(`cannot read property ${String(key)} of ${String(target)}`); + } + + const name = typeof key === "number" ? String(key) : String(key); + // Checked FIRST, on the runtime key, and before anything else looks at the + // receiver. This one test is what closes `x["constructor"]["constructor"]` + // and every spelling of it — including keys assembled at runtime, which no + // amount of source inspection can see. + if (name.startsWith("_") || FORBIDDEN_KEYS.has(name)) { + throw new UnsafeEvaluatorSource(`property '${name}' is not readable in an evaluator expression`); + } + + // A namespace global gets its own member set: `Math.abs` yes, `Math` beyond + // that no. + for (const [namespaceName, members] of Object.entries(NAMESPACE_MEMBERS)) { + if (target === NAMESPACES[namespaceName]) { + if (!members.has(name)) { + throw new UnsafeEvaluatorSource(`${namespaceName}.${name} is not available`); + } + return (NAMESPACES[namespaceName] as Record)[name]; + } + } + + // Built-in receivers expose their ALLOWLISTED methods and nothing else: their + // prototypes are shared, mutable and full of things that are not data. + if (typeof target === "string" || Array.isArray(target)) { + if (/^\d+$/.test(name)) return (target as unknown as Record)[name]; + if (name === "length") return (target as { length: number }).length; + if (!METHOD_ATTRS.has(name)) { + throw new UnsafeEvaluatorSource( + `'${name}' is not available on ${Array.isArray(target) ? "an array" : "a string"}`, + ); + } + const method = (target as unknown as Record)[name]; + if (typeof method !== "function") throw new TypeError(`'${name}' is not a method of this value`); + // Bound, so a call site cannot re-target it — `.call` is denied anyway, and + // this costs nothing to make structurally impossible as well. + return (method as (...args: unknown[]) => unknown).bind(target); + } + if (typeof target === "number" || typeof target === "boolean") { + if (!METHOD_ATTRS.has(name)) { + throw new UnsafeEvaluatorSource(`'${name}' is not available on a ${typeof target}`); + } + const method = (target as unknown as Record)[name]; + if (typeof method !== "function") throw new TypeError(`'${name}' is not a method of a number`); + return (method as (...args: unknown[]) => unknown).bind(target); + } + if (typeof target !== "object") { + throw new TypeError(`cannot read property '${name}' of a ${typeof target}`); + } + + // A plain data object — an event payload, or one the expression built. Its + // keys are the agent's own and cannot be enumerated in advance, so ANY key is + // readable here except the ones refused above. OWN properties only, so + // nothing reaches a prototype and an absent key reads as `undefined` rather + // than as whatever `Object.prototype` happens to carry. + if (!Object.prototype.hasOwnProperty.call(target, name)) return undefined; + const value = (target as Record)[name]; + if (typeof value === "function") { + // Only the transcript's own data methods are functions on this path — a + // JSON payload cannot contain one — but the allowlist still applies, so a + // future surface cannot expose something callable by accident. + if (!METHOD_ATTRS.has(name)) { + throw new UnsafeEvaluatorSource(`'${name}' is not callable in an evaluator expression`); + } + return (value as (...args: unknown[]) => unknown).bind(target); + } + return value; +} + +/** + * `==` / `!=` without ever running the operands' own code. + * + * JavaScript's loose equality calls `valueOf` and `toString` on an object + * operand, which is a call into arbitrary code — the one door this interpreter + * would otherwise leave open. Mapping `==` to `===` instead was worse in + * practice: `payload.error != null` is THE idiom for "this optional field is + * present", and under strict equality it silently stops matching `undefined`, + * so an evaluation over a payload that simply omits the key reports the + * opposite of the truth. + * + * So: nullish operands compare as JavaScript does, primitives coerce the way + * the specification says, and an object compared against a primitive is + * `false` rather than an invitation to run `valueOf`. + */ +function looselyEqual(left: unknown, right: unknown): boolean { + const leftNullish = left === null || left === undefined; + const rightNullish = right === null || right === undefined; + if (leftNullish || rightNullish) return leftNullish && rightNullish; + if (typeof left === typeof right) return left === right; + const leftPrimitive = typeof left !== "object" && typeof left !== "function"; + const rightPrimitive = typeof right !== "object" && typeof right !== "function"; + if (!leftPrimitive || !rightPrimitive) return false; + if (typeof left === "boolean") return looselyEqual(Number(left), right); + if (typeof right === "boolean") return looselyEqual(left, Number(right)); + if (typeof left === "number" && typeof right === "string") return left === Number(right); + if (typeof left === "string" && typeof right === "number") return Number(left) === right; + return false; +} + +/** The namespace objects the language exposes, frozen and minimal. */ +const NAMESPACES: Record = { + Math, + Object, + Array, + Number, + JSON, +}; + +function checkString(value: string): string { + if (value.length > MAX_STRING_LENGTH) { + throw new EvaluationBudgetExceeded( + `evaluation built a string longer than ${MAX_STRING_LENGTH} characters`, + ); + } + return value; +} + +function checkArray(value: T[]): T[] { + if (value.length > MAX_ARRAY_LENGTH) { + throw new EvaluationBudgetExceeded( + `evaluation built an array longer than ${MAX_ARRAY_LENGTH} items`, + ); + } + return value; +} + +function evaluate(node: Node, scopes: readonly Scope[], budget: Budget): unknown { + budget.step(); + switch (node.kind) { + case "literal": + return node.value; + + case "template": { + let out = ""; + for (const part of node.parts) { + out = checkString(out + part.cooked); + if (part.expression !== null) { + out = checkString(out + String(evaluate(part.expression, scopes, budget))); + } + } + return out; + } + + case "name": { + for (let i = scopes.length - 1; i >= 0; i -= 1) { + const scope = scopes[i]!; + if (scope.has(node.name)) return scope.get(node.name); + } + throw new ReferenceError(`${node.name} is not defined`); + } + + case "array": + return checkArray(node.elements.map((element) => evaluate(element, scopes, budget))); + + case "object": { + const out: Record = {}; + for (const property of node.properties) { + const key = String(evaluate(property.key, scopes, budget)); + if (key.startsWith("_") || FORBIDDEN_KEYS.has(key)) { + throw new UnsafeEvaluatorSource(`'${key}' is not a permitted object key`); + } + out[key] = evaluate(property.value, scopes, budget); + } + return out; + } + + case "member": + return readProperty( + evaluate(node.object, scopes, budget), + evaluate(node.property, scopes, budget), + ); + + case "call": { + // The receiver is resolved through `readProperty`, which already bound + // the method — so there is no separate `this` to thread and no way to + // re-target one. + const callee = evaluate(node.callee, scopes, budget); + if (typeof callee !== "function") { + throw new TypeError("attempted to call a value that is not a function"); + } + const args = node.args.map((argument) => evaluate(argument, scopes, budget)); + budget.depth += 1; + if (budget.depth > MAX_CALL_DEPTH) { + throw new EvaluationBudgetExceeded( + `evaluation exceeded a call depth of ${MAX_CALL_DEPTH}`, + ); + } + try { + const result = (callee as (...a: unknown[]) => unknown)(...args); + if (typeof result === "string") return checkString(result); + if (Array.isArray(result)) return checkArray(result); + return result; + } finally { + budget.depth -= 1; + } + } + + case "unary": { + const value = evaluate(node.argument, scopes, budget); + if (node.operator === "!") return !value; + if (node.operator === "-") return -(value as number); + return +(value as number); + } + + case "logical": { + const left = evaluate(node.left, scopes, budget); + if (node.operator === "&&") return left ? evaluate(node.right, scopes, budget) : left; + if (node.operator === "||") return left ? left : evaluate(node.right, scopes, budget); + return left === null || left === undefined ? evaluate(node.right, scopes, budget) : left; + } + + case "conditional": + return evaluate(node.test, scopes, budget) + ? evaluate(node.consequent, scopes, budget) + : evaluate(node.alternate, scopes, budget); + + case "binary": { + const left = evaluate(node.left, scopes, budget) as never; + const right = evaluate(node.right, scopes, budget) as never; + switch (node.operator) { + case "+": { + const sum = (left as unknown as number) + (right as unknown as number); + return typeof sum === "string" ? checkString(sum) : sum; + } + case "-": + return (left as number) - (right as number); + case "*": + return (left as number) * (right as number); + case "/": + return (left as number) / (right as number); + case "%": + return (left as number) % (right as number); + case "**": + return (left as number) ** (right as number); + case "<": + return left < right; + case "<=": + return left <= right; + case ">": + return left > right; + case ">=": + return left >= right; + case "===": + return left === right; + case "!==": + return left !== right; + // See `looselyEqual`: the specification's coercions, none of the + // operand's own code. + case "==": + return looselyEqual(left, right); + case "!=": + return !looselyEqual(left, right); + default: + throw new UnsafeEvaluatorSource(`unsupported operator '${node.operator}'`); + } + } + + case "arrow": { + const params = node.params; + const body = node.body; + return (...args: unknown[]): unknown => { + budget.step(); + const scope: Scope = new Map(); + params.forEach((param, index) => scope.set(param, args[index])); + return evaluate(body, [...scopes, scope], budget); + }; + } + } +} + +export interface CompiledExpression { + (globals: Record): unknown; +} + +/** + * Parse and validate `source`, returning a function that evaluates it. + * + * The parse happens once, up front, so unsafe or malformed source is rejected + * before anything runs — and the returned function contains no `eval`, no + * `Function`, and no reachable path to either. + */ +export function compileExpression( + source: string, + options: { fieldName: string; maximumBytes: number; globalNames: readonly string[] }, +): CompiledExpression { + if (typeof source !== "string" || source.trim() === "") { + fail(options.fieldName, "must not be empty"); + } + const size = Buffer.byteLength(source, "utf8"); + if (size > options.maximumBytes) { + fail(options.fieldName, `exceeds ${options.maximumBytes} bytes`); + } + const globals = new Set(options.globalNames); + const ast = Parser.parse(source, options.fieldName, globals); + + return (values: Record): unknown => { + const scope: Scope = new Map(Object.entries(values)); + // Namespace globals are bound per call rather than captured, so a future + // change that made one of them mutable could not leak across evaluations. + for (const name of Object.keys(NAMESPACES)) { + if (globals.has(name)) scope.set(name, NAMESPACES[name]); + } + return evaluate(ast, [scope], new Budget()); + }; +} + +export { METHOD_ATTRS, FORBIDDEN_KEYS, NAMESPACE_MEMBERS }; diff --git a/sdk/typescript/src/evaluator/index.ts b/sdk/typescript/src/evaluator/index.ts new file mode 100644 index 000000000..0f12a3787 --- /dev/null +++ b/sdk/typescript/src/evaluator/index.ts @@ -0,0 +1,144 @@ +/** + * Authoring and worker primitives for FailproofAI Evaluator v2. + * + * A separate entry point on purpose: a process that only emits telemetry never + * imports the evaluator's networking or sandbox machinery. + * + * import { Evaluator, EvalResult, Score } from "@failproofai/sdk/evaluator"; + * + * const app = new Evaluator({ name: "my-evals", version: "1" }); + * + * app.eval("tool_success_rate", { version: "1" }, (session) => { + * const calls = session.count("tool_use"); + * const failures = session + * .eventsOfType("tool_result") + * .filter((event) => event.payload.error != null).length; + * return new EvalResult({ + * score: new Score(calls === 0 ? 1 : 1 - failures / calls), + * }); + * }); + * + * await app.runFromEnv(); + */ + +export { + Assertion, + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, + catalogDefinitionOf, + validateKey, +} from "./authoring.js"; +export type { + CancellationFunction, + ConditionFunction, + EvalDefinition, + EvalFunction, + EvalOptions, + EvalResultOptions, + ManagedCompiler, +} from "./authoring.js"; + +export { EvaluatorAPIError, EvaluatorClient } from "./client.js"; +export type { EvaluatorClientOptions } from "./client.js"; + +export { + CLAIM_PATH, + DEFINITIONS_PATH, + DEFAULT_POLL_INTERVAL_SECONDS, + ERROR_SPECS, + EvaluatorKind, + ExecutionMode, + HEARTBEAT_INTERVAL_SECONDS, + HEARTBEAT_PATH, + LEASE_DURATION_SECONDS, + LEASE_GENERATION_HEADER, + MAX_ATTEMPTS, + MAX_CATALOG_DEFINITIONS, + MAX_CLAIM_CAPACITY, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_ERROR_CODE_BYTES, + MAX_ERROR_MESSAGE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABELS_PER_RESULT, + MAX_LABEL_BYTES, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_TRANSCRIPT_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + MAX_WORKER_ID_BYTES, + PLAN_PATH, + PROTOCOL_VERSION, + ProtocolError, + REGISTER_PATH, + RESULT_PATH, + RESULT_SCHEMA_VERSION, + ResultKind, + TRANSCRIPT_PATH, + TRANSCRIPT_SCHEMA_VERSION, + TerminalRunStatus, + UnsupportedProtocolVersion, + WORKER_ID_HEADER, + sessionTranscriptFromWire, + validateProtocolVersion, +} from "./protocol.js"; +export type { + Assignment, + AssignmentDefinition, + CatalogDefinition, + ClaimRequest, + ClaimResponse, + DefinitionsResponse, + ErrorResponse, + EvalSelection, + HeartbeatRequest, + HeartbeatResponse, + HeartbeatRun, + PlanRequest, + PlanResponse, + PlannedRun, + RegisterRequest, + RegisterResponse, + RemoteError, + ResultItem, + ResultRequest, + ResultResponse, + SessionTranscript, + SkippedEval, + TranscriptEvent, +} from "./protocol.js"; + +export { DEFAULT_EVAL_TIMEOUT_SECONDS, WorkerRuntime, workerConfigFromEnv } from "./runtime.js"; +export type { WorkerConfig } from "./runtime.js"; + +export { + DEFAULT_SANDBOX_TIMEOUT_SECONDS, + EvaluationSandboxUnavailable, + EvaluationTimeout, + MAX_CONCURRENT_SANDBOXES, + MAX_CONDITION_SOURCE_BYTES, + MAX_EVALUATOR_SOURCE_BYTES, + MAX_SANDBOX_TIMEOUT_SECONDS, + SANDBOX_GLOBAL_NAMES, + SANDBOX_MAX_RESULT_BYTES, + SANDBOX_MEMORY_MB, + UnsafeEvaluatorSource, + compileCondition, + compileEvaluator, + sourceChecksum, +} from "./source.js"; + +export { + EvaluationBudgetExceeded, + MAX_AST_NODES, + MAX_CALL_DEPTH, + MAX_POW_EXPONENT, + MAX_STEPS, + compileExpression, +} from "./expression.js"; diff --git a/sdk/typescript/src/evaluator/protocol.ts b/sdk/typescript/src/evaluator/protocol.ts new file mode 100644 index 000000000..0911c4402 --- /dev/null +++ b/sdk/typescript/src/evaluator/protocol.ts @@ -0,0 +1,747 @@ +/** + * Dependency-free wire models for the outbound Evaluator v2 protocol. + * + * Every `fromWire` is a VALIDATOR, not a cast. The server is the other side of + * a network boundary: a field that arrives as a number where a string was + * promised, or an enum value this SDK has never heard of, has to fail here with + * a message naming the field — not three frames later as an undefined property + * on an object nobody can explain. + */ + +export const PROTOCOL_VERSION = "2"; +export const TRANSCRIPT_SCHEMA_VERSION = "2"; +export const RESULT_SCHEMA_VERSION = "2"; + +export const REGISTER_PATH = "/v1/evaluator/workers/register"; +export const CLAIM_PATH = "/v1/evaluator/assignments/claim"; +export const TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript"; +export const DEFINITIONS_PATH = "/v1/evaluator/assignments/{assignment_id}/definitions"; +export const PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan"; +export const HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat"; +export const RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result"; +export const WORKER_ID_HEADER = "X-FailproofAI-Worker-Id"; +export const LEASE_GENERATION_HEADER = "X-FailproofAI-Lease-Generation"; + +export const HEARTBEAT_INTERVAL_SECONDS = 30; +export const LEASE_DURATION_SECONDS = 120; +/** + * Fallback poll cadence if the register response omits `poll_interval_seconds`. + * The worker prefers the server-advertised value; claims are normal short + * polls, never long-polls, so this only bounds idle latency, not connection + * lifetime. + */ +export const DEFAULT_POLL_INTERVAL_SECONDS = 10; +export const MAX_ATTEMPTS = 5; + +export const MAX_CATALOG_DEFINITIONS = 100; +export const MAX_CLAIM_CAPACITY = 32; +export const MAX_TRANSCRIPT_BYTES = 25 * 1024 * 1024; +export const MAX_RESULTS_PER_RUN = 25; +export const MAX_EVAL_KEY_BYTES = 128; +export const MAX_DISPLAY_NAME_BYTES = 128; +export const MAX_VERSION_BYTES = 128; +export const MAX_WORKER_ID_BYTES = 128; +export const MAX_LABEL_BYTES = 64; +export const MAX_LABELS_PER_RESULT = 20; +export const MAX_SUMMARY_BYTES = 4 * 1024; +export const MAX_REASONING_BYTES = 16 * 1024; +export const MAX_UNIT_BYTES = 64; +export const MAX_DISPLAY_VALUE_BYTES = 256; +export const MAX_DESCRIPTION_BYTES = 1_000; +export const MAX_ERROR_CODE_BYTES = 64; +export const MAX_ERROR_MESSAGE_BYTES = 4 * 1024; + +export const ERROR_SPECS: Readonly> = { + invalid_credentials: { httpStatus: 401, retryable: false }, + instance_disabled: { httpStatus: 403, retryable: false }, + insufficient_permissions: { httpStatus: 403, retryable: false }, + assignment_not_found: { httpStatus: 404, retryable: false }, + run_not_found: { httpStatus: 404, retryable: false }, + catalog_mismatch: { httpStatus: 409, retryable: false }, + lease_lost: { httpStatus: 409, retryable: false }, + plan_conflict: { httpStatus: 409, retryable: false }, + submission_conflict: { httpStatus: 409, retryable: false }, + retry_budget_exhausted: { httpStatus: 409, retryable: false }, + transcript_too_large: { httpStatus: 413, retryable: false }, + invalid_request: { httpStatus: 422, retryable: false }, + invalid_catalog: { httpStatus: 422, retryable: false }, + incomplete_plan: { httpStatus: 422, retryable: false }, + unsupported_protocol_version: { httpStatus: 426, retryable: false }, + internal_error: { httpStatus: 500, retryable: true }, +}; + +/** A local or remote evaluator protocol contract violation. */ +export class ProtocolError extends Error { + constructor(message: string) { + super(message); + this.name = "ProtocolError"; + } +} + +export class UnsupportedProtocolVersion extends ProtocolError { + readonly received: string; + constructor(received: string) { + super( + `unsupported evaluator protocol version ${JSON.stringify(received)}; ` + + `supported major version is ${PROTOCOL_VERSION}`, + ); + this.name = "UnsupportedProtocolVersion"; + this.received = received; + } +} + +export function validateProtocolVersion(version: string): void { + if (version !== PROTOCOL_VERSION) throw new UnsupportedProtocolVersion(version); +} + +export const EvaluatorKind = { MANAGED: "managed", CUSTOMER: "customer" } as const; +export type EvaluatorKind = (typeof EvaluatorKind)[keyof typeof EvaluatorKind]; + +export const ResultKind = { SCORE: "score", METRIC: "metric", ASSERTION: "assertion" } as const; +export type ResultKind = (typeof ResultKind)[keyof typeof ResultKind]; + +export const ExecutionMode = { LOCAL: "local", SANDBOX: "sandbox" } as const; +export type ExecutionMode = (typeof ExecutionMode)[keyof typeof ExecutionMode]; + +/** + * The wire value for a server-authored definition is `"python"` — the name the + * protocol was minted with, when the only worker was the Python SDK. It means + * "the server authored this source and the worker must sandbox it", which this + * SDK does in a `worker_threads` sandbox rather than a forked interpreter. The + * wire string cannot change without a protocol bump, so the constant carries + * the honest local name and this map carries the wire one. + */ +const EXECUTION_MODE_WIRE: Record = { + local: ExecutionMode.LOCAL, + python: ExecutionMode.SANDBOX, + sandbox: ExecutionMode.SANDBOX, +}; +const EXECUTION_MODE_TO_WIRE: Record = { + local: "local", + sandbox: "python", +}; + +export const TerminalRunStatus = { + SUCCEEDED: "succeeded", + FAILED: "failed", + TIMED_OUT: "timed_out", + CANCELLED: "cancelled", +} as const; +export type TerminalRunStatus = (typeof TerminalRunStatus)[keyof typeof TerminalRunStatus]; + +export type WireObject = Record; + +// --------------------------------------------------------------------------- +// Readers +// --------------------------------------------------------------------------- + +function str(data: WireObject, key: string): string { + const value = data[key]; + if (typeof value !== "string") throw new ProtocolError(`${key} must be a string`); + return value; +} + +function int(data: WireObject, key: string): number { + const value = data[key]; + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new ProtocolError(`${key} must be an integer`); + } + return value; +} + +function positiveInt(data: WireObject, key: string): number { + const value = int(data, key); + if (value <= 0) throw new ProtocolError(`${key} must be greater than zero`); + return value; +} + +function nonNegativeInt(data: WireObject, key: string): number { + const value = int(data, key); + if (value < 0) throw new ProtocolError(`${key} must not be negative`); + return value; +} + +function optionalString(data: WireObject, key: string): string | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== "string") throw new ProtocolError(`${key} must be a string or null`); + return value; +} + +function list(data: WireObject, key: string): unknown[] { + const value = data[key]; + if (!Array.isArray(value)) throw new ProtocolError(`${key} must be an array`); + return value; +} + +function object(value: unknown, fieldName: string): WireObject { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new ProtocolError(`${fieldName} must be an object`); + } + return value as WireObject; +} + +function objectList(data: WireObject, key: string): WireObject[] { + return list(data, key).map((value, index) => object(value, `${key}[${index}]`)); +} + +function stringList(data: WireObject, key: string): string[] { + const values = list(data, key); + values.forEach((value, index) => { + if (typeof value !== "string") throw new ProtocolError(`${key}[${index}] must be a string`); + }); + return values as string[]; +} + +function enumValue( + allowed: readonly T[], + data: WireObject, + key: string, +): T { + const value = str(data, key); + if (!(allowed as readonly string[]).includes(value)) { + throw new ProtocolError( + `${key} must be one of ${allowed.map((item) => JSON.stringify(item)).join(", ")}`, + ); + } + return value as T; +} + +function executionMode(data: WireObject, key: string): ExecutionMode { + // Required explicitly. Coercing a missing or unknown value to `local` + // silently runs a server-authored definition down the customer-local path + // (or vice-versa); a malformed wire value is a protocol error, not a default. + const raw = str(data, key); + const mode = EXECUTION_MODE_WIRE[raw]; + if (mode === undefined) { + throw new ProtocolError( + `${key} must be one of ${Object.keys(EXECUTION_MODE_WIRE) + .map((item) => JSON.stringify(item)) + .join(", ")}`, + ); + } + return mode; +} + +function optionalPositiveNumber(data: WireObject, key: string): number | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== "number") throw new ProtocolError(`${key} must be a number or null`); + if (!Number.isFinite(value) || value <= 0) { + throw new ProtocolError(`${key} must be finite and greater than zero`); + } + return value; +} + +function boolean_(data: WireObject, key: string, fallback?: boolean): boolean { + const value = data[key]; + if (value === undefined && fallback !== undefined) return fallback; + if (typeof value !== "boolean") throw new ProtocolError(`${key} must be a boolean`); + return value; +} + +// --------------------------------------------------------------------------- +// Models +// --------------------------------------------------------------------------- + +export interface CatalogDefinition { + evalKey: string; + displayName: string; + evalVersion: string; + resultKind: ResultKind; + labels: readonly string[]; +} + +export function catalogDefinitionToWire(value: CatalogDefinition): WireObject { + return { + eval_key: value.evalKey, + display_name: value.displayName, + eval_version: value.evalVersion, + result_kind: value.resultKind, + labels: [...value.labels], + }; +} + +export function catalogDefinitionFromWire(data: WireObject): CatalogDefinition { + return { + evalKey: str(data, "eval_key"), + displayName: str(data, "display_name"), + evalVersion: str(data, "eval_version"), + resultKind: enumValue(Object.values(ResultKind), data, "result_kind"), + labels: stringList(data, "labels"), + }; +} + +export interface RegisterRequest { + workerId: string; + sdkVersion: string; + catalogRevision: string; + maxConcurrency: number; + definitions: readonly CatalogDefinition[]; +} + +export function registerRequestToWire(value: RegisterRequest): WireObject { + return { + protocol_version: PROTOCOL_VERSION, + worker_id: value.workerId, + sdk_version: value.sdkVersion, + catalog_revision: value.catalogRevision, + max_concurrency: value.maxConcurrency, + definitions: value.definitions.map(catalogDefinitionToWire), + }; +} + +export interface RegisterResponse { + evaluatorInstanceId: string; + evaluatorKind: EvaluatorKind; + heartbeatIntervalSeconds: number; + leaseDurationSeconds: number; + pollIntervalSeconds: number; + claimLimit: number; + disabledDefinitions: readonly string[]; +} + +export function registerResponseFromWire(data: WireObject): RegisterResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { + evaluatorInstanceId: str(data, "evaluator_instance_id"), + evaluatorKind: enumValue(Object.values(EvaluatorKind), data, "evaluator_kind"), + heartbeatIntervalSeconds: int(data, "heartbeat_interval_seconds"), + leaseDurationSeconds: int(data, "lease_duration_seconds"), + pollIntervalSeconds: int(data, "poll_interval_seconds"), + claimLimit: int(data, "claim_limit"), + disabledDefinitions: stringList(data, "disabled_definitions"), + }; +} + +export interface ClaimRequest { + workerId: string; + catalogRevision: string; + capacity: number; +} + +export function claimRequestToWire(value: ClaimRequest): WireObject { + return { + protocol_version: PROTOCOL_VERSION, + worker_id: value.workerId, + catalog_revision: value.catalogRevision, + capacity: value.capacity, + }; +} + +export interface Assignment { + assignmentId: string; + leaseGeneration: number; + leaseExpiresAt: string; + sessionId: string; + sessionRevisionId: string; + agentId: string; + environment: string; + triggerReason: string; + eventCount: number; + transcriptUrl: string; + definitionsUrl: string; +} + +export function assignmentFromWire(data: WireObject): Assignment { + return { + assignmentId: str(data, "assignment_id"), + leaseGeneration: positiveInt(data, "lease_generation"), + leaseExpiresAt: str(data, "lease_expires_at"), + sessionId: str(data, "session_id"), + sessionRevisionId: str(data, "session_revision_id"), + agentId: str(data, "agent_id"), + environment: str(data, "environment"), + triggerReason: str(data, "trigger_reason"), + eventCount: nonNegativeInt(data, "event_count"), + transcriptUrl: str(data, "transcript_url"), + definitionsUrl: typeof data.definitions_url === "string" ? data.definitions_url : "", + }; +} + +export interface AssignmentDefinition { + evalKey: string; + displayName: string; + evalVersion: string; + resultKind: ResultKind; + labels: readonly string[]; + executionMode: ExecutionMode; + conditionSource: string | null; + sourceChecksum: string | null; + timeoutSeconds: number | null; +} + +export function assignmentDefinitionFromWire(data: WireObject): AssignmentDefinition { + return { + evalKey: str(data, "eval_key"), + displayName: str(data, "display_name"), + evalVersion: str(data, "eval_version"), + resultKind: enumValue(Object.values(ResultKind), data, "result_kind"), + labels: stringList(data, "labels"), + executionMode: executionMode(data, "execution_mode"), + conditionSource: optionalString(data, "condition_source"), + sourceChecksum: optionalString(data, "source_checksum"), + timeoutSeconds: optionalPositiveNumber(data, "timeout_seconds"), + }; +} + +export interface DefinitionsResponse { + assignmentId: string; + catalogRevision: string; + definitions: readonly AssignmentDefinition[]; +} + +export function definitionsResponseFromWire(data: WireObject): DefinitionsResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { + assignmentId: str(data, "assignment_id"), + catalogRevision: str(data, "catalog_revision"), + definitions: objectList(data, "definitions").map(assignmentDefinitionFromWire), + }; +} + +export interface ClaimResponse { + assignments: readonly Assignment[]; +} + +export function claimResponseFromWire(data: WireObject): ClaimResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { assignments: objectList(data, "assignments").map(assignmentFromWire) }; +} + +export interface TranscriptEvent { + id: string; + ts: string; + eventType: string; + payload: Readonly; +} + +export function transcriptEventFromWire(data: WireObject): TranscriptEvent { + return { + id: str(data, "id"), + ts: str(data, "ts"), + eventType: str(data, "event_type"), + payload: object(data.payload, "payload"), + }; +} + +export interface SessionTranscript { + assignmentId: string; + sessionId: string; + sessionRevisionId: string; + agentId: string; + environment: string; + startedAt: string; + endedAt: string; + eventCount: number; + events: readonly TranscriptEvent[]; + schemaVersion: string; + eventsOfType(eventType: string): readonly TranscriptEvent[]; + count(eventType: string): number; + toWire(): WireObject; +} + +function makeTranscript(fields: Omit): SessionTranscript { + return { + ...fields, + eventsOfType(eventType: string) { + return fields.events.filter((event) => event.eventType === eventType); + }, + count(eventType: string) { + return fields.events.reduce((total, event) => total + (event.eventType === eventType ? 1 : 0), 0); + }, + toWire(): WireObject { + return { + schema_version: fields.schemaVersion, + assignment_id: fields.assignmentId, + session_id: fields.sessionId, + session_revision_id: fields.sessionRevisionId, + agent_id: fields.agentId, + environment: fields.environment, + started_at: fields.startedAt, + ended_at: fields.endedAt, + event_count: fields.eventCount, + events: fields.events.map((event) => ({ + id: event.id, + ts: event.ts, + event_type: event.eventType, + payload: event.payload, + })), + }; + }, + }; +} + +export function sessionTranscriptFromWire(data: WireObject): SessionTranscript { + const version = str(data, "schema_version"); + if (version !== TRANSCRIPT_SCHEMA_VERSION) { + throw new ProtocolError(`unsupported transcript schema version ${JSON.stringify(version)}`); + } + const events = objectList(data, "events").map(transcriptEventFromWire); + const eventCount = nonNegativeInt(data, "event_count"); + if (eventCount !== events.length) { + throw new ProtocolError( + `event_count is ${eventCount}, but transcript contains ${events.length} events`, + ); + } + return makeTranscript({ + assignmentId: str(data, "assignment_id"), + sessionId: str(data, "session_id"), + sessionRevisionId: str(data, "session_revision_id"), + agentId: str(data, "agent_id"), + environment: str(data, "environment"), + startedAt: str(data, "started_at"), + endedAt: str(data, "ended_at"), + eventCount, + events, + schemaVersion: version, + }); +} + +/** Rebuild a transcript from `toWire()` — used across the sandbox boundary. */ +export function sessionTranscriptFromWireLoose(data: WireObject): SessionTranscript { + return sessionTranscriptFromWire(data); +} + +export interface EvalSelection { + evalKey: string; + evalVersion: string; +} + +export interface SkippedEval { + evalKey: string; + evalVersion: string; + reasonCode: string; +} + +export interface PlanRequest { + workerId: string; + leaseGeneration: number; + selected: readonly EvalSelection[]; + skipped: readonly SkippedEval[]; +} + +export function planRequestToWire(value: PlanRequest): WireObject { + return { + protocol_version: PROTOCOL_VERSION, + worker_id: value.workerId, + lease_generation: value.leaseGeneration, + selected: value.selected.map((item) => ({ + eval_key: item.evalKey, + eval_version: item.evalVersion, + })), + skipped: value.skipped.map((item) => ({ + eval_key: item.evalKey, + eval_version: item.evalVersion, + reason_code: item.reasonCode, + })), + }; +} + +export interface PlannedRun { + evaluationRunId: string; + evalKey: string; + evalVersion: string; + executionMode: ExecutionMode; + evaluatorSource: string | null; + sourceChecksum: string | null; + timeoutSeconds: number | null; +} + +export function plannedRunFromWire(data: WireObject): PlannedRun { + return { + evaluationRunId: str(data, "evaluation_run_id"), + evalKey: str(data, "eval_key"), + evalVersion: str(data, "eval_version"), + executionMode: executionMode(data, "execution_mode"), + evaluatorSource: optionalString(data, "evaluator_source"), + sourceChecksum: optionalString(data, "source_checksum"), + timeoutSeconds: optionalPositiveNumber(data, "timeout_seconds"), + }; +} + +export interface PlanResponse { + assignmentId: string; + assignmentStatus: string; + runs: readonly PlannedRun[]; + idempotentReplay: boolean; +} + +export function planResponseFromWire(data: WireObject): PlanResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { + assignmentId: str(data, "assignment_id"), + assignmentStatus: str(data, "assignment_status"), + runs: objectList(data, "runs").map(plannedRunFromWire), + idempotentReplay: boolean_(data, "idempotent_replay", false), + }; +} + +export interface HeartbeatRun { + evaluationRunId: string; + state: string; + progress?: number | null; +} + +export interface HeartbeatRequest { + workerId: string; + leaseGeneration: number; + runs: readonly HeartbeatRun[]; +} + +export function heartbeatRequestToWire(value: HeartbeatRequest): WireObject { + return { + protocol_version: PROTOCOL_VERSION, + worker_id: value.workerId, + lease_generation: value.leaseGeneration, + runs: value.runs.map((run) => ({ + evaluation_run_id: run.evaluationRunId, + state: run.state, + ...(run.progress === undefined || run.progress === null ? {} : { progress: run.progress }), + })), + }; +} + +export interface HeartbeatResponse { + leaseExpiresAt: string; + acceptedRunIds: readonly string[]; +} + +export function heartbeatResponseFromWire(data: WireObject): HeartbeatResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { + leaseExpiresAt: str(data, "lease_expires_at"), + acceptedRunIds: stringList(data, "accepted_run_ids"), + }; +} + +export interface ResultItem { + resultKey: string; + resultKind: ResultKind; + numericValue?: number | null; + boolValue?: boolean | null; + textValue?: string | null; + unit?: string; + displayValue?: string | null; + description?: string | null; + reasoning?: string | null; + labels?: readonly string[]; +} + +export function resultItemToWire(value: ResultItem): WireObject { + return { + result_key: value.resultKey, + result_kind: value.resultKind, + numeric_value: value.numericValue ?? null, + bool_value: value.boolValue ?? null, + text_value: value.textValue ?? null, + unit: value.unit ?? "", + display_value: value.displayValue ?? null, + description: value.description ?? null, + reasoning: value.reasoning ?? null, + labels: [...(value.labels ?? [])], + }; +} + +export function resultItemFromWire(data: WireObject): ResultItem { + const numeric = data.numeric_value; + if (numeric !== undefined && numeric !== null) { + if (typeof numeric !== "number") throw new ProtocolError("numeric_value must be a number or null"); + if (!Number.isFinite(numeric)) throw new ProtocolError("numeric_value must be finite"); + } + const boolean = data.bool_value; + if (boolean !== undefined && boolean !== null && typeof boolean !== "boolean") { + throw new ProtocolError("bool_value must be a boolean or null"); + } + return { + resultKey: str(data, "result_key"), + resultKind: enumValue(Object.values(ResultKind), data, "result_kind"), + numericValue: (numeric) ?? null, + boolValue: (boolean) ?? null, + textValue: optionalString(data, "text_value"), + unit: str(data, "unit"), + displayValue: optionalString(data, "display_value"), + description: optionalString(data, "description"), + reasoning: optionalString(data, "reasoning"), + labels: stringList(data, "labels"), + }; +} + +export interface ResultRequest { + submissionId: string; + workerId: string; + leaseGeneration: number; + status: TerminalRunStatus; + startedAt: string; + finishedAt: string; + durationMs: number; + summary: string | null; + results: readonly ResultItem[]; + errorCode: string | null; + errorMessage: string | null; +} + +export function resultRequestToWire(value: ResultRequest): WireObject { + return { + protocol_version: PROTOCOL_VERSION, + result_schema_version: RESULT_SCHEMA_VERSION, + submission_id: value.submissionId, + worker_id: value.workerId, + lease_generation: value.leaseGeneration, + status: value.status, + started_at: value.startedAt, + finished_at: value.finishedAt, + duration_ms: value.durationMs, + summary: value.summary, + results: value.results.map(resultItemToWire), + error_code: value.errorCode, + error_message: value.errorMessage, + }; +} + +export interface ResultResponse { + evaluationRunId: string; + submissionId: string; + status: string; + idempotentReplay: boolean; + resultCount: number; + resultChecksum: string; +} + +export function resultResponseFromWire(data: WireObject): ResultResponse { + validateProtocolVersion(str(data, "protocol_version")); + return { + evaluationRunId: str(data, "evaluation_run_id"), + submissionId: str(data, "submission_id"), + status: str(data, "status"), + idempotentReplay: boolean_(data, "idempotent_replay"), + resultCount: nonNegativeInt(data, "result_count"), + resultChecksum: str(data, "result_checksum"), + }; +} + +export interface RemoteError { + code: string; + message: string; + retryable: boolean; + requestId: string; +} + +export interface ErrorResponse { + error: RemoteError; +} + +export function errorResponseFromWire(data: WireObject): ErrorResponse { + validateProtocolVersion(str(data, "protocol_version")); + const error = object(data.error, "error"); + return { + error: { + code: str(error, "code"), + message: str(error, "message"), + retryable: boolean_(error, "retryable"), + requestId: str(error, "request_id"), + }, + }; +} + +export { EXECUTION_MODE_TO_WIRE }; diff --git a/sdk/typescript/src/evaluator/runtime.ts b/sdk/typescript/src/evaluator/runtime.ts new file mode 100644 index 000000000..8c0bb4693 --- /dev/null +++ b/sdk/typescript/src/evaluator/runtime.ts @@ -0,0 +1,930 @@ +/** + * The async worker state machine for Evaluator v2. + * + * Claim -> fetch the transcript -> run each definition's condition -> submit a + * plan -> run the planned evaluations under a heartbeat -> submit each result. + * + * ## What is different from the Python worker, and why + * + * Python runs synchronous evaluations in a sized thread pool, because a + * blocking customer function would otherwise stall its event loop. JavaScript + * has no such option: a synchronous evaluation that does not return blocks the + * one thread there is, and no timeout can fire while it does. So the contract + * here is explicit — **an evaluation must yield.** An `async` function that + * awaits, or a sandboxed managed evaluation (which runs in its own Worker and + * IS killable), can always be bounded. A synchronous busy-loop cannot be, by + * anyone, and `DEFAULT_EVAL_TIMEOUT_SECONDS` will not save a worker from one. + * + * The bound that does hold everywhere: every evaluation is raced against a + * deadline, and a run that loses the race is reported `timed_out` and its + * result submitted, so the assignment does not sit unresolved. The evaluation + * itself may still be running — a promise cannot be cancelled — which is + * counted as `evaluations_orphaned` and logged with the eval key, so a hung one + * is findable. + */ + +import { hostname } from "node:os"; +import { randomUUID } from "node:crypto"; + +import { logException, logger } from "../logger.js"; +import { VERSION } from "../version.js"; +import { EvalResult, Evaluator, ConditionResult } from "./authoring.js"; +import type { EvalDefinition } from "./authoring.js"; +import { EvaluatorAPIError, EvaluatorClient } from "./client.js"; +import { + DEFAULT_POLL_INTERVAL_SECONDS, + ExecutionMode, + MAX_CLAIM_CAPACITY, + MAX_ERROR_MESSAGE_BYTES, + MAX_WORKER_ID_BYTES, + TerminalRunStatus, +} from "./protocol.js"; +import type { + Assignment, + AssignmentDefinition, + EvalSelection, + ResultItem, + SessionTranscript, + SkippedEval, +} from "./protocol.js"; +import { + EvaluationTimeout, + UnsafeEvaluatorSource, + compileCondition, + compileEvaluator, + sourceChecksum, +} from "./source.js"; + +/** + * Reserved out of the lease for the plan request (and network jitter) so the + * pre-plan condition phase always leaves time to submit the plan before the + * lease expires. + */ +const CONDITION_PHASE_SAFETY_MARGIN_SECONDS = 5; + +/** + * Fallback wall-clock bound for an evaluation whose definition declares no + * timeout. A LOCAL (customer-authored) eval has no sandbox backstop, so without + * this a hang — a wedged `await`, an unbounded judge HTTP call — would hold its + * worker slot and the assignment lease forever. Matches the storage contract's + * 5-minute per-eval default. + */ +export const DEFAULT_EVAL_TIMEOUT_SECONDS = 300; + +function utcNow(): string { + return `${new Date().toISOString().slice(0, -1)}000Z`; +} + +function positiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined) return fallback; + const value = Number(raw); + if (!Number.isInteger(value)) throw new Error(`${name} must be an integer`); + if (value <= 0) throw new Error(`${name} must be greater than zero`); + return value; +} + +function booleanEnv(name: string, fallback = false): boolean { + const raw = process.env[name]; + if (raw === undefined) return fallback; + const normalized = raw.trim().toLowerCase(); + if (["1", "true", "yes", "on"].includes(normalized)) return true; + if (["0", "false", "no", "off"].includes(normalized)) return false; + throw new Error(`${name} must be a boolean`); +} + +export interface WorkerConfig { + serverUrl: string; + credential: string; + workerId: string; + maxConcurrency: number; + requestTimeoutSeconds: number; + drainTimeoutSeconds: number; + allowInsecureHttp: boolean; +} + +export function workerConfigFromEnv(): WorkerConfig { + const serverUrl = (process.env.FAILPROOFAI_EVALUATOR_URL ?? "").trim(); + const credential = (process.env.FAILPROOFAI_EVALUATOR_TOKEN ?? "").trim(); + if (!serverUrl) throw new Error("FAILPROOFAI_EVALUATOR_URL is required"); + if (!credential) throw new Error("FAILPROOFAI_EVALUATOR_TOKEN is required"); + + let workerId = (process.env.FAILPROOFAI_EVALUATOR_WORKER_ID ?? "").trim(); + if (!workerId) workerId = `${hostname()}-${process.pid}`; + if (Buffer.byteLength(workerId, "utf8") > MAX_WORKER_ID_BYTES) { + throw new Error(`FAILPROOFAI_EVALUATOR_WORKER_ID exceeds ${MAX_WORKER_ID_BYTES} bytes`); + } + for (const char of workerId) { + const code = char.codePointAt(0)!; + if (code < 32 || code === 127) { + throw new Error("FAILPROOFAI_EVALUATOR_WORKER_ID must not contain control characters"); + } + } + + const config: WorkerConfig = { + serverUrl, + credential, + workerId, + maxConcurrency: positiveIntEnv("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), + requestTimeoutSeconds: positiveIntEnv("FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30), + drainTimeoutSeconds: positiveIntEnv("FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60), + allowInsecureHttp: booleanEnv("FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP"), + }; + if (config.maxConcurrency > MAX_CLAIM_CAPACITY) { + throw new Error(`FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds ${MAX_CLAIM_CAPACITY}`); + } + return config; +} + +/** A deadline race that reports which side won, so the caller can count it. */ +async function withDeadline( + work: Promise, + timeoutMs: number, +): Promise<{ timedOut: false; value: T } | { timedOut: true }> { + let timer: ReturnType | undefined; + const expiry = new Promise<{ timedOut: true }>((resolve) => { + timer = setTimeout(() => { + resolve({ timedOut: true }); + }, timeoutMs); + timer.unref?.(); + }); + try { + const outcome = await Promise.race([ + work.then((value) => ({ timedOut: false as const, value })), + expiry, + ]); + return outcome; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +async function sleep(ms: number, signal?: { aborted: boolean }): Promise { + if (signal?.aborted) return; + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); +} + +/** + * Compile server-authored source LAZILY, at invocation time. + * + * Compilation can reject unsafe or malformed source. Building the definition + * with this thunk instead of a pre-compiled function routes that failure + * through the same per-run `try/catch` that turns any evaluation error into a + * bounded FAILED result — so a poison definition dead-letters cleanly as one + * failed run instead of throwing out of assignment setup, crashing the task, + * and forcing the whole assignment to be reclaimed and retried until its + * attempt budget is exhausted. + */ +function deferredManagedEval( + source: string, + timeoutSeconds: number | null, + evalKey: string, + compiler: Evaluator["managedCompiler"], +): (session: SessionTranscript) => Promise | EvalResult { + return (session: SessionTranscript) => { + const compile = compiler ?? ((s, o) => compileEvaluator(s, o)); + return compile(source, { timeoutSeconds, evalKey })(session); + }; +} + +export class WorkerRuntime { + readonly evaluator: Evaluator; + readonly config: WorkerConfig; + readonly client: EvaluatorClient; + + private stopping = false; + private stopWaiters: Array<() => void> = []; + private readonly activeAssignments = new Set>(); + private heartbeatInterval = 30; + private pollInterval: number = DEFAULT_POLL_INTERVAL_SECONDS; + private claimLimit: number; + private leaseDuration = 120; + private disabledDefinitions = new Set(); + private registered = false; + private lastServerContact: number | null = null; + private readonly metricCounts = new Map(); + private readonly slots: { inUse: number; waiters: Array<() => void> }; + + constructor(evaluator: Evaluator, config: WorkerConfig, options: { client?: EvaluatorClient } = {}) { + this.evaluator = evaluator; + this.config = config; + this.claimLimit = config.maxConcurrency; + this.client = + options.client ?? + new EvaluatorClient({ + baseUrl: config.serverUrl, + credential: config.credential, + timeoutSeconds: config.requestTimeoutSeconds, + allowInsecureHttp: config.allowInsecureHttp, + }); + this.slots = { inUse: 0, waiters: [] }; + } + + async register(): Promise { + let response; + try { + response = await this.call(() => + this.client.register({ + workerId: this.config.workerId, + sdkVersion: VERSION, + catalogRevision: this.evaluator.catalogRevision, + maxConcurrency: this.config.maxConcurrency, + definitions: this.evaluator.catalog(), + }), + ); + } catch (error) { + this.increment("registration_failure"); + throw error; + } + this.heartbeatInterval = response.heartbeatIntervalSeconds; + this.pollInterval = response.pollIntervalSeconds; + this.leaseDuration = response.leaseDurationSeconds; + this.claimLimit = Math.min(this.config.maxConcurrency, response.claimLimit); + if ( + this.heartbeatInterval <= 0 || + this.pollInterval <= 0 || + this.leaseDuration <= this.heartbeatInterval || + this.claimLimit <= 0 + ) { + this.increment("registration_failure"); + throw new Error("server returned invalid evaluator timing or claim limits"); + } + this.disabledDefinitions = new Set(response.disabledDefinitions); + this.registered = true; + this.increment("registration_success"); + } + + async runForever(): Promise { + await this.register(); + let retryDelay = 1; + try { + while (!this.stopping) { + const capacity = this.claimLimit - this.activeAssignments.size; + if (capacity <= 0) { + await this.waitForProgress(); + continue; + } + let response; + try { + response = await this.call(() => + this.client.claim({ + workerId: this.config.workerId, + catalogRevision: this.evaluator.catalogRevision, + capacity, + }), + ); + } catch (error) { + if (!(error instanceof EvaluatorAPIError)) throw error; + this.increment("claim_failures"); + logger.warn(`evaluator claim failed (code=${error.code}, retryable=${error.retryable})`); + if (!error.retryable) throw error; + // A transport failure with no status means the server is unreachable, + // not busy — back off by the whole lease rather than hammering it. + const delay = error.status === null ? this.leaseDuration : retryDelay; + await this.waitOrStop(delay * 1000); + retryDelay = Math.min(retryDelay * 2, 30); + continue; + } + retryDelay = 1; + const assignments = validatedAssignments(response.assignments, capacity); + for (const assignment of assignments) this.spawn(assignment); + this.increment("assignments_claimed", assignments.length); + if (assignments.length === 0) { + // Normal short poll: the server returns immediately, so when nothing + // is queued we wait the advertised interval instead of hot-looping. + // When work IS returned we loop straight back to drain any backlog. + await this.waitOrStop(this.pollInterval * 1000); + } + } + } finally { + await this.drain(); + } + } + + /** Claim once and finish the returned assignments; useful for jobs and tests. */ + async runOnce(): Promise { + const response = await this.call(() => + this.client.claim({ + workerId: this.config.workerId, + catalogRevision: this.evaluator.catalogRevision, + capacity: this.claimLimit, + }), + ); + const assignments = validatedAssignments(response.assignments, this.claimLimit); + this.increment("assignments_claimed", assignments.length); + await Promise.all(assignments.map((assignment) => this.processAssignment(assignment))); + return assignments.length; + } + + stop(): void { + this.stopping = true; + const waiters = this.stopWaiters; + this.stopWaiters = []; + for (const waiter of waiters) waiter(); + } + + async drain(): Promise { + if (this.activeAssignments.size === 0) return; + const outcome = await withDeadline( + Promise.allSettled([...this.activeAssignments]).then(() => undefined), + this.config.drainTimeoutSeconds * 1000, + ); + if (outcome.timedOut) { + logger.warn( + `drain timed out with ${this.activeAssignments.size} assignment(s) still running; ` + + "their results may not be submitted", + ); + } + this.activeAssignments.clear(); + } + + private spawn(assignment: Assignment): void { + const task = this.processAssignment(assignment) + .catch((error: unknown) => { + logException("evaluator assignment failed", error); + }) + .finally(() => { + this.activeAssignments.delete(task); + this.notifyProgress(); + }); + this.activeAssignments.add(task); + } + + async processAssignment(assignment: Assignment): Promise { + let session: SessionTranscript; + try { + session = await this.call(() => this.client.transcript(assignment, this.config.workerId)); + } catch (error) { + if (error instanceof EvaluatorAPIError && error.code === "transcript_too_large") { + // No runs are planned yet, so there is nothing to submit a per-run + // result for, and the error is non-retryable — re-throwing would only + // wedge the poll loop and burn the assignment's whole retry budget + // against a transcript that can never shrink. The server terminalizes + // the assignment itself. + logger.warn( + `assignment ${assignment.assignmentId} transcript is too large to evaluate; skipping`, + ); + this.increment("transcripts_too_large"); + return; + } + throw error; + } + if (session.sessionRevisionId !== assignment.sessionRevisionId) { + throw new Error("transcript session revision does not match assignment"); + } + + const descriptors = await this.assignmentDefinitions(assignment); + // Every descriptor the assignment carries, keyed for reconstruction: on an + // idempotent replay the server re-serves the first attempt's run set, which + // may include a run this attempt's re-derived plan would have skipped. + const descriptorByKey = new Map( + descriptors.map((item) => [`${item.evalKey}@${item.evalVersion}`, item]), + ); + const localDefinitions = new Map( + this.evaluator.definitions.map((item) => [`${item.evalKey}@${item.evalVersion}`, item]), + ); + + const selected: Array<[AssignmentDefinition, EvalDefinition | null]> = []; + const skipped: SkippedEval[] = []; + // The assignment lease is fixed at claim time and cannot be renewed until + // the plan is submitted (the server only extends a lease for a *planned* + // assignment with running runs). A slow condition phase can therefore burn + // the whole lease and get the plan fenced as `lease_lost`, so every + // condition is bounded by the lease it must leave time to plan within. + const conditionDeadline = this.conditionPhaseDeadline(assignment); + + for (const descriptor of descriptors) { + const key = `${descriptor.evalKey}@${descriptor.evalVersion}`; + const local = localDefinitions.get(key) ?? null; + if (descriptor.executionMode === ExecutionMode.LOCAL && local === null) { + throw new Error("server requested a definition absent from this worker"); + } + if (this.disabledDefinitions.has(descriptor.evalKey)) { + skipped.push(skippedOf(descriptor, "disabled_by_server")); + this.increment("conditions_skipped"); + continue; + } + + let applicable: boolean; + let reasonCode = "condition_false"; + try { + // Whose condition decides applicability follows the EXECUTION MODE: a + // LOCAL definition's condition is client-authored; a sandboxed + // (server-authored) definition's is server-authored and MUST govern + // even when the worker also registered the same key/version locally. + // Keying `local` on key+version alone means a managed definition can + // collide with a local one; selecting the local condition there would + // let it override the server's rule and run the managed evaluator + // against the operator's intent. + let condition: ((s: SessionTranscript) => unknown) | null = null; + let managedConditionSource: string | null = null; + if (descriptor.executionMode === ExecutionMode.LOCAL) { + condition = local?.condition ?? null; + } else if (descriptor.conditionSource) { + managedConditionSource = descriptor.conditionSource; + } + if (condition === null && managedConditionSource === null) { + selected.push([descriptor, local]); + continue; + } + const budget = this.conditionBudget(conditionDeadline, descriptor.timeoutSeconds); + if (budget <= 0) { + // Not enough lease left to evaluate this condition and still submit + // the plan in time; skip it (and, as the loop proceeds, every later + // condition) rather than do work the server will fence as + // `lease_lost` and reclaim in a loop. + skipped.push(skippedOf(descriptor, "lease_exhausted")); + this.increment("conditions_skipped"); + this.increment("conditions_lease_exhausted"); + continue; + } + // Compiled INSIDE the try, and only once the lease budget is known: a + // managed condition the sandbox rejects must dead-letter as + // `condition_error`, not throw out of the plan loop and strand the + // whole assignment until its retry budget is exhausted. + if (managedConditionSource !== null) { + condition = compileCondition(managedConditionSource, { timeoutSeconds: budget }); + } + const outcome = await withDeadline( + Promise.resolve(condition!(session)), + budget * 1000, + ); + if (outcome.timedOut) throw new EvaluationTimeout("condition exceeded its budget"); + const value = outcome.value; + if (value instanceof ConditionResult) { + applicable = value.applicable; + reasonCode = value.reasonCode; + } else if (typeof value === "boolean") { + applicable = value; + } else { + throw new TypeError("condition must return a boolean or a ConditionResult"); + } + } catch (error) { + logger.warn( + `evaluator condition failed (assignment=${assignment.assignmentId}, ` + + `error=${error instanceof Error ? error.name : typeof error})`, + ); + skipped.push(skippedOf(descriptor, "condition_error")); + this.increment("conditions_skipped"); + continue; + } + + if (applicable) { + selected.push([descriptor, local]); + this.increment("conditions_selected"); + } else { + skipped.push(skippedOf(descriptor, reasonCode)); + this.increment("conditions_skipped"); + } + } + + const plan = await this.call(() => + this.client.plan(assignment.assignmentId, { + workerId: this.config.workerId, + leaseGeneration: assignment.leaseGeneration, + selected: selected.map( + ([item]): EvalSelection => ({ evalKey: item.evalKey, evalVersion: item.evalVersion }), + ), + skipped, + }), + ); + if (plan.assignmentId !== assignment.assignmentId) { + throw new Error("server returned a plan for a different assignment"); + } + // On an idempotent replay the server's status is authoritative: this + // attempt may have selected a different set than the first, so a mismatch + // against our own `selected` is expected, not an error. + if (!plan.idempotentReplay) { + const expected = selected.length > 0 ? "planned" : "skipped"; + if (plan.assignmentStatus !== expected) { + throw new Error("server returned an inconsistent assignment status"); + } + } + + const pending = new Map( + selected.map(([item, local]) => [`${item.evalKey}@${item.evalVersion}`, [item, local]]), + ); + const runDefinitions: Array<[string, EvalDefinition]> = []; + const runIds = new Set(); + for (const run of plan.runs) { + if (runIds.has(run.evaluationRunId)) { + throw new Error("server returned a duplicate evaluation run id"); + } + runIds.add(run.evaluationRunId); + const key = `${run.evalKey}@${run.evalVersion}`; + let entry = pending.get(key); + pending.delete(key); + if (entry === undefined) { + // On an idempotent replay the server's run set is AUTHORITATIVE — it + // re-serves the first attempt's runs even for a definition this + // attempt's condition phase would have skipped. Reconstruct from the + // assignment's descriptors rather than throwing and dead-lettering an + // assignment that could otherwise never converge. + const replay = plan.idempotentReplay ? descriptorByKey.get(key) : undefined; + if (replay === undefined) throw new Error("server returned an unrequested evaluation run"); + entry = [replay, localDefinitions.get(key) ?? null]; + } + const [descriptor, local] = entry; + if (run.executionMode !== descriptor.executionMode) { + throw new Error("server changed the evaluation execution mode"); + } + + let definition: EvalDefinition; + if (run.executionMode === ExecutionMode.LOCAL) { + if (local === null) throw new Error("local evaluation definition is unavailable"); + definition = local; + } else { + if (!run.evaluatorSource || !run.sourceChecksum) { + throw new Error("server omitted managed evaluation source"); + } + const expected = sourceChecksum(descriptor.conditionSource, run.evaluatorSource); + if ( + expected !== run.sourceChecksum || + (descriptor.sourceChecksum && descriptor.sourceChecksum !== run.sourceChecksum) + ) { + throw new Error("managed evaluation source checksum mismatch"); + } + const timeoutSeconds = run.timeoutSeconds ?? descriptor.timeoutSeconds; + definition = { + evalKey: descriptor.evalKey, + displayName: descriptor.displayName, + evalVersion: descriptor.evalVersion, + resultKind: descriptor.resultKind, + labels: descriptor.labels, + function: deferredManagedEval( + run.evaluatorSource, + timeoutSeconds, + descriptor.evalKey, + this.evaluator.managedCompiler, + ), + condition: null, + onCancel: null, + timeoutSeconds, + }; + } + runDefinitions.push([run.evaluationRunId, definition]); + } + if (pending.size > 0 && !plan.idempotentReplay) { + throw new Error("server omitted a selected evaluation run"); + } + + const states = new Map(); + const tasks = runDefinitions.map(([runId, definition]) => { + const state = { done: false, cancelled: false }; + states.set(runId, state); + return this.executeRun(assignment, runId, definition, session, state).finally(() => { + state.done = true; + }); + }); + const heartbeat = this.heartbeat(assignment, states); + try { + const outcomes = await Promise.allSettled(tasks); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason as Error; + } + } finally { + heartbeat.cancel(); + await heartbeat.finished; + } + } + + private async acquireSlot(): Promise { + if (this.slots.inUse < this.config.maxConcurrency) { + this.slots.inUse += 1; + return; + } + await new Promise((resolve) => this.slots.waiters.push(resolve)); + this.slots.inUse += 1; + } + + private releaseSlot(): void { + this.slots.inUse -= 1; + const waiter = this.slots.waiters.shift(); + if (waiter) waiter(); + } + + private async executeRun( + assignment: Assignment, + runId: string, + definition: EvalDefinition, + session: SessionTranscript, + state: { done: boolean; cancelled: boolean }, + ): Promise { + await this.acquireSlot(); + try { + await this.executeRunInSlot(assignment, runId, definition, session, state); + } finally { + this.releaseSlot(); + } + } + + private async executeRunInSlot( + assignment: Assignment, + runId: string, + definition: EvalDefinition, + session: SessionTranscript, + state: { done: boolean; cancelled: boolean }, + ): Promise { + const startedAt = utcNow(); + const started = Date.now(); + let items: readonly ResultItem[] = []; + let status: TerminalRunStatus; + let summary: string | null = null; + let errorCode: string | null = null; + let errorMessage: string | null = null; + + try { + if (state.cancelled) throw new EvaluationCancelled(); + // Always bound the evaluation. A definition with no declared timeout + // falls back to the default rather than awaiting unbounded — an unbounded + // local eval that hangs would wedge its worker slot and hold the lease + // forever. + const timeoutSeconds = definition.timeoutSeconds ?? DEFAULT_EVAL_TIMEOUT_SECONDS; + const outcome = await withDeadline( + Promise.resolve(definition.function(session)), + timeoutSeconds * 1000, + ); + if (outcome.timedOut) throw new EvaluationTimeout("evaluation exceeded its timeout"); + const result = outcome.value; + if (!(result instanceof EvalResult)) throw new TypeError("evaluation must return an EvalResult"); + items = result.resultItems(definition.evalKey); + if ( + !items.some( + (item) => item.resultKey === definition.evalKey && item.resultKind === definition.resultKind, + ) + ) { + throw new Error("evaluation result does not contain its declared primary result"); + } + status = TerminalRunStatus.SUCCEEDED; + summary = result.summary ?? null; + } catch (error) { + await this.cancelHook(definition, session); + if (error instanceof EvaluationCancelled || state.cancelled) { + status = TerminalRunStatus.CANCELLED; + errorCode = "lease_lost"; + errorMessage = "the assignment lease was lost before this run finished"; + } else if (error instanceof EvaluationTimeout) { + // A SANDBOXED evaluation that loses this race has already been + // terminated by the worker sandbox, so nothing is left running. A LOCAL + // one cannot be: a promise has no cancel, so the customer's function + // keeps going. Count it and name the eval so a hung one is findable. + this.increment("evaluations_orphaned"); + logger.warn( + `evaluation ${JSON.stringify(definition.evalKey)} exceeded its timeout ` + + `(assignment=${assignment.assignmentId}). If it is a local evaluation it is still ` + + "running — a promise cannot be cancelled — and its worker slot is freed only when " + + "it settles.", + ); + status = TerminalRunStatus.TIMED_OUT; + errorCode = "eval_timeout"; + errorMessage = "evaluation exceeded its configured timeout"; + } else if (error instanceof UnsafeEvaluatorSource) { + // Surface the REASON for a rejected server-authored definition. This is + // deliberately narrower than the generic branch below: + // `UnsafeEvaluatorSource` is raised by our own validator before any + // customer source runs, and its message is SDK-authored text about the + // source's shape — it embeds no transcript content, so it is safe to + // send back over the wire. Without it the author sees only "evaluation + // raised UnsafeEvaluatorSource" on every session, with no way to learn + // what was wrong. + status = TerminalRunStatus.FAILED; + errorCode = "eval_error"; + const detail = error.message.trim(); + errorMessage = truncateUtf8( + detail ? `evaluator source rejected: ${detail}` : "evaluator source rejected by the validator", + MAX_ERROR_MESSAGE_BYTES, + ); + } else { + // Type name ONLY on the wire. A customer eval's error text can quote + // the transcript it was reading, and this field is persisted and shown + // in the dashboard. But log the FULL error LOCALLY: this runs on the + // customer's own pod over their own data, and without it an author + // whose eval throws sees only "evaluation raised TypeError" in the + // dashboard and nothing at all in their logs. + logException( + `evaluation ${JSON.stringify(definition.evalKey)} threw; reported to the server as a failed run`, + error, + ); + status = TerminalRunStatus.FAILED; + errorCode = "eval_error"; + errorMessage = `evaluation raised ${error instanceof Error ? error.name : typeof error}`; + } + } + + await this.call(() => + this.client.submitResult(runId, { + submissionId: randomUUID(), + workerId: this.config.workerId, + leaseGeneration: assignment.leaseGeneration, + status, + startedAt, + finishedAt: utcNow(), + durationMs: Math.max(0, Math.round(Date.now() - started)), + summary, + results: items, + errorCode, + errorMessage, + }), + ); + this.increment(`runs_${status}`); + } + + private async cancelHook(definition: EvalDefinition, session: SessionTranscript): Promise { + if (definition.onCancel === null) return; + try { + await definition.onCancel(session); + } catch (error) { + logger.warn( + `evaluator cancellation hook failed (${error instanceof Error ? error.name : typeof error})`, + ); + } + } + + private heartbeat( + assignment: Assignment, + states: Map, + ): { cancel: () => void; finished: Promise } { + let cancelled = false; + const finished = (async () => { + // Beat IMMEDIATELY, before the first sleep. The pre-plan condition phase + // may have consumed most of the claim-time lease, and the server only + // renews a planned assignment's lease on heartbeat — so sleeping a full + // interval here can let the lease expire after the runs have started, + // cancelling every one of them. + let first = true; + while (!cancelled) { + if (!first) await sleep(this.heartbeatInterval * 1000); + if (cancelled) return; + first = false; + const active = [...states.entries()] + .filter(([, state]) => !state.done) + .map(([runId]) => ({ evaluationRunId: runId, state: "running" })); + if (active.length === 0) return; + try { + const response = await this.call(() => + this.client.heartbeat({ + workerId: this.config.workerId, + leaseGeneration: assignment.leaseGeneration, + runs: active, + }), + ); + const accepted = new Set(response.acceptedRunIds); + for (const [runId, state] of states) { + if (!state.done && !accepted.has(runId)) state.cancelled = true; + } + } catch (error) { + if (error instanceof EvaluatorAPIError && error.code === "lease_lost") { + this.increment("leases_lost"); + for (const state of states.values()) state.cancelled = true; + return; + } + logger.warn( + `evaluator heartbeat failed (assignment=${assignment.assignmentId}, ` + + `${error instanceof EvaluatorAPIError ? `code=${error.code}` : `error=${error instanceof Error ? error.name : typeof error}`})`, + ); + this.increment("heartbeat_failures"); + } + } + })(); + return { + cancel: () => { + cancelled = true; + }, + finished, + }; + } + + /** + * The clock reading by which the pre-plan condition phase must end. + * + * The real `leaseExpiresAt` is used when it is in the future (production); a + * past or unparseable value (clock skew, or a replayed transcript in a test) + * falls back to the negotiated lease duration measured from now, so the bound + * never fires spuriously on a stale deadline. + */ + private conditionPhaseDeadline(assignment: Assignment): number { + let remaining = this.leaseDuration; + const expires = Date.parse(assignment.leaseExpiresAt); + if (Number.isFinite(expires)) { + const parsed = (expires - Date.now()) / 1000; + if (parsed > 0) remaining = parsed; + } + return Date.now() + remaining * 1000; + } + + /** + * Seconds a single condition may run: the lease left before the + * plan-submission margin, capped by the definition's own timeout. + */ + private conditionBudget(deadline: number, timeoutSeconds: number | null): number { + let remaining = (deadline - Date.now()) / 1000 - CONDITION_PHASE_SAFETY_MARGIN_SECONDS; + if (timeoutSeconds !== null) remaining = Math.min(remaining, timeoutSeconds); + return remaining; + } + + private async assignmentDefinitions(assignment: Assignment): Promise { + if (assignment.definitionsUrl) { + const response = await this.call(() => + this.client.definitions(assignment, this.config.workerId), + ); + if (response.assignmentId !== assignment.assignmentId) { + throw new Error("server returned definitions for another assignment"); + } + return [...response.definitions]; + } + return this.evaluator.definitions.map((item) => ({ + evalKey: item.evalKey, + displayName: item.displayName, + evalVersion: item.evalVersion, + resultKind: item.resultKind, + labels: item.labels, + executionMode: ExecutionMode.LOCAL, + conditionSource: null, + sourceChecksum: null, + timeoutSeconds: item.timeoutSeconds, + })); + } + + private progressWaiters: Array<() => void> = []; + + private notifyProgress(): void { + const waiters = this.progressWaiters; + this.progressWaiters = []; + for (const waiter of waiters) waiter(); + } + + private async waitForProgress(): Promise { + if (this.activeAssignments.size === 0) return; + await new Promise((resolve) => { + this.progressWaiters.push(resolve); + this.stopWaiters.push(resolve); + }); + } + + private async waitOrStop(ms: number): Promise { + if (this.stopping) return; + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + this.stopWaiters.push(() => { + clearTimeout(timer); + resolve(); + }); + }); + } + + private async call(work: () => Promise): Promise { + const result = await work(); + this.lastServerContact = Date.now(); + return result; + } + + private increment(name: string, amount = 1): void { + this.metricCounts.set(name, (this.metricCounts.get(name) ?? 0) + amount); + } + + metrics(): Record { + return Object.fromEntries(this.metricCounts); + } + + isReady(): boolean { + if (this.stopping || !this.registered || this.lastServerContact === null) return false; + return (Date.now() - this.lastServerContact) / 1000 <= Math.max(this.leaseDuration, 60); + } +} + +class EvaluationCancelled extends Error { + constructor() { + super("evaluation cancelled"); + this.name = "EvaluationCancelled"; + } +} + +function skippedOf(definition: AssignmentDefinition, reasonCode: string): SkippedEval { + return { evalKey: definition.evalKey, evalVersion: definition.evalVersion, reasonCode }; +} + +function validatedAssignments( + assignments: readonly Assignment[], + capacity: number, +): Assignment[] { + if (assignments.length > capacity) { + throw new Error("server returned more assignments than requested"); + } + const ids = assignments.map((item) => item.assignmentId); + if (new Set(ids).size !== ids.length) throw new Error("server returned duplicate assignments"); + return [...assignments]; +} + +/** Cut to `maximum` BYTES without splitting a multi-byte character. */ +function truncateUtf8(value: string, maximum: number): string { + const encoded = Buffer.from(value, "utf8"); + if (encoded.byteLength <= maximum) return value; + return new TextDecoder("utf-8", { fatal: false }).decode(encoded.subarray(0, maximum)).replace( + /�$/, + "", + ); +} diff --git a/sdk/typescript/src/evaluator/sandbox-worker.ts b/sdk/typescript/src/evaluator/sandbox-worker.ts new file mode 100644 index 000000000..2f1752f0a --- /dev/null +++ b/sdk/typescript/src/evaluator/sandbox-worker.ts @@ -0,0 +1,171 @@ +/** + * The sandbox worker entry: evaluates ONE piece of server-authored source and + * posts the result back. + * + * Started by `source.ts` as a fresh `worker_threads` Worker per evaluation, + * with V8 heap limits, an empty environment and a wall-clock `terminate()` + * above it. Nothing here is trusted to enforce those — a worker cannot bound + * itself, which is exactly why the bound lives in the parent. + * + * What this file IS responsible for: + * + * * **Re-validating the source.** The parent already parsed it. Parsing it + * again here means a worker can never evaluate source the parent has not + * vetted, whatever reaches `workerData`. + * * **Bounding the result before it crosses back.** A permitted expression can + * build a result far larger than any real evaluation; measuring it here means + * the parent never has to hold it to find that out. + * * **Reporting failures as data, not as a crash.** An evaluation that throws + * is an ordinary outcome — a failed run — and has to arrive as one, with the + * original error's name preserved so the parent can rebuild its semantics. + */ + +import { parentPort, workerData } from "node:worker_threads"; + +import { Assertion, ConditionResult, EvalResult, Metric, Score } from "./authoring.js"; +import { + MAX_CONDITION_SOURCE_BYTES, + MAX_EVALUATOR_SOURCE_BYTES, + SANDBOX_GLOBAL_NAMES, +} from "./source-limits.js"; +import { compileExpression } from "./expression.js"; +import { sessionTranscriptFromWire } from "./protocol.js"; +import type { WireObject } from "./protocol.js"; + +interface SandboxRequest { + kind: "condition" | "evaluator"; + source: string; + session: WireObject; + evalKey: string | null; + maxResultBytes: number; +} + +type SandboxOutcome = + | { ok: true; value: unknown } + | { ok: false; name: string; message: string }; + +/** + * The constructors the language exposes, as plain functions. + * + * Callable WITHOUT `new`, so an expression reads the way the Python SDK's does + * (`EvalResult(score=Score(0.9))` there, `EvalResult({score: Score(0.9)})` + * here). The interpreter has no `new` in its grammar at all, so a class would + * simply be uncallable. + */ +function sandboxGlobals(session: unknown): Record { + return { + session, + EvalResult: (options?: ConstructorParameters[0]) => new EvalResult(options), + Score: (value: number, options?: ConstructorParameters[1]) => + new Score(value, options), + Metric: (value: number, options?: ConstructorParameters[1]) => + new Metric(value, options), + Assertion: (passed: boolean, options?: ConstructorParameters[1]) => + new Assertion(passed, options), + ConditionResult: (applicable: boolean, reasonCode?: string) => + new ConditionResult(applicable, reasonCode), + }; +} + +function serializeEvalResult(result: EvalResult): Record { + const metrics: Record = {}; + for (const [key, metric] of Object.entries(result.metrics)) { + metrics[key] = + metric instanceof Metric + ? { + value: metric.value, + unit: metric.unit, + displayValue: metric.displayValue, + description: metric.description, + } + : metric; + } + const assertions: Record = {}; + for (const [key, assertion] of Object.entries(result.assertions)) { + assertions[key] = + assertion instanceof Assertion + ? { passed: assertion.passed, description: assertion.description } + : assertion; + } + return { + score: + result.score === undefined + ? undefined + : { + value: result.score.value, + passed: result.score.passed, + unit: result.score.unit, + displayValue: result.score.displayValue, + description: result.score.description, + }, + metrics, + assertions, + reasoning: result.reasoning, + summary: result.summary, + labels: [...result.labels], + }; +} + +function run(request: SandboxRequest): unknown { + const isCondition = request.kind === "condition"; + const compiled = compileExpression(request.source, { + fieldName: isCondition ? "condition_source" : "evaluator_source", + maximumBytes: isCondition ? MAX_CONDITION_SOURCE_BYTES : MAX_EVALUATOR_SOURCE_BYTES, + globalNames: SANDBOX_GLOBAL_NAMES, + }); + + const session = sessionTranscriptFromWire(request.session); + const value = compiled(sandboxGlobals(session)); + + if (isCondition) { + if (typeof value === "boolean") return value; + if (value instanceof ConditionResult) { + return { applicable: value.applicable, reasonCode: value.reasonCode }; + } + throw new TypeError("condition_source must return a boolean or a ConditionResult"); + } + + if (!(value instanceof EvalResult)) { + throw new TypeError("evaluator_source must return an EvalResult"); + } + // Validate the item count HERE, where the result is still cheap to discard. + // The parent would reject it too, but only after paying to receive it. + if (request.evalKey !== null) value.resultItems(request.evalKey); + return serializeEvalResult(value); +} + +function main(): void { + const port = parentPort; + if (port === null) { + // Started as a main module rather than as a worker. There is nothing to + // report to and nothing to evaluate; exiting quietly is the only sensible + // behaviour, and a non-zero code says it was not a normal run. + process.exitCode = 2; + return; + } + const request = workerData as SandboxRequest; + + let outcome: SandboxOutcome; + try { + const value = run(request); + const encoded = Buffer.byteLength(JSON.stringify(value ?? null), "utf8"); + if (encoded > request.maxResultBytes) { + outcome = { + ok: false, + name: "EvaluationBudgetExceeded", + message: `evaluation result is ${encoded} bytes; the limit is ${request.maxResultBytes}`, + }; + } else { + outcome = { ok: true, value }; + } + } catch (error) { + outcome = + error instanceof Error + ? { ok: false, name: error.name || "Error", message: error.message } + : { ok: false, name: "Error", message: String(error) }; + } + + port.postMessage(outcome); +} + +main(); diff --git a/sdk/typescript/src/evaluator/source-limits.ts b/sdk/typescript/src/evaluator/source-limits.ts new file mode 100644 index 000000000..1d9bbb251 --- /dev/null +++ b/sdk/typescript/src/evaluator/source-limits.ts @@ -0,0 +1,27 @@ +/** + * The handful of constants the sandbox worker and its parent must agree on. + * + * They live apart from `source.ts` so that `sandbox-worker.ts` can read them + * without importing it — and importing it would drag `node:worker_threads`'s + * `Worker`, the semaphore and the spawn logic into every sandbox child, giving + * each evaluation the machinery for starting more of them. A worker that cannot + * spawn a worker is a smaller thing to reason about. + */ + +export const MAX_CONDITION_SOURCE_BYTES = 16 * 1024; +export const MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024; + +/** The names an evaluation expression may reference. */ +export const SANDBOX_GLOBAL_NAMES = [ + "session", + "EvalResult", + "Score", + "Metric", + "Assertion", + "ConditionResult", + "Math", + "Object", + "Array", + "Number", + "JSON", +] as const; diff --git a/sdk/typescript/src/evaluator/source.ts b/sdk/typescript/src/evaluator/source.ts new file mode 100644 index 000000000..7c396ff32 --- /dev/null +++ b/sdk/typescript/src/evaluator/source.ts @@ -0,0 +1,509 @@ +/** + * Compiling and sandboxing server-authored ("managed") evaluations. + * + * Two independent boundaries, and they guard different things: + * + * 1. **The language** (`expression.ts`). Parsed and interpreted, never `eval`'d + * and never handed to `node:vm`, so tenant source has no reachable path to a + * function constructor, a module, `process`, or anything else in the worker. + * This is the CORRECTNESS boundary and it holds on its own. + * + * 2. **A `worker_threads` sandbox** (this file). A fresh Worker per evaluation + * with V8 `resourceLimits` on the heap, a wall-clock `terminate()`, a cap on + * the bytes the result may occupy, and a semaphore bounding how many run at + * once. This is the RESOURCE boundary: it stops a permitted expression from + * eating the worker even though every operation in it is individually legal. + * + * This is the same division the Python SDK draws between its AST allowlist and + * its fork+RLIMIT subprocess, with one difference that matters: there, the + * subprocess is the real bound because `eval` with empty builtins is porous. + * Here the language is genuinely closed, so the sandbox is defence in depth + * rather than the only thing holding. + * + * **It still fails closed.** If the sandbox cannot be established — no + * `worker_threads`, no resolvable worker entry — managed source is REFUSED + * rather than run unbounded in the worker's own thread. An evaluation that + * cannot be terminated is not an evaluation we are willing to start. + */ + +import { createHash } from "node:crypto"; +import { Worker } from "node:worker_threads"; + +import { resolveFrom } from "../node-require.js"; +import { Assertion, ConditionResult, EvalResult, Metric, Score } from "./authoring.js"; +import { UnsafeEvaluatorSource, compileExpression } from "./expression.js"; +import type { SessionTranscript } from "./protocol.js"; +import { + MAX_CONDITION_SOURCE_BYTES, + MAX_EVALUATOR_SOURCE_BYTES, + SANDBOX_GLOBAL_NAMES, +} from "./source-limits.js"; + +export { UnsafeEvaluatorSource }; +export { MAX_CONDITION_SOURCE_BYTES, MAX_EVALUATOR_SOURCE_BYTES, SANDBOX_GLOBAL_NAMES }; + +/** Wall-clock and heap budget for ONE sandboxed evaluation. */ +export const DEFAULT_SANDBOX_TIMEOUT_SECONDS = 30; +/** + * The effective budget is CLAMPED to this ceiling regardless of the + * (server-set) per-definition timeout, so a large `timeout_seconds` can never + * remove the execution bound. + */ +export const MAX_SANDBOX_TIMEOUT_SECONDS = 60; +/** + * Per-sandbox heap cap. A managed eval works over a transcript (<= 25 MiB) and + * returns a small result, so this is generous; it also stops an allocation bomb + * before it can return a valid result. + */ +export const SANDBOX_MEMORY_MB = 512; +/** + * A per-worker cap alone does not bound the HOST: a worker with + * `maxConcurrency: 32` could run 32 sandboxes at once. Capping how many run + * concurrently makes the AGGREGATE bounded independently of the claim + * concurrency; extra evaluations queue on the semaphore rather than pile up + * memory. + */ +export const MAX_CONCURRENT_SANDBOXES = 4; +/** + * The result crossing back is bounded so that a permitted expression which + * builds a huge result cannot exhaust the parent even though the child's heap + * limit let it construct one. A valid result (<= 25 items, bounded fields) is + * far under this. + */ +export const SANDBOX_MAX_RESULT_BYTES = 1024 * 1024; + +/** + * A sandboxed evaluation exceeded its CPU/memory/wall-clock budget. + * + * Distinct from an eval that *returned* an error: the computation was forcibly + * terminated because it could not be allowed to keep running. + */ +export class EvaluationTimeout extends Error { + constructor(message: string) { + super(message); + this.name = "EvaluationTimeout"; + } +} + +/** + * The killable sandbox could not be established. + * + * Thrown instead of running server-authored source unsandboxed — if the worker + * cannot be started there is no way to bound or terminate the evaluation, so we + * fail closed. + */ +export class EvaluationSandboxUnavailable extends Error { + constructor(message: string) { + super(message); + this.name = "EvaluationSandboxUnavailable"; + } +} + +export function sourceChecksum( + conditionSource: string | null | undefined, + evaluatorSource: string, +): string { + const payload = Buffer.concat([ + Buffer.from(conditionSource ?? "", "utf8"), + Buffer.from([0]), + Buffer.from(evaluatorSource, "utf8"), + ]); + return `sha256:${createHash("sha256").update(payload).digest("hex")}`; +} + +/** + * The wall-clock/heap budget for one evaluation: a positive value no larger + * than `MAX_SANDBOX_TIMEOUT_SECONDS`. Server-provided timeouts cannot exceed it. + */ +function clampBudget(timeoutSeconds: number | null | undefined): number { + const requested = Number(timeoutSeconds ?? DEFAULT_SANDBOX_TIMEOUT_SECONDS); + const positive = + Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_SANDBOX_TIMEOUT_SECONDS; + return Math.min(positive, MAX_SANDBOX_TIMEOUT_SECONDS); +} + +// --------------------------------------------------------------------------- +// Locating the worker entry +// --------------------------------------------------------------------------- + +/** + * Where the sandbox worker's compiled entry lives. + * + * `import.meta.url` would answer this in one line and is deliberately not used: + * it is a syntax error in the CommonJS half of this package's dual build, and a + * telemetry SDK that only works from one module system is a telemetry SDK half + * its users cannot install. The resolution order instead is: + * + * 1. `FAILPROOFAI_SDK_SANDBOX_WORKER` — an explicit path. Used by this + * repository's own tests, and the supported answer for anyone bundling + * this package into a single file, where the published layout is gone. + * 2. The package's own `./sandbox-worker` export, resolved through the + * consuming application. This is the installed case and needs no setup. + * + * Failure names the variable rather than falling back to an unsandboxed run. + */ +function sandboxWorkerPath(): string { + const override = process.env.FAILPROOFAI_SDK_SANDBOX_WORKER; + if (override) return override; + const resolved = resolveFrom("@failproofai/sdk/sandbox-worker"); + if (resolved !== null) return resolved; + throw new EvaluationSandboxUnavailable( + "could not locate the evaluator sandbox worker. This happens when @failproofai/sdk has " + + "been bundled and its published file layout is gone. Set " + + "FAILPROOFAI_SDK_SANDBOX_WORKER to the path of the package's sandbox-worker entry, " + + "or run managed evaluations from an unbundled install. Refusing to evaluate " + + "server-authored source without a sandbox.", + ); +} + +// --------------------------------------------------------------------------- +// The concurrency gate +// --------------------------------------------------------------------------- + +class Semaphore { + private available: number; + private readonly waiters: Array<(granted: boolean) => void> = []; + + constructor(permits: number) { + this.available = permits; + } + + /** + * Acquire within `timeoutMs`, or resolve false. + * + * The timeout is not decoration: the runtime awaits an evaluation with its + * own deadline, and a queue wait that ignored it would let a thread launch a + * sandbox after its run had already been reported timed out — twenty-eight + * of them queued behind four long sandboxes, all still to come. + */ + acquire(timeoutMs: number): Promise { + if (this.available > 0) { + this.available -= 1; + return Promise.resolve(true); + } + if (timeoutMs <= 0) return Promise.resolve(false); + return new Promise((resolve) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + const index = this.waiters.indexOf(waiter); + if (index !== -1) this.waiters.splice(index, 1); + resolve(false); + }, timeoutMs); + timer.unref?.(); + const waiter = (granted: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(granted); + }; + this.waiters.push(waiter); + }); + } + + release(): void { + const waiter = this.waiters.shift(); + if (waiter === undefined) { + this.available += 1; + return; + } + waiter(true); + } +} + +const slots = new Semaphore(MAX_CONCURRENT_SANDBOXES); + +// --------------------------------------------------------------------------- +// Running one evaluation in a sandbox +// --------------------------------------------------------------------------- + +interface SandboxRequest { + kind: "condition" | "evaluator"; + source: string; + session: Record; + evalKey: string | null; + maxResultBytes: number; +} + +type SandboxOutcome = + | { ok: true; value: unknown } + | { ok: false; name: string; message: string }; + +async function runSandboxed( + request: Omit, + budgetSeconds: number, +): Promise { + const workerPath = sandboxWorkerPath(); + const deadline = Date.now() + budgetSeconds * 1000; + + // One deadline covers BOTH the queue wait and the execution, so a run cannot + // spend its whole budget waiting and then start anyway. + const granted = await slots.acquire(deadline - Date.now()); + if (!granted) throw new EvaluationTimeout("evaluation timed out waiting for a sandbox slot"); + + try { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + // Slot acquired exactly at the deadline: a worker started now could only + // be killed immediately, so do not start one at all. + throw new EvaluationTimeout("evaluation timed out waiting for a sandbox slot"); + } + return await runWorker(workerPath, { ...request, maxResultBytes: SANDBOX_MAX_RESULT_BYTES }, remaining); + } finally { + slots.release(); + } +} + +function runWorker( + workerPath: string, + request: SandboxRequest, + timeoutMs: number, +): Promise { + return new Promise((resolve, reject) => { + let worker: Worker; + try { + worker = new Worker(workerPath, { + workerData: request, + // Nothing from the parent's environment crosses. On a hosted worker + // `FAILPROOFAI_EVALUATOR_TOKEN` is the credential the fleet + // authenticates with; the language cannot reach `process` at all, so + // this is defence in depth rather than a fix for a live escape — it + // means a future gap could not be escalated into credential theft. + env: {}, + argv: [], + execArgv: [], + resourceLimits: { + maxOldGenerationSizeMb: SANDBOX_MEMORY_MB, + maxYoungGenerationSizeMb: 64, + codeRangeSizeMb: 32, + stackSizeMb: 8, + }, + stdin: false, + stdout: true, + stderr: true, + }); + } catch (error) { + reject( + new EvaluationSandboxUnavailable( + `could not start the evaluation sandbox: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + return; + } + + let settled = false; + const finish = (run: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + void worker.terminate(); + run(); + }; + + const timer = setTimeout(() => { + finish(() => { + reject(new EvaluationTimeout("evaluation exceeded its wall-clock budget")); + }); + }, timeoutMs); + timer.unref?.(); + + worker.on("message", (outcome: SandboxOutcome) => { + finish(() => { + if (outcome.ok) { + resolve(outcome.value); + return; + } + // Preserve the child's original error SEMANTICS, so an eval's + // TypeError and the sandbox's own UnsafeEvaluatorSource read the same + // as they would in-process. + if (outcome.name === "UnsafeEvaluatorSource") { + reject(new UnsafeEvaluatorSource(outcome.message)); + return; + } + if (outcome.name === "EvaluationBudgetExceeded") { + reject(new EvaluationTimeout(outcome.message)); + return; + } + reject(rebuildError(outcome.name, outcome.message)); + }); + }); + + worker.on("error", (error: Error & { code?: string }) => { + finish(() => { + // A heap-limit kill arrives here, not as a message. It is a resource + // failure, not an evaluation that returned something. + if (error.code === "ERR_WORKER_OUT_OF_MEMORY") { + reject(new EvaluationTimeout("evaluation exceeded its memory budget")); + return; + } + reject(error); + }); + }); + + worker.on("exit", (code) => { + finish(() => { + reject( + new EvaluationTimeout( + `evaluation was terminated before producing a result (exit code ${code})`, + ), + ); + }); + }); + }); +} + +/** + * The built-in error types an evaluation can realistically produce. + * + * Reconstructing the REAL constructor, rather than an `Error` with its `name` + * reassigned, is what keeps `instanceof TypeError` true on this side of the + * worker boundary. An author debugging their own evaluation — or a `catch` in a + * custom managed compiler — cannot tell a sandboxed run from an in-process one + * otherwise, which turns every sandbox-only bug into a mystery. + */ +const BUILTIN_ERRORS: Record = { + Error, + TypeError: TypeError, + RangeError: RangeError, + ReferenceError: ReferenceError, + SyntaxError: SyntaxError, + EvalError: EvalError, + URIError: URIError, +}; + +function rebuildError(name: string, message: string): Error { + const Builtin = BUILTIN_ERRORS[name]; + if (Builtin !== undefined) return new Builtin(message); + const error = new Error(message); + error.name = name; + return error; +} + +// --------------------------------------------------------------------------- +// Rebuilding results on this side of the boundary +// --------------------------------------------------------------------------- + +interface WireScore { + value: number; + passed?: boolean; + unit?: string; + displayValue?: string; + description?: string; +} + +interface WireEvalResult { + score?: WireScore; + metrics?: Record; + assertions?: Record; + reasoning?: string; + summary?: string; + labels?: string[]; +} + +function rebuildEvalResult(value: unknown): EvalResult { + const wire = value as WireEvalResult; + if (typeof wire !== "object" || wire === null) { + throw new TypeError("evaluator source must return an EvalResult"); + } + const metrics: Record = {}; + for (const [key, metric] of Object.entries(wire.metrics ?? {})) { + metrics[key] = + typeof metric === "number" + ? metric + : new Metric(metric.value, { + unit: metric.unit, + displayValue: metric.displayValue, + description: metric.description, + }); + } + const assertions: Record = {}; + for (const [key, assertion] of Object.entries(wire.assertions ?? {})) { + assertions[key] = + typeof assertion === "boolean" + ? assertion + : new Assertion(assertion.passed, { description: assertion.description }); + } + return new EvalResult({ + score: + wire.score === undefined + ? undefined + : new Score(wire.score.value, { + passed: wire.score.passed, + unit: wire.score.unit, + displayValue: wire.score.displayValue, + description: wire.score.description, + }), + metrics, + assertions, + reasoning: wire.reasoning, + summary: wire.summary, + labels: wire.labels ?? [], + }); +} + +function rebuildCondition(value: unknown): boolean | ConditionResult { + if (typeof value === "boolean") return value; + const wire = value as { applicable?: unknown; reasonCode?: unknown }; + if (typeof wire?.applicable !== "boolean") { + throw new TypeError("condition source must return a boolean or a ConditionResult"); + } + return new ConditionResult( + wire.applicable, + typeof wire.reasonCode === "string" ? wire.reasonCode : undefined, + ); +} + +// --------------------------------------------------------------------------- +// Public compile API +// --------------------------------------------------------------------------- + +/** + * Validate `source` here, in the PARENT, so unsafe or malformed source is + * rejected before any worker is started — and return a function that runs it in + * the sandbox. The worker parses it again as defence in depth, so a worker can + * never evaluate source the parent has not vetted. + */ +function validateLocally(source: string, kind: "condition" | "evaluator"): void { + compileExpression(source, { + fieldName: kind === "condition" ? "condition_source" : "evaluator_source", + maximumBytes: kind === "condition" ? MAX_CONDITION_SOURCE_BYTES : MAX_EVALUATOR_SOURCE_BYTES, + globalNames: SANDBOX_GLOBAL_NAMES, + }); +} + +export function compileCondition( + source: string, + options: { timeoutSeconds?: number | null } = {}, +): (session: SessionTranscript) => Promise { + validateLocally(source, "condition"); + const budget = clampBudget(options.timeoutSeconds); + + return async (session: SessionTranscript): Promise => { + // Conditions are sandboxed exactly like evaluators. A condition has no + // runtime-level timeout above it, so an unbounded one would block the + // worker with no deadline at all. + const value = await runSandboxed( + { kind: "condition", source, session: session.toWire(), evalKey: null }, + budget, + ); + return rebuildCondition(value); + }; +} + +export function compileEvaluator( + source: string, + options: { timeoutSeconds?: number | null; evalKey?: string | null } = {}, +): (session: SessionTranscript) => Promise { + validateLocally(source, "evaluator"); + const budget = clampBudget(options.timeoutSeconds); + const evalKey = options.evalKey ?? null; + + return async (session: SessionTranscript): Promise => { + const value = await runSandboxed( + { kind: "evaluator", source, session: session.toWire(), evalKey }, + budget, + ); + return rebuildEvalResult(value); + }; +} diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts new file mode 100644 index 000000000..4f7e411f1 --- /dev/null +++ b/sdk/typescript/src/events.ts @@ -0,0 +1,879 @@ +import * as context from "./context.js"; +import { logger } from "./logger.js"; +import { + agentEndEvent, + agentPauseEvent, + agentResumeEvent, + agentStartEvent, + errorEvent, + hookCompletedEvent, + hookTriggeredEvent, + humanInputEvent, + humanInterruptEvent, + humanPauseEvent, + humanWaitEvent, + modelRequestEvent, + modelResponseEvent, + toolResultEvent, + toolUseEvent, +} from "./schema.js"; +import type { EventWriter } from "./writer.js"; +import { formatMicros, nowMicros } from "./clock.js"; +import { fatalSuffix, onProcessExit, type OpenItem } from "./exit.js"; + +/** + * `sessionId` and `agentId` are named options on every method, so a caller + * cannot pass them as extras. `timestamp` and `type` are not, so they would + * land in the extras and are caught here — and `environment` with them, since + * an extra of that name would overwrite the one the schema sets. + */ +const RESERVED: ReadonlySet = new Set([ + "timestamp", + "session_id", + "agent_id", + "type", + "environment", +]); + +/** + * Payload keys ingest lifts out of the JSON blob into unsigned 32-bit columns + * via `pu32()`. Everything else is stored as-is and can be any shape, but these + * three are read with a typed accessor that returns null on a mismatch — and a + * null there is written as NULL under a 200 OK. Nothing is logged, nothing is + * rejected, and the row still arrives, so the only symptom is a column that is + * empty for some events and not others. + * + * `durationMs` is refused outright on the four events that MEASURE it. These + * checks cover the other way in: any of the three passed as a custom field on + * an event that does not name it, plus `modelResponse`'s own two options, which + * are the ones a caller is most likely to fill straight from a provider's usage + * object. + */ +const PROMOTED_NUMERIC: ReadonlySet = new Set([ + "duration_ms", + "input_tokens", + "output_tokens", +]); + +/** + * Payload keys ingest lifts into an INDEXED STRING column via `ps()`, which + * reads JSON strings and stores NULL for anything else — the same + * silent-at-200 failure `PROMOTED_NUMERIC` guards. `undefined` is the realistic + * way in: `toolName: tool?.name` is ordinary code, and it produces a row that + * is invisible to every tool-name filter and whose `tool_result` never pairs. + */ +const PROMOTED_STRING: ReadonlySet = new Set([ + "tool_name", + "tool_call_id", + "hook_name", + "hook_id", + "input_id", + "pause_id", + "error_type", + "model", +]); + +const U32_MAX = 2 ** 32 - 1; + +/** A value for an error message, without rendering an object as `[object Object]`. */ +function describe(value: unknown): string { + try { + return JSON.stringify(value) ?? typeof value; + } catch { + return typeof value; + } +} + +/** + * Reject anything `pu32()` would silently turn into NULL. + * + * Rejecting rather than coercing, and at the boundary rather than in the + * writer, for the same reason `validatedInterval` does: this is the last point + * where the caller still has a stack trace pointing at their own call. A + * non-integer is a mistake worth hearing about — the server drops it whole + * rather than rounding it — and rounding it here would hide that from the one + * person who could fix the source of it. + */ +export function validatePromotedNumeric(name: string, value: unknown): void { + if (value === undefined || value === null) return; + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new TypeError( + `${name} must be an integer (the server reads it as an unsigned 32-bit integer and ` + + `stores NULL for anything else), got ${typeof value}: ${describe(value)}`, + ); + } + if (value < 0 || value > U32_MAX) { + throw new RangeError( + `${name} must be between 0 and ${U32_MAX} (an unsigned 32-bit integer), got ${value}`, + ); + } +} + +/** + * Reject anything `ps()` would silently turn into NULL. + * + * Same reasoning and the same boundary. The schema copies the identity block + * verbatim, so an absent value here would reach the wire as an explicit JSON + * `null`, the row would be accepted at 200 OK, and the column would be empty + * for some events and not others with nothing logged anywhere. + * + * Extras no longer reach this holding a nullish value: `validateFields` drops + * the key and warns, so an optional column the caller simply does not have + * costs a log line rather than the whole event. The throw below stays as the + * backstop for the declared options. + */ +export function validatePromotedString(name: string, value: unknown): void { + if (value === undefined || value === null) { + throw new TypeError( + `${name} must be a string (the server lifts it into an indexed column and stores NULL ` + + `for anything else, so the event would be accepted at 200 OK and be invisible to ` + + `every filter on ${name}), got ${value === null ? "null" : "undefined"}`, + ); + } + if (typeof value !== "string") { + throw new TypeError( + `${name} must be a string (the server lifts it into an indexed column and stores NULL ` + + `for anything else), got ${typeof value}`, + ); + } +} + +/** + * Reject an id the server will skip, at the point the caller can see it. + * + * `session_id` and `agent_id` are on every one of the 15 event types and are + * what everything downstream groups by. Ingest requires each to be a JSON + * string: hand it anything else and the row is SKIPPED — and the response is + * `200 OK` with `{"accepted": 0, "skipped": 1}`, so nothing upstream learns. + * The SDK reports success, the collector deletes the batch, and the event is + * gone. Verified against the live server for number, null and object. + * + * Empty and whitespace-only are refused as well, and those the server DOES + * accept. That is the worse outcome of the two: every event lands, grouped + * under one blank id, so the data looks present and is silently merged. + */ +function validateIdentity(name: string, value: unknown): asserts value is string { + if (typeof value !== "string") { + throw new TypeError( + `${name} must be a string — the server skips any event whose ${name} is not a JSON ` + + `string, and answers 200 as though it stored it. Got ${value === null ? "null" : typeof value}`, + ); + } + if (value.trim() === "") { + throw new Error( + `${name} must not be empty — the server accepts it, so every event sent this way is ` + + "silently grouped under one blank id.", + ); + } +} + +/** + * Fill an omitted `sessionId`/`agentId` from the ambient scope, then validate. + * + * ORDER MATTERS. The validation runs on the RESOLVED value, not the argument. + * Validating first would reject every ambient call; resolving without + * validating would put the silent-skip back: ingest drops an event whose + * `session_id` is not a JSON string and answers `200 OK` with + * `{"accepted":0,"skipped":1}`, so a run with nothing bound would vanish rather + * than fail. + * + * Called AFTER `validateFields`, deliberately. A reserved extra is a fault in + * the call itself and reads identically from anywhere, so reporting it first + * gives a stable, reproducible message; the identity error depends on where the + * call was made from, and is the less useful of the two to hear when both are + * true. + * + * `agentId` falls back to `DEFAULT_AGENT_ID` rather than throwing — an event + * emitted inside `session()` with no `agent()` around it lands somewhere + * sensible. `sessionId` has no such default: inventing one would scatter a run + * across as many sessions as it has emit sites. + */ +function resolveIdentity( + sessionId: string | null | undefined, + agentId: string | null | undefined, +): [string, string] { + let sid: string | null | undefined = sessionId; + let aid: string | null | undefined = agentId; + if (sid === undefined || sid === null) sid = context.sessionId(); + if (aid === undefined || aid === null) aid = context.agentId(); + + if (sid === undefined || sid === null) { + throw new TypeError( + "sessionId is required and nothing is bound. Pass sessionId, or wrap the call in " + + "`await failproofai.session(fn)` / `await failproofai.agent('name', fn)`. A callback " + + "stored in one run and invoked from another does not inherit the ambient scope — " + + "hand it over with `failproofai.propagate(fn)`.", + ); + } + validateIdentity("sessionId", sid); + validateIdentity("agentId", aid); + return [sid, aid]; +} + +/** + * Whole milliseconds between a paired start and end, or undefined. + * + * `pu32()` reads `duration_ms` as an unsigned 32-bit integer and stores NULL + * for anything outside it, at `200 OK`, so an out-of-range duration is not an + * error anywhere: the row lands with an empty column and nothing says why. + * + * Two ways to leave the range, both reachable without anything being wrong with + * the caller: + * + * * **over.** 2**32 ms is ~49.7 days. A `humanWait` answered after a long + * weekend, or an `agentPause` resumed a month later, is an ordinary lifetime + * for these pairs, not an abuse of them. + * * **under.** These are wall-clock readings, so an NTP step backwards between + * start and end yields a NEGATIVE interval, and a negative into an unsigned + * column is the same silent NULL. + * + * Omitted rather than clamped. A clamped 49.7 days is indistinguishable from a + * measurement, and the whole reason `duration_ms` is computed here instead of + * accepted from the caller is that a reported duration is unfalsifiable. An + * absent field is at least honest, and the timestamps are still on both events + * for anyone who wants to do the subtraction themselves. + */ +function measuredDurationMs(start: number | undefined, end: number): number | undefined { + if (start === undefined) return undefined; + const ms = Math.round((end - start) / 1000); + if (ms < 0 || ms > U32_MAX) { + logger.warn( + `omitted duration_ms=${ms}: outside the unsigned 32-bit range the server stores it in ` + + `(0..${U32_MAX}). The event is unaffected.`, + ); + return undefined; + } + return ms; +} + +/** + * Hard cap on the correlation map. Orphaned starts (a `toolUse` with no + * `toolResult`, a `humanWait` the user never answers) would otherwise grow it + * unbounded in a long-running process. At the cap the oldest entry is evicted — + * a `Map` preserves insertion order. + */ +const PENDING_CAP = 10_000; + +/** + * Every pairing is keyed by what it pairs and by the SESSION it belongs to — + * and deliberately NOT by the agent. + * + * The rule: include what makes the id unique, exclude what can legitimately + * change between the start event and the end event. + * + * * KIND belongs in the key. Tool pairs keyed on the bare `toolCallId` and hook + * pairs on the bare `hookId` would share one flat keyspace, so a caller whose + * tool call and hook happened to share an id — not exotic, both are + * frequently the harness's own step id — would get a `hook_completed` that + * consumed the `tool_use` timestamp, and then a `tool_result` with no + * duration at all. + * + * * SESSION belongs in the key. This map lives on one process-wide namespace, + * so two sessions in one process — a supervisor running agents concurrently, + * the ordinary multi-agent shape — would collide on any shared step id. + * + * * AGENT DOES NOT. This is the tempting third component and it is wrong. Once + * a framework runs tools inside sub-agents, a `tool_use` opened under + * `planner` and closed under `worker` is routine — LangGraph does it — and an + * agent-scoped key makes those pairs miss entirely, silently dropping + * `duration_ms` for exactly the nested runs that most need it. A session + * cannot change under a pair; an agent can. + * + * These are correlation keys only; they are never emitted and never leave the + * process, so the shape changes no wire format. + */ +const toolKey = (sessionId: string, toolCallId: string): string => + `tool:${sessionId}:${toolCallId}`; +const hookKey = (sessionId: string, hookId: string): string => `hook:${sessionId}:${hookId}`; +const pauseKey = (sessionId: string, pauseId: string): string => `pause:${sessionId}:${pauseId}`; +const humanKey = (sessionId: string, inputId: string): string => `human:${sessionId}:${inputId}`; + +export interface BaseEventOptions { + sessionId?: string | null; + agentId?: string | null; + /** + * Any other key is a custom payload field, merged verbatim onto the event — + * the direct analogue of Python's `**fields`. Namespace anything + * framework-specific `fw_*`; a name that collides with a declared field is + * refused. + */ + [field: string]: unknown; +} + +export interface ToolUseOptions extends BaseEventOptions { + toolName: string; + toolCallId: string; + input?: Record | null; +} + +export interface ToolResultOptions extends BaseEventOptions { + toolName: string; + toolCallId: string; + output?: unknown; + error?: string | null; +} + +export interface ModelRequestOptions extends BaseEventOptions { + model?: string | null; + messages?: Array> | null; + system?: unknown; + tools?: Array> | null; + requestId?: string | null; +} + +export interface ModelResponseOptions extends BaseEventOptions { + model?: string | null; + stopReason?: string | null; + inputTokens?: number | null; + outputTokens?: number | null; + content?: unknown; + role?: string | null; + requestId?: string | null; +} + +export interface AgentStartOptions extends BaseEventOptions { + goal?: string | null; + parentId?: string | null; +} + +export interface AgentEndOptions extends BaseEventOptions { + outcome?: string | null; + summary?: string | null; +} + +export interface AgentPauseOptions extends BaseEventOptions { + pauseId: string; + reason?: string | null; + userId?: string | null; +} + +export interface AgentResumeOptions extends BaseEventOptions { + pauseId: string; + reason?: string | null; + userId?: string | null; +} + +export interface HookTriggeredOptions extends BaseEventOptions { + hookName: string; + hookId: string; + triggerEvent?: string | null; + input?: unknown; +} + +export interface HookCompletedOptions extends BaseEventOptions { + hookName: string; + hookId: string; + outcome?: string | null; + output?: unknown; + error?: string | null; +} + +export interface ErrorOptions extends BaseEventOptions { + errorType: string; + message: string; + traceback?: string | null; +} + +export interface HumanWaitOptions extends BaseEventOptions { + inputId: string; + prompt?: string | null; + options?: string[] | null; + reason?: string | null; +} + +export interface HumanInputOptions extends BaseEventOptions { + inputId: string; + response?: string | null; +} + +export interface HumanPauseOptions extends BaseEventOptions { + reason?: string | null; + userId?: string | null; +} + +export interface HumanInterruptOptions extends BaseEventOptions { + reason?: string | null; + userId?: string | null; + atStep?: string | null; +} + +type Extras = Record; + +/** A tool call, hook or model call opened and not yet closed. */ +interface OpenLeaf { + kind: "tool" | "hook" | "model"; + sessionId: string; + agentId: string; + id: string; + /** Tool or hook name; the model, for a model call. */ + name: string | undefined; + /** When it opened: the exit path's ordering, and a model call's duration. */ + startedMicros: number; +} + +export class EventNamespace { + private readonly writer: EventWriter; + private readonly pending = new Map(); + /** + * Tool calls, hooks and model calls opened and not yet closed, keyed + * `::`, so the exit path can close them — whoever opened + * them: a hand-written `event.*` call, a `toolCall()` scope, or an adapter. + * Every one of those goes through this namespace, which is why this is the + * one place it is done. + */ + private readonly openLeaves = new Map(); + /** Open `agent_pause`s per session: a run waiting on a human is not abandoned. */ + private readonly pausedSessions = new Map(); + + constructor(writer: EventWriter) { + this.writer = writer; + const self = new WeakRef(this); + const unregister = onProcessExit(() => { + const namespace = self.deref(); + if (namespace === undefined) { + unregister(); + return []; + } + return namespace.openAtExit(); + }); + } + + private openLeaf(key: string, leaf: OpenLeaf): void { + if (this.openLeaves.size >= PENDING_CAP) { + const oldest = this.openLeaves.keys().next(); + if (!oldest.done) this.openLeaves.delete(oldest.value); + } + this.openLeaves.set(key, leaf); + } + + /** + * The process is exiting: every open tool call, hook and model call, for the + * exit path to close in one most-recent-first order with everything else + * still open (`exit.ts`) — except in a session paused on a human, which + * another process may resume. + */ + openAtExit(): OpenItem[] { + const items: OpenItem[] = []; + for (const [key, leaf] of this.openLeaves) { + if ((this.pausedSessions.get(leaf.sessionId) ?? 0) > 0) continue; + items.push({ opened: leaf.startedMicros, close: (exitCode) => this.closeLeaf(key, leaf, exitCode) }); + } + return items; + } + + private closeLeaf(key: string, leaf: OpenLeaf, exitCode: number): void { + if (!this.openLeaves.delete(key)) return; // closed normally in the meantime + const why = (what: string) => + `ProcessExit: the process exited (code ${exitCode})${fatalSuffix()} while ${what} was still running`; + const identity = { sessionId: leaf.sessionId, agentId: leaf.agentId }; + if (leaf.kind === "tool") { + this.toolResult({ ...identity, toolName: leaf.name ?? "tool", toolCallId: leaf.id, error: why(`tool ${JSON.stringify(leaf.name)}`) }); + } else if (leaf.kind === "hook") { + this.hookCompleted({ ...identity, hookName: leaf.name ?? "hook", hookId: leaf.id, outcome: "failed", error: why(`hook ${JSON.stringify(leaf.name)}`) }); + } else { + // Model calls are not timed by the SDK (a caller passes duration_ms); + // one closed here is, so it matches the tools and hooks closed beside it. + const elapsed = Math.round((nowMicros() - leaf.startedMicros) / 1000); + this.modelResponse({ + ...identity, + requestId: leaf.id, + model: leaf.name, + stopReason: "error", + error: why("the model call"), + ...(elapsed >= 0 ? { duration_ms: elapsed } : {}), + }); + } + } + + + private trackPending(key: string, ts: number): void { + // No lock and no tolerance for a concurrent evictor, unlike the Python SDK: + // JavaScript runs this on one thread, so `size` / `keys().next()` / `delete` + // cannot interleave with another emit. The cap is exact here rather than + // approximate. + if (this.pending.size >= PENDING_CAP) { + const oldest = this.pending.keys().next(); + if (!oldest.done) this.pending.delete(oldest.value); + } + this.pending.set(key, ts); + } + + private takePending(key: string): number | undefined { + const value = this.pending.get(key); + if (value !== undefined) this.pending.delete(key); + return value; + } + + private validateFields(fields: Extras): void { + const bad = Object.keys(fields).filter((key) => RESERVED.has(key)); + if (bad.length > 0) { + throw new Error( + `Reserved field names cannot be used as custom fields: ${JSON.stringify(bad.sort())}`, + ); + } + // The schema merges extras verbatim, so a promoted key left nullish would + // reach the wire as an explicit JSON null — accepted at 200 OK, stored as + // NULL, invisible to every filter on that column. For a promoted column "no + // value" has to mean "no key", so drop it here, the one place holding the + // caller's own object. Warned rather than silent: passing a nullish value + // is still a mistake worth hearing about, it just must not cost the event. + for (const name of Object.keys(fields)) { + if (!PROMOTED_NUMERIC.has(name) && !PROMOTED_STRING.has(name)) continue; + if (fields[name] === undefined || fields[name] === null) { + logger.warn( + `${name} was passed as ${fields[name] === null ? "null" : "undefined"} and has been ` + + "omitted from the event; pass a value, or omit the key entirely to silence this.", + ); + delete fields[name]; + } + } + for (const name of Object.keys(fields)) { + if (PROMOTED_NUMERIC.has(name)) validatePromotedNumeric(name, fields[name]); + if (PROMOTED_STRING.has(name)) validatePromotedString(name, fields[name]); + } + } + + private refuseDuration(fields: Extras): void { + if ("duration_ms" in fields || "durationMs" in fields) { + throw new Error( + "duration_ms is auto-computed by the SDK and cannot be passed by the caller", + ); + } + } + + /** Epoch microseconds, strictly increasing in the process — see `clock.ts`. */ + private now(): number { + return nowMicros(); + } + + /** + * `2026-09-23T12:34:56.123004Z` — six fractional digits, matching the Python + * SDK's `%f` and the format the ingest endpoint parses. The last three are an + * ordering sequence inside the millisecond, not a measurement (`clock.ts`). + */ + private fmtTs(micros: number): string { + return formatMicros(micros); + } + + toolUse(options: ToolUseOptions): void { + const { sessionId, agentId, toolName, toolCallId, input, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + if (typeof toolCallId === "string") { + this.openLeaf(`tool:${sid}:${toolCallId}`, { kind: "tool", sessionId: sid, agentId: aid, id: toolCallId, name: toolName, startedMicros: nowMicros() }); + } + const ts = this.now(); + this.trackPending(toolKey(sid, toolCallId), ts); + this.writer.submit( + toolUseEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + toolName, + toolCallId, + input, + extraFields: fields, + }), + ); + } + + toolResult(options: ToolResultOptions): void { + const { sessionId, agentId, toolName, toolCallId, output, error, ...fields } = options; + this.refuseDuration(fields); + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.openLeaves.delete(`tool:${sid}:${toolCallId}`); + const ts = this.now(); + const durationMs = measuredDurationMs(this.takePending(toolKey(sid, toolCallId)), ts); + this.writer.submit( + toolResultEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + toolName, + toolCallId, + output, + error, + durationMs, + extraFields: fields, + }), + ); + } + + modelRequest(options: ModelRequestOptions = {}): void { + const { sessionId, agentId, model, messages, system, tools, requestId, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + if (typeof requestId === "string" && requestId !== "") { + this.openLeaf(`model:${sid}:${requestId}`, { kind: "model", sessionId: sid, agentId: aid, id: requestId, name: model ?? undefined, startedMicros: nowMicros() }); + } + this.writer.submit( + modelRequestEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + model, + messages, + system, + tools, + requestId, + extraFields: fields, + }), + ); + } + + modelResponse(options: ModelResponseOptions = {}): void { + const { + sessionId, + agentId, + model, + stopReason, + inputTokens, + outputTokens, + content, + role, + requestId, + ...fields + } = options; + // Named options, so they never reach `validateFields`. They are also the + // likeliest of the three to arrive wrong: a caller reading them off a + // provider's usage object gets whatever that object holds. + validatePromotedNumeric("inputTokens", inputTokens); + validatePromotedNumeric("outputTokens", outputTokens); + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + if (typeof requestId === "string") this.openLeaves.delete(`model:${sid}:${requestId}`); + this.writer.submit( + modelResponseEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + model, + stopReason, + inputTokens, + outputTokens, + content, + role, + requestId, + extraFields: fields, + }), + ); + } + + agentStart(options: AgentStartOptions = {}): void { + const { sessionId, agentId, goal, parentId, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.writer.submit( + agentStartEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + goal, + parentId, + extraFields: fields, + }), + ); + } + + agentEnd(options: AgentEndOptions = {}): void { + const { sessionId, agentId, outcome, summary, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.writer.submit( + agentEndEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + outcome, + summary, + extraFields: fields, + }), + ); + } + + agentPause(options: AgentPauseOptions): void { + const { sessionId, agentId, pauseId, reason, userId, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.pausedSessions.set(sid, (this.pausedSessions.get(sid) ?? 0) + 1); + const ts = this.now(); + this.trackPending(pauseKey(sid, pauseId), ts); + this.writer.submit( + agentPauseEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + pauseId, + reason, + userId, + extraFields: fields, + }), + ); + } + + agentResume(options: AgentResumeOptions): void { + const { sessionId, agentId, pauseId, reason, userId, ...fields } = options; + this.refuseDuration(fields); + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + { + const open = (this.pausedSessions.get(sid) ?? 0) - 1; + if (open > 0) this.pausedSessions.set(sid, open); + else this.pausedSessions.delete(sid); + } + const ts = this.now(); + const durationMs = measuredDurationMs(this.takePending(pauseKey(sid, pauseId)), ts); + this.writer.submit( + agentResumeEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + pauseId, + durationMs, + reason, + userId, + extraFields: fields, + }), + ); + } + + hookTriggered(options: HookTriggeredOptions): void { + const { sessionId, agentId, hookName, hookId, triggerEvent, input, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + if (typeof hookId === "string") { + this.openLeaf(`hook:${sid}:${hookId}`, { kind: "hook", sessionId: sid, agentId: aid, id: hookId, name: hookName, startedMicros: nowMicros() }); + } + const ts = this.now(); + this.trackPending(hookKey(sid, hookId), ts); + this.writer.submit( + hookTriggeredEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + hookName, + hookId, + triggerEvent, + input, + extraFields: fields, + }), + ); + } + + hookCompleted(options: HookCompletedOptions): void { + const { sessionId, agentId, hookName, hookId, outcome, output, error, ...fields } = options; + this.refuseDuration(fields); + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + const ts = this.now(); + this.openLeaves.delete(`hook:${sid}:${hookId}`); + const durationMs = measuredDurationMs(this.takePending(hookKey(sid, hookId)), ts); + this.writer.submit( + hookCompletedEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + hookName, + hookId, + outcome, + output, + error, + durationMs, + extraFields: fields, + }), + ); + } + + error(options: ErrorOptions): void { + const { sessionId, agentId, errorType, message, traceback, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.writer.submit( + errorEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + errorType, + message, + traceback, + extraFields: fields, + }), + ); + } + + humanWait(opts: HumanWaitOptions): void { + const { sessionId, agentId, inputId, prompt, options, reason, ...fields } = opts; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + const ts = this.now(); + this.trackPending(humanKey(sid, inputId), ts); + this.writer.submit( + humanWaitEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + inputId, + prompt, + options, + reason, + extraFields: fields, + }), + ); + } + + humanInput(options: HumanInputOptions): void { + const { sessionId, agentId, inputId, response, ...fields } = options; + this.refuseDuration(fields); + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + const ts = this.now(); + const durationMs = measuredDurationMs(this.takePending(humanKey(sid, inputId)), ts); + this.writer.submit( + humanInputEvent({ + timestamp: this.fmtTs(ts), + sessionId: sid, + agentId: aid, + inputId, + response, + durationMs, + extraFields: fields, + }), + ); + } + + humanPause(options: HumanPauseOptions = {}): void { + const { sessionId, agentId, reason, userId, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.writer.submit( + humanPauseEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + reason, + userId, + extraFields: fields, + }), + ); + } + + humanInterrupt(options: HumanInterruptOptions = {}): void { + const { sessionId, agentId, reason, userId, atStep, ...fields } = options; + this.validateFields(fields); + const [sid, aid] = resolveIdentity(sessionId, agentId); + this.writer.submit( + humanInterruptEvent({ + timestamp: this.fmtTs(this.now()), + sessionId: sid, + agentId: aid, + reason, + userId, + atStep, + extraFields: fields, + }), + ); + } +} + +export { RESERVED, PROMOTED_NUMERIC, PROMOTED_STRING, U32_MAX, PENDING_CAP }; diff --git a/sdk/typescript/src/exit.ts b/sdk/typescript/src/exit.ts new file mode 100644 index 000000000..b3ccc420b --- /dev/null +++ b/sdk/typescript/src/exit.ts @@ -0,0 +1,117 @@ +/** + * Close what is still open when the process exits. + * + * A run killed mid-flight — every rolling deploy, every `docker stop`, a + * Kubernetes eviction — used to leave its `agent_start` without an `agent_end` + * and its `tool_use` without a `tool_result`. The dashboard renders those as + * running forever and the run is never handed to evaluation. Python gets this + * for free: its SIGTERM recipe raises `SystemExit`, which unwinds every + * `with agent(...)` block. JavaScript has no unwinding through a pending + * `await`, so the event namespace, the scopes and the adapters' trackers + * register closers here and the writer's `exit` listener runs them before its + * final synchronous flush. + * + * Everything closes most recently opened first, across every owner — the + * order it would have closed in had it returned. + * + * Only on `exit`. A `flushSync()` called while the process carries on closes + * nothing — the runs it would close are still running. + */ + +/** Something still open when the process exits, and how to close it. */ +export interface OpenItem { + /** When it opened, in the event clock's microseconds (`clock.ts`). */ + opened: number; + close(exitCode: number): void; +} + +/** Returns what its owner still has open. Called once, at exit. */ +export type ExitCollector = () => OpenItem[]; + +const collectors = new Set(); + +/** + * The uncaught exception that is taking the process down, if one is. Without + * it, a run killed by a crash was closed with "the process exited (code 1)" + * and the exception itself was nowhere in the trace. Read through + * `uncaughtExceptionMonitor`, which observes without handling: the process + * still crashes exactly as it would have. + */ +let fatal: { error: unknown } | null = null; +let monitoring = false; + +function monitorFatal(): void { + if (monitoring) return; + monitoring = true; + process.on("uncaughtExceptionMonitor", (error) => { + fatal ??= { error }; + }); +} + +/** The uncaught exception the process is exiting on, or undefined. */ +export function fatalError(): unknown { + return fatal?.error; +} + +/** @internal For tests. */ +export function resetFatal(): void { + fatal = null; +} + +/** Register a collector; returns its unregister function. */ +export function onProcessExit(collector: ExitCollector): () => void { + monitorFatal(); + collectors.add(collector); + return () => { + collectors.delete(collector); + }; +} + +/** + * Close everything still open, most recently opened first, across every + * owner. One ordering, not one per owner: a tool that ran a sub-agent opened + * before that sub-agent, so the sub-agent — and its model call, opened after + * it — must close first. Closing all tools before all agents showed a + * `delegate_writer` tool finishing while the writer it started still ran. + */ +export function runExitClosers(exitCode: number): void { + const items: OpenItem[] = []; + for (const collector of collectors) { + try { + items.push(...collector()); + } catch { + // A collector that throws must not cost the others. + } + } + items.sort((a, b) => b.opened - a.opened); + for (const item of items) { + try { + item.close(exitCode); + } catch { + // Swallowed: an exception thrown inside `process.on("exit")` prints a + // stack into the host's stderr and would skip the final flush. + } + } +} + +/** `: : ` of the fatal exception, for the closing messages. */ +export function fatalSuffix(): string { + if (fatal === null) return ""; + const error = fatal.error; + if (error instanceof Error) { + const name = error.name && error.name !== "Error" ? error.name : error.constructor?.name || "Error"; + return ` on an uncaught ${name}: ${error.message}`; + } + return ` on an uncaught ${String(error)}`; +} + +/** The error an agent is closed with when the process exits under it. */ +export class ProcessExit extends Error { + readonly exitCode: number; + + constructor(exitCode: number, what: string) { + super(`the process exited (code ${exitCode})${fatalSuffix()} while ${what} was still running`); + this.name = "ProcessExit"; + this.exitCode = exitCode; + } +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts new file mode 100644 index 000000000..c32437283 --- /dev/null +++ b/sdk/typescript/src/index.ts @@ -0,0 +1,186 @@ +/** + * Telemetry for AI agents: emit events, spool them, let the daemon ship them. + * + * Three surfaces, in the order most people meet them: + * + * * **Scopes** — `session()`, `agent()`, `toolCall()`. They bind run identity + * and, for the latter two, bracket a run with its own events. Callback form + * by default; a `using`-compatible `.open()` for the cases a callback cannot + * express. + * * **Adapters** — `instrument()`. Wires LangChain/LangGraph, the Vercel AI + * SDK, Mastra and LlamaIndex.TS to the scopes above. + * * **`event.*`** — the 15 event methods, for anything the adapters do not + * cover. + * + * `sessionId` and `agentId` are optional on every event method: omitted, they + * resolve from the enclosing scope. Nothing bound and nothing passed is an + * error, never a silent drop — ingest skips an event with no session and + * answers 200. + * + * ## Shutdown + * + * Buffered events are flushed on `process.on("exit")` automatically. A process + * that is killed by a signal never reaches that, and Node's default for SIGTERM + * is to terminate without running exit handlers — so a containerised agent + * loses whatever the last interval had not yet written. Installing a signal + * handler from a library would change the process's behaviour (a listener + * suppresses Node's default termination), so this package will not do it for + * you. Two lines, at your own startup: + * + * for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]] as const) { + * process.once(signal, () => { failproofai.flushSync(); process.exit(code); }); + * } + * + * 128 + the signal number, so an orchestrator sees a termination rather than a + * success. On the way out, anything still open — a tool, a hook, a model call, + * an agent, including those an adapter opened — is closed with a `ProcessExit` + * error, so an interrupted run never renders as running forever. + */ + +import { setEnvironment, rejectComma } from "./environment.js"; +import type { EventNamespace } from "./events.js"; +import { setBaseDir } from "./resolver.js"; +import { runtime } from "./runtime.js"; +import { VERSION } from "./version.js"; +import { flushAllNow, flushAllSync, validatedInterval } from "./writer.js"; + +export { VERSION as version }; + +export { current, propagate, DEFAULT_AGENT_ID } from "./context.js"; +export type { Identity } from "./context.js"; + +export { session, agent, toolCall, AUTO, ToolCall } from "./scopes.js"; +export type { + AgentOptions, + AgentScope, + ParentId, + SessionOptions, + SessionScope, + ToolCallOptions, + ToolCallScope, +} from "./scopes.js"; + +export type { + AgentEndOptions, + AgentPauseOptions, + AgentResumeOptions, + AgentStartOptions, + ErrorOptions, + HookCompletedOptions, + HookTriggeredOptions, + HumanInputOptions, + HumanInterruptOptions, + HumanPauseOptions, + HumanWaitOptions, + ModelRequestOptions, + ModelResponseOptions, + ToolResultOptions, + ToolUseOptions, +} from "./events.js"; + +export { setLogger, setLogLevel } from "./logger.js"; +export type { Logger, LogLevel } from "./logger.js"; + +export { + available as availableFrameworks, + activeFrameworks, + instrument, + uninstrument, +} from "./integrations/index.js"; +export type { FrameworkName, InstrumentOptions } from "./integrations/index.js"; + +export interface ConfigureOptions { + /** + * Override the spool root. Omitted, it resolves to + * `~/.failproofai/custom-agents` (honouring `$FAILPROOFAI_HOME`, which moves + * the umbrella but cannot take the spool outside it — the `custom-agents` + * segment is always appended). + * + * This is the ONLY way to spool anywhere else. No environment variable + * redirects it, because a redirect with no confirmation and no error means + * batches land in a directory nothing reads, and an unread spool is + * indistinguishable from an idle one. + */ + baseDir?: string | null; + /** Seconds between flush cycles. Default 0.5 (500 ms). */ + flushInterval?: number; + /** + * Deployment environment label (e.g. "production", "staging"). Can also be + * set with `AGENTEYE_ENVIRONMENT`. Defaults to "dev" when neither is set. + */ + environment?: string | null; +} + +/** + * Configure the SDK. Call once at startup, before any `event.*` call. + * + * Throws if `flushInterval` is not a finite number greater than zero, or if + * `environment` contains a comma. + * + * BOTH validations run before ANY of it is applied, so a rejected call leaves + * the SDK exactly as it was rather than with a new `baseDir` and the old + * interval. Applying as we went meant + * `configure({ baseDir, environment: "prod,eu" })` threw having already moved + * the spool — precisely the half-applied state a caller who wraps startup in a + * `try/catch` (a reasonable thing to do for a telemetry library that must not + * crash the agent) was left shipping from. + */ +export function configure(options: ConfigureOptions = {}): void { + const flushInterval = validatedInterval(options.flushInterval ?? 0.5); + if (options.environment) rejectComma(options.environment, "configure({ environment })"); + setBaseDir(options.baseDir ?? null); + runtime.writer.setFlushInterval(flushInterval); + setEnvironment(options.environment ?? null); +} + +/** + * The 15 event methods. + * + * A forwarding view of the process-wide namespace rather than a direct + * reference, so a test can replace `runtime.event` with a recording namespace + * and every call site — including the adapters and the scopes — picks it up. + */ +export const event: EventNamespace = new Proxy(Object.create(null) as EventNamespace, { + get(_target, property) { + const value = (runtime.event as unknown as Record)[property]; + return typeof value === "function" + ? (value as (...args: unknown[]) => unknown).bind(runtime.event) + : value; + }, + has(_target, property) { + return property in (runtime.event as unknown as object); + }, +}); + +/** + * Write everything buffered, now. + * + * Awaitable, so a short-lived script or a serverless handler can guarantee its + * events reached disk before returning — the interval alone does not, and a + * function that returns immediately after its last event routinely exits before + * the next cycle. + */ +export async function flush(): Promise { + await flushAllNow(); +} + +/** + * The synchronous flush, for a signal handler or an `exit` listener, where + * awaiting is not possible. Blocks; use `flush()` anywhere else. + */ +export function flushSync(): void { + flushAllSync(); +} + +// The per-framework call-site helpers live on subpaths rather than here: +// +// import { telemetry, wrapModel } from "@failproofai/sdk/ai"; +// import { wrapTool, workflow } from "@failproofai/sdk/mastra"; +// import { langchainHandler } from "@failproofai/sdk/langchain"; +// import { Evaluator } from "@failproofai/sdk/evaluator"; +// +// Re-exporting them here would make `import "@failproofai/sdk"` evaluate every +// adapter module in the package — cheap, since none of them imports its +// framework at module scope, but it is exactly the creep that ends with the +// root import pulling in half the ecosystem. A subpath costs one line at the +// call site and keeps that impossible. diff --git a/sdk/typescript/src/integrations/ai.ts b/sdk/typescript/src/integrations/ai.ts new file mode 100644 index 000000000..ebe7fb127 --- /dev/null +++ b/sdk/typescript/src/integrations/ai.ts @@ -0,0 +1,1566 @@ +/** + * The Vercel AI SDK (`ai`), majors 4 through 7. + * + * ## The mapping + * + * Python's rule (`sdk/python/skill/references/frameworks.md`): a construct is + * an agent if and only if it owns an LLM decision loop and has its own goal. + * In the AI SDK that is exactly one thing — a top-level operation. + * + * | AI SDK | FailproofAI | + * |----------------------------------------------------------|-----------------------------------------------| + * | `generateText` / `streamText` / `generateObject` / `streamObject` call — including one made by an agent class (`Experimental_Agent`, `ToolLoopAgent`) | `agent_start` / `agent_end`; `agent_id` = `functionId`, else the operation name (`ai.generateText`) — never a call or span id. An agent class's own `id` is dropped by the SDK before telemetry sees it: set `functionId` in its telemetry settings | + * | `embed` / `embedMany` with nothing enclosing it | its own run, like a bare `wrapModel` call: `agent_start`, a model pair per provider call, `agent_end` | + * | `embed` / `embedMany` inside `failproofai.agent()` or a tool | model pairs of the enclosing agent — no nested agent (it owns no decision loop) | + * | a model step (one provider call; a tool loop makes several) | `model_request` / `model_response`, paired on `request_id` | + * | a tool execution | `tool_use` / `tool_result`, with the MODEL's own `toolCallId` | + * | an operation inside `failproofai.agent()` / a tool | nested: `parent_id` = the enclosing agent | + * | a bare `wrapModel` call with no enclosing agent | its own run: `agent_start` named after the model, the model pair, `agent_end` | + * + * `model_response` always carries integer `input_tokens` / `output_tokens`, a + * STRING `stop_reason`, the model id and an integer `duration_ms`. Every major + * spells those differently — v4 `promptTokens`, v5 `inputTokens: 11`, v6+ + * `inputTokens: { total: 11 }`; a finish reason is `"stop"` up to v5 and + * `{ unified: "stop", raw }` from v6 — and every spelling is read here. + * + * **A failure is recorded once, where it happened.** A model that throws closes + * its `model_response` with `error` and `stop_reason: "error"`; a tool that + * throws closes its `tool_result` with `error`. The enclosing agent then ends + * `failed` WITHOUT a separate `error` event — one is emitted only when the + * failure happened at the operation itself (schema validation, say) and no + * leaf carried it. v5+ hands a tool failure back to the model as a tool-error + * result and the loop carries on, so there the agent ends `success`: it + * recovered. v4 throws it out of `generateText`, so there it ends `failed`. + * + * **A stream that does not finish ends `cancelled`.** An aborted stream closes + * its open model call with `stop_reason: "cancelled"` on every major. A stream + * whose reader goes away — a client disconnecting from a route that returns + * `toUIMessageStreamResponse()`, a stream nobody reads, a v4–v6 provider + * stream that breaks mid-way — is never ended by the SDK at all; on v4–v6 the + * adapter closes it when its root span is garbage-collected (`fw_abandoned`). + * v7 reports none of these to an integration and hands it nothing to watch, + * so there the agent stays open (bounded by `MAX_OPEN_CALLS`) unless the call + * passes an `abortSignal` — the request's own, in a route handler. + * + * ## Where it attaches + * + * The AI SDK's surface is module-level functions exported from an ES module, + * whose namespace is immutable by specification — there is nothing to patch. + * So this adapter uses the extension points the SDK itself documents: + * + * * **v4–v6: an OpenTelemetry `Tracer`.** The SDK opens a span per operation + * (`ai.generateText`), per model step (`ai.generateText.doGenerate`) and per + * tool (`ai.toolCall`) on whatever tracer `experimental_telemetry.tracer` + * names, else on the global one. `FailproofTracer` implements just enough of + * the OTel `Tracer` interface to translate those spans — structurally, with + * no dependency on `@opentelemetry/api`. + * * **v7: a `Telemetry` integration.** v7 removed the `tracer` option and the + * OpenTelemetry dependency and replaced them with lifecycle callbacks + * (`onStart`, `onLanguageModelCallStart`/`End`, `onToolExecutionStart`/`End`, + * `onEnd`, `onError`, …) passed per call in `telemetry.integrations` or + * registered process-wide on `globalThis.AI_SDK_TELEMETRY_INTEGRATIONS` + * (which is all `registerTelemetry()` does). `integration` below is one. + * + * `telemetry()` returns BOTH — `{ isEnabled, functionId, tracer, integrations }` + * — so one call site works on every major: v4–v6 read `tracer`, v7 reads + * `integrations`. v6 also knows `integrations` (an earlier, smaller interface), + * but its events carry no `callId`, and the integration ignores any event + * without one, so v6 is recorded by the tracer alone and never twice. + * + * `instrument("ai")` registers the integration on v7's global list, so v7 + * records EVERY call (its telemetry is on by default once an integration is + * registered). That list is additive and per-call `integrations` replace it, + * so it takes nothing from anybody else. + * + * On v4–v6 `instrument("ai")` records nothing by itself, and says so once. The + * process-wide hook there is the global OpenTelemetry tracer provider, a + * single slot that OpenTelemetry refuses to hand over once taken: registering + * ours would silently refuse the customer's own `NodeSDK.start()` later in + * startup and route their http/database spans to a tracer that exports + * nothing. So taking it is opt-in — `instrument("ai", { registerGlobalTracer: + * true })`, for a process with no OpenTelemetry of its own — and even then + * only when the slot is still empty. (v4–v6 consult the global tracer only for + * calls that pass `experimental_telemetry: { isEnabled: true }`; that is the + * SDK's rule, not ours.) The call-site `telemetry()` and `wrapModel()` are the + * recommended paths on v4–v6. + * + * * **`middleware()` / `wrapModel()`** see model calls only — tools run above + * the model layer. Combined with `telemetry()` or `instrument()` they defer, + * so each call is recorded once: the tracer marks a model span's callback, + * and the integration its `executeLanguageModelCall`, with an + * `AsyncLocalStorage` flag the middleware checks. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; + +import { current as currentIdentity } from "../context.js"; +import { logger } from "../logger.js"; +import { resolveEsm, resolveFrom, tryRequire } from "../node-require.js"; +import * as compat from "./compat.js"; +import * as core from "./core.js"; +import type { Adapter } from "./core.js"; + +const NAME = "ai"; +const PACKAGE = "ai"; + +let tracker: core.RunTracker | null = null; + +/** + * Set while the tracer or the integration is recording a MODEL call, so + * `middleware()` knows this call is already covered and stays quiet. Scoped to + * the model call, not the whole operation: a wrapped model called from inside + * a tool is a different call and must still be recorded. + */ +const recordingModelCall = new AsyncLocalStorage(); + +function ensureTracker(options: Record = {}): core.RunTracker { + tracker ??= new core.RunTracker(NAME, { + baseFields: core.frameworkFields(NAME, PACKAGE), + fieldLimit: typeof options.captureLimit === "number" ? options.captureLimit : undefined, + }); + return tracker; +} + +// --------------------------------------------------------------------------- +// Reading every major's spelling +// --------------------------------------------------------------------------- + +function asInt(value: unknown): number | undefined { + const n = typeof value === "string" && value.trim() !== "" ? Number(value) : value; + return typeof n === "number" && Number.isFinite(n) && n >= 0 ? Math.round(n) : undefined; +} + +/** `11` (v5, v7's normalised usage) or `{ total: 11, … }` (v6+ provider usage). */ +function tokenCount(value: unknown): number | undefined { + if (typeof value === "object" && value !== null) return asInt((value as { total?: unknown }).total); + return asInt(value); +} + +/** Input/output tokens from any major's usage object. */ +export function usageTokens(usage: unknown): { inputTokens?: number; outputTokens?: number } { + if (typeof usage !== "object" || usage === null) return {}; + const u = usage as Record; + return { + inputTokens: tokenCount(u.inputTokens ?? u.promptTokens ?? u.tokens), + outputTokens: tokenCount(u.outputTokens ?? u.completionTokens), + }; +} + +/** + * A finish reason as a string. v6 made it `{ unified, raw }`; the column is a + * string, and an object there is unfilterable. + */ +export function stopReasonOf(value: unknown): string | undefined { + if (typeof value === "string") return value || undefined; + if (Array.isArray(value)) return stopReasonOf(value[0]); + if (typeof value === "object" && value !== null) { + const v = value as { unified?: unknown; type?: unknown; raw?: unknown }; + return stopReasonOf(v.unified ?? v.type ?? v.raw); + } + return undefined; +} + +/** Attribute values arrive as JSON strings; a non-JSON string is itself. */ +function parseMaybeJson(value: unknown): unknown { + if (typeof value !== "string") return value; + const trimmed = value.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return value; + try { + return JSON.parse(trimmed) as unknown; + } catch { + return value; + } +} + +/** `""` is not content: a tool-call step has empty text and a real answer elsewhere. */ +function nonEmpty(value: unknown): unknown { + if (value === "" || value === null || value === undefined) return undefined; + if (Array.isArray(value) && value.length === 0) return undefined; + return value; +} + +function toolCallOf(part: Record): Record { + return { + toolCallId: part.toolCallId, + toolName: part.toolName, + input: parseMaybeJson(part.input ?? part.args), + }; +} + +/** + * A step's tool calls in the one shape every path emits — + * `{ toolCallId, toolName, input }` with `input` parsed. The tracer reads them + * from `ai.response.toolCalls`, which each major spells its own way: v4 + * `{ toolCallType, args: "" }`, v5/v6 `generateText` `{ input: "" }`, + * v5/v6 `streamText` `{ type: "tool-call", input: {…} }`. Passed through, the + * same call read three different ways on the dashboard, its input a JSON + * string inside JSON. + * + * @internal Exported for the unit tests. + */ +export function toolCallsOf(value: unknown): Array> | undefined { + const parsed = parseMaybeJson(value); + if (!Array.isArray(parsed)) return undefined; + const calls = parsed + .filter((call): call is Record => typeof call === "object" && call !== null) + .map(toolCallOf); + return calls.length > 0 ? calls : undefined; +} + +/** + * What the model said: its text, or — for a step that only called tools — the + * calls. Reads v4's `{ text, toolCalls }` and v5+'s `content: [parts]`. + */ +export function responseContent(result: unknown): unknown { + if (typeof result !== "object" || result === null) return undefined; + const r = result as { content?: unknown; text?: unknown; toolCalls?: unknown }; + if (Array.isArray(r.content)) { + const parts = r.content as Array>; + const text = parts + .filter((p) => p.type === "text" && typeof p.text === "string") + .map((p) => p.text as string) + .join(""); + if (text) return text; + const calls = parts.filter((p) => p.type === "tool-call").map(toolCallOf); + return calls.length > 0 ? calls : undefined; + } + const text = nonEmpty(r.text); + if (text !== undefined) return text; + if (Array.isArray(r.toolCalls) && r.toolCalls.length > 0) { + return (r.toolCalls as Array>).map(toolCallOf); + } + return undefined; +} + +function errorOf(error: unknown): { type: string; message: string; stack?: string } { + if (error instanceof Error) { + return { type: error.name || "Error", message: error.message, stack: error.stack }; + } + if (typeof error === "object" && error !== null && typeof (error as { message?: unknown }).message === "string") { + const e = error as { name?: unknown; message: string; stack?: unknown }; + return { + type: typeof e.name === "string" && e.name ? e.name : "Error", + message: e.message, + stack: typeof e.stack === "string" ? e.stack : undefined, + }; + } + return { type: typeof error, message: String(error) }; +} + +const errorText = (error: unknown): string => { + const detail = errorOf(error); + return `${detail.type}: ${detail.message}`; +}; + +/** An operation's default agent name — stable, low-cardinality, never an id. */ +function agentName(functionId: unknown, operation: unknown): string { + if (typeof functionId === "string" && functionId) return functionId; + return typeof operation === "string" && operation ? operation : "ai"; +} + +// --------------------------------------------------------------------------- +// v4–v6: the tracer +// --------------------------------------------------------------------------- + +type Attributes = Record; + +/** + * An OpenTelemetry attribute value. Spelled out rather than imported so the + * public types check against `ai`'s own `TelemetrySettings.metadata` without + * this package depending on `@opentelemetry/api`. + */ +export type AttributeValue = + | string + | number + | boolean + | Array + | Array + | Array; + +/** The operations the SDK opens a ROOT span for. Each becomes an agent. */ +const ROOT_OPERATIONS = new Set([ + "ai.generateText", + "ai.streamText", + "ai.generateObject", + "ai.streamObject", + "ai.embed", + "ai.embedMany", +]); + +/** + * The embedding operations. An embedding call owns no decision loop, so it is + * an agent only when nothing encloses it — a bare `embed()` in an indexing + * script is its own run, the way a bare `wrapModel` call is. Inside an agent + * (a `failproofai.agent()` scope, or a tool of a traced operation) it is a + * model call of THAT agent: a nested `ai.embed` agent per retrieval drowned + * the real agents in the `agent_id` facet. + */ +const EMBED_OPERATIONS = new Set(["ai.embed", "ai.embedMany"]); + +/** Whether an operation starting now has an agent above it to belong to. */ +const enclosedByAgent = (parentKey: unknown): boolean => + parentKey !== undefined || currentIdentity().agentId !== null; + +/** The operations that ARE the model call. Each becomes a request/response pair. */ +const MODEL_OPERATIONS = new Set([ + "ai.generateText.doGenerate", + "ai.streamText.doStream", + "ai.generateObject.doGenerate", + "ai.streamObject.doStream", + "ai.embed.doEmbed", + "ai.embedMany.doEmbed", +]); + +let spanCounter = 0; + +/** A model or tool span still open: what closing it takes, held apart from the span. */ +interface OpenLeaf { + id: string; + parentId: string | undefined; + /** Set for a model call (its model id, possibly undefined); unset for a tool. */ + model?: { id: string | undefined }; + tool?: { toolName: string; toolCallId: string }; + started: number; + closed: boolean; +} + +/** + * What an operation still owes the trace — its `agent_end` and the leaves + * under it that never closed. Plain data, deliberately holding no span, so a + * `FinalizationRegistry` can settle it once the root span itself is gone. + */ +interface Ledger { + id: string; + operation: string; + started: number; + tracker: core.RunTracker; + /** False for an enclosed embedding operation, which opened no agent. */ + agent: boolean; + leaves: Map; + done: boolean; +} + +/** Close a leaf that will never end by itself, as cancelled. */ +function closeLeaf(t: core.RunTracker, leaf: OpenLeaf): void { + leaf.closed = true; + if (leaf.model !== undefined) { + t.emit("modelResponse", leaf.id, { + parentKey: leaf.parentId, + model: leaf.model.id, + stopReason: "cancelled", + role: "assistant", + requestId: leaf.id, + ...core.fwFields({ duration_ms: core.ms(Date.now() - leaf.started) }), + }); + } else if (leaf.tool !== undefined) { + t.emit("toolResult", leaf.id, { + parentKey: leaf.parentId, + ...leaf.tool, + error: "cancelled: the operation ended before the tool returned", + }); + } + t.unlink(leaf.id); +} + +/** Close every leaf still open under an operation; whether there were any. */ +function settleLeaves(t: core.RunTracker, ledger: Ledger): boolean { + const leaves = [...ledger.leaves.values()]; + ledger.leaves.clear(); + for (const leaf of leaves) closeLeaf(t, leaf); + return leaves.length > 0; +} + +/** + * Operations whose root span was garbage-collected before anything ended it. + * + * The SDK ends a stream's root span from its result stream's `flush()`, and a + * `TransformStream` never flushes when its reader cancels or its source + * errors. So a client that disconnects from a route returning + * `toUIMessageStreamResponse()`, a stream nobody reads, and (on v4) a provider + * stream that breaks mid-way all leave the root span open forever — for the + * SDK's own OpenTelemetry exporters as much as for this adapter — and the + * agent reads as still running. Once nothing can reach the root span nothing + * can end it either, so that is exactly when it is safe to say it never will: + * the agent ends `cancelled`, marked `fw_abandoned`. + */ +const abandonedOperations = + typeof FinalizationRegistry === "function" + ? new FinalizationRegistry((ledger) => { + if (ledger.done) return; + ledger.done = true; + const t = tracker; + // A tracker since replaced (uninstall, reinstall) no longer owns this run. + if (t === null || t !== ledger.tracker) return; + core.callSafely( + () => { + settleLeaves(t, ledger); + if (ledger.agent) { + t.endAgent(ledger.id, { + outcome: "cancelled", + ...core.fwFields({ + operation: ledger.operation, + abandoned: true, + duration_ms: core.ms(Date.now() - ledger.started), + }), + }); + } + t.unlink(ledger.id); + }, + [], + `${NAME}.span.abandoned`, + ); + }) + : null; + +/** + * One OpenTelemetry span, translated as it opens and as it ends. + * + * The START events are emitted when the span OPENS, not when it ends: the SDK + * sets a model span's prompt and a tool span's arguments as start attributes, + * and in a stream the tool a model asked for runs while the model's span is + * still open — emitting the request at end time put it after the tool it + * caused. + */ +export class FailproofSpan { + readonly attributes: Attributes = {}; + /** The ROOT span of this operation, where "a leaf already reported the failure" is kept. */ + readonly root: FailproofSpan | undefined; + private spanName: string; + private readonly id: string; + private readonly parentId: string | undefined; + private readonly started = Date.now(); + private ended = false; + private failure: unknown = undefined; + private leafFailed = false; + /** An enclosed embedding operation: its model calls belong to the agent above, it is no agent itself. */ + private passThrough = false; + /** + * On a root: what the operation still owes. The SDK does not end a model + * span when a stream is aborted — only the root — so the root closes + * whatever is left when it ends, and the ledger survives the span for the + * case where nothing ends the root at all. + */ + private ledger: Ledger | undefined; + /** On a model or tool span: its entry in the root's ledger. */ + private leaf: OpenLeaf | undefined; + + constructor(spanName: string, parent: FailproofSpan | undefined, attributes: Attributes = {}) { + this.spanName = spanName; + spanCounter += 1; + this.id = `ai-${String(spanCounter)}-${randomUUID().slice(0, 8)}`; + this.parentId = parent?.spanId; + Object.assign(this.attributes, attributes); + this.root = ROOT_OPERATIONS.has(this.operation()) && !parent?.root ? this : parent?.root; + core.callSafely(() => { + this.open(); + }, [], `${NAME}.span.start`); + core.callSafely(() => { + this.track(); + }, [], `${NAME}.span.track`); + } + + /** Enter this span in its operation's ledger (or, for a root, open one). */ + private track(): void { + const t = tracker; + if (t === null || this.root === undefined) return; + if (this.root === this) { + this.ledger = { + id: this.id, + operation: this.operation(), + started: this.started, + tracker: t, + agent: !this.passThrough, + leaves: new Map(), + done: false, + }; + abandonedOperations?.register(this, this.ledger, this.ledger); + } else if (this.isLeaf && this.root.ledger !== undefined) { + this.leaf = { + id: this.id, + parentId: this.parentId, + ...(this.isModelCall ? { model: { id: this.model() } } : { tool: this.tool() }), + started: this.started, + closed: false, + }; + this.root.ledger.leaves.set(this.id, this.leaf); + } + } + + /** A model or tool span: one that opens a request/tool pair it must close. */ + private get isLeaf(): boolean { + return this.isModelCall || this.operation() === "ai.toolCall"; + } + + get spanId(): string { + return this.id; + } + + /** Whether this span is the model call itself. */ + get isModelCall(): boolean { + return MODEL_OPERATIONS.has(this.operation()); + } + + private operation(): string { + const id = this.attributes["ai.operationId"]; + return typeof id === "string" ? id : this.spanName; + } + + private open(): void { + const t = tracker; + if (t === null) return; + const a = this.attributes; + const operation = this.operation(); + if (EMBED_OPERATIONS.has(operation) && enclosedByAgent(this.parentId)) { + this.passThrough = true; + t.link(this.id, this.parentId); + return; + } + if (ROOT_OPERATIONS.has(operation)) { + t.startAgent(this.id, { + agentId: agentName(a["ai.telemetry.functionId"], operation), + parentKey: this.parentId, + ...core.fwFields({ operation }), + }); + return; + } + if (MODEL_OPERATIONS.has(operation)) { + const messages = parseMaybeJson(a["ai.prompt.messages"] ?? a["ai.prompt"]); + const tools = Array.isArray(a["ai.prompt.tools"]) + ? (a["ai.prompt.tools"] as unknown[]).map(parseMaybeJson) + : undefined; + t.emit("modelRequest", this.id, { + parentKey: this.parentId, + model: this.model(), + messages: Array.isArray(messages) ? (messages as Array>) : undefined, + tools: tools as Array> | undefined, + requestId: this.id, + ...core.fwFields({ + provider: a["ai.model.provider"] ?? a["gen_ai.system"], + operation, + prompt: Array.isArray(messages) ? undefined : messages, + }), + }); + } else if (operation === "ai.toolCall") { + const { toolName, toolCallId } = this.tool(); + t.emit("toolUse", this.id, { + parentKey: this.parentId, + toolName, + toolCallId, + input: parseMaybeJson(a["ai.toolCall.args"] ?? a["ai.toolCall.input"]) ?? undefined, + }); + } else { + // Any other span still has to lead its children to the agent above it. + t.link(this.id, this.parentId); + } + } + + private model(): string | undefined { + const a = this.attributes; + const id = a["ai.model.id"] ?? a["gen_ai.request.model"] ?? a["ai.response.model"]; + return typeof id === "string" ? id : undefined; + } + + private tool(): { toolName: string; toolCallId: string } { + const a = this.attributes; + return { + toolName: typeof a["ai.toolCall.name"] === "string" ? a["ai.toolCall.name"] : "tool", + toolCallId: typeof a["ai.toolCall.id"] === "string" ? a["ai.toolCall.id"] : this.id, + }; + } + + setAttribute(key: string, value: unknown): this { + this.attributes[key] = value; + return this; + } + + setAttributes(attributes: Attributes): this { + Object.assign(this.attributes, attributes); + return this; + } + + addEvent(): this { + return this; + } + + addLink(): this { + return this; + } + + addLinks(): this { + return this; + } + + setStatus(status: { code?: number; message?: string }): this { + // OpenTelemetry's SpanStatusCode.ERROR is 2. + if (status?.code === 2 && this.failure === undefined) { + this.failure = new Error(status.message ?? "span reported an error status"); + } + return this; + } + + recordException(error: unknown): void { + this.failure = error; + } + + updateName(name: string): this { + this.spanName = name; + return this; + } + + isRecording(): boolean { + return !this.ended; + } + + spanContext(): { traceId: string; spanId: string; traceFlags: number } { + return { traceId: this.id.padEnd(32, "0").slice(0, 32), spanId: this.id.slice(0, 16), traceFlags: 1 }; + } + + end(): void { + if (this.ended) return; + this.ended = true; + if (this.leaf !== undefined) { + // Closed as cancelled when its operation ended; a late end is not news. + if (this.leaf.closed) return; + this.root?.ledger?.leaves.delete(this.id); + } + if (this.ledger !== undefined) { + this.ledger.done = true; + abandonedOperations?.unregister(this.ledger); + } + const t = tracker; + if (t === null) return; + core.callSafely(() => { + this.emitEnd(t); + }, [], `${NAME}.span.end`); + } + + private emitEnd(t: core.RunTracker): void { + const a = this.attributes; + const operation = this.operation(); + const failed = this.failure !== undefined; + if (failed && this.root && this.root !== this) this.root.leafFailed = true; + + if (MODEL_OPERATIONS.has(operation)) { + const toolCalls = toolCallsOf(a["ai.response.toolCalls"]); + t.emit("modelResponse", this.id, { + parentKey: this.parentId, + model: (typeof a["ai.response.model"] === "string" ? a["ai.response.model"] : undefined) ?? this.model(), + stopReason: failed + ? "error" + : stopReasonOf(a["ai.response.finishReason"] ?? a["gen_ai.response.finish_reasons"]), + inputTokens: asInt( + a["ai.usage.inputTokens"] ?? a["ai.usage.promptTokens"] ?? a["ai.usage.tokens"] ?? a["gen_ai.usage.input_tokens"], + ), + outputTokens: asInt( + a["ai.usage.outputTokens"] ?? a["ai.usage.completionTokens"] ?? a["gen_ai.usage.output_tokens"], + ), + content: failed + ? undefined + : (nonEmpty(a["ai.response.text"]) ?? toolCalls ?? parseMaybeJson(nonEmpty(a["ai.response.object"]))), + role: "assistant", + requestId: this.id, + error: failed ? errorText(this.failure) : undefined, + ...core.fwFields({ + duration_ms: core.ms(Date.now() - this.started), + tool_calls: failed ? undefined : toolCalls, + response_id: a["ai.response.id"], + }), + }); + } else if (operation === "ai.toolCall") { + const { toolName, toolCallId } = this.tool(); + t.emit("toolResult", this.id, { + parentKey: this.parentId, + toolName, + toolCallId, + output: failed ? undefined : parseMaybeJson(a["ai.toolCall.result"] ?? a["ai.toolCall.output"]), + error: failed ? errorText(this.failure) : undefined, + }); + } else if (ROOT_OPERATIONS.has(operation)) { + const cutOff = this.ledger !== undefined && settleLeaves(t, this.ledger); + // The failure is already on the model_response / tool_result that raised + // it; a second `error` here would count it twice. + if (failed && !this.leafFailed) { + const detail = errorOf(this.failure); + t.emit("error", this.id, { + ...(this.passThrough ? { parentKey: this.parentId } : {}), + errorType: detail.type, + message: detail.message, + traceback: detail.stack, + }); + } + if (!this.passThrough) { + // An aborted stream ends its root with a model call still open and no + // finish reason (v5/v6 end only the root; v4 the same when the + // provider honours the signal). It did not succeed: it was cancelled. + const finishReason = stopReasonOf(a["ai.response.finishReason"]); + const cancelled = cutOff || (operation === "ai.streamText" && finishReason === undefined); + t.endAgent(this.id, { + outcome: failed ? "failed" : cancelled ? "cancelled" : "success", + ...core.fwFields({ + operation, + duration_ms: core.ms(Date.now() - this.started), + finish_reason: finishReason, + }), + }); + } + } + // Every span kind leaves a parent link behind (`emit` with a `parentKey`, + // `startAgent` under a parent, `link` for any other span). Its last event + // is out, so forget it: a completed span's link kept forever is a leak, + // and at the tracker's FIFO cap it evicts a LIVE run's links instead. + t.unlink(this.id); + } +} + +const activeSpan = new AsyncLocalStorage(); + +type SpanCallback = (span: FailproofSpan) => unknown; +interface SpanOptions { + attributes?: Attributes; +} + +/** + * Just enough of OpenTelemetry's `Tracer` for the AI SDK — structurally + * assignable to it, which is what lets `experimental_telemetry.tracer` accept + * it with no OpenTelemetry dependency in this package. + * + * `startActiveSpan` NEVER ends the span. That is OpenTelemetry's contract, and + * the SDK relies on it: its stream spans are opened with `endWhenDone: false` + * and ended by the SDK itself when the stream finishes. Ending on the + * callback's promise closed a `streamText` span the moment the stream object + * was returned, before a single token — and dropped half the events. + */ +export class FailproofTracer { + startSpan(name: string, options?: SpanOptions): FailproofSpan { + return new FailproofSpan(name, activeSpan.getStore(), options?.attributes ?? {}); + } + + startActiveSpan(name: string, fn: F): ReturnType; + startActiveSpan(name: string, options: SpanOptions, fn: F): ReturnType; + startActiveSpan(name: string, options: SpanOptions, context: unknown, fn: F): ReturnType; + startActiveSpan(name: string, ...rest: unknown[]): unknown { + const callback = rest[rest.length - 1] as SpanCallback; + const options = (rest.length > 1 && typeof rest[0] === "object" && rest[0] !== null ? rest[0] : {}) as SpanOptions; + const span = new FailproofSpan(name, activeSpan.getStore(), options.attributes ?? {}); + const run = (): unknown => callback(span); + return activeSpan.run(span, () => (span.isModelCall ? recordingModelCall.run(true, run) : run())); + } +} + +/** What `telemetry()` returns: every major's telemetry settings at once. */ +export interface AiTelemetry { + isEnabled: true; + functionId?: string; + metadata?: Record; + /** Read by ai v4–v6. */ + tracer: FailproofTracer; + /** Read by ai v7. */ + integrations: AiTelemetryIntegration[]; +} + +/** + * The value to hand the SDK's telemetry option, on any major: + * + * const { text } = await generateText({ + * model, + * prompt, + * telemetry: telemetry({ functionId: "answer-question" }), // ai 4–6: experimental_telemetry + * }); + * + * `functionId` names the agent; without one it is named after the operation + * (`ai.generateText`). Keep it low-cardinality — it lands in `agent_id`, the + * primary dashboard facet. + */ +export function telemetry( + options: { functionId?: string; metadata?: Record } = {}, +): AiTelemetry { + ensureTracker(); + return { + isEnabled: true, + ...(options.functionId === undefined ? {} : { functionId: options.functionId }), + ...(options.metadata === undefined ? {} : { metadata: options.metadata }), + tracer: new FailproofTracer(), + integrations: [integration], + }; +} + +/** The bare tracer, for `experimental_telemetry: { isEnabled: true, tracer }` (ai v4–v6). */ +export function tracer(): FailproofTracer { + ensureTracker(); + return new FailproofTracer(); +} + +// --------------------------------------------------------------------------- +// v7: the Telemetry integration +// --------------------------------------------------------------------------- + +/** + * The subset of ai v7's `Telemetry` interface this adapter implements. + * + * Parameters are `unknown` so the object is assignable to v7's `Telemetry` and + * v6's `TelemetryIntegration` alike without naming either; every field is read + * defensively below. + */ +export interface AiTelemetryIntegration { + onStart: (event: unknown) => void; + onLanguageModelCallStart: (event: unknown) => void; + onLanguageModelCallEnd: (event: unknown) => void; + onObjectStepStart: (event: unknown) => void; + onObjectStepEnd: (event: unknown) => void; + onEmbedStart: (event: unknown) => void; + onEmbedEnd: (event: unknown) => void; + onToolExecutionStart: (event: unknown) => void; + onToolExecutionEnd: (event: unknown) => void; + onEnd: (event: unknown) => void; + onAbort: (event: unknown) => void; + onError: (event: unknown) => void; + executeLanguageModelCall: (options: { execute: () => PromiseLike; callId?: string }) => Promise; + executeTool: (options: { execute: () => PromiseLike; callId?: string }) => Promise; +} + +interface Call { + callId: string; + key: string; + operation: string; + /** Model calls started and not yet ended, oldest first — one at a time in practice. */ + pending: Array<{ id: string; started: number; model: string | undefined }>; + /** Tool keys whose `tool_use` is out and whose `tool_result` is not. */ + tools: Set; + sequence: number; + /** A leaf already recorded the failure; the agent's end must not repeat it. */ + leafFailed: boolean; + /** + * Whether this call is an agent. An embedding call inside one is not: its + * model calls are recorded on the enclosing agent and it opens no run. + */ + agent: boolean; +} + +const MAX_OPEN_CALLS = 10_000; +const calls = new Map(); + +/** + * The agent a tool's `execute` runs under, so an operation started INSIDE a + * tool — a sub-agent — nests under the operation that called the tool. v7 + * calls `executeTool` around exactly that function, which is the one place an + * `AsyncLocalStorage` scope covers the nested call and nothing else. + */ +const enclosingCall = new AsyncLocalStorage(); + +type Event7 = Record; + +/** v7 events carry `callId`; v6's do not, and are recorded by the tracer instead. */ +function callOf(event: unknown): Call | undefined { + const id = (event as Event7 | undefined)?.callId; + return typeof id === "string" ? calls.get(id) : undefined; +} + +function startModel(event: Event7, fields: Record): void { + const t = tracker; + const call = callOf(event); + if (t === null || call === undefined) return; + call.sequence += 1; + const id = `${call.key}:m${String(call.sequence)}`; + call.pending.push({ id, started: Date.now(), model: typeof event.modelId === "string" ? event.modelId : undefined }); + t.emit("modelRequest", id, { + parentKey: call.key, + model: typeof event.modelId === "string" ? event.modelId : undefined, + requestId: id, + ...fields, + ...core.fwFields({ provider: event.provider, operation: call.operation }), + }); +} + +function endModel(event: Event7 | undefined, call: Call, fields: Record): void { + const t = tracker; + const open = call.pending.shift(); + if (t === null || open === undefined) return; + const performance = event?.performance as { responseTimeMs?: unknown } | undefined; + t.emit("modelResponse", open.id, { + parentKey: call.key, + // An aborted or failed call is closed with no end event: keep the model it started with. + model: typeof event?.modelId === "string" ? event.modelId : open.model, + role: "assistant", + requestId: open.id, + ...fields, + ...core.fwFields({ + duration_ms: core.ms(asInt(performance?.responseTimeMs) ?? Date.now() - open.started), + response_id: event?.responseId, + }), + }); + t.unlink(open.id); +} + +/** Close the oldest open model call as failed. */ +function failModel(call: Call, error: unknown): void { + if (call.pending.length === 0) return; + call.leafFailed = true; + endModel(undefined, call, { stopReason: "error", error: errorText(error) }); +} + +function finishCall(call: Call, outcome: string, error?: unknown, fields: Record = {}): void { + const t = tracker; + calls.delete(call.callId); + if (t === null) return; + while (call.pending.length > 0) { + // An abort closes the model call it interrupted as cancelled, like the + // middleware does; "incomplete" is left for an operation that ended with + // a call the SDK never reported back. + if (error === undefined) endModel(undefined, call, { stopReason: outcome === "cancelled" ? "cancelled" : "incomplete" }); + else failModel(call, error); + } + if (error !== undefined && !call.leafFailed) { + const detail = errorOf(error); + t.emit("error", call.key, { errorType: detail.type, message: detail.message, traceback: detail.stack }); + } + if (call.agent) t.endAgent(call.key, { outcome, ...fields }); + // Forget every link this call left: its own (a nested call's), and any tool + // the operation abandoned mid-run (an abort, an error). + t.unlink(call.key); + for (const key of call.tools) t.unlink(key); + call.tools.clear(); +} + +const safe = (site: string, fn: (event: Event7) => void) => (event: unknown): void => { + if (typeof event !== "object" || event === null) return; + core.callSafely(fn, [event], `${NAME}.${site}`); +}; + +/** + * One shared, stateless-per-call object: per-call state is keyed by v7's + * `callId`, so the same instance serves every call, the per-call + * `integrations` list and the global one. + */ +export const integration: AiTelemetryIntegration = { + onStart: safe("onStart", (event: Event7) => { + const callId = event?.callId; + if (typeof callId !== "string" || calls.has(callId)) return; + // The integration is exported, so it can be registered by hand + // (`registerTelemetry(integration)`) without telemetry() or instrument(). + const t = ensureTracker(); + while (calls.size >= MAX_OPEN_CALLS) { + const oldest = calls.keys().next(); + if (oldest.done) break; + calls.delete(oldest.value); + } + const operation = typeof event.operationId === "string" ? event.operationId : "ai"; + const key = `ai7:${callId}`; + const parentKey = enclosingCall.getStore(); + const agent = !(EMBED_OPERATIONS.has(operation) && enclosedByAgent(parentKey)); + calls.set(callId, { callId, key, operation, pending: [], tools: new Set(), sequence: 0, leafFailed: false, agent }); + if (!agent) { + t.link(key, parentKey); + return; + } + t.startAgent(key, { + agentId: agentName(event.functionId, operation), + parentKey, + ...core.fwFields({ operation, call_id: callId }), + }); + }), + + onLanguageModelCallStart: safe("onLanguageModelCallStart", (event: Event7) => { + startModel(event, { + messages: Array.isArray(event.messages) ? (event.messages as Array>) : undefined, + system: event.system, + tools: Array.isArray(event.tools) ? (event.tools as Array>) : undefined, + }); + }), + + onLanguageModelCallEnd: safe("onLanguageModelCallEnd", (event: Event7) => { + const call = callOf(event); + if (call === undefined) return; + const content = responseContent(event); + endModel(event, call, { + stopReason: stopReasonOf(event.finishReason), + ...usageTokens(event.usage), + content, + ...core.fwFields({ tool_calls: Array.isArray(content) ? content : undefined }), + }); + }), + + onObjectStepStart: safe("onObjectStepStart", (event: Event7) => { + startModel(event, { + messages: Array.isArray(event.promptMessages) ? (event.promptMessages as Array>) : undefined, + }); + }), + + onObjectStepEnd: safe("onObjectStepEnd", (event: Event7) => { + const call = callOf(event); + if (call === undefined) return; + endModel(event, call, { + stopReason: stopReasonOf(event.finishReason), + ...usageTokens(event.usage), + content: parseMaybeJson(nonEmpty(event.objectText)), + }); + }), + + onEmbedStart: safe("onEmbedStart", (event: Event7) => { + startModel(event, { ...core.fwFields({ values: Array.isArray(event.values) ? event.values.length : undefined }) }); + }), + + onEmbedEnd: safe("onEmbedEnd", (event: Event7) => { + const call = callOf(event); + if (call === undefined) return; + endModel(event, call, { ...usageTokens(event.usage) }); + }), + + onToolExecutionStart: safe("onToolExecutionStart", (event: Event7) => { + const t = tracker; + const call = callOf(event); + const toolCall = event.toolCall as Record | undefined; + if (t === null || call === undefined || !toolCall) return; + const toolCallId = String(toolCall.toolCallId); + call.tools.add(`${call.key}:${toolCallId}`); + t.emit("toolUse", `${call.key}:${toolCallId}`, { + parentKey: call.key, + toolName: typeof toolCall.toolName === "string" ? toolCall.toolName : "tool", + toolCallId, + input: parseMaybeJson(toolCall.input ?? toolCall.args) ?? undefined, + }); + }), + + onToolExecutionEnd: safe("onToolExecutionEnd", (event: Event7) => { + const t = tracker; + const call = callOf(event); + const toolCall = event.toolCall as Record | undefined; + if (t === null || call === undefined || !toolCall) return; + const toolCallId = String(toolCall.toolCallId); + const output = event.toolOutput as { type?: unknown; output?: unknown; error?: unknown } | undefined; + const failed = output?.type === "tool-error"; + if (failed) call.leafFailed = true; + t.emit("toolResult", `${call.key}:${toolCallId}`, { + parentKey: call.key, + toolName: typeof toolCall.toolName === "string" ? toolCall.toolName : "tool", + toolCallId, + output: failed ? undefined : output?.output, + error: failed ? errorText(output?.error) : undefined, + }); + call.tools.delete(`${call.key}:${toolCallId}`); + t.unlink(`${call.key}:${toolCallId}`); + }), + + onEnd: safe("onEnd", (event: Event7) => { + const call = callOf(event); + if (call === undefined) return; + // streamObject reports a schema failure here rather than throwing. + const failure = event.error; + finishCall(call, failure === undefined ? "success" : "failed", failure, { + ...core.fwFields({ operation: call.operation, finish_reason: stopReasonOf(event.finishReason) }), + }); + }), + + onAbort: safe("onAbort", (event: Event7) => { + const call = callOf(event); + if (call !== undefined) finishCall(call, "cancelled"); + }), + + onError: safe("onError", (event: Event7) => { + const call = callOf(event); + if (call === undefined) return; + failModel(call, event.error); + finishCall(call, "failed", event.error, { ...core.fwFields({ operation: call.operation }) }); + }), + + async executeLanguageModelCall(options: { execute: () => PromiseLike; callId?: string }): Promise { + try { + return await recordingModelCall.run(true, () => options.execute()); + } catch (error) { + // A provider call that throws never reaches `onLanguageModelCallEnd`, + // and a retried one starts again: close THIS attempt here, as failed. + const call = callOf(options); + if (call !== undefined) core.callSafely(failModel, [call, error], `${NAME}.executeLanguageModelCall`); + throw error; + } + }, + + async executeTool(options: { execute: () => PromiseLike; callId?: string }): Promise { + const key = callOf(options)?.key; + return key === undefined ? await options.execute() : await enclosingCall.run(key, () => options.execute()); + }, +}; + +// --------------------------------------------------------------------------- +// The middleware +// --------------------------------------------------------------------------- + +interface ModelLike { + modelId?: string; + provider?: string; +} + +/** + * A language-model middleware, typed structurally so it is assignable to + * `LanguageModelV1Middleware` (ai 4), `V2` (ai 5), `V3` (ai 6) and v7's + * `LanguageModelMiddleware` without naming any of them. The result type is a + * type parameter: whatever the model's `doGenerate` resolves to is what comes + * back. + */ +export interface AiMiddleware { + /** + * ai 6 requires the literal `"v3"`; ai 4, 5 and 7 do not look at it. The + * wrapping behaviour is identical across the four specifications. + */ + readonly specificationVersion: "v3"; + wrapGenerate(options: { doGenerate: () => PromiseLike; params: unknown; model?: unknown }): Promise; + wrapStream(options: { doStream: () => PromiseLike; params: unknown; model?: unknown }): Promise; +} + +/** Where a middleware-observed call's events go: an enclosing agent, or a run of its own. */ +interface Placement { + requestId: string; + run: string | null; +} + +/** + * Open the model call. With an enclosing `failproofai.agent()` the call is one + * step of that agent; without one it is its own run — `agent_start` named after + * the model — the same answer the LangChain adapter gives a bare chat-model + * call. Dropping it (no session) or pinning it to a phantom `main` agent with + * no `agent_start` were the two previous answers, and both lost it. + */ +function openModelCall(t: core.RunTracker, params: Record, model: ModelLike | undefined, streaming: boolean): Placement { + const requestId = randomUUID(); + let run: string | null = null; + if (currentIdentity().agentId === null) { + run = `ai-model:${requestId}`; + t.startAgent(run, { + agentId: typeof model?.modelId === "string" && model.modelId ? model.modelId : "model", + ...core.fwFields({ provider: model?.provider }), + }); + } + const prompt = params.prompt; + t.emit("modelRequest", requestId, { + parentKey: run ?? undefined, + model: model?.modelId, + messages: Array.isArray(prompt) ? (prompt as Array>) : undefined, + tools: Array.isArray(params.tools) ? (params.tools as Array>) : undefined, + requestId, + ...core.fwFields({ provider: model?.provider, streaming: streaming || undefined }), + }); + return { requestId, run }; +} + +function closeModelCall( + t: core.RunTracker, + placement: Placement, + model: ModelLike | undefined, + started: number, + fields: Record, + outcome: string = fields.error === undefined ? "success" : "failed", +): void { + t.emit("modelResponse", placement.requestId, { + parentKey: placement.run ?? undefined, + model: model?.modelId, + role: "assistant", + requestId: placement.requestId, + ...fields, + ...core.fwFields({ duration_ms: core.ms(Date.now() - started) }), + }); + t.unlink(placement.requestId); + if (placement.run !== null) t.endAgent(placement.run, { outcome }); +} + +/** + * A language-model middleware that records `model_request` / `model_response`. + * + * const model = wrapLanguageModel({ model: openai("gpt-4o"), middleware: middleware() }); + * + * Defers when `telemetry()` / `instrument()` is already recording the call, so + * combining them records each model call once. + */ +export function middleware(options: Record = {}): AiMiddleware { + const t = ensureTracker(options); + + const record = async ( + params: unknown, + rawModel: unknown, + run: () => PromiseLike, + streaming: boolean, + ): Promise => { + if (recordingModelCall.getStore()) return await run(); + const model = (typeof rawModel === "object" && rawModel !== null ? rawModel : undefined) as ModelLike | undefined; + const started = Date.now(); + const placement = core.callSafely( + openModelCall, + [t, (params ?? {}) as Record, model, streaming], + `${NAME}.middleware.request`, + ); + if (placement === undefined) return await run(); + + let result: R; + try { + result = await run(); + } catch (error) { + core.callSafely( + closeModelCall, + [t, placement, model, started, { stopReason: "error", error: errorText(error) }], + `${NAME}.middleware.error`, + ); + throw error; + } + + if (streaming) return instrumentStream(t, result, placement, model, started); + core.callSafely( + closeModelCall, + [ + t, + placement, + model, + started, + { + stopReason: stopReasonOf((result as { finishReason?: unknown }).finishReason), + ...usageTokens((result as { usage?: unknown }).usage), + content: responseContent(result), + }, + ], + `${NAME}.middleware.response`, + ); + return result; + }; + + return { + specificationVersion: "v3", + wrapGenerate: ({ doGenerate, params, model }) => record(params, model, doGenerate, false), + wrapStream: ({ doStream, params, model }) => record(params, model, doStream, true), + }; +} + +/** + * Observe a model stream so the response event carries what was actually produced. + * + * A stream's usage and finish reason only exist in its FINAL part, so emitting + * at `doStream` return time would record every streaming call with no tokens + * and no finish reason — the fields the dashboard's cost and failure views are + * built on. Every chunk passes through untouched, and the model call closes + * exactly once, however the stream stops: + * + * * it finishes — the response carries the text, tool calls, usage and finish + * reason read off the parts; + * * the consumer cancels it (a client that disconnected, a `streamText` that + * was aborted) — `stop_reason: "cancelled"` with whatever had arrived, and a + * standalone run ends `cancelled`; the cancel is passed on to the provider's + * stream so its connection is released; + * * it errors — `stop_reason: "error"` with the error, and a standalone run + * ends `failed`. + * + * `pipeThrough(new TransformStream())` saw only the first: a transformer's + * `flush` never runs on a cancel or an error, so those calls stayed open + * forever — no `model_response`, and no `agent_end` for a standalone run. + */ +function instrumentStream( + t: core.RunTracker, + result: R, + placement: Placement, + model: ModelLike | undefined, + started: number, +): R { + const value = result as { stream?: ReadableStream }; + if (typeof value?.stream?.getReader !== "function" || typeof ReadableStream !== "function") { + core.callSafely(closeModelCall, [t, placement, model, started, {}], `${NAME}.middleware.stream`); + return result; + } + + let text = ""; + let finishReason: string | undefined; + let usage: { inputTokens?: number; outputTokens?: number } = {}; + let failure: unknown = undefined; + const toolCalls: Array> = []; + + const onPart = (chunk: unknown): void => { + const part = chunk as Record; + if (part.type === "text-delta" || part.type === "text") { + // v5+ `delta`, v4 `textDelta`. + const delta = part.delta ?? part.textDelta ?? part.text; + if (typeof delta === "string") text += delta; + } else if (part.type === "tool-call") { + toolCalls.push(toolCallOf(part)); + } else if (part.type === "finish") { + finishReason = stopReasonOf(part.finishReason); + usage = usageTokens(part.usage); + } else if (part.type === "error") { + failure = part.error; + } + }; + + const done = (error: unknown, cancelled: boolean): void => { + const produced = { + content: text || (toolCalls.length > 0 ? toolCalls : undefined), + ...core.fwFields({ streaming: true, tool_calls: toolCalls.length > 0 ? toolCalls : undefined }), + }; + if (cancelled) { + closeModelCall(t, placement, model, started, { stopReason: "cancelled", ...usage, ...produced }, "cancelled"); + return; + } + // A thrown stream error, else an in-band `error` part. + const problem = error ?? failure; + const fields: Record = + problem === undefined + ? { stopReason: finishReason, ...usage, ...produced } + : { stopReason: "error", error: errorText(problem), ...usage }; + closeModelCall(t, placement, model, started, fields); + }; + + return { + ...(result as object), + stream: core.observeStream(value.stream, onPart, done, `${NAME}.middleware.stream`), + } as R; +} + +/** + * Wrap a model so every call through it is recorded. + * + * const model = await wrapModel(openai("gpt-4o")); + * + * Uses the SDK's own `wrapLanguageModel`, so the returned value is exactly + * what the SDK expects. + */ +export async function wrapModel(model: T, options: Record = {}): Promise { + const sdk = (await compat.requireModule("ai", "npm install ai")) as { + wrapLanguageModel?: (arg: { model: T; middleware: unknown }) => T; + experimental_wrapLanguageModel?: (arg: { model: T; middleware: unknown }) => T; + }; + const wrap = sdk.wrapLanguageModel ?? sdk.experimental_wrapLanguageModel; + if (typeof wrap !== "function") { + throw new Error( + "this version of `ai` does not export wrapLanguageModel; pass " + + "`experimental_telemetry: telemetry()` instead " + + '(`import { telemetry } from "@failproofai/sdk/ai"`).', + ); + } + return wrap({ model, middleware: middleware(options) }); +} + +/** + * Wrap one tool's `execute` so its call and result are recorded. + * + * Only needed when you are NOT using `telemetry()` — the SDK reports every + * tool it runs, which already becomes a `tool_use`/`tool_result` pair. + */ +export function wrapTool unknown }>( + toolName: string, + tool: T, +): T { + const execute = tool.execute; + if (typeof execute !== "function") return tool; + const t = ensureTracker(); + + const wrapped = core.wrapCallable( + execute, + { + before: (input: unknown, meta: unknown) => { + const toolCallId = + (meta as { toolCallId?: string } | undefined)?.toolCallId ?? randomUUID(); + t.emit("toolUse", toolCallId, { + toolName, + toolCallId, + input: (input) ?? undefined, + }); + return toolCallId; + }, + after: (ctx, output) => { + t.emit("toolResult", ctx, { toolName, toolCallId: String(ctx), output }); + }, + onError: (ctx, error) => { + t.emit("toolResult", ctx, { + toolName, + toolCallId: String(ctx), + error: errorText(error), + }); + }, + }, + NAME, + ); + return { ...tool, execute: wrapped }; +} + +/** `wrapTool` across a `{ name: tool }` record, as `generateText({ tools })` takes. */ +export function wrapTools unknown }>>( + tools: T, +): T { + const out: Record = {}; + for (const [name, tool] of Object.entries(tools)) out[name] = wrapTool(name, tool); + return out as T; +} + +// --------------------------------------------------------------------------- +// instrument("ai") +// --------------------------------------------------------------------------- + +/** The global v7 reads; `registerTelemetry()` only ever pushes onto it. */ +const GLOBAL_INTEGRATIONS = "AI_SDK_TELEMETRY_INTEGRATIONS"; + +function globalIntegrations(): unknown[] | undefined { + const value = (globalThis as Record)[GLOBAL_INTEGRATIONS]; + return Array.isArray(value) ? value : undefined; +} + +function registerIntegration(): void { + const list = globalIntegrations(); + if (list === undefined) { + (globalThis as Record)[GLOBAL_INTEGRATIONS] = [integration]; + } else if (!list.includes(integration)) { + list.push(integration); + } +} + +function unregisterIntegration(): void { + const list = globalIntegrations(); + if (list === undefined) return; + const index = list.indexOf(integration); + if (index !== -1) list.splice(index, 1); +} + +interface OtelApi { + trace: { + setGlobalTracerProvider(provider: unknown): boolean; + getTracerProvider(): unknown; + disable(): void; + }; + ProxyTracerProvider?: new () => { getDelegate(): unknown }; +} + +const provider = { getTracer: (): FailproofTracer => sharedTracer }; +const sharedTracer = new FailproofTracer(); +let registeredWith: OtelApi | null = null; + +/** + * `@opentelemetry/api` as the AI SDK sees it: resolved from `ai`'s own + * location first — under pnpm the application cannot see a transitive + * dependency, and the global registry is shared between copies anyway — then + * from the application. + */ +function loadOtel(): OtelApi | null { + for (const anchor of [resolveFrom(PACKAGE), resolveEsm(PACKAGE)]) { + if (anchor === null) continue; + try { + return createRequire(anchor)("@opentelemetry/api") as OtelApi; + } catch { + // Not beside `ai` (v7 dropped the dependency); try the next anchor. + } + } + return tryRequire("@opentelemetry/api"); +} + +/** Whether the "instrument('ai') does not cover v4–v6 by itself" note has been logged. */ +let advised = false; + +/** + * Register our tracer as the process-wide OpenTelemetry tracer (ai v4–v6). + * Only ever on `instrument("ai", { registerGlobalTracer: true })`. + * + * Only when no provider is registered yet: taking over a customer's own + * tracing is the opposite of what an observability library should do to + * somebody else's observability. `getTracerProvider()` ALWAYS returns the + * API's proxy, so the question is asked of the proxy's DELEGATE — a no-op + * provider until somebody registers one. + * + * "Not yet" is not "never", which is why this is opt-in: a provider the + * customer registers AFTER this call is refused by OpenTelemetry ("duplicate + * registration"), and theirs is the one that exports. Composing instead — + * handing their provider our spans too — is not possible from here: the + * registration that would have to be wrapped has not happened yet, and one + * that already has is held by instrumentations as a cached delegate that a + * re-registration does not reach. + */ +function registerGlobalTracer(): "registered" | "absent" | "taken" { + const api = loadOtel(); + if (api === null || typeof api.trace?.setGlobalTracerProvider !== "function") return "absent"; + const current = api.trace.getTracerProvider() as { getDelegate?: () => unknown } | undefined; + const delegate = typeof current?.getDelegate === "function" ? current.getDelegate() : current; + if (delegate === provider) { + registeredWith = api; + return "registered"; + } + const noop = api.ProxyTracerProvider ? (new api.ProxyTracerProvider().getDelegate() as object) : undefined; + if (noop !== undefined && (delegate as object | undefined)?.constructor !== noop.constructor) return "taken"; + if (!api.trace.setGlobalTracerProvider(provider)) return "taken"; + registeredWith = api; + return "registered"; +} + +export const adapter: Adapter = { + name: NAME, + + install(options: Record = {}): void { + compat.checkVersion(NAME, PACKAGE, { + minimum: "4.0.0", + below: "8.0.0", + reason: "the tracer spans, telemetry integration and middleware below are the v4–v7 shapes", + }); + ensureTracker(options); + + // v7: harmless on older majors, whose events carry no callId. The global + // list is additive — v7 dispatches every event to every integration on it, + // and a per-call `integrations` option replaces the list for that call — + // so registering ours takes nothing from anybody else's. + registerIntegration(); + + const major = compat.versionTuple(PACKAGE)?.[0]; + if (major !== undefined && major >= 7) return; + + // v4–v6 read spans from the ONE process-wide OpenTelemetry tracer + // provider, and OpenTelemetry refuses every registration after the first. + // Taking that slot here would silently refuse the customer's own + // `NodeSDK.start()` later on, and send their http/database/framework spans + // to a tracer that exports nothing. So it is opt-in, never the default. + if (options.registerGlobalTracer !== true) { + if (options.registerGlobalTracer === undefined && major !== undefined && !advised) { + advised = true; + logger.warn( + `instrument("ai") on ai ${String(major)}.x does not register a global OpenTelemetry ` + + "tracer — that slot belongs to your own tracing — so by itself it records nothing " + + "on ai 4–6 (it covers ai 7). Record calls at the call site with " + + "`experimental_telemetry: telemetry()`, or wrap the model once with " + + "`await wrapModel(model)` — both `import { telemetry, wrapModel } from " + + '"@failproofai/sdk/ai"`. If this process runs no OpenTelemetry of its own, ' + + '`instrument("ai", { registerGlobalTracer: true })` records every call that passes ' + + "`experimental_telemetry: { isEnabled: true }`. Pass `registerGlobalTracer: false` " + + "to silence this.", + ); + } + return; + } + const outcome = registerGlobalTracer(); + if (outcome === "registered" || major === undefined) return; + compat.warn( + (outcome === "taken" + ? "an OpenTelemetry tracer provider is already registered, so the `ai` adapter left it alone. " + : "`@opentelemetry/api` is not importable, so the `ai` adapter cannot register a global tracer. ") + + 'Add telemetry at the call site (import { telemetry, wrapModel } from "@failproofai/sdk/ai") —\n' + + " experimental_telemetry: telemetry()\n" + + "or wrap the model once —\n" + + " const model = await wrapModel(openai('gpt-4o'))", + `${NAME}:global-tracer`, + ); + }, + + uninstall(): void { + unregisterIntegration(); + if (registeredWith !== null) { + try { + registeredWith.trace.disable(); + } catch (error) { + logger.debug(`could not unregister the OpenTelemetry tracer provider: ${String(error)}`); + } + registeredWith = null; + } + advised = false; + tracker?.closeOpenAgents(); + tracker?.reset(); + tracker = null; + calls.clear(); + }, +}; + +/** + * The adapter's bookkeeping, for this package's own unit tests. + * + * @internal Not part of the public API. + */ +export const _internals = { + tracker: (): core.RunTracker | null => tracker, + openCalls: (): number => calls.size, +}; diff --git a/sdk/typescript/src/integrations/compat.ts b/sdk/typescript/src/integrations/compat.ts new file mode 100644 index 000000000..814518a6d --- /dev/null +++ b/sdk/typescript/src/integrations/compat.ts @@ -0,0 +1,322 @@ +/** + * Version and capability probes for the framework adapters. + * + * Peer dependencies express a *floor*, not enforcement: most users already have + * the framework and will never read our peer range. So the real check happens + * here, at `instrument()` time, in three tiers: + * + * 1. **framework not importable** -> a hard error whose message contains the + * literal install command. Instrumenting is an explicit user action, so + * silently doing nothing is never the right answer. + * 2. **importable but outside the declared range** -> a warning, once, then + * best-effort. A ceiling exists because without one a clean install a year + * from now pulls the next major, the callback API shifts, and the adapter + * stops receiving events *while throwing nothing*. + * 3. **a capability probe fails** -> warn and no-op **that hook only**, never + * the whole adapter. + * + * `FAILPROOFAI_SDK_STRICT_INTEGRATIONS=1` promotes every warning here to an + * exception. Warn-by-default is only defensible because there is a supported + * way to make it fail loudly. + * + * ## Why the version comparison is naive + * + * This package is contractually zero-dependency, so it cannot use `semver`. + * `parseVersion` reads the **leading numeric components only** and stops at the + * first component that is not purely numeric: + * + * "1.5.2" -> [1, 5, 2] + * "2.0.0-beta.1" -> [2, 0, 0] # pre-release suffix ignored + * "0.14.23+build" -> [0, 14, 23] # build metadata ignored + * + * That means `2.0.0-beta.1` compares **equal** to `2.0.0`, so a pre-release of + * a major we have declared a ceiling against will not be flagged. That is + * deliberate: the alternative is shipping a semver parser, and being wrong + * about a release candidate is much cheaper than a runtime dependency. + */ + +import { join, sep } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { logger } from "../logger.js"; +import { + appImportsReachCommonJs, + importModule, + isRequired, + nodeRequire, + resolveEsm, + resolveFrom, +} from "../node-require.js"; + +/** A framework is outside the range this adapter was written against. */ +export class FailproofAICompatError extends Error { + constructor(message: string) { + super(message); + this.name = "FailproofAICompatError"; + } +} + +const TRUTHY = new Set(["1", "true", "yes", "on"]); + +/** Read a boolean env var. Shared with `core.ts` so both flags parse alike. */ +export function envFlag(name: string): boolean { + return TRUTHY.has((process.env[name] ?? "").trim().toLowerCase()); +} + +// Cached, because it is read on every warning and every `safe()` failure, and +// resettable, because a test that cannot flip the switch cannot test the policy. +let strictIntegrationsValue: boolean | null = null; + +export function strictIntegrations(): boolean { + strictIntegrationsValue ??= envFlag("FAILPROOFAI_SDK_STRICT_INTEGRATIONS"); + return strictIntegrationsValue; +} + +/** Override the flag. `null` re-reads the environment variable. */ +export function setStrictIntegrations(value: boolean | null): void { + strictIntegrationsValue = value; +} + +const warned = new Set(); + +/** Forget which warnings have already fired (tests; also `uninstrument()`). */ +export function resetWarnings(): void { + warned.clear(); +} + +/** + * Warn once per `key`, or throw if strict. + * + * Deduplicated because these fire from `install()` *and* from hot callbacks: a + * per-call warning on a chatty framework is its own outage. + */ +export function warn(message: string, key?: string): void { + if (strictIntegrations()) throw new FailproofAICompatError(message); + const dedup = key ?? message; + if (warned.has(dedup)) return; + warned.add(dedup); + logger.warn(message); +} + +/** Leading numeric components of a version string. See the module comment. */ +export function parseVersion(text: string): number[] { + const parts: number[] = []; + for (const chunk of String(text).split(".")) { + let digits = ""; + for (const character of chunk) { + if (character < "0" || character > "9") break; + digits += character; + } + if (digits === "") break; + parts.push(Number.parseInt(digits, 10)); + if (digits.length !== chunk.length) { + // A partially numeric component ("0-beta", "3+build") ends the numeric + // prefix — everything after it is a pre-release or build segment. + break; + } + } + return parts; +} + +function compare(left: readonly number[], right: readonly number[]): number { + const length = Math.max(left.length, right.length); + for (let i = 0; i < length; i += 1) { + const a = left[i] ?? 0; + const b = right[i] ?? 0; + if (a !== b) return a < b ? -1 : 1; + } + return 0; +} + +/** + * The installed version of a package, or null if it is not installed. + * + * Reads `package.json` rather than a `version` export: that export is not + * guaranteed to exist and most of the frameworks we target do not define one. + * Resolution is anchored at the consuming APPLICATION (see `node-require.ts`), + * so it finds a framework installed beside the app even when this SDK itself + * lives somewhere isolated — a pnpm store, a workspace link. + */ +export function versionString(pkg: string): string | null { + const manifest = resolveFrom(`${pkg}/package.json`); + if (manifest !== null) { + try { + const parsed = nodeRequire(manifest) as { version?: unknown }; + if (typeof parsed.version === "string") return parsed.version; + } catch { + // A manifest that will not load is not a reason to refuse to instrument. + } + } + // `package.json` is not always in a package's `exports` map. Fall back to the + // package root's own resolution and walk up to the manifest beside it, rather + // than reporting "not installed" for a package that is merely strict about + // what it exports. + const entry = resolveFrom(pkg); + if (entry === null) return null; + const marker = `${sep}node_modules${sep}`; + const index = entry.lastIndexOf(marker + pkg.split("/")[0]); + if (index === -1) return null; + const root = join(entry.slice(0, index + marker.length), ...pkg.split("/")); + try { + const parsed = nodeRequire(join(root, "package.json")) as { version?: unknown }; + return typeof parsed.version === "string" ? parsed.version : null; + } catch { + return null; + } +} + +export function versionTuple(pkg: string): number[] | null { + const text = versionString(pkg); + return text ? parseVersion(text) : null; +} + +/** + * Import a framework module or throw with the literal install command. + * + * Tier 1. `instrument("langchain")` on a machine without LangChain is a mistake + * the user can fix in one command, so we hand them the command. + */ +export async function requireModule(specifier: string, install: string): Promise { + const copies = await requireModuleCopies(specifier, install); + return copies[0]!; +} + +/** + * Every in-memory copy of `specifier` the application can be using — the one + * it loads first. Adapters that patch or subscribe must do so on EACH. + * + * ## Why there can be two + * + * A dual-published framework ships an ES-module build and a CommonJS build of + * every module, and Node loads them as two unrelated module instances: two + * `CallbackManager` classes, two `Agent.prototype`s, two `Settings` singletons. + * Patching one does nothing to the other. The first release of these adapters + * resolved with `createRequire` and so always patched the CommonJS copy — in an + * ES-module application, which is the default for a new TypeScript project, + * `instrument()` returned success and not one event was ever recorded. + * + * ## Which copies + * + * 1. The copy the application's own imports reach: CommonJS when the process + * entry point is CommonJS, the ES-module build otherwise — and always the + * ES-module build inside a Next.js server, whose CommonJS launcher + * `import()`s every external package (`appImportsReachCommonJs`). A + * framework Next BUNDLES is a third copy no resolution can reach; only + * the call-site helpers see it. Loaded if it is + * not loaded yet — instrumenting before the framework's first import is + * the documented order, and loading it then is exactly what the + * application is about to do anyway. + * 2. The CommonJS copy as well, but ONLY if something has already + * `require`d it — an ES-module application with a CommonJS dependency + * that pulls the framework in. It is never loaded speculatively: a second + * copy that nothing uses costs memory at best, and at worst the framework + * notices — LlamaIndex prints "llamaindex was already imported. This + * breaks constructor checks" to the customer's terminal. + * + * The one arrangement this cannot see is the mirror image of (2): a CommonJS + * application whose ESM-only dependency imports the framework's ES-module + * build. Node keeps no inspectable registry of loaded ES modules, so there is + * nothing to detect it with short of a loader hook. Call-site helpers + * (`langchainHandler()`, `telemetry()`) cover that case, as they cover a + * bundled application where neither copy in `node_modules` is the one running. + */ +export async function requireModuleCopies(specifier: string, install: string): Promise { + // Resolve against the APPLICATION. A bare `import(specifier)` resolves + // relative to this file, which under pnpm or a workspace cannot see the + // caller's dependencies at all — so the framework the user definitely has + // installed reports as missing. + const cjsPath = resolveFrom(specifier); + const esmPath = resolveEsm(specifier); + const dual = cjsPath !== null && esmPath !== null && esmPath !== cjsPath; + + const load = async (path: string | null, viaRequire: boolean): Promise => + viaRequire && path !== null + ? nodeRequire(path) + : await importModule(path === null ? specifier : pathToFileURL(path).href); + + const copies: unknown[] = []; + try { + if (!dual) { + copies.push(await load(cjsPath ?? esmPath, false)); + } else if (appImportsReachCommonJs()) { + copies.push(await load(cjsPath, true)); + } else { + copies.push(await load(esmPath, false)); + if (isRequired(cjsPath)) copies.push(nodeRequire(cjsPath)); + } + } catch (error) { + throw new Error( + `cannot instrument ${JSON.stringify(specifier)} because it is not importable. ` + + `Install it with: ${install}`, + { cause: error }, + ); + } + return [...new Set(copies)]; +} + +/** + * Tier 2. True when `pkg` is inside [minimum, below); warns once if not. + * + * Returns true (best effort) for an unknown version too — a framework installed + * from a git checkout or a workspace link has no usable manifest, and refusing + * to instrument it would be a worse answer than trying. + */ +export function checkVersion( + framework: string, + pkg: string, + options: { minimum?: string; below?: string; reason?: string } = {}, +): boolean { + const found = versionString(pkg); + if (found === null) return true; + const got = parseVersion(found); + if (got.length === 0) return true; + + if (options.minimum !== undefined && compare(got, parseVersion(options.minimum)) < 0) { + warn( + `${pkg} ${found} is older than the ${options.minimum} this ${framework} adapter was ` + + `written against${options.reason ? ` (${options.reason})` : ""}. ` + + "Instrumenting anyway; some events may be missing.", + `${framework}:${pkg}:min`, + ); + return false; + } + if (options.below !== undefined && compare(got, parseVersion(options.below)) >= 0) { + warn( + `${pkg} ${found} is newer than the <${options.below} this ${framework} adapter was ` + + "written against. Instrumenting anyway, but a callback API change would make it " + + "stop recording silently — please report this.", + `${framework}:${pkg}:max`, + ); + return false; + } + return true; +} + +/** + * Tier 3. Run a capability probe; on failure warn and disable ONE hook. + * + * A missing capability is never a reason to abandon the whole adapter: the + * other 90% of the events are still correct and still worth having. + */ +export function probe(framework: string, hook: string, check: () => unknown): boolean { + let ok: boolean; + try { + ok = Boolean(check()); + } catch (error) { + warn( + `${framework} capability probe for ${JSON.stringify(hook)} failed ` + + `(${error instanceof Error ? error.message : String(error)}); that hook is disabled, ` + + "the rest of the adapter is unaffected.", + `${framework}:${hook}`, + ); + return false; + } + if (!ok) { + warn( + `${framework} does not provide ${JSON.stringify(hook)} in this version; that hook is ` + + "disabled, the rest of the adapter is unaffected.", + `${framework}:${hook}`, + ); + } + return ok; +} diff --git a/sdk/typescript/src/integrations/core.ts b/sdk/typescript/src/integrations/core.ts new file mode 100644 index 000000000..0ec9c2649 --- /dev/null +++ b/sdk/typescript/src/integrations/core.ts @@ -0,0 +1,1321 @@ +/** + * The parts every framework adapter shares: failure policy, patching, identity. + * + * An adapter under `integrations/` is supposed to be a **translation table** + * and nothing else. Everything that is genuinely hard — never throwing into the + * customer's call stack, restoring exactly what we replaced, mapping a + * framework's run ids onto FailproofAI identity, keeping payloads inside the + * store's patience — lives here, in one copy. If an adapter needs something + * added to this module, that is a signal the core is wrong, not that the + * adapter is special. + * + * Three things in here are load-bearing and easy to "fix" into a bug: + * + * * `safe()` must also catch a REJECTED PROMISE, not just a synchronous throw. + * Half of every framework's callback surface is `async`, and a try/catch does + * not see a rejection. + * * `RunTracker` never touches `AsyncLocalStorage`. A callback surface whose + * start and end are separate calls has no single async subtree to bind in, so + * identity is carried in a map and passed EXPLICITLY on every emit. + * * `fwFields()` is a safety rule, not a style rule. The schema merges extra + * fields **last**, so an extra named `tool_name` silently overwrites the + * declared one and changes the promoted column. + */ + +import { randomUUID } from "node:crypto"; + +import { DEFAULT_AGENT_ID, current as currentIdentity } from "../context.js"; +import type { Identity } from "../context.js"; +import { nowMicros } from "../clock.js"; +import { fatalSuffix, onProcessExit, type OpenItem } from "../exit.js"; +import { logException, logger } from "../logger.js"; +import { runtime } from "../runtime.js"; +import { DECLARED_FIELD_NAMES } from "../schema.js"; +import { VERSION } from "../version.js"; +import { envFlag, versionString } from "./compat.js"; + +// --------------------------------------------------------------------------- +// The adapter contract +// --------------------------------------------------------------------------- + +/** + * What `integrations/.ts` must export as `adapter`. + * + * `install()` must save the **original attribute object** it replaces (use + * `Patcher`), and `uninstall()` must restore that saved object rather than + * re-importing or reconstructing it. + */ +export interface Adapter { + readonly name: string; + install(options?: Record): Promise | void; + uninstall(): void; +} + +// --------------------------------------------------------------------------- +// Failure policy +// --------------------------------------------------------------------------- + +/** + * Everything under `integrations/` obeys one rule: never throw into the + * customer's call stack. Observability that takes the process down with it is + * worse than no observability. `FAILPROOFAI_SDK_STRICT=1` inverts that for + * tests and for debugging an adapter that has gone quiet — without it you can + * only ever prove "it didn't crash", never "it swallowed the right thing". + */ +let strictValue: boolean | null = null; + +export function strict(): boolean { + strictValue ??= envFlag("FAILPROOFAI_SDK_STRICT"); + return strictValue; +} + +/** Override the flag. `null` re-reads `FAILPROOFAI_SDK_STRICT`. */ +export function setStrict(value: boolean | null): void { + strictValue = value; +} + +/** + * After this many failures at one call site we stop calling it. A broken + * adapter should cost one log line, not 40% of the process and a full disk. + */ +const MAX_FAILURES = 3; + +const failures = new Map(); +const disabled = new Set(); + +/** Re-enable every degraded call site (tests; also `uninstrument()`). */ +export function resetFailures(): void { + failures.clear(); + disabled.clear(); +} + +export function isDegraded(site: string): boolean { + return disabled.has(site); +} + +function recordFailure(site: string, error: unknown): void { + const count = (failures.get(site) ?? 0) + 1; + failures.set(site, count); + const newlyDisabled = count >= MAX_FAILURES && !disabled.has(site); + if (newlyDisabled) disabled.add(site); + + if (count === 1) { + // Logged once per site, with the stack. Repeats are silent: a hook that + // fails on every token of a streaming response would otherwise become the + // log volume. + logException( + `instrumentation hook ${site} failed; the instrumented call was not affected. ` + + "Set FAILPROOFAI_SDK_STRICT=1 to re-throw.", + error, + ); + } else { + logger.debug(`instrumentation hook ${site} failed again (${count})`); + } + if (newlyDisabled) { + logger.error( + `instrumentation hook ${site} failed ${count} times and is now disabled for the rest ` + + "of this process. Events from it will be missing.", + ); + } +} + +/** + * Call `fn`, swallowing any failure and degrading a repeatedly failing site. + * + * A REJECTED PROMISE counts. Most framework callback surfaces are `async`, so a + * bare try/catch sees nothing at all when the body fails — the rejection lands + * as an unhandled rejection in the customer's process instead, which in Node 15+ + * terminates it by default. A telemetry hook must never be able to do that. + */ +export function callSafely(fn: (...args: never[]) => T, args: unknown[], site: string): T | undefined { + if (disabled.has(site)) return undefined; + let result: T; + try { + result = (fn as (...a: unknown[]) => T)(...args); + } catch (error) { + if (strict()) throw error; + recordFailure(site, error); + return undefined; + } + if ( + typeof result === "object" && + result !== null && + typeof (result as unknown as PromiseLike).then === "function" + ) { + return (result as unknown as Promise).catch((error: unknown) => { + if (strict()) throw error; + recordFailure(site, error); + return undefined; + }) as unknown as T; + } + return result; +} + +function siteOf(fn: unknown, namespace: string): string { + const named = (fn as { name?: unknown }).name; + return `${namespace}.${typeof named === "string" && named !== "" ? named : "anonymous"}`; +} + +/** Wrap a callback an adapter exposes so it can never throw into the framework. */ +export function safe( + namespace: string, + fn: (...args: Args) => Result, +): (...args: Args) => Result | undefined { + const site = siteOf(fn, namespace); + const wrapped = (...args: Args): Result | undefined => + callSafely(fn as unknown as (...a: never[]) => Result, args, site); + Object.defineProperty(wrapped, "name", { value: fn.name, configurable: true }); + return wrapped; +} + +function safeCall(fn: ((...args: unknown[]) => unknown) | undefined, args: unknown[], site: string): unknown { + if (fn === undefined) return undefined; + return callSafely(fn as (...a: never[]) => unknown, args, site); +} + +// --------------------------------------------------------------------------- +// Shape A — wrapper surfaces +// --------------------------------------------------------------------------- + +export interface WrapHooks { + before?: (...args: unknown[]) => unknown; + after?: (ctx: unknown, result: unknown) => unknown; + onError?: (ctx: unknown, error: unknown) => unknown; +} + +const WRAPPED = Symbol.for("failproofai.wrapped"); + +/** + * Wrap a framework callable so start and end are one frame. + * + * The structural guarantee, which is the whole reason this is a function and + * not hand-written try/catch in four adapters: **the user's call sits in + * exactly one `try`, whose only job is to re-throw.** Nothing we do can change + * what the wrapped callable returns or throws, because every one of our own + * calls is outside that block and inside `callSafely`. + * + * Async is handled explicitly rather than by luck: when the original returns a + * thenable we attach our hooks to ITS settlement and hand the caller back a + * promise that settles exactly as theirs did — same value, same rejection + * reason, same identity. + */ +export function wrapCallable unknown>( + original: T, + hooks: WrapHooks, + namespace = "wrap", +): T { + const site = `${namespace}.${original.name || "anonymous"}`; + const wrapper = function failproofaiWrapper(this: unknown, ...args: unknown[]): unknown { + const ctx = safeCall(hooks.before, args, site); + let result: unknown; + try { + result = (original as unknown as (...a: unknown[]) => unknown).apply(this, args); + } catch (error) { + safeCall(hooks.onError, [ctx, error], site); + throw error; + } + if ( + typeof result === "object" && + result !== null && + typeof (result as PromiseLike).then === "function" + ) { + return (result as PromiseLike).then( + (value) => { + safeCall(hooks.after, [ctx, value], site); + return value; + }, + (error: unknown) => { + safeCall(hooks.onError, [ctx, error], site); + throw error; + }, + ); + } + safeCall(hooks.after, [ctx, result], site); + return result; + }; + Object.defineProperty(wrapper, "name", { + value: original.name, + configurable: true, + }); + (wrapper as unknown as Record)[WRAPPED] = original; + return wrapper as unknown as T; +} + +export function isWrapped(value: unknown): boolean { + return ( + typeof value === "function" && (value as unknown as Record)[WRAPPED] !== undefined + ); +} + +/** The object we replaced, or `value` itself if we never wrapped it. */ +export function unwrap(value: T): T { + if (typeof value !== "function") return value; + const original = (value as unknown as Record)[WRAPPED]; + return (original as T) ?? value; +} + +// --------------------------------------------------------------------------- +// Observing a stream without owning it +// --------------------------------------------------------------------------- + +/** + * Pass `source` through untouched, handing each part to `onPart` and calling + * `done` exactly once — when the stream ends, when it errors, or when the + * reader cancels it. + * + * `done(undefined, false)` is a clean end; `done(error, false)` a stream that + * errored; `done(reason, true)` a consumer that cancelled (the reason is + * whatever it passed to `cancel()`, or a generic error when it passed nothing). + * The cancel is forwarded to `source`, so a provider connection is released. + * + * A pull-based re-stream rather than `pipeThrough(new TransformStream())`: a + * transformer's `flush` runs only on a clean end, with no hook for a consumer + * that walks away or a source that errors — and a model call observed that + * way stays open forever in exactly the cases worth recording. + * + * `onPart` and `done` run under `callSafely(site)`: an observer that throws + * never breaks the caller's stream. + */ +export function observeStream( + source: ReadableStream, + onPart: (part: T) => void, + done: (error: unknown, cancelled: boolean) => void, + site: string, +): ReadableStream { + const reader = source.getReader(); + let finished = false; + const finish = (error: unknown, cancelled: boolean): void => { + if (finished) return; + finished = true; + callSafely(done, [error, cancelled], site); + }; + return new ReadableStream({ + async pull(controller) { + let chunk: { done: boolean; value?: T }; + try { + chunk = await reader.read(); + } catch (error) { + finish(error, false); + controller.error(error); + return; + } + if (chunk.done) { + finish(undefined, false); + controller.close(); + return; + } + callSafely(onPart, [chunk.value], site); + controller.enqueue(chunk.value as T); + }, + cancel(reason) { + finish(reason ?? new Error("stream cancelled"), true); + return reader.cancel(reason); + }, + }); +} + +// --------------------------------------------------------------------------- +// Install / uninstall discipline +// --------------------------------------------------------------------------- + +interface PatchRecord { + target: object; + property: string; + original: unknown; + installed: unknown; + existed: boolean; +} + +/** + * Records what an `install()` replaced so `uninstall()` can put it back. + * + * Two rules, both of which exist because instrumentation libraries are + * routinely installed alongside each other: + * + * 1. **Restore the saved object, never a re-import.** Re-importing to restore + * hands back whatever the *current* value of the attribute's source is, + * which is how two instrumentation libraries silently un-patch each other. + * 2. **If the attribute is no longer ours, leave it alone.** Somebody patched + * on top of us; restoring would delete their patch. We log at WARN and keep + * our record, so the customer can see it happened. + */ +export class Patcher { + private records: PatchRecord[] = []; + + /** Set `target[property] = replacement`, remembering the exact object replaced. */ + patch(target: object, property: string, replacement: unknown): boolean { + const existed = property in target; + const original = (target as Record)[property]; + // An ESM namespace object and a frozen class both refuse assignment — + // silently in sloppy mode, loudly here. Reporting it lets the caller fall + // back to a supported wrapping API instead of believing it installed. + const descriptor = Object.getOwnPropertyDescriptor(target, property); + if (descriptor && !descriptor.configurable && !descriptor.writable) return false; + try { + (target as Record)[property] = replacement; + } catch { + return false; + } + if ((target as Record)[property] !== replacement) return false; + this.records.push({ target, property, original, installed: replacement, existed }); + return true; + } + + /** Undo every patch, newest first. Never throws. */ + restoreAll(): void { + const records = [...this.records].reverse(); + this.records = []; + for (const { target, property, original, installed, existed } of records) { + try { + const currentValue = (target as Record)[property]; + if (currentValue !== installed) { + logger.warn( + `not restoring ${describeTarget(target)}.${property} — it is no longer the object ` + + "this SDK installed (something else patched on top). Leaving the current value " + + "in place rather than deleting their patch.", + ); + continue; + } + if (existed) { + (target as Record)[property] = original; + } else { + delete (target as Record)[property]; + } + } catch (error) { + logException(`failed to restore ${describeTarget(target)}.${property}`, error); + } + } + } + + get size(): number { + return this.records.length; + } +} + +function describeTarget(target: object): string { + const named = target as { name?: unknown; constructor?: { name?: unknown } }; + if (typeof named.name === "string") return named.name; + if (typeof named.constructor?.name === "string") return named.constructor.name; + return "object"; +} + +// --------------------------------------------------------------------------- +// Payload discipline +// --------------------------------------------------------------------------- + +export const TRUNCATION_MARKER = "…[truncated]"; +export const FIELD_LIMIT = 8192; + +/** + * How many MAX-SIZE fields one event may carry before `payload()` starts + * dropping keys. The budget is DERIVED from the field limit rather than being a + * second independent number, because the two are not independent: raising one + * without the other silently changes how much survives. + * + * This matters because of HOW `payload()` runs out: past the budget it does not + * shorten the next field, it OMITS THE KEY. A caller raising `fieldLimit` + * therefore has to raise the budget in step or it trades shortened values for + * missing ones, which is strictly worse — the event stops saying that anything + * is absent. + */ +const FIELDS_PER_EVENT = 16; + +export const EVENT_BUDGET = FIELD_LIMIT * FIELDS_PER_EVENT; +const MAX_ITEMS = 100; +const MAX_DEPTH = 6; + +/** Mutable "did we cut anything" flag, threaded through the recursion. */ +class Cut { + hit = false; +} + +/** Remaining bytes for a whole event, spent as `truncateValue` emits. */ +class Budget { + remaining: number; + constructor(total: number) { + this.remaining = total; + } + spend(n: number): void { + this.remaining -= n; + } + get spentOut(): boolean { + return this.remaining <= 0; + } +} + +/** + * Shrink a payload value to something a column store will tolerate. + * + * Framework payloads are prompts, retrieved documents and tool outputs — the + * three largest strings in the process. None of these are promoted columns, so + * querying them means a JSON extraction over the payload, which has already + * caused a production memory blowup in the events store. Payload discipline is + * not optional. + */ +export function truncate(value: unknown, limit: number = FIELD_LIMIT): unknown { + return truncateValue(value, limit, new Cut(), 0); +} + +function truncateValue( + value: unknown, + limit: number, + cut: Cut, + depth: number, + budget?: Budget, +): unknown { + if (value === null || value === undefined) { + budget?.spend(8); + return value === undefined ? null : value; + } + const kind = typeof value; + if (kind === "boolean" || kind === "number") { + budget?.spend(8); + return value; + } + if (kind === "bigint") { + budget?.spend(8); + return (value as bigint).toString(); + } + if (kind === "function" || kind === "symbol") { + return truncateValue(render(value), limit, cut, MAX_DEPTH, budget); + } + if (kind === "string") { + let text = value as string; + if (text.length > limit) { + cut.hit = true; + text = text.slice(0, Math.max(limit - TRUNCATION_MARKER.length, 0)) + TRUNCATION_MARKER; + } + // The per-field limit bounds ONE string; the budget bounds the whole event. + // A structure whose leaves each fit under the limit would otherwise sail + // past the budget entirely. + if (budget) { + if (text.length > budget.remaining) { + cut.hit = true; + const keep = Math.max(budget.remaining - TRUNCATION_MARKER.length, 0); + text = text.slice(0, keep) + TRUNCATION_MARKER; + } + budget.spend(text.length); + } + return text; + } + if (depth >= MAX_DEPTH) { + cut.hit = true; + return truncateValue(render(value), limit, cut, MAX_DEPTH, budget); + } + if (value instanceof Date) { + return truncateValue( + Number.isNaN(value.getTime()) ? null : value.toISOString(), + limit, + cut, + depth, + budget, + ); + } + if (Array.isArray(value) || value instanceof Set) { + const items = Array.isArray(value) ? value : [...value]; + const out: unknown[] = []; + for (const [i, item] of items.slice(0, MAX_ITEMS).entries()) { + if (budget?.spentOut) { + cut.hit = true; + out.push(`[${items.length - i} more items truncated]`); + return out; + } + out.push(truncateValue(item, limit, cut, depth + 1, budget)); + } + if (items.length > MAX_ITEMS) { + cut.hit = true; + out.push(`[${items.length - MAX_ITEMS} more items truncated]`); + } + return out; + } + const mapping = asMapping(value); + if (mapping !== null) { + const entries = Object.entries(mapping); + const out: Record = {}; + for (const [i, [key, item]] of entries.entries()) { + if (i >= MAX_ITEMS) { + cut.hit = true; + out["…"] = `[${entries.length - MAX_ITEMS} more keys truncated]`; + return out; + } + if (budget) { + if (budget.spentOut) { + cut.hit = true; + out["…"] = `[${entries.length - i} more keys truncated]`; + return out; + } + budget.spend(key.length); + } + out[key] = truncateValue(item, limit, cut, depth + 1, budget); + } + return out; + } + // An object with no JSON shape is rendered, not cut — `fw_truncated` means + // "data was lost", and a rendering that fits has lost nothing a JSON encoder + // would have kept. + return truncateValue(render(value), limit, cut, MAX_DEPTH, budget); +} + +/** + * A plain object view of `value`, or null. + * + * Shallow on purpose. A deep clone would duplicate the whole tree before + * `truncateValue` gets to decide it only wanted the first 8 KB. Reading the top + * level and handing it back lets the existing walk apply the field limit, the + * item cap and the depth cap on the way down. + * + * Everything here can execute the caller's own code — a getter, a `toJSON`, a + * Zod schema's accessor — so all of it is guarded, and a failure falls through + * to `render`. + */ +function asMapping(value: unknown): Record | null { + if (typeof value !== "object" || value === null) return null; + try { + if (value instanceof Map) { + const out: Record = {}; + for (const [key, item] of value) out[String(key)] = item; + return out; + } + if (value instanceof Error) { + return { name: value.name, message: value.message }; + } + if (ArrayBuffer.isView(value)) return null; + const toJSON = (value as { toJSON?: unknown }).toJSON; + if (typeof toJSON === "function") { + const dumped: unknown = (toJSON as () => unknown).call(value); + return typeof dumped === "object" && dumped !== null && !Array.isArray(dumped) + ? (dumped as Record) + : null; + } + // Own enumerable properties only. Walking the prototype chain would pull in + // a framework class's accessors, half of which are lazy and some of which + // make network calls. + return { ...(value as Record) }; + } catch { + return null; + } +} + +function render(value: unknown): string { + try { + if (typeof value === "object" && value !== null) { + const name = value.constructor?.name; + return name && name !== "Object" ? `[${name}]` : "[object]"; + } + if (typeof value === "symbol") return value.toString(); + if (typeof value === "function") return `[function ${value.name || "anonymous"}]`; + return String(value); + } catch { + return "[unrenderable]"; + } +} + +function sizeOf(value: unknown, depth = 0): number { + if (value === null || value === undefined) return 8; + const kind = typeof value; + if (kind === "boolean" || kind === "number" || kind === "bigint") return 8; + if (kind === "string") return (value as string).length; + if (depth >= MAX_DEPTH) return render(value).length; + try { + if (Array.isArray(value)) { + let total = 0; + for (const item of value) total += sizeOf(item, depth + 1); + return total; + } + if (value instanceof Set) { + let total = 0; + for (const item of value) total += sizeOf(item, depth + 1); + return total; + } + const mapping = asMapping(value); + if (mapping !== null) { + let total = 0; + for (const [key, item] of Object.entries(mapping)) total += key.length + sizeOf(item, depth + 1); + return total; + } + } catch { + return 16; + } + return render(value).length; +} + +/** + * Apply the per-field limit and the per-event budget to a set of extras. + * + * Anything cut sets `fw_truncated=true`, so a surprising-looking payload in the + * dashboard is self-explaining rather than a mystery. + */ +export function payload( + fields: Record, + options: { limit?: number; budget?: number; cut?: Cut } = {}, +): Record { + const limit = options.limit ?? FIELD_LIMIT; + // Shared with the caller when it also truncated something — the tracker cuts + // the DECLARED parameters itself, and `fw_truncated` has to mean "this event + // lost data", not "one of its metadata extras did". + const cut = options.cut ?? new Cut(); + const spend = new Budget(options.budget ?? EVENT_BUDGET); + + // SMALLEST FIRST, spent in that order and emitted in the caller's. The budget + // binds either way, but insertion order decides WHICH keys survive it, and + // the adapters put the big payload before the metadata: an oversized + // `fw_inputs` would consume the whole event and take `fw_run_id` and + // `fw_node` with it — the two fields that say which run the payload belongs + // to. Sizing first costs a walk over the node count, not the character count. + const sized = Object.entries(fields) + .map(([key, value]) => ({ key, value, size: sizeOf(value) + key.length })) + .sort((a, b) => a.size - b.size); + + const kept = new Map(); + for (const { key, value } of sized) { + if (spend.spentOut) { + // Past the budget a field does not arrive short, it does not arrive at + // all — which is why the limit and the budget cannot move independently. + cut.hit = true; + continue; + } + spend.spend(key.length); + kept.set(key, truncateValue(value, limit, cut, 0, spend)); + } + + const out: Record = {}; + for (const key of Object.keys(fields)) { + if (kept.has(key)) out[key] = kept.get(key); + } + if (cut.hit) out.fw_truncated = true; + return out; +} + +/** + * Deliberate exceptions: these are top-level by design. `duration_ms` is how an + * adapter reports a model call's real latency; `usage` is read by both the + * server summary and the dashboard as a token fallback; `request_id` pairs + * model events; `framework*` label every event. + */ +export const ALLOWED_TOP_LEVEL: ReadonlySet = new Set([ + "request_id", + "duration_ms", + "usage", + "traceback", + "framework", + "framework_version", + "integration_version", +]); + +/** + * An extra whose name collides with a declared field SILENTLY OVERWRITES it: + * the schema ends with a merge of the extras. An adapter reflecting a + * framework's options into extras would then change `tool_name`, `model`, + * `outcome` or `input_tokens` — i.e. the promoted columns and the server's + * computed summary — and every test would still pass. + */ +export const FORBIDDEN_EXTRAS: ReadonlySet = new Set( + [...DECLARED_FIELD_NAMES].filter((name) => !ALLOWED_TOP_LEVEL.has(name)), +); + +const FW_PREFIX = "fw_"; + +/** + * Build the `fw_*` extra-field namespace. + * + * fwFields({ run_id: runId, node: "retrieve", tags: undefined }) + * -> { fw_run_id: "...", fw_node: "retrieve" } + * + * Keys are prefixed unless they already are, or are one of the deliberate + * top-level names. Nullish values are dropped (the schema omits absent + * optionals anyway, and an extra explicitly set to null would still occupy a + * key, reach the wire as JSON null, and NULL out a promoted column). + * + * Flat only — the server's payload key expression is single-level, so a nested + * object is not queryable. + * + * Values are NOT truncated here. Doing it at this layer would pin every `fw_*` + * extra at the module-level default while `instrument(..., { captureLimit })` + * raised the ceiling for the declared fields — half the event honouring the + * option and half not, with nothing saying which. `payload()`, which receives + * the tracker's real limit, is the one place that bounds them. + */ +export function fwFields(fields: Record): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (value === null || value === undefined) continue; + const name = ALLOWED_TOP_LEVEL.has(key) || key.startsWith(FW_PREFIX) ? key : FW_PREFIX + key; + out[name] = value; + } + return guardExtras(out); +} + +/** + * Strip (or, in strict mode, reject) extras that would shadow a real field. + * + * Called on every emit, so even an adapter that builds its extras by hand + * cannot silently rewrite a promoted column. + */ +export function guardExtras(fields: Record): Record { + const bad = Object.keys(fields).filter((key) => FORBIDDEN_EXTRAS.has(key)); + if (bad.length === 0) return fields; + const message = + `extra fields ${JSON.stringify(bad.sort())} would overwrite declared event fields ` + + "(the schema merges extras last). Namespace them as fw_* instead."; + if (strict()) throw new Error(message); + logger.warn(`${message} Dropping them.`); + const out: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (!FORBIDDEN_EXTRAS.has(key)) out[key] = value; + } + return out; +} + +/** + * The `framework` / `framework_version` / `integration_version` triple. + * + * Payload-only, so **not** server-side filterable; promoting it later is a + * hand-mirrored change across several files, so it is done on demand, not + * speculatively. + */ +export function frameworkFields(name: string, pkg?: string): Record { + const out: Record = { framework: name, integration_version: VERSION }; + const version = pkg ? versionString(pkg) : null; + if (version) out.framework_version = version; + return out; +} + +const ID_SEPARATORS = /[\s\-_.:/]+/; +const EMBEDDED_UUID = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g; +const UUID_EXACT = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/; +const HEX_ONLY = /^[0-9a-fA-F]+$/; +const AGENT_ID_LIMIT = 64; + +/** + * Turn a framework's label into something safe for `agentId`. + * + * `agent_id` is a `LowCardinality(String)` column and the primary facet on + * every dashboard surface. A UUID in it poisons that facet permanently — + * LowCardinality degrades, and the filter dropdown fills with one entry per + * run. So a value that looks like an id becomes `fallback` and the real id goes + * to `fw_agent_id` / `fw_run_id` where it belongs. + */ +export function normalizeAgentId(raw: unknown, fallback: string = DEFAULT_AGENT_ID): string { + if (raw === null || raw === undefined) return fallback; + const text = (typeof raw === "string" ? raw : render(raw)).split(/\s+/).filter(Boolean).join(" "); + if (text === "") return fallback; + if (looksLikeId(text)) return fallback; + const stripped = stripEmbeddedId(text); + if (stripped === "") return fallback; + return stripped.slice(0, AGENT_ID_LIMIT); +} + +/** True for UUIDs and long bare hex strings. */ +function looksLikeId(text: string): boolean { + if (UUID_EXACT.test(text)) return true; + const bare = text.replaceAll("-", "").replaceAll("_", ""); + return bare.length >= 16 && HEX_ONLY.test(bare); +} + +/** + * Drop a per-run id that a readable prefix is carrying. + * + * `looksLikeId` only fires on a value that is an id ALL THE WAY THROUGH, so it + * catches a bare UUID and misses `agent-`, `crew_`, `task-3f9a1c…` + * — a readable name with a per-run suffix, which is the shape frameworks + * actually produce and precisely the one the docs warn against. Those would go + * through untouched, one distinct value per run, into a `LowCardinality(String)` + * column that is the primary facet on every dashboard surface. + * + * Stripping rather than falling back: `agent-` still knows it is an + * agent, and collapsing every such label to `main` would throw away the one + * readable thing in it. A segment is dropped only if it is a UUID or a hex run + * of 16+ characters, so a name like `agent-v2` or `step-3` is untouched. + */ +function stripEmbeddedId(text: string): string { + // Dashed UUIDs first, and as a substring: splitting on separators would break + // `task-3f9a1c2b-...` into five segments none of which is an id on its own, + // so the most standard shape of all would survive the segment pass. + const stripped = text.replace(EMBEDDED_UUID, " "); + const parts = stripped.split(ID_SEPARATORS).filter(Boolean); + const kept = parts.filter((part) => !looksLikeId(part)); + // Nothing was an id: hand back the ORIGINAL, separators and all. Rejoining on + // spaces would rewrite every `node_a_b` in the process into `node a b`, which + // is a rename of the primary facet in exchange for nothing. + if (stripped === text && kept.length === parts.length) return text; + return kept.join(" ").trim(); +} + +/** + * Whole milliseconds, as an integer. + * + * The server stores `duration_ms` as a u32 and its JSON parser drops + * non-integers, so a float silently NULLs the column: the dashboard then shows + * no duration and nobody sees an error. Negative deltas (clock adjustments, a + * framework handing us an end before its start) clamp to 0. + */ +export function ms(deltaMs: number): number { + return Math.max(Math.round(deltaMs), 0); +} + +// --------------------------------------------------------------------------- +// Shape B — callback surfaces +// --------------------------------------------------------------------------- + +interface Run { + identity: Identity; + parentKey: unknown; + /** + * The run joined an enclosing `agent()` scope of the same name instead of + * opening its own agent, so it emits neither `agent_start` nor `agent_end`. + */ + joined?: boolean; + /** When it opened, in the event clock's microseconds: the exit path's ordering. */ + opened?: number; +} + +export type EventMethod = + | "toolUse" + | "toolResult" + | "modelRequest" + | "modelResponse" + | "agentStart" + | "agentEnd" + | "agentPause" + | "agentResume" + | "hookTriggered" + | "hookCompleted" + | "error" + | "humanWait" + | "humanInput" + | "humanPause" + | "humanInterrupt"; + +/** + * Maps a framework's own run ids onto FailproofAI identity. + * + * This is Shape B: the surface where a start and its end are **separate + * callbacks**, possibly on different async branches. Such an adapter can never + * use `AsyncLocalStorage` — there is no single subtree to run the pair inside, + * and `enterWith` in the start callback would bind identity into whatever + * unrelated context happened to dispatch it. Instead we keep the mapping here + * and pass `sessionId` / `agentId` **explicitly** on every emit. + * + * Bounded (`maxOpen`, FIFO eviction) because orphaned starts are normal: a + * crashed run, a stream nobody consumed, a framework that forgot an end + * callback. Unbounded, that is a memory leak in a long-lived server. + */ +/** The event methods that end the run they are keyed on. */ +const CLOSING_METHODS: ReadonlySet = new Set(["toolResult", "modelResponse", "hookCompleted"]); + + +export class RunTracker { + readonly name: string; + private readonly maxOpen: number; + private readonly baseFields: Record; + private readonly fieldLimit: number; + private readonly budget: number; + private readonly runs = new Map(); + private readonly links = new Map(); + /** + * Open pauses per agent key. A run paused on a human (a LangGraph interrupt, + * a suspended Mastra workflow) is deliberately left open: another process + * may take the answer and resume it from the checkpoint. Closing it at this + * process's exit would end a run that is not over. + */ + private readonly pauses = new Map(); + private warned = false; + + constructor( + name: string, + options: { + maxOpen?: number; + baseFields?: Record; + fieldLimit?: number; + } = {}, + ) { + this.name = name; + this.maxOpen = options.maxOpen ?? 10_000; + this.baseFields = { ...(options.baseFields ?? {}) }; + // One place decides how much of a value survives, for both halves of an + // event: the declared parameters (`input`, `output`, `messages`) and the + // `fw_*` extras. Two different rules would mean raising the adapter's limit + // changed only half the event. + this.fieldLimit = options.fieldLimit ?? FIELD_LIMIT; + this.budget = this.fieldLimit * FIELDS_PER_EVENT; + // A framework run still open when the process exits (a deploy's SIGTERM + // mid-graph) would otherwise render as running forever. Held weakly, so a + // tracker an adapter drops is not kept alive by this registration. + const self = new WeakRef(this); + const unregister = onProcessExit(() => { + const tracker = self.deref(); + if (tracker === undefined) { + unregister(); + return []; + } + return tracker.openAtExit(); + }); + } + + // -- identity --------------------------------------------------------- + + /** + * Resolve a run to an identity, in this order: + * + * 1. the exact `key`; + * 2. the `parentKey` chain, walked through every link we have seen — a + * framework's own parent run id is a *better* parent chain than an + * ambient stack, because it survives async hops; + * 3. **`failproofai.current()`** — this is the whole interop story. An + * adapter running inside a hand-written `agent("planner", ...)` joins that + * same session and gets `parentId: "planner"`, so mixing the manual API + * and an adapter produces one tree, not two; + * 4. otherwise the event is dropped and we log **once**. + */ + identity(key: unknown, parentKey?: unknown, warnOnMiss = true): Identity | null { + if (key !== undefined && key !== null) { + const run = this.runs.get(key); + if (run) return run.identity; + } + const walked = this.walk(parentKey); + if (walked) return walked; + + const ambient = RunTracker.ambient(true); + if (ambient) return ambient; + + if (warnOnMiss) this.warnUnresolved(key); + return null; + } + + /** + * Step 3: the identity a hand-written scope has bound, if any. + * + * When resolving an event we coerce a missing agent id to `main`, but when + * resolving a *parent* we must not: inside a bare `session(...)` there is no + * open agent, and claiming `parentId: "main"` would point at an agent that + * never emitted an `agent_start` — which makes the dashboard synthesize a + * never-ending root span that stays `ongoing` forever. + */ + private static ambient(coerceAgent: boolean): Identity | null { + const identity = currentIdentity(); + if (identity.sessionId === null) return null; + return { + sessionId: identity.sessionId, + agentId: identity.agentId ?? (coerceAgent ? DEFAULT_AGENT_ID : null), + parentId: identity.parentId, + depth: identity.depth, + }; + } + + private walk(parentKey: unknown): Identity | null { + const seen = new Set(); + let key = parentKey; + while (key !== undefined && key !== null && !seen.has(key)) { + seen.add(key); + const run = this.runs.get(key); + if (run) return run.identity; + key = this.links.get(key); + } + return null; + } + + private warnUnresolved(key: unknown): void { + if (this.warned) return; + this.warned = true; + logger.warn( + `${this.name} could not resolve a session for run ${String(key)} and is dropping its ` + + "events. Wrap the call in `await failproofai.session(fn)` (or " + + "`await failproofai.agent('name', fn)`) if you want them attributed. This is logged " + + "once per tracker.", + ); + } + + /** + * Record a run's parent without making it an agent. + * + * Intermediate framework runs (a LangChain chain, a Mastra step) do not + * become spans, but their children still need to find the agent above them. + * This is what makes step 2 of `identity()` work more than one hop up. + */ + link(key: unknown, parentKey: unknown): void { + if (key === undefined || key === null) return; + if (parentKey === undefined || parentKey === null || key === parentKey) return; + // Delete first so a re-link refreshes the entry's age: the FIFO cap below + // must evict the runs that have been around longest, not the ones that + // happened to be linked first. + this.links.delete(key); + this.evict(this.links); + this.links.set(key, parentKey); + } + + /** + * Forget a run's link. Call it when the run ENDS. + * + * A link is only needed while the run is live — it is how that run's own + * closing event, and its children's, find the agent above. Kept past that, + * the table fills to its cap with runs that finished long ago, and the FIFO + * cap then evicts the links of runs that are STILL RUNNING: on a busy server + * a model call that outlived ~10k other runs lost its `model_response` to + * "could not resolve a session". `emit()` does this itself for the closing + * event types; an adapter that links a run which emits nothing (an + * intermediate chain) must call it. + */ + unlink(key: unknown): void { + this.links.delete(key); + } + + /** FIFO — a `Map` keeps insertion order. */ + private evict(table: Map): void { + while (table.size >= this.maxOpen) { + const oldest = table.keys().next(); + if (oldest.done) break; + table.delete(oldest.value); + } + } + + // -- agents ----------------------------------------------------------- + + /** Register a run as an agent and emit `agent_start`. */ + startAgent( + key: unknown, + options: { + agentId: string; + parentKey?: unknown; + sessionId?: string; + goal?: string; + } & Record, + ): Identity { + const { agentId, parentKey, sessionId, goal, ...fields } = options; + const parent = this.resolveParent(parentKey); + const sid = sessionId ?? parent?.sessionId ?? randomUUID().replace(/-/g, ""); + const aid = normalizeAgentId(agentId); + // `await agent("support", () => graph.invoke(...))` around a graph also + // named "support" is the user saying "this run IS my agent", not "my agent + // contains an agent of the same name". Opening a second one gave every run + // two agent_start/agent_end pairs and an agent listed as its own parent. + // So a framework root that lands directly inside a hand-written scope of + // the same name, in the same session, joins it. Differently named, it + // still nests — that is a real tree. + if ( + parent !== null && + parent.agentId === aid && + parent.sessionId === sid && + this.walk(parentKey) === null + ) { + this.runs.delete(key); + this.evict(this.runs); + this.runs.set(key, { identity: parent, parentKey, joined: true }); + this.link(key, parentKey); + return parent; + } + const identity: Identity = { + sessionId: sid, + agentId: aid, + parentId: parent?.agentId ?? null, + depth: parent ? parent.depth + 1 : 1, + }; + this.runs.delete(key); + this.evict(this.runs); + this.runs.set(key, { identity, parentKey, opened: nowMicros() }); + this.link(key, parentKey); + this.emitWith("agentStart", identity, { + goal: goal === undefined ? undefined : truncate(goal, this.fieldLimit), + parentId: identity.parentId, + ...fields, + }); + return identity; + } + + /** + * Emit `agent_end` and forget the run. + * + * `outcome` is `"failed"`, never `"failure"` — the server only counts + * `error|failed|timeout|rejected` as a failure. + */ + endAgent( + key: unknown, + options: { outcome?: string; summary?: string } & Record = {}, + ): void { + const { outcome = "success", summary, ...fields } = options; + const run = this.runs.get(key); + this.runs.delete(key); + this.pauses.delete(key); + const identity = run?.identity ?? this.identity(key); + this.links.delete(key); + if (identity === null) return; + // A joined run's agent belongs to the enclosing scope, which ends it — and + // records the failure, if the error propagates out of the framework call. + if (run?.joined === true) return; + this.emitWith("agentEnd", identity, { + outcome, + summary: summary === undefined ? undefined : truncate(summary, this.fieldLimit), + ...fields, + }); + } + + openAgents(): unknown[] { + return [...this.runs.keys()]; + } + + /** Whether `key` is an open agent. O(1) — `openAgents()` copies every key. */ + isOpen(key: unknown): boolean { + return this.runs.has(key); + } + + /** + * Drop an agent and its link WITHOUT emitting anything. + * + * For an agent this process will never close but must stop holding: a run + * paused on a human and resumed by another worker. Closing it here would put + * a second `agent_end` into a session the other worker ends; keeping it + * would hold a slot in the table live runs need. + */ + forget(key: unknown): void { + this.runs.delete(key); + this.links.delete(key); + this.pauses.delete(key); + } + + /** @internal Table sizes, for the tests that prove nothing is retained. */ + stats(): { runs: number; links: number } { + return { runs: this.runs.size, links: this.links.size }; + } + + /** + * Close every still-open agent, newest first. + * + * A session that dies with an open `agent_start` renders as `ongoing` + * forever, so teardown closes what it opened. + */ + closeOpenAgents(outcome = "cancelled"): void { + for (const key of this.openAgents().reverse()) this.endAgent(key, { outcome }); + } + + /** + * The process is exiting: end every open agent as `failed`, newest first — + * except one paused on a human, which is waiting, not abandoned (`pauses`). + */ + closeAtExit(exitCode = 0): void { + for (const item of this.openAtExit().sort((x, y) => y.opened - x.opened)) item.close(exitCode); + } + + /** + * Every open agent, for the exit path to close in one most-recent-first + * order with everything else still open (`exit.ts`) — except one paused on a + * human, which is waiting, not abandoned (`pauses`), and one that joined a + * hand-written scope, which that scope ends. + */ + openAtExit(): OpenItem[] { + const items: OpenItem[] = []; + for (const [key, run] of this.runs) { + if ((this.pauses.get(key) ?? 0) > 0 || run.joined === true) continue; + items.push({ + opened: run.opened ?? 0, + close: (exitCode) => { + const current = this.runs.get(key); + if (current === undefined) return; // ended normally in the meantime + const message = `the process exited (code ${exitCode})${fatalSuffix()} while this run was still running`; + // `error` strictly before `agent_end`, as a scope that threw would. + this.emitWith("error", current.identity, { errorType: "ProcessExit", message }); + this.endAgent(key, { outcome: "failed", summary: "the process exited while this run was open" }); + }, + }); + } + return items; + } + + + reset(): void { + this.runs.clear(); + this.links.clear(); + this.pauses.clear(); + this.warned = false; + } + + // -- everything else -------------------------------------------------- + + /** + * Emit any `failproofai.event.*` method against a run's identity. + * + * Drops the event (with one warning) when nothing resolves, rather than + * inventing a session id: a synthesized session splits one run into many. + */ + emit( + method: EventMethod, + key: unknown, + fields: { parentKey?: unknown } & Record = {}, + ): void { + const { parentKey, ...rest } = fields; + if (parentKey !== undefined && parentKey !== null) this.link(key, parentKey); + if (method === "agentPause") this.pauses.set(key, (this.pauses.get(key) ?? 0) + 1); + if (method === "agentResume") { + const open = (this.pauses.get(key) ?? 0) - 1; + if (open > 0) this.pauses.set(key, open); + else this.pauses.delete(key); + } + const identity = this.identity(key, parentKey); + // A closing event ends the run it is keyed on, so its link is done with. + // Its children closed before it, and anything later that still names it + // resolves through its parent chain or the ambient scope — see `unlink`. + if (CLOSING_METHODS.has(method)) this.links.delete(key); + if (identity === null) return; + this.emitWith(method, identity, rest); + } + + private emitWith(method: EventMethod, identity: Identity, fields: Record): void { + callSafely( + () => { + this.emitNow(method, identity, fields); + }, + [], + `${this.name}.${method}`, + ); + } + + private emitNow(method: EventMethod, identity: Identity, fields: Record): void { + // Two kinds of key here, and the split is by NAME, not by meaning: `fw_*` + // (plus whatever the adapter set as base fields) are payload extras and go + // through the guard and the size budget; everything else is a real option + // of the `event.*` method — `toolName`, `input`, `outcome` — and is passed + // straight through. Those still get truncated, because `input`, `output`, + // `messages` and `content` are exactly the fields a framework fills with a + // 200 KB prompt. + const declared: Record = {}; + const extras: Record = {}; + // ONE `Cut` across both halves. A throwaway one per declared field would + // set `fw_truncated` — the only machine-readable "this event lost data" + // signal — when a small `fw_*` extra was cut and NOT when the prompt or the + // completion was. Exactly the wrong way round: `output` is cut on + // essentially every real tool loop. + const cut = new Cut(); + for (const [key, value] of Object.entries(fields)) { + if (value === undefined || value === null) continue; + if (key.startsWith(FW_PREFIX) || ALLOWED_TOP_LEVEL.has(key)) { + extras[key] = value; + } else { + declared[key] = truncateValue(value, this.fieldLimit, cut, 0); + } + } + const merged = payload(guardExtras({ ...this.baseFields, ...extras }), { + limit: this.fieldLimit, + budget: this.budget, + cut, + }); + // A base field named like a real option would be a duplicate key; the + // explicit value wins. + for (const key of Object.keys(declared)) delete merged[key]; + + // eslint-disable-next-line @typescript-eslint/unbound-method -- called with `.call` below, so `this` is explicit + const emit = runtime.event[method] as (options: Record) => void; + emit.call(runtime.event, { + sessionId: identity.sessionId, + agentId: identity.agentId, + ...declared, + ...merged, + }); + } + + private resolveParent(parentKey: unknown): Identity | null { + const walked = this.walk(parentKey); + if (walked) return walked; + // Same three steps as `identity()`, minus the exact-key lookup (a run + // cannot be its own parent) and minus the warning (a root agent with no + // ambient scope is normal, not a dropped event). + return RunTracker.ambient(false); + } +} diff --git a/sdk/typescript/src/integrations/index.ts b/sdk/typescript/src/integrations/index.ts new file mode 100644 index 000000000..e2b9181b7 --- /dev/null +++ b/sdk/typescript/src/integrations/index.ts @@ -0,0 +1,355 @@ +/** + * Framework adapters: the registry behind `failproofai.instrument()`. + * + * import * as failproofai from "@failproofai/sdk"; + * await failproofai.instrument(); // every framework we can find + * await failproofai.instrument("langchain"); // exactly one + * failproofai.uninstrument(); // put everything back + * + * **This module imports no adapter until asked.** The registry maps a name to a + * dynamic `import()` thunk, so `import "@failproofai/sdk"` never pulls in + * LangChain. + * + * ## Detection is by RESOLVABILITY, and that is a real difference from Python + * + * The Python SDK detects by reading `sys.modules` — "is this framework already + * imported in this process" — precisely so that auto-detection never imports a + * framework the user is not using. Node exposes no equivalent for ESM: there is + * no public view of the module registry, and `require.cache` only covers the + * CommonJS half. So detection here asks whether the package **resolves**, which + * is a weaker question: a framework that is installed but unused will be + * imported and patched by a bare `instrument()`. + * + * That is cheap and side-effect-free for all four adapters (each patches a + * class prototype or subscribes to a callback registry; none of them changes + * behaviour on its own), but it is not free. A process that cares names its + * framework: `instrument("langchain")` imports exactly one thing. + * + * ## Writing an adapter + * + * A module registered here must export a const named `adapter` implementing + * `Adapter` from `./core.js`: + * + * name: string + * install(options): Promise | void + * uninstall(): void + * + * `install()` must save the **original attribute object** it replaces — use + * `core.Patcher`, which also does the "somebody patched on top of us" check — + * and `uninstall()` must restore that saved object. Never re-import to restore: + * that hands back whatever the current value happens to be, which is how two + * instrumentation libraries silently un-patch each other. + * + * Every callback an adapter hands to the framework goes through `core.safe`, + * and every event goes through a `core.RunTracker`. Adapters own the + * translation table and nothing else. + */ + +import { logException, logger } from "../logger.js"; +import { NEXT_EXTERNALS_ENV } from "../next.js"; +import { isNextServer, loadedModulePaths, resolveFrom } from "../node-require.js"; +import * as compat from "./compat.js"; +import type { Adapter } from "./core.js"; +import * as core from "./core.js"; + +export type FrameworkName = "langchain" | "ai" | "mastra" | "llamaindex"; + +const REGISTRY: Record Promise<{ adapter: Adapter }>> = { + langchain: () => import("./langchain.js"), + ai: () => import("./ai.js"), + mastra: () => import("./mastra.js"), + llamaindex: () => import("./llamaindex.js"), +}; + +/** + * Spellings people actually type. LangGraph is served by the LangChain adapter + * because LangGraph.js runs on `@langchain/core`'s callback manager. + */ +const ALIASES: Record = { + langgraph: "langchain", + "@langchain/core": "langchain", + langchainjs: "langchain", + "aisdk": "ai", + "ai-sdk": "ai", + vercel: "ai", + "vercelai": "ai", + "@ai-sdk/provider": "ai", + "@mastra/core": "mastra", + "llama_index": "llamaindex", + "llama-index": "llamaindex", + "llamaindexts": "llamaindex", +}; + +/** + * name -> the packages whose presence means "this framework is in use". Kept + * here rather than read off the adapter so that detection imports nothing at + * all, not even our own adapter module. + */ +const DETECT: Record = { + langchain: ["@langchain/core", "langchain", "@langchain/langgraph"], + ai: ["ai"], + mastra: ["@mastra/core"], + llamaindex: ["llamaindex", "@llamaindex/core"], +}; + +const active = new Map(); + +/** Every registry name that can be instrumented, aliases excluded. */ +export function available(): FrameworkName[] { + return (Object.keys(REGISTRY) as FrameworkName[]).sort(); +} + +/** Names currently instrumented. */ +export function activeFrameworks(): FrameworkName[] { + return [...active.keys()].sort(); +} + +function canonical(name: string): FrameworkName { + const key = name.trim().toLowerCase().replaceAll(" ", ""); + const resolved = (ALIASES[key] ?? key) as FrameworkName; + if (!(resolved in REGISTRY)) { + const valid = [...new Set([...Object.keys(REGISTRY), ...Object.keys(ALIASES)])].sort().join(", "); + throw new Error( + `unknown framework ${JSON.stringify(name)}. Valid names are: ${valid}. ` + + "(Call instrument() with no argument to auto-detect.)", + ); + } + return resolved; +} + +function resolvable(pkg: string): boolean { + if (resolveFrom(pkg) !== null) return true; + // `require.cache` is the one authoritative "already loaded" signal Node gives + // us, and it only covers CommonJS. Checking it costs nothing and turns a + // false negative above — a package whose `exports` map hides its root — into + // a true positive for CommonJS consumers. + const marker = `node_modules/${pkg}/`; + return loadedModulePaths().some((path) => path.replaceAll("\\", "/").includes(marker)); +} + +function detected(): FrameworkName[] { + return available().filter((name) => DETECT[name].some((pkg) => resolvable(pkg))); +} + +async function load(name: FrameworkName): Promise { + const module = await REGISTRY[name](); + const adapter = module.adapter; + for (const method of ["install", "uninstall"] as const) { + if (typeof adapter?.[method] !== "function") { + throw new TypeError( + `adapter ${JSON.stringify(name)} does not implement ${method}() — see ` + + "integrations/core.ts's Adapter interface.", + ); + } + } + return adapter; +} + +export interface InstrumentOptions { + /** Per-field truncation ceiling for this adapter's events. */ + captureLimit?: number; + [option: string]: unknown; +} + +/** + * Install the adapters. Returns the names newly instrumented. + * + * With no argument, instruments every framework this process can resolve. + * Instrumenting something already active is a no-op that returns `[]`, so + * calling this from two code paths (or from a reloading dev server) cannot + * double-record. + * + * An unknown name throws, listing the valid ones — a typo that silently records + * nothing is the worst outcome available. An adapter whose `install()` throws + * is logged and skipped; the others still install, because a broken LlamaIndex + * should not cost you LangGraph. `FAILPROOFAI_SDK_STRICT=1` turns that skip + * back into a throw. + */ +export async function instrument( + framework?: string | null, + options: InstrumentOptions = {}, +): Promise { + let names: FrameworkName[]; + if (framework === undefined || framework === null) { + names = detected(); + if (names.length === 0) { + // WARN, not debug. This fires only when somebody explicitly asked for + // instrumentation and got none, and the result is a process that records + // nothing at all with the adapter "installed" and the docs followed. + logger.warn( + "instrument() found no supported framework, so NOTHING was instrumented. " + + "Install the framework you are using, or name one explicitly: " + + available() + .map((name) => `instrument(${JSON.stringify(name)})`) + .join(", ") + + ".", + ); + } + } else { + names = [canonical(framework)]; + } + + const installed: FrameworkName[] = []; + for (const name of names) { + if (active.has(name)) continue; + let adapter: Adapter | null = null; + try { + adapter = await load(name); + await adapter.install(options); + } catch (error) { + // An install is NOT atomic, so a failure part-way through leaves global + // state behind — a patched prototype, a registered listener. Catching + // without rolling back would leave the adapter fully patched and never + // recorded in `active`, so `activeFrameworks()` would deny it existed and + // `uninstrument()` — which iterates `active` — could never undo it. It + // would record for the life of the process and could not be removed. + if (adapter !== null) { + try { + adapter.uninstall(); + } catch (rollbackError) { + logger.debug( + `rollback of a failed ${name} install did not complete cleanly: ${String(rollbackError)}`, + ); + } + } + if (core.strict()) throw error; + // `FAILPROOFAI_SDK_STRICT_INTEGRATIONS=1` is documented as the supported + // way to make a compat problem loud. Swallowing it here would give the + // operator neither behaviour — not the throw the flag promises, and not + // the best-effort instrumentation the warning text promises. + if (error instanceof compat.FailproofAICompatError && compat.strictIntegrations()) { + throw error; + } + logException( + `could not instrument ${JSON.stringify(name)}; the rest of your process is ` + + "unaffected and other adapters still installed. Set FAILPROOFAI_SDK_STRICT=1 to " + + "throw instead.", + error, + ); + continue; + } + active.set(name, adapter); + installed.push(name); + warnIfNextBundles(name); + } + return installed; +} + +/** + * What a framework's adapter needs a Next.js server to load from + * `node_modules`. The Vercel AI SDK needs nothing: ai 7 reads its telemetry + * integrations from a global and `telemetry()` is passed at the call site, so + * both reach a bundled copy. + */ +const NEXT_REQUIRES: Partial> = { + langchain: ["@failproofai/sdk", "@langchain/core"], + mastra: ["@failproofai/sdk", "@mastra/core"], + llamaindex: ["@failproofai/sdk", "@llamaindex/core", "@llamaindex/workflow"], +}; + +const nextWarned = new Set(); + +/** + * The packages `name` needs externalized that this Next.js server does not + * externalize — empty when configured, `null` when that cannot be determined. + * + * Sources, most direct first: the list `withFailproofai` records when Next + * evaluates the config (`next start` / `next dev`); the resolved config a + * standalone server stores in `__NEXT_PRIVATE_STANDALONE_CONFIG`; and a + * hand-set `FAILPROOFAI_NEXT_EXTERNALS=1`, which trusts the app. + * + * @internal Exported for tests. + */ +export function nextExternalsGap(name: FrameworkName): string[] | null { + const required = NEXT_REQUIRES[name]; + if (required === undefined) return []; + const marker = process.env[NEXT_EXTERNALS_ENV]; + if (marker !== undefined && marker.trim() !== "") { + if (["1", "true", "yes"].includes(marker.trim().toLowerCase())) return []; + const listed = new Set(marker.split(",").map((entry) => entry.trim())); + return required.filter((pkg) => !listed.has(pkg)); + } + const standalone = process.env.__NEXT_PRIVATE_STANDALONE_CONFIG; + if (standalone) { + try { + const config = JSON.parse(standalone) as { serverExternalPackages?: unknown }; + const listed = new Set(Array.isArray(config.serverExternalPackages) ? config.serverExternalPackages : []); + return required.filter((pkg) => !listed.has(pkg)); + } catch { + return null; + } + } + return null; +} + +/** + * Warn, once per framework, when a Next.js server bundles what `instrument()` + * attaches to — the one arrangement where it reports success and records + * nothing. Node runtime only: an Edge route gets the SDK's no-op build. + */ +function warnIfNextBundles(name: FrameworkName): void { + if (!isNextServer() || process.env.NEXT_RUNTIME !== "nodejs" || nextWarned.has(name)) return; + const gap = nextExternalsGap(name); + if (gap !== null && gap.length === 0) return; + nextWarned.add(name); + const missing = gap === null ? NEXT_REQUIRES[name]! : gap; + logger.warn( + `instrument(${JSON.stringify(name)}) is running under Next.js, which bundles ` + + `${missing.join(", ")} into its server output unless told not to — and then this adapter ` + + "records NOTHING while reporting success. Wrap your next.config: " + + '`import { withFailproofai } from "@failproofai/sdk/next"; export default withFailproofai(config)`, ' + + `or add ${missing.join(", ")} to serverExternalPackages yourself and set ` + + `${NEXT_EXTERNALS_ENV}=1 to silence this.`, + ); +} + +/** @internal Tests only: forget which Next.js warnings have fired. */ +export function resetNextWarnings(): void { + nextWarned.clear(); +} + +/** + * Reverse `instrument()`. Returns the names removed. Never throws. + * + * With no argument, removes everything. An unknown name, or a name that was + * never instrumented, is a no-op — teardown that can fail is teardown people + * stop calling. + */ +export function uninstrument(framework?: string | null): FrameworkName[] { + let names: FrameworkName[]; + if (framework === undefined || framework === null) { + names = [...active.keys()]; + } else { + try { + names = [canonical(framework)]; + } catch (error) { + logger.warn(error instanceof Error ? error.message : String(error)); + return []; + } + } + + const removed: FrameworkName[] = []; + for (const name of names) { + const adapter = active.get(name); + if (adapter === undefined) continue; + active.delete(name); + try { + adapter.uninstall(); + } catch (error) { + logException( + `${JSON.stringify(name)} did not uninstall cleanly; it is no longer registered, ` + + "but some patches may remain.", + error, + ); + } + removed.push(name); + } + if (active.size === 0) { + // Nothing is instrumented any more, so a later instrument() starts from a + // clean slate rather than inheriting a degraded call site or a warning that + // has "already been shown". + core.resetFailures(); + compat.resetWarnings(); + } + return removed; +} diff --git a/sdk/typescript/src/integrations/langchain.ts b/sdk/typescript/src/integrations/langchain.ts new file mode 100644 index 000000000..4a6c7aab6 --- /dev/null +++ b/sdk/typescript/src/integrations/langchain.ts @@ -0,0 +1,2340 @@ +/** + * LangChain.js and LangGraph.js. + * + * The TypeScript twin of the Python SDK's `integrations/langchain.py`, and it + * is held to that adapter's output: both SDKs write into one pipe and the + * dashboard cannot tell which language wrote an event, so for the same program + * they must draw the same tree. Every mapping decision below is Python's, and + * where JavaScript forced a different mechanism the comment says why. Verified + * against `@langchain/core` 0.3.80 + `@langchain/langgraph` 0.4.10 and + * `@langchain/core` 1.2.12 + `@langchain/langgraph` 1.4.17 — the two fixtures + * under `integration/fixtures/langchain-*`, which run this file from the packed + * tarball as both ES module and CommonJS. + * + * ## The mapping + * + * LangChain / LangGraph FailproofAI + * --------------------------- ------------------------------------------ + * root run (no parent) agent_start / agent_end + * LangGraph node hook_triggered / hook_completed, + * trigger_event="graph_node" + * compiled subgraph nested agent_start ("root/node") + * tool run tool_use / tool_result (the MODEL's id) + * retriever run tool_use / tool_result (summarised) + * chat model / LLM run model_request / model_response + * interrupt() human_wait + agent_pause + * Command({ resume }) agent_resume + human_input + * intermediate chains nothing (see `includeChains`) + * + * **A LangGraph node is a hook, not a nested agent.** `agent_id` is a + * `LowCardinality(String)` column and the primary facet on every dashboard + * surface, and a session is labelled with the first `agent_id` it saw — so + * promoting `retrieve`, `grade_documents` and `should_continue` to agents would + * drown the facet and name the session after whichever node ran first. Hook + * spans draw identically on the timeline, and `/hooks` becomes a per-node + * latency page for free. (The first release of this adapter made every node an + * agent, which is exactly the trace this paragraph exists to prevent.) + * + * ## Where it attaches + * + * `CallbackManager.configure` and `CallbackManager._configureSync` — the two + * functions every runnable calls to build the manager for an invocation, and + * the one LangGraph's Pregel loop calls for the graph itself. Patching them + * attaches the handler to every `invoke`/`stream`/`batch` in the process + * without the caller passing `callbacks:` anywhere. LangChain.js has no + * supported global-handler registry (Python has `register_configure_hook`), so + * this is the only placement that works without editing call sites. Both are + * patched on EVERY loaded copy of `@langchain/core` — see + * `compat.requireModuleCopies` for why there can be two — and on every copy + * nested under a dependency that pinned its own (see `nestedManagers`). + * + * `langchainHandler()` is the patch-free path: the same handler, passed + * explicitly. It works with or without `instrument()`, and the two together do + * not double-record, because attaching is idempotent on a marker the handler + * carries rather than on object identity. + * + * ## Why the handler is awaited + * + * `awaitHandlers: true` is not optional, for the same reason Python sets + * `run_inline = True`. Without it LangChain queues every callback on a + * background promise queue: callbacks land after the call that caused them + * returned, possibly after the process flushed its spool, and — the part that + * is not recoverable — in whatever async context the queue happens to run in, + * so an enclosing `failproofai.agent()` scope is invisible to them and a graph + * that should nest under it becomes a second, unrelated session. Every + * callback here is synchronous bookkeeping plus an in-memory `submit`, so + * awaiting it costs nothing measurable. + * + * ## Control flow is not failure + * + * LangGraph reports an `interrupt()` through the same `handleChainError` as a + * genuine failure — the node run "errors" with a `GraphInterrupt`. Reporting it + * would paint a red error plus `agent_end(outcome="failed")` on every human + * approval. Anything LangGraph marks `is_bubble_up` (`GraphInterrupt`, + * `NodeInterrupt`, `ParentCommand`, `GraphDrained`), or whose name says it is + * one, is treated as control flow and emits the human-in-the-loop pairs + * instead. + */ + +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { sessionId as ambientSessionId } from "../context.js"; +import { logger } from "../logger.js"; +import { isCancellation as isScopeCancellation } from "../scopes.js"; +import { + entryIsCommonJs, + importModule, + isRequired, + nestedCopies, + nodeRequire, + resolveExportsAt, + resolveFrom, +} from "../node-require.js"; +import * as compat from "./compat.js"; +import * as core from "./core.js"; +import type { Adapter } from "./core.js"; + +const NAME = "langchain"; +const PACKAGE = "@langchain/core"; +const GRAPH_PACKAGE = "@langchain/langgraph"; + +/** + * The documented per-call session key — the same string the Python SDK reads, + * so one config object means the same thing to both: + * + * graph.invoke(input, { metadata: { failproofai_sdk_session_id: requestId } }) + */ +export const SESSION_METADATA_KEY = "failproofai_sdk_session_id"; + +/** + * Checked in order after the explicit key. `thread_id` is last because a thread + * is a *conversation*: two turns on one thread are two runs, and a caller who + * wants them merged says so with one of the earlier keys. + */ +const SESSION_METADATA_FALLBACKS = ["session_id", "conversation_id", "thread_id"] as const; + +/** + * LangSmith's convention for "machinery, not user-visible work". Demoted rather + * than dropped: never a span, but kept in the parent chain so its children + * still find the agent above them. + */ +const HIDDEN_TAG = "langsmith:hidden"; + +/** + * A run of one of these types is a leaf — a model call, a tool call, a + * retrieval — and is never a LangGraph node's own run: whatever is handed to + * `addNode`, the node's own run is a chain run and the thing passed runs as its + * child. + */ +const LEAF_RUN_TYPES: ReadonlySet = new Set(["llm", "chat_model", "tool", "retriever"]); + +/** LangChain tags every step of a `RunnableSequence` `seq:step:N`; a node is `graph:step:N`. */ +const INNER_STEP_TAG = "seq:step:"; + +/** + * Name-based fallback for `is_bubble_up`. Getting this wrong is expensive and + * silent — a red error on every human approval — so it is worth a second check. + */ +const CONTROL_FLOW_NAMES: ReadonlySet = new Set([ + "GraphBubbleUp", + "GraphInterrupt", + "NodeInterrupt", + "ParentCommand", + "GraphDrained", +]); + +/** + * JavaScript's own cancellation: an `AbortSignal` firing inside a run, which is + * how a stream whose consumer went away and a request whose client + * disconnected both end. The analogue of Python's `GeneratorExit` / + * `CancelledError`, and for the same reason: a stopped stream is not a crashed + * one, and must not flip a healthy session to failed. + */ +const CANCELLATION_NAMES: ReadonlySet = new Set(["AbortError"]); + +/** + * LangGraph.js (>= 1.x) routes interrupt/resume lifecycle events to any handler + * carrying this marker — its `GraphCallbackHandler.isInstance` is a duck-typed + * check on exactly this registered symbol. Setting it on a plain object means + * the handler gets `handleInterrupt`/`handleResume` without this module ever + * importing LangGraph. On 0.x the symbol means nothing and the exception-path + * fallback produces the same pairs. + */ +const GRAPH_CALLBACK_HANDLER = Symbol.for("langgraph.graph_callback_handler"); + +/** + * Marks our handler, so "is it already attached?" is answered by what the + * handler IS rather than which object it is. Identity is not enough: a manager + * built from `callbacks: [langchainHandler()]` and then passed through the + * patched `configure` would otherwise carry it once per path. + */ +const HANDLER_MARK = Symbol.for("failproofai.langchain.handler"); + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +/** Everything `instrument("langchain", ...)` and `langchainHandler()` accept. */ +export interface LangChainOptions { + /** Pin every run to this session id (step 1 of the resolution order). */ + sessionId?: string; + /** Drop prompts, messages and outputs; keep structure, durations and tokens. */ + captureContent?: boolean; + /** Record these intermediate chains, by run name, as `trigger_event="pipeline"` hooks. */ + includeChains?: string | Iterable; + /** LangGraph interrupt/resume lifecycle callbacks (LangGraph.js >= 1). Default on. */ + graphCallbacks?: boolean; + /** Per-value truncation ceiling. Default: the core field limit. */ + captureLimit?: number | string; +} + +interface Options { + sessionId: string | null; + includeChains: ReadonlySet; + captureContent: boolean; + graphCallbacks: boolean; + captureLimit: number; +} + +const KNOWN_OPTIONS: ReadonlySet = new Set([ + "sessionId", + "includeChains", + "captureContent", + "graphCallbacks", + "captureLimit", +]); + +function defaultOptions(): Options { + return { + sessionId: null, + includeChains: new Set(), + captureContent: true, + graphCallbacks: true, + captureLimit: core.FIELD_LIMIT, + }; +} + +/** + * Validate `captureLimit`, falling back rather than throwing. + * + * `instrument()` with no name installs every detected adapter with the same + * options object, so a value meant for — or mistyped for — another framework + * must never take this one down. `Infinity`, the obvious spelling of "capture + * everything", is not an integer and falls back too (it is what broke the + * Python adapter's startup under strict mode). + */ +export function captureLimitOf(value: unknown): number { + if (value === undefined || value === null) return core.FIELD_LIMIT; + const limit = typeof value === "string" && value.trim() !== "" ? Number(value) : value; + if (typeof limit !== "number" || !Number.isInteger(limit)) { + logger.warn(`langchain captureLimit=${display(value)} is not an integer; using ${core.FIELD_LIMIT}`); + return core.FIELD_LIMIT; + } + if (limit < 1) { + logger.warn(`langchain captureLimit=${limit} must be >= 1; using ${core.FIELD_LIMIT}`); + return core.FIELD_LIMIT; + } + return limit; +} + +/** `LangChainOptions` (or the shared `instrument()` bag) -> the validated `Options`. */ +export function readOptions(options: Record = {}): Options { + const unknown = Object.keys(options).filter((key) => !KNOWN_OPTIONS.has(key)); + if (unknown.length > 0) { + // Not fatal: a bare `instrument()` hands every adapter the same options, + // so one meant for another framework legitimately arrives here. + logger.debug(`langchain adapter ignoring options ${JSON.stringify(unknown.sort())}`); + } + const include = options.includeChains; + let chains: string[] = []; + if (typeof include === "string") chains = [include]; + else if (include !== null && typeof include === "object" && Symbol.iterator in include) { + chains = [...(include as Iterable)].map(String); + } + const sid = options.sessionId; + return { + sessionId: sid === undefined || sid === null || sid === "" ? null : display(sid), + includeChains: new Set(chains), + captureContent: options.captureContent === undefined ? true : Boolean(options.captureContent), + graphCallbacks: options.graphCallbacks === undefined ? true : Boolean(options.graphCallbacks), + captureLimit: captureLimitOf(options.captureLimit), + }; +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/** + * One FailproofAI session, which may outlive a single `.invoke()`. + * + * It has to: a human-in-the-loop graph runs `invoke()`, interrupts, and is + * resumed by a *second* `invoke()` minutes later. Both are the same session and + * the same root agent, and the agent stays open across the gap so that + * `agent_pause` -> `agent_resume` measures the wait. + */ +interface Session { + sessionId: string; + agentKey: string; + agentId: string; + /** pause id -> the prompt it asked, for `human_input.fw_prompt`. */ + openPauses: Map; + reportedError: boolean; + /** When its root ended with a pause still open; null while running. */ + pausedAt: number | null; +} + +/** + * Bookkeeping for a resume whose pause was opened in ANOTHER process. Set on + * the ROOT run only, and only when this process has no open pause of its own to + * close. See `closeRemotePause`. + */ +interface RemoteResume { + value: unknown; + /** level key (checkpoint-ns prefix) -> the `langgraph_step` of the first node seen there. */ + levels: Map; + /** The deepest level LangGraph said is resuming (`handleResume`, LangGraph.js >= 1). */ + deepest: string | null; + done: Set; +} + +type RunType = "chain" | "llm" | "chat_model" | "tool" | "retriever"; +type Kind = "" | "root" | "node" | "tool" | "retriever" | "model" | "chain" | "subgraph"; + +/** What we need about a LangChain run after its start callback returns. */ +interface RunInfo { + id: string; + parent: string | null; + name: string; + runType: RunType; + started: number; + hidden: boolean; + kind: Kind; + /** Set only on a ROOT run that is itself a leaf — a bare `model.invoke()`, a standalone tool. */ + leafKind: "" | "model" | "tool" | "retriever"; + root: string | null; + session: Session | null; + node: string | null; + toolCallId: string | null; + model: string | null; + ttftMs: number | null; + chunks: number; + remote: RemoteResume | null; + tags: string[]; + meta: Record; + /** Chain inputs, kept for `goal` and for recovering a tool call's id on core 0.3. */ + inputs: unknown; + /** Tool-call ids already handed to a child tool run (0.3 id recovery). */ + claimed: Set | null; + /** ROOT only: bumped by every callback under this root (see `reapIfAbandoned`). */ + activity: number; +} + +interface StartArgs { + id: string; + parent: string | null; + name: string; + runType: RunType; + tags: string[]; + meta: Record; + inputs: unknown; + /** The chat model's messages, still as `BaseMessage` objects. */ + messages?: unknown[]; + invocationParams?: Record; + toolCallId?: string; +} + +interface EndArgs { + outputs?: unknown; + error?: unknown; + response?: unknown; +} + +const MAX_RUNS = 10_000; +const MAX_SESSIONS = 1_000; + +/** + * How long this process keeps a run that paused on a human. + * + * An interrupted graph deliberately leaves its agent open so the resume can + * continue it — but the resume usually lands on ANOTHER worker, which already + * handles it (`closeRemotePause`), and then this process would hold the agent + * forever: a slot in the tracker that live runs need, plus a linear cost on + * every lookup that scanned open agents. After this long it is forgotten + * without emitting anything. A resume that does arrive here later takes the + * same path a cross-worker resume does, so nothing is lost but the in-process + * shortcut. + */ +export const PAUSED_SESSION_TTL_MS = 15 * 60_000; + +/** + * All cross-callback state, module level on purpose: one handler object serves + * every callback manager in the process, and a start and its end are separate + * calls on possibly different async branches. + */ +class State { + /** + * The kill switch `uninstrument()` flips. Restoring `configure` stops new + * managers getting the handler, but a handler object somebody already holds + * (from `langchainHandler()`) or a manager already built would keep + * recording; this makes every entry point a no-op instead. + */ + enabled = false; + installed = false; + options: Options = defaultOptions(); + tracker: core.RunTracker = State.newTracker(core.FIELD_LIMIT); + runs = new Map(); + sessions = new Map(); + + static newTracker(limit: number): core.RunTracker { + return new core.RunTracker(NAME, { baseFields: baseFields(), fieldLimit: limit }); + } + + configure(options: Options): void { + this.options = options; + this.tracker = State.newTracker(options.captureLimit); + this.runs.clear(); + this.sessions.clear(); + } + + reset(): void { + this.tracker.reset(); + this.runs.clear(); + this.sessions.clear(); + } + + /** + * FIFO; a `Map` keeps insertion order. Orphaned entries are normal — a + * cancelled stream, a crashed node, a framework that skipped an end callback + * — and unbounded, each table is a memory leak in a long-lived server. + */ + evict(): void { + while (this.runs.size >= MAX_RUNS) { + const oldest = this.runs.keys().next(); + if (oldest.done) break; + this.runs.delete(oldest.value); + this.tracker.unlink(oldest.value); + } + while (this.sessions.size >= MAX_SESSIONS) { + const oldest = this.sessions.keys().next(); + if (oldest.done) break; + this.dropSession(oldest.value); + } + } + + /** + * Forget sessions paused longer than `PAUSED_SESSION_TTL_MS`. + * + * Paused sessions are the only ones that outlive their root, and a `Map` + * keeps insertion order, so a sweep from the front stops at the first one + * that is either not paused or not yet stale. + */ + sweepPaused(now: number): void { + for (const [id, session] of this.sessions) { + if (session.pausedAt === null) continue; + if (now - session.pausedAt <= PAUSED_SESSION_TTL_MS) break; + this.dropSession(id); + } + } + + /** Remove a session; a paused one's agent is forgotten, never closed. */ + dropSession(id: string): void { + const session = this.sessions.get(id); + this.sessions.delete(id); + if (session !== undefined && session.pausedAt !== null) this.tracker.forget(session.agentKey); + } +} + +function baseFields(): Record { + const fields = core.frameworkFields(NAME, PACKAGE); + const graphVersion = compat.versionString(GRAPH_PACKAGE); + if (graphVersion) fields.fw_langgraph_version = graphVersion; + return fields; +} + +const state = new State(); +let patcher: core.Patcher | null = null; + +// --------------------------------------------------------------------------- +// Small readers +// --------------------------------------------------------------------------- + +type Loose = Record; + +const isObject = (value: unknown): value is Loose => typeof value === "object" && value !== null; + +function isPlainObject(value: unknown): value is Loose { + if (!isObject(value)) return false; + const proto = Object.getPrototypeOf(value) as unknown; + return proto === Object.prototype || proto === null; +} + +/** `runName` first, then the serialized `name`, then the last `id` segment (the class name). */ +function runNameOf(serialized: unknown, runName: unknown, fallback: string): string { + if (typeof runName === "string" && runName) return runName; + const value = serialized as { name?: unknown; id?: unknown } | undefined; + if (typeof value?.name === "string" && value.name) return value.name; + const id = value?.id; + if (Array.isArray(id) && id.length > 0) { + const last = id[id.length - 1] as unknown; + if (typeof last === "string" && last) return last; + } + return fallback; +} + +/** A LangChain message's type (`human`, `ai`, `tool`, `system`), or null for anything else. */ +function messageType(value: unknown): string | null { + if (!isObject(value)) return null; + for (const method of ["getType", "_getType"]) { + const fn: unknown = value[method]; + if (typeof fn === "function") { + try { + const type = (fn as () => unknown).call(value); + if (typeof type === "string" && type) return type; + } catch { + // A message whose type getter throws is still not worth failing over. + } + } + } + return null; +} + +/** + * The payload view of a LangChain value: messages as `{type, content, ...}`, + * anything else `Serializable` as its constructor kwargs. + * + * `truncate()` would otherwise reach these through `toJSON()`, which LangChain + * defines as its SERIALISATION envelope — `{lc: 1, type: "constructor", id: + * [...], kwargs}` — so every graph-state payload would render as class paths + * with the content buried one level down. The Python adapter gets the + * equivalent of this view from pydantic's `model_dump`. + */ +function plain(value: unknown, depth = 0): unknown { + if (!isObject(value) || depth > 6) return value; + if (Array.isArray(value)) return value.map((item) => plain(item, depth + 1)); + const type = messageType(value); + if (type !== null) { + const out: Loose = { type, content: plain(value.content, depth + 1) }; + for (const key of ["name", "id", "tool_call_id", "status"]) { + if (value[key] !== undefined && value[key] !== null) out[key] = value[key]; + } + for (const key of ["tool_calls", "usage_metadata"]) { + const field = value[key]; + if (field !== undefined && field !== null && !(Array.isArray(field) && field.length === 0)) { + out[key] = plain(field, depth + 1); + } + } + return out; + } + if (isPlainObject(value)) { + const out: Loose = {}; + for (const [key, item] of Object.entries(value)) out[key] = plain(item, depth + 1); + return out; + } + const kwargs = (value as Loose).lc_kwargs; + if (isObject(kwargs)) return plain(kwargs, depth + 1); + // A LangGraph `Command` / `Send` — what every `createAgent` model node + // returns. Not `Serializable`, so `truncate()` would dump it through its own + // `toJSON()` and every message inside through LangChain's envelope. Take the + // dump here instead and give its contents the same payload view. + const dump = (value as { toJSON?: unknown }).toJSON; + if (typeof dump === "function") { + try { + const dumped: unknown = (dump as () => unknown).call(value); + if (isPlainObject(dumped) && dumped.lc === undefined) return plain(dumped, depth + 1); + } catch { + // A throwing `toJSON` is the value's problem; `truncate` renders it. + } + } + return value; +} + +const limit = (): number => state.options.captureLimit; + +/** Payload discipline for the big three: inputs, outputs, graph state. */ +function shrink(value: unknown): unknown { + if (!state.options.captureContent || value === undefined) return undefined; + return core.truncate(plain(value), limit()); +} + +function tagsOf(tags: unknown): string[] { + return Array.isArray(tags) ? tags.filter((tag): tag is string => typeof tag === "string") : []; +} + +function metaOf(metadata: unknown): Record { + return isObject(metadata) ? { ...metadata } : {}; +} + +/** + * The LangGraph node name iff this run is the node's OWN run. + * + * Every inner runnable inherits `langgraph_node` from the node that contains + * it, so the metadata alone matches the node, the chat model inside it, the + * tool it called and each conditional-edge function. Only the node's own run + * is NAMED after the node — but the name is the user's to choose on both + * sides, and the Python adapter verified three collisions that each deleted + * the most valuable event in the trace: a node named after its tool swallowed + * the tool pair, a node named after its model swallowed the model pair, and an + * inner runnable carrying the node's name doubled the node's visits. So the run + * must also be SHAPED like a node's own run: a non-leaf run type, and not an + * inner step of a `RunnableSequence`. Both are exclusions, so if LangGraph ever + * stops emitting `seq:step:` tags this degrades to a duplicate span rather than + * to no spans. + */ +export function nodeOf( + run: { name: string; runType: string; tags: readonly string[] }, + meta: Record, +): string | null { + const node = meta.langgraph_node; + if (typeof node !== "string" || !node || node !== run.name) return null; + if (LEAF_RUN_TYPES.has(run.runType)) return null; + if (run.tags.some((tag) => tag.startsWith(INNER_STEP_TAG))) return null; + return node; +} + +/** + * `langgraph_checkpoint_ns` split into its `name:uuid` segments. One segment + * for a top-level node, `child:uuid|inner:uuid` inside a compiled subgraph: the + * number beyond the first is the nesting depth, and the leading segments name + * the subgraphs — which is how nested agents get their ids without recognising + * a compiled graph from a callback. + */ +function nsParts(meta: Record): string[] { + const ns = meta.langgraph_checkpoint_ns; + return typeof ns === "string" && ns ? ns.split("|") : []; +} + +function errorName(error: unknown): string { + if (isObject(error)) { + if (typeof error.name === "string" && error.name) return error.name; + const ctor = (error as { constructor?: { name?: unknown } }).constructor?.name; + if (typeof ctor === "string" && ctor) return ctor; + } + return "Error"; +} + +function errorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + if (isObject(error) && typeof error.message === "string") return error.message; + return String(error); +} + +/** Names on the error AND its class chain — a subclass keeps its parent's meaning. */ +function errorNames(error: unknown): string[] { + const names = [errorName(error)]; + let proto: unknown = isObject(error) ? Object.getPrototypeOf(error) : null; + while (isObject(proto)) { + const ctor = (proto as { constructor?: { name?: unknown } }).constructor?.name; + if (typeof ctor === "string") names.push(ctor); + proto = Object.getPrototypeOf(proto); + } + return names; +} + +export function isControlFlow(error: unknown): boolean { + if (!isObject(error)) return false; + if (error.is_bubble_up === true) return true; + return errorNames(error).some((name) => CONTROL_FLOW_NAMES.has(name)); +} + +export function isCancellation(error: unknown): boolean { + if (!isObject(error)) return false; + if (isScopeCancellation(error)) return true; + if (errorNames(error).some((name) => CANCELLATION_NAMES.has(name))) return true; + // LangChain's own abort sentinels, bare `Error`s identified only by their + // text: core's `raceWithSignal` rejects with "Aborted" when the signal + // carries no reason of its own, and LangGraph.js 0.x fails the graph's root + // run with "Abort". + return ( + error instanceof Error && + error.name === "Error" && + (error.message === "Aborted" || error.message === "Abort") + ); +} + +function outcomeOf(error: unknown): string { + if (error === undefined || error === null) return "success"; + if (isControlFlow(error)) return "paused"; + // Before `failed`, or an abandoned stream reads as a crash. + if (isCancellation(error)) return "cancelled"; + return "failed"; +} + +/** + * The error as one short line — `"Error: model exploded"` — never the stack. + * The stack belongs on an `error` event's `traceback`; inline in + * `tool_result.error` it is unreadable. + */ +function errorText(error: unknown): string | undefined { + if (error === undefined || error === null || isControlFlow(error)) return undefined; + return core.truncate(`${errorName(error)}: ${errorMessage(error)}`, limit()) as string; +} + +// --------------------------------------------------------------------------- +// Session resolution +// --------------------------------------------------------------------------- + +/** + * Pick the session id for a root run, first that produces a value wins: + * + * 1. `instrument("langchain", { sessionId })`; + * 2. `metadata.failproofai_sdk_session_id` on the call — the documented + * per-call key, the one to use in a web service; + * 3. an enclosing `failproofai.session()` / `failproofai.agent()` scope, so a + * hand-written outer bracket and the adapter produce ONE session; + * 4. `metadata.session_id | conversation_id | thread_id`; + * 5. the root run id. + * + * Never synthesised from scratch: a made-up id splits one run into many + * sessions, a silent wrong answer rather than a loud one. + */ +function resolveSessionId(id: string, meta: Record): string { + if (state.options.sessionId) return state.options.sessionId; + const explicit = meta[SESSION_METADATA_KEY]; + if (explicit !== undefined && explicit !== null && explicit !== "") return display(explicit); + const ambient = ambientSessionId(); + if (ambient) return ambient; + for (const key of SESSION_METADATA_FALLBACKS) { + const value = meta[key]; + if (value !== undefined && value !== null && value !== "") return display(value); + } + return id; +} + +// --------------------------------------------------------------------------- +// Emission helpers +// --------------------------------------------------------------------------- + +function emit(method: core.EventMethod, info: RunInfo, fields: Record): void { + state.tracker.emit(method, info.id, { parentKey: info.parent, ...fields }); +} + +function emitOnAgent(session: Session, method: core.EventMethod, fields: Record): void { + state.tracker.emit(method, session.agentKey, fields); +} + +/** + * The `fw_*` extras every event from this adapter carries. Namespaced as a + * SAFETY rule: the schema merges extras last, so an extra called `tool_name` + * or `outcome` would silently overwrite the declared field. + */ +function fwCommon(info: RunInfo): Record { + const meta = info.meta; + return core.fwFields({ + run_id: info.id, + parent_run_id: info.parent ?? undefined, + node: info.node ?? meta.langgraph_node, + step: meta.langgraph_step, + checkpoint_ns: meta.langgraph_checkpoint_ns, + thread_id: meta.thread_id, + tags: info.tags.length > 0 ? info.tags : undefined, + hidden: info.hidden ? true : undefined, + }); +} + +// --------------------------------------------------------------------------- +// Start +// --------------------------------------------------------------------------- + +function onStart(args: StartArgs): void { + if (!state.enabled) return; + state.evict(); + if (args.parent === null) state.sweepPaused(Date.now()); + const info: RunInfo = { + id: args.id, + parent: args.parent, + name: args.name, + runType: args.runType, + started: Date.now(), + hidden: args.tags.includes(HIDDEN_TAG), + kind: "", + leafKind: "", + root: null, + session: null, + node: null, + toolCallId: null, + model: null, + ttftMs: null, + chunks: 0, + remote: null, + tags: args.tags, + meta: args.meta, + inputs: args.runType === "chain" ? args.inputs : undefined, + claimed: null, + activity: 0, + }; + state.runs.set(info.id, info); + // Every run is linked, span or not: a tool three runnables deep still finds + // the agent above it by walking the chain, and an intermediate chain that + // emits nothing would otherwise break the walk. + state.tracker.link(info.id, info.parent); + + if (info.parent === null) { + // A LangGraph node never runs outside its graph. One arriving as a root + // means the graph's own run began before the callback was installed. + if (nodeOf(info, info.meta) !== null) warnOrphan(info); + startRoot(info, args); + return; + } + + const holder = state.runs.get(info.parent); + if (holder === undefined) warnOrphan(info); + info.root = holder?.root ?? null; + info.session = holder?.session ?? null; + touch(info.root); + + const node = nodeOf(info, info.meta); + if (node !== null) { + info.kind = "node"; + info.node = node; + startNode(info, args); + return; + } + if (info.runType === "llm" || info.runType === "chat_model") { + info.kind = "model"; + startModel(info, args); + return; + } + if (info.runType === "tool") { + info.kind = "tool"; + startTool(info, args); + return; + } + if (info.runType === "retriever") { + info.kind = "retriever"; + startRetriever(info, args); + return; + } + // Everything else — RunnableSequence, prompt templates, output parsers, + // conditional-edge functions, a compiled subgraph's own run. Linked above and + // otherwise invisible unless explicitly allowlisted. + if (info.name && state.options.includeChains.has(info.name) && !info.hidden) { + info.kind = "chain"; + emit("hookTriggered", info, { + hookName: info.name, + hookId: info.id, + triggerEvent: "pipeline", + input: shrink(args.inputs), + ...fwCommon(info), + }); + } +} + +/** + * A run whose parent this adapter never saw start. The usual cause is an + * `instrument()` that was not awaited: the graph's root run began before the + * callback was installed, so its children arrive with a parent nobody knows + * and a node or a model call ends up as the session's agent — a wrong trace, + * with nothing said. Warned once per process; the trace itself cannot be + * repaired after the fact. + */ +let warnedOrphan = false; + +function warnOrphan(info: RunInfo): void { + if (warnedOrphan || info.hidden) return; + warnedOrphan = true; + logger.warn( + `a LangChain run (${JSON.stringify(info.name)}) started under a parent run the langchain ` + + "adapter never saw (or, being a graph node, with no parent at all), so its trace is " + + "missing the root. Most often `instrument()` was not " + + "awaited before the run began — `await failproofai.instrument()` at startup, before the " + + "first invoke/stream.", + ); +} + +/** @internal Re-arm the once-per-process orphan warning, for tests. */ +export function resetOrphanWarning(): void { + warnedOrphan = false; +} + +/** + * The root run becomes the session's agent — and its FIRST event. A session is + * labelled by the first `agent_id` it saw, and the dashboard parents every leaf + * to the open agent with the same `agent_id`, synthesising a never-ending root + * span when there is none — so this must never be skipped or preceded. + */ +function startRoot(info: RunInfo, args: StartArgs): void { + info.kind = "root"; + info.root = info.id; + const sessionId = resolveSessionId(info.id, info.meta); + + const existing = state.sessions.get(sessionId); + if ( + existing !== undefined && + existing.openPauses.size > 0 && + state.tracker.isOpen(existing.agentKey) && + isContinuation(args.inputs, info.meta) + ) { + // A resume: the previous `.invoke()` interrupted, we deliberately left its + // agent open, and this continues it. Both halves of the test are needed. + // "The session's agent is still open" is also true of two roots that merely + // OVERLAP under one session id — `.batch()`, two requests on one + // conversation id — and reading those as a resume folds one root into the + // other and drops its events. And an open pause bounds the window without + // closing it: any other run on the same session id while a human thinks — + // a different graph, a background summariser — would be recorded as the + // human's answer. LangGraph only continues an interrupted thread through a + // `Command` or a `null` input, so that is what is required. + info.session = existing; + state.tracker.link(info.id, existing.agentKey); + resume(existing, args.inputs); + return; + } + + const identity = state.tracker.startAgent(info.id, { + agentId: core.normalizeAgentId(info.name, "agent"), + sessionId, + goal: goalOf(args), + ...fwCommon(info), + }); + const session: Session = { + sessionId: identity.sessionId ?? sessionId, + agentKey: info.id, + agentId: identity.agentId ?? "agent", + openPauses: new Map(), + reportedError: false, + pausedAt: null, + }; + info.session = session; + state.sessions.set(session.sessionId, session); + + // A resume, but nothing in THIS process is paused — so the pause was opened + // by another process. That is the ordinary deployment shape (one worker + // serves the interrupt, whichever worker picks up the approval resumes + // against the shared checkpointer), and without this its human_wait and + // agent_pause stay open forever. See `closeRemotePause`. + const answer = resumeValue(args.inputs); + if (answer !== undefined && answer !== null) { + info.remote = { value: answer, levels: new Map(), deepest: null, done: new Set() }; + } + + // A root run that is ITSELF a leaf is recorded as one too. A bare + // `model.invoke()` handled only as a root produced agent_start/agent_end and + // nothing else — no model name, no tokens, no latency — while the trace + // still looked populated. The agent span stays; the leaf pair lands inside. + if (info.runType === "llm" || info.runType === "chat_model") { + info.leafKind = "model"; + startModel(info, args); + } else if (info.runType === "tool") { + info.leafKind = "tool"; + startTool(info, args); + } else if (info.runType === "retriever") { + info.leafKind = "retriever"; + startRetriever(info, args); + } +} + +function goalOf(args: StartArgs): string | undefined { + if (!state.options.captureContent) return undefined; + // A chat model's input is a list of message BATCHES; its goal is the last + // message of the last one, exactly as a graph's is the last of its state. + const batches = args.messages; + const lastBatch = Array.isArray(batches) && Array.isArray(batches[batches.length - 1]) + ? (batches[batches.length - 1] as unknown[]) + : batches; + const inputs = args.runType === "chat_model" ? { messages: lastBatch } : args.inputs; + if (inputs === undefined || inputs === null) return undefined; + if (isObject(inputs) && Array.isArray(inputs.messages) && inputs.messages.length > 0) { + const last = inputs.messages[inputs.messages.length - 1] as unknown; + const content = isObject(last) ? last.content : undefined; + if (typeof content === "string" && content) return core.truncate(content, 512) as string; + } + if (typeof inputs === "string") return core.truncate(inputs, 512) as string; + try { + return core.truncate(JSON.stringify(plain(inputs)), 512) as string; + } catch { + return undefined; + } +} + +/** + * A LangGraph node -> `hook_triggered`. Also where a compiled SUBGRAPH becomes + * a nested agent: a node whose checkpoint namespace is more than one segment + * deep runs inside one, and its parent run IS the subgraph's own run. Deriving + * it here nests to any depth without recognising a compiled graph. + */ +function startNode(info: RunInfo, args: StartArgs): void { + const parts = nsParts(info.meta); + if (parts.length > 1 && info.parent !== null) ensureSubgraphAgent(info, parts.slice(0, -1)); + + const remote = remoteOf(info); + if (remote !== null) { + // First node seen at a level wins: LangGraph re-runs the interrupted tasks + // in the level's first superstep and nothing else, so anything at a later + // step is ordinary downstream work. + const level = parts.slice(0, -1).join("|"); + if (!remote.levels.has(level)) remote.levels.set(level, info.meta.langgraph_step); + } + + if (info.hidden) return; + emit("hookTriggered", info, { + hookName: info.node, + hookId: info.id, + triggerEvent: "graph_node", + input: shrink(args.inputs), + ...fwCommon(info), + }); +} + +function ensureSubgraphAgent(info: RunInfo, prefix: string[]): void { + const key = info.parent; + if (key === null || state.tracker.isOpen(key)) return; + const holder = state.runs.get(key); + const session = info.session; + if (holder === undefined || session === null) return; + const names = prefix.filter(Boolean).map((part) => part.split(":")[0]!); + state.tracker.startAgent(key, { + agentId: [session.agentId, ...names].join("/"), + parentKey: holder.parent, + sessionId: session.sessionId, + ...core.fwFields({ run_id: key, subgraph: names[names.length - 1], kind: "subgraph" }), + }); + holder.kind = "subgraph"; + holder.session = session; +} + +function startTool(info: RunInfo, args: StartArgs): void { + // The MODEL's tool-call id when there is one, so a tool_use joins to the + // `tool_calls[]` entry that asked for it and to the provider's own logs. + info.toolCallId = args.toolCallId ?? recoverToolCallId(info, args.inputs) ?? info.id; + if (info.hidden) return; + emit("toolUse", info, { + toolName: info.name || "tool", + toolCallId: info.toolCallId, + input: state.options.captureContent ? toolInput(args.inputs) : undefined, + ...fwCommon(info), + }); +} + +/** `handleToolStart` hands over a string; the tool's arguments are JSON inside it. */ +function toolInput(input: unknown): Record | undefined { + if (input === undefined || input === null) return undefined; + if (isPlainObject(input)) return core.truncate(plain(input), limit()) as Record; + return { input: core.truncate(plain(input), limit()) }; +} + +function parseToolInput(input: unknown): unknown { + if (typeof input !== "string") return input; + const text = input.trim(); + if (!text.startsWith("{")) return input; + try { + return JSON.parse(text) as unknown; + } catch { + return input; + } +} + +/** Order-insensitive JSON, for comparing a tool call's `args` with the tool's parsed input. */ +function canonical(value: unknown): string { + const sort = (item: unknown): unknown => { + if (Array.isArray(item)) return item.map(sort); + if (isPlainObject(item)) { + return Object.fromEntries( + Object.keys(item) + .sort() + .map((key) => [key, sort(item[key])]), + ); + } + return item; + }; + try { + return JSON.stringify(sort(value)) ?? ""; + } catch { + return ""; + } +} + +/** + * The model's tool-call id, on a core that does not pass one. + * + * `@langchain/core` 1.x hands `handleToolStart` the id as its eighth argument; + * 0.3 does not, and neither does it put it anywhere a callback can read at + * start. Without it every `tool_use` on 0.3 carried the tool's RUN id, which + * joins to nothing: not the assistant message's `tool_calls[]`, not the + * provider's logs. + * + * It is recoverable, exactly, in the case that matters — a tool run under a + * `ToolNode` or any runnable fed the conversation. The nearest ancestor whose + * input carries messages holds the assistant message that asked for this call, + * and its `tool_calls[]` entry names the tool and carries the same arguments + * the tool was invoked with. Each id is claimed once per ancestor, so two calls + * to one tool with identical arguments still get two different ids. When there + * is no such ancestor, or no entry matches, the run id stands, as before. + */ +function recoverToolCallId(info: RunInfo, input: unknown): string | null { + const args = canonical(parseToolInput(input)); + let key = info.parent; + const seen = new Set(); + while (key !== null && !seen.has(key)) { + seen.add(key); + const holder = state.runs.get(key); + if (holder === undefined) return null; + const inputs = holder.inputs; + const messages = Array.isArray(inputs) ? inputs : isObject(inputs) ? inputs.messages : undefined; + if (Array.isArray(messages)) { + holder.claimed ??= new Set(); + const claimed = holder.claimed; + for (let i = messages.length - 1; i >= 0; i -= 1) { + const calls = (messages[i] as { tool_calls?: unknown } | undefined)?.tool_calls; + if (!Array.isArray(calls) || calls.length === 0) continue; + const open = (calls as Array<{ id?: unknown; name?: unknown; args?: unknown }>).filter( + (call) => typeof call.id === "string" && call.name === info.name && !claimed.has(call.id), + ); + const match = open.find((call) => canonical(call.args) === args) ?? (open.length === 1 ? open[0] : undefined); + if (match === undefined) return null; + claimed.add(match.id as string); + return match.id as string; + } + return null; + } + key = holder.parent; + } + return null; +} + +function startRetriever(info: RunInfo, args: StartArgs): void { + info.toolCallId = info.id; + if (info.hidden) return; + emit("toolUse", info, { + toolName: `retriever:${info.name || "retriever"}`, + toolCallId: info.toolCallId, + input: state.options.captureContent ? { query: core.truncate(args.inputs, limit()) } : undefined, + ...fwCommon(info), + }); +} + +function startModel(info: RunInfo, args: StartArgs): void { + info.model = modelNameOf(info, args.invocationParams); + if (info.hidden) return; + const messages = + args.runType === "chat_model" + ? normalizeMessages(args.messages) + : promptsAsMessages((args.inputs as { prompts?: unknown } | undefined)?.prompts); + emit("modelRequest", info, { + // The correlation id the dashboard pairs a request with its response on. + requestId: info.id, + model: info.model, + messages: state.options.captureContent ? messages : undefined, + tools: toolsOf(args.invocationParams), + ...fwCommon(info), + }); +} + +/** + * `ls_model_name` first — the LangSmith standard key a real provider + * integration sets — then the invocation params, then the model's own name. + * The fallbacks are load-bearing: fakes and some community integrations set no + * `ls_model_name` at all. + */ +function modelNameOf(info: RunInfo, params: Record | undefined): string { + const name = info.meta.ls_model_name; + if (typeof name === "string" && name) return name; + for (const key of ["model_name", "model", "modelName", "model_id", "deployment_name"]) { + const value = params?.[key]; + if (typeof value === "string" && value) return value; + } + return info.name || "unknown"; +} + +function toolsOf(params: Record | undefined): Array> | undefined { + const tools = params?.tools; + if (!Array.isArray(tools) || tools.length === 0) return undefined; + return core.truncate(tools, limit()) as Array>; +} + +const ROLES: Record = { human: "user", ai: "assistant", system: "system", tool: "tool" }; + +/** + * Chat messages as `{role, content, tool_calls?}`, from the LAST batch. They + * arrive as real `BaseMessage` objects here, which is the one place their + * roles are still intact. + */ +export function normalizeMessages(batches: unknown): Array> | undefined { + if (!Array.isArray(batches) || batches.length === 0) return undefined; + const last = batches[batches.length - 1] as unknown; + const batch = Array.isArray(last) ? last : batches; + return batch.map((message: unknown) => { + const kind = messageType(message) ?? (isObject(message) && typeof message.role === "string" ? message.role : ""); + const value = isObject(message) ? message : {}; + const entry: Record = { + role: ROLES[kind] ?? (kind || "user"), + content: core.truncate(plain(value.content ?? ""), limit()), + }; + if (Array.isArray(value.tool_calls) && value.tool_calls.length > 0) { + entry.tool_calls = core.truncate(plain(value.tool_calls), limit()); + } + return entry; + }); +} + +function promptsAsMessages(prompts: unknown): Array> | undefined { + if (!Array.isArray(prompts)) return undefined; + return prompts.map((prompt: unknown) => ({ role: "user", content: core.truncate(prompt, limit()) })); +} + +// --------------------------------------------------------------------------- +// End +// --------------------------------------------------------------------------- + +function onEnd(id: string, end: EndArgs): void { + const info = state.runs.get(id); + if (info === undefined) return; + const error = end.error; + touch(info.root); + + if (info.kind === "root") { + // Close the leaf pair first when the root was also a leaf: the dashboard + // closes the agent span at `agent_end`, so a `model_response` after it is + // attributed to nothing. + if (info.leafKind) { + endLeaf(info.leafKind, info, end); + // ...and that leaf OWNS the failure, exactly as a nested one does. + // Without this a failing top-level `tool.invoke()` counted twice: once as + // `tool_result.error` and again as a standalone `error` event. + if (info.session !== null && error !== undefined && error !== null && !isControlFlow(error)) { + info.session.reportedError = true; + } + } + endRoot(info, error); + return; + } + + state.runs.delete(id); + + if (info.kind === "subgraph" || state.tracker.isOpen(id)) { + state.tracker.endAgent(id, { outcome: outcomeOf(error), summary: errorText(error) }); + } + // `owned` records whether a SPAN was actually emitted for this run, which is + // what decides `reportedError` below. + let owned = false; + if (info.kind === "node" || info.kind === "chain") { + endHook(info, end); + owned = !info.hidden; + } else if (info.kind === "tool" || info.kind === "retriever" || info.kind === "model") { + endLeaf(info.kind, info, end); + owned = !info.hidden; + } + + if (info.kind === "node") { + // Strictly BEFORE the suspend below: a node that answers one interrupt and + // raises the next must close the old pause before opening the new one. + closeRemotePause(info); + } + + // The exception-path HITL: every LangGraph interrupt surfaces here, as the + // node's `handleChainError`. Outside the span handling on purpose, so it + // still fires for a hidden node and for a subgraph that bubbled the + // interrupt up. `suspend` dedups on the interrupt id, so this and + // `handleInterrupt` cannot double-emit. + const interrupts = interruptsOf(error); + if (interrupts.length > 0 && info.session !== null) suspend(info.session, interrupts); + + if (owned && error !== undefined && error !== null && !isControlFlow(error) && info.session !== null) { + // Only when a span reported it. The failures nobody owned — a + // RunnableSequence step, an output parser, a hidden run — must still reach + // the root's one standalone `error` event; the ones a span DID report must + // not be counted twice. + info.session.reportedError = true; + } + + if (isCancellation(error) && info.root !== null) reapIfAbandoned(info.root); + // Last, after every event above has resolved through it. See `RunTracker.unlink`. + state.tracker.unlink(id); +} + +function touch(rootId: string | null): void { + const root = rootId !== null ? state.runs.get(rootId) : undefined; + if (root !== undefined) root.activity += 1; +} + +/** How long an aborted root may stay silent before it is closed for LangGraph. */ +export const ABANDONED_ROOT_GRACE_MS = 3_000; + +/** + * Close a root that was aborted and will never be told so. + * + * An `AbortSignal` firing inside `graph.invoke()` on LangGraph.js 1.x ends the + * node's run with an `AbortError` and then abandons the graph's OWN run: the + * invoke races the signal and returns, the stream generator is never resumed, + * and no `handleChainEnd` or `handleChainError` ever arrives for the root + * (VERIFIED on 1.4.17; 0.4.10 does report it, as `Error("Abort")`). Without + * this the session reads as running forever — the JavaScript face of the + * `GeneratorExit` case the Python adapter closes as `cancelled`. + * + * It cannot be closed at the node's error: a node's own `AbortError` — its + * fetch timed out — can be retried, and the graph carries on. So the root is + * closed only if, a grace period later, it is still open, nothing under it is + * still running, and NOTHING under it has happened since: any retry, any new + * node, any end callback bumps `activity` and the reap stands down. The timer + * is `unref`'d, so it never holds a process open; a script that exits first + * leaves the root open exactly as a crash would. + */ +function reapIfAbandoned(rootId: string): void { + const root = state.runs.get(rootId); + if (root === undefined) return; + const seen = root.activity; + const timer = setTimeout(() => { + core.callSafely( + () => { + if (!state.enabled || state.runs.get(rootId) !== root || root.activity !== seen) return; + for (const info of state.runs.values()) if (info.root === rootId && info.id !== rootId) return; + const abort = new Error("the run was aborted and LangGraph never closed it"); + abort.name = "AbortError"; + endRoot(root, abort); + }, + [], + `${NAME}.reapIfAbandoned`, + ); + }, ABANDONED_ROOT_GRACE_MS); + timer.unref?.(); +} + +function endLeaf(kind: "tool" | "retriever" | "model", info: RunInfo, end: EndArgs): void { + if (info.hidden) return; + if (kind === "tool") endTool(info, end); + else if (kind === "retriever") endRetriever(info, end); + else endModel(info, end); +} + +function endHook(info: RunInfo, end: EndArgs): void { + if (info.hidden) return; + emit("hookCompleted", info, { + hookName: info.node ?? info.name, + hookId: info.id, + // "paused" for an interrupt: the node did not fail, it stopped to ask a + // human. "failed", never "failure" — the server counts only + // error|failed|timeout|rejected. + outcome: outcomeOf(end.error), + output: shrink(end.outputs), + error: errorText(end.error), + ...fwCommon(info), + }); +} + +/** + * The tool's actual result, plus an error when it failed quietly. + * + * A tool invoked with a `ToolCall` — what every tool loop does — returns a + * `ToolMessage`, not a string, and rendering that object is not the result. + * `status: "error"` is the second half: a tool whose exception the framework + * turned into a message for the model has no error anywhere else, so without + * this the failure had no representation at all. + */ +export function toolOutput(output: unknown): { output: unknown; failed?: string } { + if (messageType(output) !== "tool") return { output }; + const message = output as { content?: unknown; status?: unknown }; + if (message.status === "error") { + const text = typeof message.content === "string" ? message.content : JSON.stringify(message.content); + return { output: message.content, failed: core.truncate(text, limit()) as string }; + } + return { output: message.content }; +} + +function endTool(info: RunInfo, end: EndArgs): void { + const { output, failed } = toolOutput(end.outputs); + emit("toolResult", info, { + toolName: info.name || "tool", + toolCallId: info.toolCallId ?? info.id, + output: shrink(output), + error: errorText(end.error) ?? failed, + ...fwCommon(info), + }); +} + +function endRetriever(info: RunInfo, end: EndArgs): void { + emit("toolResult", info, { + toolName: `retriever:${info.name || "retriever"}`, + toolCallId: info.toolCallId ?? info.id, + output: summarizeDocuments(end.outputs), + error: errorText(end.error), + ...fwCommon(info), + }); +} + +/** + * `{n, sources}` — never the document text. Twenty 4 KB chunks per retrieval + * would put 80 KB of prose into one event on every hop of every RAG loop. The + * count is structure and survives `captureContent: false`; the sources do not, + * because a source is a document path and on regulated data that path is + * content. + */ +export function summarizeDocuments(documents: unknown): Record | undefined { + if (!Array.isArray(documents)) return undefined; + if (!state.options.captureContent) return { n: documents.length }; + const sources = documents.slice(0, 10).map((doc: unknown, index) => { + const meta = isObject(doc) && isObject(doc.metadata) ? doc.metadata : {}; + const source = meta.source ?? meta.id ?? meta.file_path; + return core.truncate(source ? display(source) : `doc[${index}]`, 256); + }); + return { n: documents.length, sources }; +} + +function endModel(info: RunInfo, end: EndArgs): void { + const usage = usageOf(end.response); + const completion = completionOf(end.response); + const extras: Record = { ...fwCommon(info) }; + if (info.chunks > 0) { + Object.assign(extras, core.fwFields({ streamed: true, chunks: info.chunks, ttft_ms: info.ttftMs ?? 0 })); + } + emit("modelResponse", info, { + requestId: info.id, + model: info.model, + stopReason: end.error !== undefined && end.error !== null ? "error" : completion.stopReason, + content: state.options.captureContent ? completion.content : undefined, + role: completion.role, + inputTokens: usage?.input_tokens, + outputTokens: usage?.output_tokens, + // Shipped as an object as well: both server-side summaries fall back to + // `payload.usage` for tokens. + usage, + error: errorText(end.error), + // ALWAYS set, always an integer. The dashboard prefers the closing event's + // duration, which is what keeps model durations honest when concurrent + // calls pair up by arrival, and a float would NULL the u32 column. + duration_ms: core.ms(Date.now() - info.started), + ...extras, + }); +} + +interface Usage { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + input_token_details?: unknown; + output_token_details?: unknown; +} + +const asInt = (value: unknown): number | undefined => + typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; + +function firstGeneration(response: unknown): Loose | undefined { + const generations = (response as { generations?: unknown } | undefined)?.generations; + if (!Array.isArray(generations) || generations.length === 0) return undefined; + const first = generations[0] as unknown; + const generation = Array.isArray(first) ? (first[0] as unknown) : first; + return isObject(generation) ? generation : undefined; +} + +/** + * Token counts from wherever this provider put them. + * + * `usage_metadata` on the generated message is primary — LangChain's standard + * shape and the only one carrying cache/reasoning detail. The fallbacks are + * `llmOutput`: `tokenUsage` / `estimatedTokenUsage` (camelCase, the JS OpenAI + * integration) and `token_usage` / `usage` (snake_case). Reading one place only + * is how an adapter ends up with an empty token column for half the providers, + * at 200 OK, with nothing logged. + */ +export function usageOf(response: unknown): Usage | undefined { + const message = firstGeneration(response)?.message; + const data = isObject(message) ? message.usage_metadata : undefined; + if (isObject(data) && Object.keys(data).length > 0) { + const usage: Usage = { + input_tokens: asInt(data.input_tokens), + output_tokens: asInt(data.output_tokens), + total_tokens: asInt(data.total_tokens), + }; + for (const key of ["input_token_details", "output_token_details"] as const) { + if (isObject(data[key]) && Object.keys(data[key]).length > 0) usage[key] = { ...data[key] }; + } + return compact(usage); + } + const output = (response as { llmOutput?: unknown } | undefined)?.llmOutput; + if (!isObject(output)) return undefined; + for (const raw of [output.tokenUsage, output.estimatedTokenUsage, output.token_usage, output.usage]) { + if (!isObject(raw)) continue; + const usage: Usage = { + input_tokens: asInt(raw.promptTokens ?? raw.prompt_tokens ?? raw.input_tokens), + output_tokens: asInt(raw.completionTokens ?? raw.completion_tokens ?? raw.output_tokens), + total_tokens: asInt(raw.totalTokens ?? raw.total_tokens), + }; + if (usage.input_tokens === undefined && usage.output_tokens === undefined) continue; + usage.total_tokens ??= (usage.input_tokens ?? 0) + (usage.output_tokens ?? 0); + return compact(usage); + } + return undefined; +} + +function compact(usage: Usage): Usage | undefined { + const out = Object.fromEntries(Object.entries(usage).filter(([, value]) => value !== undefined)) as Usage; + return Object.keys(out).length > 0 ? out : undefined; +} + +function completionOf(response: unknown): { content?: unknown; role?: string; stopReason?: string } { + const generation = firstGeneration(response); + if (generation === undefined) return {}; + const message = isObject(generation.message) ? generation.message : undefined; + const content = message?.content ?? generation.text; + let stopReason: string | undefined; + for (const source of [generation.generationInfo, message?.response_metadata]) { + if (!isObject(source)) continue; + for (const key of ["finish_reason", "stop_reason", "finishReason", "stopReason"]) { + const value = source[key]; + if (typeof value === "string" && value) { + stopReason = value; + break; + } + } + if (stopReason !== undefined) break; + } + return { + content: core.truncate(plain(content), limit()), + role: message !== undefined ? "assistant" : undefined, + stopReason, + }; +} + +function endRoot(info: RunInfo, error: unknown): void { + const session = info.session; + state.runs.delete(info.id); + closeOpenLeaves(info.id); + // A resumed root is linked to the session's agent rather than being one, so + // `endAgent` below would not clear its link. + state.tracker.unlink(info.id); + if (session === null) return; + + if (session.openPauses.size > 0) { + // Interrupted, waiting on a human. Deliberately no `agent_end`: closing the + // agent force-closes the open pause, zeroing the one interval that + // measures how long the human took. The resuming `.invoke()` closes it — + // here, or on another worker, in which case this process forgets it after + // `PAUSED_SESSION_TTL_MS` (see `State.sweepPaused`). + session.pausedAt = Date.now(); + return; + } + + const cancelled = isCancellation(error); + const failed = error !== undefined && error !== null && !isControlFlow(error) && !cancelled; + + if (failed && !session.reportedError) { + // Nothing below reported this failure, so nobody owns it — a standalone + // `error` is the only way it reaches the Errors surface. Strictly before + // `agent_end`, which closes the span it would be attributed to. + emitOnAgent(session, "error", { + errorType: errorName(error), + // The bare message: the server renders `: `, and the + // prefixed form would read "Error: Error: ...". + message: core.truncate(errorMessage(error), limit()) || errorName(error), + traceback: error instanceof Error && error.stack ? core.truncate(error.stack, core.FIELD_LIMIT) : undefined, + ...fwCommon(info), + }); + } + + state.tracker.endAgent(session.agentKey, { + outcome: cancelled ? "cancelled" : failed ? "failed" : "success", + summary: failed ? errorText(error) : undefined, + ...fwCommon(info), + }); + if (state.sessions.get(session.sessionId) === session) state.sessions.delete(session.sessionId); +} + +/** + * Close every leaf still open under this root. `agent_end` force-closes open + * pauses but not tools, models or hooks, so a run that dies mid-tool — a hard + * cancellation, a killed stream, a framework that skipped an end callback — + * would otherwise leave the session `ongoing` forever. + */ +function closeOpenLeaves(rootId: string): void { + const stale = [...state.runs.values()].filter((info) => info.root === rootId && info.id !== rootId); + for (const info of stale.reverse()) { + state.runs.delete(info.id); + if (info.hidden || !info.kind) { + state.tracker.unlink(info.id); + continue; + } + const marker = core.fwFields({ incomplete: true }); + core.callSafely( + () => { + if (info.kind === "tool" || info.kind === "retriever") { + emit("toolResult", info, { + toolName: info.kind === "retriever" ? `retriever:${info.name || "retriever"}` : info.name || "tool", + toolCallId: info.toolCallId ?? info.id, + ...marker, + }); + } else if (info.kind === "node" || info.kind === "chain") { + emit("hookCompleted", info, { + hookName: info.node ?? info.name, + hookId: info.id, + outcome: "cancelled", + ...marker, + }); + } else if (info.kind === "model") { + emit("modelResponse", info, { + requestId: info.id, + model: info.model, + stopReason: "incomplete", + duration_ms: core.ms(Date.now() - info.started), + ...marker, + }); + } else if (info.kind === "subgraph") { + state.tracker.endAgent(info.id, { outcome: "cancelled", ...marker }); + } + }, + [], + `${NAME}.closeOpenLeaves`, + ); + state.tracker.unlink(info.id); + } +} + +// --------------------------------------------------------------------------- +// Human in the loop +// --------------------------------------------------------------------------- + +interface InterruptLike { + id?: unknown; + value?: unknown; +} + +/** + * The interrupts a `GraphInterrupt` carries. `ParentCommand` and + * `GraphDrained` are bubble-ups too but carry a command / a reason, which is + * what the `value` check keeps out. + */ +function interruptsOf(error: unknown): InterruptLike[] { + if (!isControlFlow(error)) return []; + const interrupts = (error as { interrupts?: unknown }).interrupts; + if (!Array.isArray(interrupts)) return []; + return interrupts.filter((item): item is InterruptLike => isObject(item) && "value" in item); +} + +/** + * `human_wait` + `agent_pause`, one pair per interrupt, in that order. Both are + * required: only `agent_pause` -> `agent_resume` feeds the session's paused + * time, and only `human_wait` -> `human_input` carries the prompt and the + * answer. + */ +function suspend(session: Session, interrupts: readonly InterruptLike[]): void { + interrupts.forEach((interrupt, index) => { + const pauseId = + typeof interrupt.id === "string" && interrupt.id ? interrupt.id : `${session.agentKey}:${index}`; + if (session.openPauses.has(pauseId)) return; + const { prompt, options } = promptOf(interrupt.value); + session.openPauses.set(pauseId, prompt); + // `captureContent: false` covers these: in a real HITL graph the interrupt + // payload IS the record being approved, and the answer is a human's free + // text — the two most sensitive strings in the run. + const capture = state.options.captureContent; + emitOnAgent(session, "humanWait", { + inputId: pauseId, + prompt: capture ? prompt : undefined, + options: capture ? options : undefined, + reason: "langgraph_interrupt", + ...core.fwFields({ interrupt_id: pauseId, kind: "interrupt" }), + }); + emitOnAgent(session, "agentPause", { + pauseId, + reason: "langgraph_interrupt", + ...core.fwFields({ interrupt_id: pauseId }), + }); + }); +} + +/** A value as text: strings as-is, everything else as JSON of its payload view. */ +function display(value: unknown): string { + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") { + return value.toString(); + } + try { + return JSON.stringify(plain(value)) ?? String(value); + } catch { + return String(value); + } +} + +export function promptOf(value: unknown): { prompt?: string; options?: string[] } { + if (value === undefined || value === null) return {}; + if (isPlainObject(value)) { + const prompt = value.prompt ?? value.question ?? value.message; + const options = Array.isArray(value.options) ? value.options.map((option: unknown) => display(option)) : undefined; + const text = prompt !== undefined && prompt !== null ? display(prompt) : display(value); + return { prompt: core.truncate(text, limit()) as string, options }; + } + return { prompt: core.truncate(display(value), limit()) as string }; +} + +/** `agent_resume` + `human_input`, in that order, one pair per open pause. */ +function resume(session: Session, inputs: unknown): void { + session.pausedAt = null; + if (session.openPauses.size === 0) return; + const answers = resumeValue(inputs); + const capture = state.options.captureContent; + for (const [pauseId, prompt] of [...session.openPauses]) { + session.openPauses.delete(pauseId); + emitOnAgent(session, "agentResume", { + pauseId, + reason: "langgraph_resume", + ...core.fwFields({ interrupt_id: pauseId }), + }); + emitOnAgent(session, "humanInput", { + inputId: pauseId, + response: capture ? answerFor(answers, pauseId) : undefined, + ...core.fwFields({ interrupt_id: pauseId, prompt: capture ? prompt : undefined }), + }); + } +} + +const MISSING = Symbol("missing"); + +function isCommand(value: unknown): value is { resume?: unknown } { + if (!isObject(value)) return false; + if (value.lg_name === "Command") return true; + // Duck-typed fallback for a moved or renamed `Command`. + return "resume" in value && "goto" in value; +} + +/** + * What `.invoke()` was called with, when it was NOT fresh state. + * + * LangGraph.js hands the root's `handleChainStart` a `Command` itself (it is an + * object, so `_coerceToDict` passes it through), and wraps anything that is not + * an object under a single `input` key — `{input: null}` for + * `invoke(null, config)`. Fresh state arrives as the state object. So a + * `Command`, or an `input` key, is what separates "steering an existing + * checkpointed run" from "starting a new one". + */ +function steeringValue(inputs: unknown): unknown { + if (isCommand(inputs)) return inputs; + if (isPlainObject(inputs) && "input" in inputs) return inputs.input; + return MISSING; +} + +/** Is this run a LangGraph invocation at all? A resume always is. */ +function isGraphRun(meta: Record): boolean { + return ( + "langgraph_checkpoint_ns" in meta || + "checkpoint_ns" in meta || + "thread_id" in meta || + "langgraph_step" in meta + ); +} + +/** + * True when this root run continues an interrupted thread: a `Command`, or a + * `null` input to a graph. A bare `null` alone is not enough — any runnable + * invoked with no argument produces the same `{input: null}` shape, and + * reading an unrelated heartbeat as the human's answer would fabricate an + * approval nobody gave. + */ +function isContinuation(inputs: unknown, meta: Record): boolean { + const value = steeringValue(inputs); + if (value === MISSING) return false; + if (value === undefined || value === null) return isGraphRun(meta); + return isCommand(value); +} + +/** The value handed to `Command({ resume })`, read off the root run's input. */ +function resumeValue(inputs: unknown): unknown { + const value = steeringValue(inputs); + return value !== MISSING && isCommand(value) ? value.resume : undefined; +} + +function answerFor(answers: unknown, pauseId: string): string | undefined { + if (answers === undefined || answers === null) return undefined; + if (isPlainObject(answers) && pauseId in answers) { + return core.truncate(display(answers[pauseId]), limit()) as string; + } + return core.truncate(display(answers), limit()) as string; +} + +// --------------------------------------------------------------------------- +// Human in the loop, resumed by a DIFFERENT PROCESS +// --------------------------------------------------------------------------- +// +// Everything above keys the pause on the interrupt object this process saw, +// which assumes the process that paused is the one that resumes. Real HITL is +// not shaped like that: one worker serves the interrupt, a human answers later, +// and any worker may pick the approval up. The resuming process has no session +// and no open pauses, so nothing correlated and the pause stayed open forever. +// +// It is recoverable, exactly, because an interrupt's id is not random: +// LangGraph.js's `interrupt()` sets it to `XXH3(checkpoint_ns)` — a pure +// function of the interrupted task's namespace, which is +// `metadata.langgraph_checkpoint_ns` on the node's run and identical across the +// two invocations. So the resuming process can rebuild the id the pausing +// process used with no shared state. +// +// Which node re-ran BECAUSE it was interrupted: only the first superstep of a +// level re-runs interrupted tasks, and the deepest resuming level is the graph +// that actually paused — which excludes a subgraph's HOST node, a normal node +// one level up. LangGraph.js >= 1 names that level in `handleResume`; on 0.x, +// where there is no such event, the same answer is read off the runs +// themselves: decided at node END, a host node has always seen its subgraph's +// deeper nodes by then. + +type Hash = (input: string) => string; +let xxh3: Hash | null | undefined; + +/** + * LangGraph's own XXH3, loaded from the installed package on first use. + * + * Not an export — LangGraph keeps it internal — so this reads the file beside + * its `package.json`. That is the price of an id that matches the one + * `interrupt()` produced byte for byte; a reimplementation would have to match + * too, and would not be told when LangGraph changed. A missing or changed file + * disables ONLY the cross-process resume: the probe below checks the function + * still returns the 32-hex-digit shape `interrupt()` stamps. + */ +function interruptHash(): Hash | null { + if (xxh3 !== undefined) return xxh3; + xxh3 = null; + const manifest = resolveFrom(`${GRAPH_PACKAGE}/package.json`); + if (manifest === null) return xxh3; + compat.probe(NAME, "langgraph interrupt ids", () => { + const module = nodeRequire(join(manifest, "..", "dist", "hash.cjs")) as { XXH3?: unknown }; + const fn = module.XXH3; + if (typeof fn !== "function") return false; + const sample = String((fn as (text: string) => unknown)("failproofai")); + if (!/^[0-9a-f]{32}$/.test(sample)) return false; + xxh3 = (text: string) => String((fn as (value: string) => unknown)(text)); + return true; + }); + return xxh3; +} + +/** The id LangGraph's `interrupt()` gave a task with this checkpoint namespace. */ +export function interruptIdOf(ns: string): string | null { + if (!ns) return null; + const hash = interruptHash(); + if (hash === null) return null; + try { + return hash(ns); + } catch { + return null; + } +} + +function remoteOf(info: RunInfo): RemoteResume | null { + return info.root !== null ? (state.runs.get(info.root)?.remote ?? null) : null; +} + +/** `agent_resume` + `human_input` for a pause this process never opened. */ +function closeRemotePause(info: RunInfo): void { + const remote = remoteOf(info); + const session = info.session; + if (remote === null || session === null) return; + const parts = nsParts(info.meta); + const level = parts.slice(0, -1).join("|"); + if (remote.deepest !== null) { + if (level !== remote.deepest) return; + } else { + // No lifecycle event named the resuming level (LangGraph.js 0.x). A level + // strictly deeper than this one has run, so this node is a subgraph's host, + // not the task that paused. + const depth = parts.length - 1; + for (const seen of remote.levels.keys()) { + if ((seen ? seen.split("|").length : 0) > depth) return; + } + } + if (info.meta.langgraph_step !== remote.levels.get(level)) return; + const ns = info.meta.langgraph_checkpoint_ns; + const pauseId = typeof ns === "string" ? interruptIdOf(ns) : null; + if (pauseId === null || remote.done.has(pauseId)) return; + remote.done.add(pauseId); + const marker = core.fwFields({ interrupt_id: pauseId, resumed_elsewhere: true }); + emitOnAgent(session, "agentResume", { pauseId, reason: "langgraph_resume", ...marker }); + emitOnAgent(session, "humanInput", { + inputId: pauseId, + response: state.options.captureContent ? answerFor(remote.value, pauseId) : undefined, + ...marker, + }); +} + +/** LangGraph.js >= 1 lifecycle: an interrupt, delivered with the root's run id. */ +function onInterrupt(event: unknown): void { + if (!state.enabled || !isObject(event)) return; + const info = typeof event.runId === "string" ? state.runs.get(event.runId) : undefined; + if (info?.session == null) return; + const interrupts = Array.isArray(event.interrupts) ? event.interrupts : []; + suspend( + info.session, + interrupts.filter((item): item is InterruptLike => isObject(item) && "value" in item), + ); +} + +/** + * LangGraph.js >= 1 lifecycle: a Pregel level is resuming. Normally a no-op for + * the in-process case — the resuming root already closed the pause at start — + * and load-bearing for the cross-process one, because it names the level that + * is resuming, once per level, deepest last. + */ +function onResume(event: unknown): void { + if (!state.enabled || !isObject(event)) return; + const info = typeof event.runId === "string" ? state.runs.get(event.runId) : undefined; + if (info === undefined) return; + const remote = remoteOf(info); + if (remote !== null) { + const ns = Array.isArray(event.checkpointNs) ? event.checkpointNs.map(String) : []; + const level = ns.join("|"); + if (remote.deepest === null || ns.length >= (remote.deepest ? remote.deepest.split("|").length : 0)) { + remote.deepest = level; + } + } + if (info.session !== null) resume(info.session, undefined); +} + +function onToken(id: string): void { + const info = state.runs.get(id); + if (info === undefined) return; + // Folded into the closing `model_response`, NEVER an event: a 500-token + // response would otherwise be 500 stored rows. + info.chunks += 1; + info.ttftMs ??= core.ms(Date.now() - info.started); +} + +// --------------------------------------------------------------------------- +// The handler +// --------------------------------------------------------------------------- + +type Handler = Record; + +function nullable(value: unknown): string | null { + return typeof value === "string" && value ? value : null; +} + +/** + * The one handler. A plain object in LangChain's `BaseCallbackHandler` shape + * rather than a subclass, so this module never imports LangChain — the class + * would have to come from the application's copy, of which there can be two. + * + * Every method is a thin argument-order adapter onto `onStart`/`onEnd`, wrapped + * in `core.safe`. The argument order is the one `CallbackManager` DISPATCHES + * with, which is not always the order its own `.d.ts` declares (1.x's + * `handleChainStart` declaration puts `runType` fourth; the call site passes + * `parentRunId` fourth, as 0.3 did). + */ +function buildHandler(): Handler { + const wrap = (fn: (...args: Args) => void): ((...args: Args) => void) => + core.safe(NAME, fn); + + return { + name: "failproofai", + [HANDLER_MARK]: true, + // See the module comment: without this LangChain backgrounds every + // callback, out of our async context and possibly past the final flush. + awaitHandlers: true, + ignoreLLM: false, + ignoreChain: false, + ignoreAgent: false, + ignoreRetriever: false, + // Python's adapter records no custom events; neither does this one. + ignoreCustomEvent: true, + // Normally false so an adapter bug can never take down the graph. Under + // FAILPROOFAI_SDK_STRICT it follows strict mode — otherwise LangChain's own + // handler firewall swallows the re-raise and the escape hatch does nothing. + get raiseError(): boolean { + return core.strict(); + }, + get [GRAPH_CALLBACK_HANDLER](): boolean { + return state.options.graphCallbacks; + }, + + handleChainStart: wrap(function handleChainStart( + serialized: unknown, + inputs: unknown, + runId: string, + parentRunId?: string, + tags?: unknown, + metadata?: unknown, + _runType?: unknown, + runName?: unknown, + ) { + onStart({ + id: runId, + parent: nullable(parentRunId), + name: runNameOf(serialized, runName, "chain"), + runType: "chain", + tags: tagsOf(tags), + meta: metaOf(metadata), + inputs, + }); + }), + handleChainEnd: wrap(function handleChainEnd(outputs: unknown, runId: string) { + onEnd(runId, { outputs }); + }), + handleChainError: wrap(function handleChainError(error: unknown, runId: string) { + onEnd(runId, { error: error ?? new Error("chain failed") }); + }), + + handleLLMStart: wrap(function handleLLMStart( + serialized: unknown, + prompts: unknown, + runId: string, + parentRunId?: string, + extraParams?: unknown, + tags?: unknown, + metadata?: unknown, + runName?: unknown, + ) { + onStart({ + id: runId, + parent: nullable(parentRunId), + name: runNameOf(serialized, runName, "llm"), + runType: "llm", + tags: tagsOf(tags), + meta: metaOf(metadata), + inputs: { prompts }, + invocationParams: (extraParams as { invocation_params?: Record } | undefined) + ?.invocation_params, + }); + }), + handleChatModelStart: wrap(function handleChatModelStart( + serialized: unknown, + messages: unknown, + runId: string, + parentRunId?: string, + extraParams?: unknown, + tags?: unknown, + metadata?: unknown, + runName?: unknown, + ) { + onStart({ + id: runId, + parent: nullable(parentRunId), + name: runNameOf(serialized, runName, "chat_model"), + runType: "chat_model", + tags: tagsOf(tags), + meta: metaOf(metadata), + inputs: messages, + messages: Array.isArray(messages) ? messages : undefined, + invocationParams: (extraParams as { invocation_params?: Record } | undefined) + ?.invocation_params, + }); + }), + handleLLMNewToken: wrap(function handleLLMNewToken(_token: unknown, _idx: unknown, runId: string) { + if (state.enabled) onToken(runId); + }), + handleLLMEnd: wrap(function handleLLMEnd(output: unknown, runId: string) { + onEnd(runId, { response: output }); + }), + handleLLMError: wrap(function handleLLMError(error: unknown, runId: string) { + onEnd(runId, { error: error ?? new Error("model call failed") }); + }), + + handleToolStart: wrap(function handleToolStart( + serialized: unknown, + input: unknown, + runId: string, + parentRunId?: string, + tags?: unknown, + metadata?: unknown, + runName?: unknown, + toolCallId?: unknown, + ) { + onStart({ + id: runId, + parent: nullable(parentRunId), + name: runNameOf(serialized, runName, "tool"), + runType: "tool", + tags: tagsOf(tags), + meta: metaOf(metadata), + inputs: parseToolInput(input), + toolCallId: nullable(toolCallId) ?? undefined, + }); + }), + handleToolEnd: wrap(function handleToolEnd(output: unknown, runId: string) { + onEnd(runId, { outputs: output }); + }), + handleToolError: wrap(function handleToolError(error: unknown, runId: string) { + onEnd(runId, { error: error ?? new Error("tool failed") }); + }), + + handleRetrieverStart: wrap(function handleRetrieverStart( + serialized: unknown, + query: unknown, + runId: string, + parentRunId?: string, + tags?: unknown, + metadata?: unknown, + name?: unknown, + ) { + onStart({ + id: runId, + parent: nullable(parentRunId), + name: runNameOf(serialized, name, "retriever"), + runType: "retriever", + tags: tagsOf(tags), + meta: metaOf(metadata), + inputs: query, + }); + }), + handleRetrieverEnd: wrap(function handleRetrieverEnd(documents: unknown, runId: string) { + onEnd(runId, { outputs: documents }); + }), + handleRetrieverError: wrap(function handleRetrieverError(error: unknown, runId: string) { + onEnd(runId, { error: error ?? new Error("retriever failed") }); + }), + + handleInterrupt: wrap(function handleInterrupt(event: unknown) { + onInterrupt(event); + }), + handleResume: wrap(function handleResume(event: unknown) { + onResume(event); + }), + }; +} + +let handler: Handler | null = null; + +function theHandler(): Handler { + handler ??= buildHandler(); + return handler; +} + +interface CallbackManagerLike { + handlers?: unknown[]; + addHandler?: (handler: unknown, inherit?: boolean) => void; +} + +interface CallbackManagerCtor { + new (): CallbackManagerLike; + configure?: (...args: unknown[]) => unknown; + _configureSync?: (...args: unknown[]) => unknown; +} + +function isOurs(value: unknown): boolean { + return isObject(value) && (value as Record)[HANDLER_MARK] === true; +} + +/** + * Attach our handler to a manager `configure` just built — unless one is + * already there. LangChain calls `configure` for every invocation and a child + * manager inherits its parent's handlers, so a blind `addHandler` would emit + * every event once per attachment. + */ +function attach(manager: unknown): unknown { + const value = manager as CallbackManagerLike | undefined | null; + if (!value || typeof value.addHandler !== "function") return manager; + if (!state.enabled) return manager; + if (Array.isArray(value.handlers) && value.handlers.some(isOurs)) { + return manager; + } + value.addHandler(theHandler(), true); + return manager; +} + +/** + * `configure`'s first argument — the inheritable handlers — with ours added. + * An array (or nothing) gets the handler appended; a `CallbackManager` is left + * alone and the manager `configure` derives from it is handled by `attach`. + */ +function withHandler(inheritable: unknown): unknown { + if (inheritable === undefined || inheritable === null) return [theHandler()]; + if (Array.isArray(inheritable)) { + return inheritable.some(isOurs) ? inheritable : [...(inheritable as unknown[]), theHandler()]; + } + return inheritable; +} + +// --------------------------------------------------------------------------- +// Install / uninstall +// --------------------------------------------------------------------------- + +/** Close every span still open at teardown, leaves before agents. */ +function closeEverything(): void { + const roots = new Set(); + for (const info of state.runs.values()) if (info.root !== null) roots.add(info.root); + for (const root of roots) closeOpenLeaves(root); + state.tracker.closeOpenAgents("cancelled"); +} + +/** + * The `CallbackManager` of every OTHER installed copy of `@langchain/core` — + * the ones nested under a dependency that pinned its own version. + * + * That layout is ordinary: a provider or community package declaring + * `@langchain/core` as a hard dependency on a range the application's copy does + * not satisfy gets its own copy at `node_modules//node_modules/ + * @langchain/core`, and everything it exports — its chat model, its tools, its + * retrievers — is built on that copy. Resolution from the application can never + * reach it. A run such a class starts INSIDE one of the application's runs was + * always recorded: the child is handed the parent's manager, handler included. + * But one it starts as a ROOT — `providerModel.invoke()`, a provider's tool or + * retriever called directly — went through the nested copy's own, unpatched + * `configure`, and was recorded nowhere, silently, while `instrument()` + * reported success (VERIFIED: `integration/fixtures/langchain-dup-core`). + * + * LangChain offers no cross-copy hook to use instead. Its one registration + * point, `registerConfigureHook`, keys its list on a module-private + * `Symbol("lc:configure_hooks")` — each copy reads only its own — and stores + * it in the current async context, not globally. So the copies are found on + * disk (`nestedCopies`) and loaded the way the application will load them: the + * build its module system reaches, plus the CommonJS build if something has + * already `require`d it — `requireModuleCopies`' rule, for the same reasons. A + * copy outside the declared range is left alone rather than patched blind. + * + * The one arrangement this cannot see is the one `requireModuleCopies` cannot: + * an ES-module application whose CommonJS-only dependency `require`s its nested + * copy AFTER `instrument()`. `langchainHandler()` covers it. + */ +async function nestedManagers(): Promise { + const found: CallbackManagerCtor[] = []; + for (const root of nestedCopies(PACKAGE)) { + await core.callSafely( + async () => { + const version = (nodeRequire(join(root, "package.json")) as { version?: unknown }).version; + const parts = compat.parseVersion(typeof version === "string" ? version : ""); + if (parts.length === 0 || (parts[0] ?? 0) >= 2 || ((parts[0] ?? 0) === 0 && (parts[1] ?? 0) < 3)) { + logger.debug(`langchain adapter leaving ${root} (${String(version)}) alone: outside >=0.3.0 <2.0.0`); + return; + } + const cjs = resolveExportsAt(root, "./callbacks/manager", "require"); + const esm = resolveExportsAt(root, "./callbacks/manager", "import"); + const modules: unknown[] = []; + if (esm === null || entryIsCommonJs()) { + if (cjs !== null) modules.push(nodeRequire(cjs)); + } else { + modules.push(await importModule(pathToFileURL(esm).href)); + if (cjs !== null && cjs !== esm && isRequired(cjs)) modules.push(nodeRequire(cjs)); + } + for (const module of modules) { + const CallbackManager = (module as { CallbackManager?: unknown }).CallbackManager; + if (typeof CallbackManager === "function") found.push(CallbackManager as CallbackManagerCtor); + } + }, + [], + `${NAME}.nestedManagers`, + ); + } + return found; +} + +export const adapter: Adapter = { + name: NAME, + + async install(options: Record = {}): Promise { + // Every loaded copy: the ES-module and CommonJS builds of @langchain/core + // are two different CallbackManager classes. See `requireModuleCopies`. + const primary = ( + (await compat.requireModuleCopies( + "@langchain/core/callbacks/manager", + "npm install @langchain/core", + )) as Array<{ CallbackManager?: CallbackManagerCtor }> + ).map((module) => module.CallbackManager); + if (primary.some((CallbackManager) => typeof CallbackManager !== "function")) { + throw new Error("@langchain/core/callbacks/manager does not export CallbackManager"); + } + // ...and every copy nested under a dependency. See `nestedManagers`. + const managers = [...new Set([...primary, ...(await nestedManagers())])] as CallbackManagerCtor[]; + + compat.checkVersion(NAME, PACKAGE, { + minimum: "0.3.0", + below: "2.0.0", + reason: "the callback argument order and run metadata below are the 0.3+ shape", + }); + // LangGraph is optional — plain LangChain is instrumented without it — so + // only an INSTALLED LangGraph outside the range warns. + compat.checkVersion(NAME, GRAPH_PACKAGE, { + minimum: "0.4.0", + below: "2.0.0", + reason: "the node metadata and interrupt shape below are LangGraph.js 0.4+", + }); + + state.configure(readOptions(options)); + state.enabled = true; + state.installed = true; + patcher = new core.Patcher(); + + // Both entry points, and at least one must take. A version that routed + // through the other would install cleanly and record nothing — the single + // most expensive failure an adapter can have, because everything looks fine. + let patched = 0; + for (const CallbackManager of managers) { + for (const method of ["configure", "_configureSync"] as const) { + const original = CallbackManager[method]; + if (typeof original !== "function") continue; + if ( + !compat.probe(NAME, `CallbackManager.${method}`, () => + Object.getOwnPropertyDescriptor(CallbackManager, method)?.writable !== false, + ) + ) { + continue; + } + const replacement = function failproofaiConfigure(this: unknown, ...args: unknown[]): unknown { + // Our handler goes IN, as one of the inheritable handlers, rather + // than onto whatever comes out. `configure` returns undefined when + // there are no handlers at all — the ordinary case for an un-traced + // process, exactly the process we are here to trace — and it only + // applies the call's tags and metadata to a manager it builds. The + // first release attached to a bare `new CallbackManager()` in that + // case, which carried the handler and none of the metadata: every + // `thread_id` and `failproofai_sdk_session_id` was silently dropped, + // so neither could ever choose the session. + if (state.enabled) args[0] = withHandler(args[0]); + const built = original.apply(this, args); + if (typeof (built as PromiseLike | undefined)?.then === "function") { + return (built as PromiseLike).then(attach); + } + return attach(built); + }; + if (patcher.patch(CallbackManager, method, replacement)) patched += 1; + } + } + + if (patched === 0) { + throw new Error( + "could not patch CallbackManager.configure — this build of @langchain/core exposes " + + "neither a writable `configure` nor `_configureSync`. Pass the handler explicitly " + + "instead: `chain.invoke(input, { callbacks: [langchainHandler()] })`.", + ); + } + logger.debug(`langchain adapter attached to ${patched} callback-manager entry point(s)`); + }, + + uninstall(): void { + // The switch goes FIRST: teardown below emits through the tracker directly, + // and nothing may re-enter `onStart` while it does. + state.enabled = false; + state.installed = false; + patcher?.restoreAll(); + patcher = null; + closeEverything(); + state.reset(); + state.options = defaultOptions(); + }, +}; + +let warnedOptions = false; + +/** + * The raw handler, for passing explicitly instead of — or as well as — + * `instrument()`: + * + * import { langchainHandler } from "@failproofai/sdk/langchain"; + * await graph.invoke(input, { callbacks: [langchainHandler()] }); + * + * Works without `instrument()`: the patch-free path, and the documented + * fallback for a bundled application where the `@langchain/core` in + * `node_modules` is not the copy that runs. Takes the same options as + * `instrument("langchain", options)`; while `instrument()` is active its + * options govern, and passing different ones here warns once. + * + * Using it alongside `instrument()` does not double-record: the patched + * `configure` sees the handler is already on the manager and adds nothing. + * `uninstrument()` disables it too, until it is asked for again. + */ +/** @internal Table sizes, for the tests that prove a finished request leaves nothing behind. */ +export function _stats(): { runs: number; sessions: number; tracker: { runs: number; links: number } } { + return { runs: state.runs.size, sessions: state.sessions.size, tracker: state.tracker.stats() }; +} + +export function langchainHandler(options?: LangChainOptions): Record { + if (state.installed) { + if (options !== undefined && !warnedOptions) { + warnedOptions = true; + logger.warn( + "langchainHandler() options are ignored while instrument('langchain') is active; " + + "the options passed to instrument() govern.", + ); + } + } else if (!state.enabled) { + state.configure(readOptions((options ?? {}) as Record)); + state.enabled = true; + } else if (options !== undefined) { + // Already recording through an earlier call: new options, same runs. + state.options = readOptions(options as Record); + } + return theHandler(); +} diff --git a/sdk/typescript/src/integrations/llamaindex.ts b/sdk/typescript/src/integrations/llamaindex.ts new file mode 100644 index 000000000..a00ce9b06 --- /dev/null +++ b/sdk/typescript/src/integrations/llamaindex.ts @@ -0,0 +1,2111 @@ +/** + * LlamaIndex.TS (`llamaindex`, `@llamaindex/core`, `@llamaindex/workflow`). + * + * The TypeScript counterpart of the Python SDK's `integrations/llama_index.py`, + * and it must draw the same tree for the same program: + * + * | LlamaIndex.TS | FailproofAI | + * |---------------------------------------|-----------------------------------------------| + * | `AgentWorkflow` run (`agent()`) | session + `agent_start`/`agent_end` | + * | nested run (inside a tool, a scope) | nested `agent_start`/`agent_end` | + * | `multiAgent()` handoff | nested agent per agent holding the turn | + * | workflow step | `hook_triggered`/`hook_completed`, `trigger_event="workflow_step"` | + * | legacy `LLMAgent` / `AgentRunner` task| `agent_start`/`agent_end` across ALL its steps | + * | `createWorkflow()` workflow (core ≥1.1)| `agent_start`/`agent_end` (`"Workflow"`), steps as hooks | + * | LLM chat | `model_request`/`model_response` on `request_id` | + * | tool call | `tool_use`/`tool_result`, the model's call id | + * | retrieval | `tool_use`/`tool_result` named after the retriever's class, output summarised | + * | top-level chat engine / query engine / retriever / `llm.chat()` / tool | its own root run, named after its class | + * + * `agent_id` is the agent's `name` (`"Agent"` when unnamed, as in Python), the + * class name for a multi-agent workflow or a legacy runner — never an id. + * + * ## Two extension points, because LlamaIndex.TS has two halves + * + * **The callback bus** (`Settings.callbackManager`, from `@llamaindex/core/global`) + * carries everything below the agent: `llm-start`/`llm-stream`/`llm-end`, + * `llm-tool-call`/`llm-tool-result`, `retrieve-*`, `query-*`, and the legacy + * runner's `agent-start`/`agent-end`. Subscribing is the whole integration for + * that half, and uninstrumenting is unsubscribing. + * + * **The workflow runtime** (`@llamaindex/workflow` ≥1.1, on + * `@llamaindex/workflow-core`) emits NOTHING on that bus: an `agent().run()` + * has no run boundary and no step events there at all. What it does have is the + * middleware surface its own first-party middleware is built on — a context's + * `__internal__call_context` (wraps every step handler invocation; this is how + * `withTraceEvents` works) and `__internal__call_send_event` (sees every event + * a step sends; this is how `withState` works). So `AgentWorkflow.prototype.runStream` + * — which `run()` also goes through — is wrapped to open the run and to attach + * those two subscriptions to the context it creates. That is a prototype patch, + * so it records agents built before `instrument()` too. + * + * Three smaller hooks fill what neither half says, each on one object and each + * undone by `uninstrument()`: + * + * * **Invocation boundaries.** A chat engine's `chat()` dispatches nothing of + * its own, so its retrieval and its model call used to become two root runs + * in two sessions. Every `@wrapEventCaller` method runs as + * `storage.run(new EventCaller(...), fn)` on ONE module-private storage in + * `@llamaindex/core/global`; an own `run` on that storage (found through the + * exported `getEventCaller()`, see `eventCallerStorage`) sees each + * invocation start and return. A top-level invocation is then one run named + * after its class that ends when it returns, and an invocation that THROWS + * is the failure signal `wrapLLMEvent` and the legacy runner lack. + * * **Retriever names.** `retrieve-start` carries only the query; the patched + * `BaseRetriever.prototype.retrieve` binds the retriever for it. + * * **Plain workflows.** workflow-core ≥1.1 runs every step handler as + * `AsyncContext.Variable#run(handlerContext, …)`, a class it exports; its + * prototype `run` sees every step of every context. See `plainStep` for why + * such a run ends when its context goes idle. + * + * ## Correlation, which LlamaIndex.TS does not give us + * + * The bus events carry an id that pairs start with end and nothing that says + * which run they belong to. Two signals do, and we use both: + * + * * our own `AsyncLocalStorage` frame, bound around every workflow step we + * wrap. The bus dispatches in a `queueMicrotask`, which Node runs in the + * dispatcher's async context, so a handler sees the step that caused it; + * * LlamaIndex's own `EventCaller` (`event.reason`), set by `@wrapEventCaller` + * — on `AgentRunner.chat`, `BaseQueryEngine.query` and every first-party + * provider's `chat`. `withEventCaller` binds a FRESH `EventCaller` per + * invocation in LlamaIndex's own `AsyncLocalStorage`, chained through + * `.parent` to the invocation it ran inside. A legacy task or a query run is + * registered under the `EventCaller` of the invocation that opened it, and + * an event belongs to it when that exact object is on the event's chain. + * + * The invocation, NOT the object that owns it. One query engine (or one + * `LLMAgent`) built at startup and serving every request is the normal + * deployment, so the owner is the same object for every concurrent call; keyed + * by owner, request B's `query-start` found A's run, treated itself as nested + * and recorded nothing of its own. The `EventCaller` is per call and already + * flows through the async context, which is why it is used rather than a + * prototype patch of `query`/`chat`: `@wrapEventCaller` binds the method onto + * each INSTANCE at construction (`this.query = (...) => withEventCaller(...)`), + * so a prototype patch would miss every engine built before `instrument()`. + * + * Only a bus that carries no `EventCaller` (a build without one; the unit + * tests' stand-ins) falls back to matching the owner objects in + * `computedCallers`, and there a run owned by the object STARTING a new run is + * never taken as its parent: without the chain a concurrent sibling on a shared + * object is indistinguishable from re-entry, and the sibling is the common case. + * + * When both our frame and a caller match, the deeper run wins. When neither + * does, the call is a root run of its own — the LangChain/Python precedent for a + * bare model call — and nests under an enclosing `failproofai.session()`/ + * `agent()` scope if there is one. + * + * ## Known gaps, each one the framework's and each one documented, not faked + * + * * A model call has no failure signal of its own: `wrapLLMEvent` has no error + * path (no `llm-end` when `chat()` throws). A provider whose `chat` is + * `@wrapEventCaller` (every first-party one) fails with its invocation, and + * inside a workflow the failed step closes it. A STREAM that fails while it + * is being read, outside a workflow, has returned already: that leaf stays + * open until the reaper (`staleAfter`) or `uninstrument()`. + * * A provider error mid-stream inside `agent().run()` escapes LlamaIndex's + * workflow runtime as an UNHANDLED rejection and `run()` never settles — + * with or without this SDK. The failed step still closes the run, failed. + * * `callTool` dispatches no `llm-tool-result` when a tool throws. In a + * workflow the runtime's own tool-result event closes it with the error; in + * a legacy agent the next model call does, from the tool-result message. + * * No embedding events exist on the TS bus, so `embeddings: true` has + * nothing to record. + * * Streamed calls carry token usage only when the provider sends it: + * `@llamaindex/openai` requests it only with + * `additionalChatOptions: { stream_options: { include_usage: true } }`. + * * A plain `createWorkflow()` workflow is a run only on workflow-core ≥1.1 + * resolvable from the application (not the floor's `@llama-flow/core`, not + * an unhoisted pnpm layout): elsewhere its model calls are loose root runs. + * Its run ends when the context goes idle, so a workflow that waits for an + * event from outside records each burst as its own run, and it is named + * `"Workflow"` — the runtime has no name to give it (wrap it in + * `failproofai.agent("name", …)` to name the parent). + * * No human-in-the-loop pairs: the TS runtime has no waiting-for-event signal. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; +import { performance } from "node:perf_hooks"; + +import { logger } from "../logger.js"; +import * as compat from "./compat.js"; +import * as core from "./core.js"; +import type { Adapter } from "./core.js"; + +const NAME = "llamaindex"; +const PACKAGE = "llamaindex"; +const CORE_PACKAGE = "@llamaindex/core"; +const WORKFLOW_PACKAGE = "@llamaindex/workflow"; +const ASYNC_CONTEXT_MODULE = "@llamaindex/workflow-core/async-context"; +const INSTALL = "npm install llamaindex"; + +/** + * 0.11.4 is a CAPABILITY floor: it is the first `llamaindex` whose agent API + * (`agent()` / `multiAgent()`) runs on the `@llamaindex/workflow` 1.1 runtime + * this adapter wraps. 0.9–0.11.3 ship workflow 1.0, a different class-based + * runtime with none of the surfaces above — their workflow agents would record + * no run and no steps, only loose model and tool calls. + */ +export const MIN_VERSION = "0.11.4"; +export const BELOW_VERSION = "1.0.0"; +const WORKFLOW_MIN = "1.1.0"; +const WORKFLOW_BELOW = "2.0.0"; + +const MAX_NODES_IN_SUMMARY = 5; +/** + * Open runs and leaves are bounded, oldest evicted first: orphans are normal (a + * stream nobody consumed, a legacy task whose step threw and so never sent + * `agent-end`) and a long-lived server must not keep every one of them. + */ +const MAX_OPEN = 10_000; + +// Token key aliases, widest first. LlamaIndex normalises nothing, so this is +// the union of what the provider packages actually put in `raw` — the Python +// adapter's list, plus the camelCase spellings TS providers use. +const INPUT_TOKEN_KEYS = [ + "prompt_tokens", + "input_tokens", + "inputTokens", + "promptTokens", + "prompt_token_count", + "promptTokenCount", +] as const; +const OUTPUT_TOKEN_KEYS = [ + "completion_tokens", + "output_tokens", + "outputTokens", + "completionTokens", + "candidates_token_count", + "candidatesTokenCount", +] as const; + +// --------------------------------------------------------------------------- +// Pure helpers — no framework import in any of these +// --------------------------------------------------------------------------- + +type Json = Record; + +function isObject(value: unknown): value is Json { + return typeof value === "object" && value !== null; +} + +/** A property read that cannot throw (getters on framework classes can). */ +function read(target: unknown, key: string): unknown { + if (!isObject(target) && typeof target !== "function") return undefined; + try { + return (target as Json)[key]; + } catch { + return undefined; + } +} + +function nonEmpty(value: unknown): string | undefined { + return typeof value === "string" && value !== "" ? value : undefined; +} + +/** + * A correlation id out of a payload field whose type the framework does not + * promise. An object would render as `[object Object]` and correlate with every + * other one; a fresh id leaves the pair merely unpaired instead. + */ +function asId(...candidates: unknown[]): string { + for (const candidate of candidates) { + if (typeof candidate === "string" && candidate !== "") return candidate; + if (typeof candidate === "number" && Number.isFinite(candidate)) return String(candidate); + } + return randomUUID(); +} + +function firstInt(source: Json, keys: readonly string[]): number | undefined { + for (const key of keys) { + const value = source[key]; + if (typeof value === "number" && Number.isInteger(value) && value >= 0) return value; + } + return undefined; +} + +export interface Usage { + usage?: Json; + inputTokens?: number; + outputTokens?: number; +} + +/** + * Token usage from a `ChatResponse`, streaming or not. + * + * Conservative on purpose, like the Python adapter: the token numbers are set + * ONLY when a key we recognise is present, while the raw usage object always + * ships as `usage` so a provider that names its counters something new still + * reports something the server can fall back to. + * + * A streamed response is the case that used to lose everything: `wrapLLMEvent` + * hands `llm-end` a `raw` that is the ARRAY of chunks, and providers put the + * usage on one chunk — OpenAI on the last, content-less one — so reading + * `raw.usage` found nothing on every streamed call, which is every + * `FunctionAgent` call. + */ +export function usageOf(response: unknown): Usage { + const raw = read(response, "raw"); + const candidates: unknown[] = []; + const fromChunk = (chunk: unknown): void => { + const chunkRaw = read(chunk, "raw"); + candidates.push( + read(chunkRaw, "usage"), + read(chunkRaw, "usage_metadata"), + read(chunkRaw, "usageMetadata"), + read(read(chunk, "options"), "usage"), + ); + }; + if (Array.isArray(raw)) { + // Newest chunk first: providers that report running totals end with the final one. + for (let i = raw.length - 1; i >= 0; i -= 1) fromChunk(raw[i]); + } else { + candidates.push(read(raw, "usage"), read(raw, "usage_metadata"), read(raw, "usageMetadata")); + } + candidates.push(read(read(read(response, "message"), "options"), "usage"), read(response, "usage")); + for (const candidate of candidates) { + if (!isObject(candidate) || Array.isArray(candidate) || Object.keys(candidate).length === 0) continue; + return { + usage: candidate, + inputTokens: firstInt(candidate, INPUT_TOKEN_KEYS), + outputTokens: firstInt(candidate, OUTPUT_TOKEN_KEYS), + }; + } + return {}; +} + +function stopReasonOf(response: unknown): string | undefined { + const raw = read(response, "raw"); + const sources = Array.isArray(raw) ? [...raw].reverse().map((chunk) => read(chunk, "raw")) : [raw]; + for (const source of sources) { + const choice = (read(source, "choices") as unknown[] | undefined)?.[0]; + const value = + nonEmpty(read(choice, "finish_reason")) ?? + nonEmpty(read(source, "stop_reason")) ?? + nonEmpty(read(source, "finishReason")); + if (value) return value; + } + return undefined; +} + +/** A retrieval result small enough to store: count, scores, a prefix of the top few. */ +export function summarizeNodes(nodes: unknown): { num_nodes: number; top: Json[] } { + const items = Array.isArray(nodes) ? nodes : []; + const top = items.slice(0, MAX_NODES_IN_SUMMARY).map((item) => { + const node = read(item, "node") ?? item; + let text: unknown; + const getContent = read(node, "getContent"); + if (typeof getContent === "function") { + try { + text = (getContent as () => unknown).call(node); + } catch { + text = undefined; + } + } + text ??= read(node, "text"); + const score = read(item, "score"); + return { + id: nonEmpty(read(node, "id_")) ?? nonEmpty(read(node, "id")), + score: typeof score === "number" ? score : undefined, + text: core.truncate(typeof text === "string" ? text : "", 200), + }; + }); + return { num_nodes: items.length, top }; +} + +function messagesOf(messages: unknown): Json[] | undefined { + if (!Array.isArray(messages)) return undefined; + return messages.map((message) => ({ + role: nonEmpty(read(message, "role")) ?? "user", + content: read(message, "content"), + })); +} + +/** The text of a query or a `QueryBundle`. */ +function queryText(query: unknown): unknown { + if (typeof query === "string") return query; + return read(query, "query") ?? read(query, "queryStr") ?? query; +} + +/** The text of whatever a run returned — `EngineResponse`, a message, a string. */ +function textOf(value: unknown): string | undefined { + if (typeof value === "string") return value; + const content = read(read(value, "message"), "content") ?? read(value, "response") ?? read(value, "content"); + if (typeof content === "string") return content; + if (Array.isArray(content)) { + const parts = content.map((part) => read(part, "text")).filter((part) => typeof part === "string"); + if (parts.length > 0) return parts.join(""); + } + return undefined; +} + +function errorText(error: unknown): string { + if (error instanceof Error) return `${error.name || "Error"}: ${error.message}`; + return String(error); +} + +function className(value: unknown): string | undefined { + const name = read(read(value, "constructor"), "name"); + return typeof name === "string" && name !== "" && name !== "Object" && name !== "Function" + ? name + : undefined; +} + +/** `event.reason.computedCallers` — the objects inside whose `@wrapEventCaller` calls this ran. */ +function callersOf(event: unknown): unknown[] { + const callers = read(read(event, "reason"), "computedCallers"); + return Array.isArray(callers) ? callers : []; +} + +/** + * The `EventCaller` chain of an event's `reason`, innermost first — or `null` + * when the reason is not an `EventCaller` (no `caller` field), in which case + * only the owner objects in `computedCallers` are known. + */ +function invocationChain(reason: unknown): object[] | null { + if (!isObject(reason)) return null; + try { + if (!("caller" in reason)) return null; + } catch { + return null; + } + const chain: object[] = []; + const seen = new Set(); + let node: unknown = reason; + while (isObject(node) && !seen.has(node)) { + seen.add(node); + chain.push(node); + node = read(node, "parent"); + } + return chain; +} + +/** Where an event came from: LlamaIndex's invocation chain, and the owner objects on it. */ +interface Origin { + chain: object[] | null; + callers: unknown[]; +} + +function originOf(event: unknown): Origin { + return { chain: invocationChain(read(event, "reason")), callers: callersOf(event) }; +} + +/** + * The model name for a bus event. + * + * `llm-start` carries only `{id, messages}`: `wrapLLMEvent` never passes the + * model. Every first-party provider decorates `chat` with `@wrapEventCaller` + * too, so the LLM instance itself is the nearest caller, and its + * `metadata.model` is the name. A legacy runner above it has the LLM as `.llm`. + */ +function modelFromCallers(event: unknown): string | undefined { + for (const caller of callersOf(event)) { + const model = nonEmpty(read(read(caller, "metadata"), "model")); + if (model) return model; + const viaLlm = nonEmpty(read(read(read(caller, "llm"), "metadata"), "model")); + if (viaLlm) return viaLlm; + } + return undefined; +} + +function detail(event: unknown): Json { + // A `CustomEvent`, so the payload is on `.detail` — with a fallback to the + // event itself, for a build that dispatched the payload directly. + const payload = read(event, "detail") ?? event; + return isObject(payload) ? payload : {}; +} + +function numberOption(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/** One agent span we opened: a workflow run, a sub-agent, a legacy task, a bare call. */ +interface Run { + key: string; + agentId: string; + depth: number; + /** The run this one is nested in, when it is a multi-agent sub-agent. */ + root: Run | null; + leaves: Set; + usedToolIds: Map; + /** Sub-agent currently holding the turn (multi-agent workflows only). */ + sub: Run | null; + subSeq: number; + /** The object whose `@wrapEventCaller` calls belong to this run, if any. */ + owner: object | null; + /** LlamaIndex's `EventCaller` for the ONE invocation that opened this run, if any. */ + caller: object | null; + /** Workflow step keys linked to this run and not yet ended. */ + steps: Set; + /** Legacy runs have no end signal until their last step; bare runs end with their leaf. */ + bare: boolean; + /** + * Ends when the LlamaIndex invocation in `caller` returns — a chat engine, a + * bare model call — because nothing on the bus marks its end. (Query and + * legacy runs have their own end event and take only a FAILURE from there.) + */ + byInvocation: boolean; + /** Plain-workflow runs: step handlers still running, and the last step's output text. */ + inFlight: number; + output?: string; + /** A run whose end arrived while a streamed leaf was still open. */ + ending: { outcome: string; summary?: string } | null; + lastContent?: string; + model?: string; + ended: boolean; + /** `performance.now()` of the last event that touched this run; what the reaper reads. */ + lastActivity: number; + /** Drops this run from whichever index found it (legacy tasks, queries). */ + forget?: () => void; +} + +interface Leaf { + key: string; + kind: "model" | "tool" | "retrieval"; + run: Run; + parentKey: string; + name: string; + callId: string; + rawId?: string; + /** Who opened a tool leaf: the callback bus, or the workflow runtime's own event. */ + source?: "bus" | "workflow"; + started: number; + model?: string; + firstChunk?: number; +} + +/** What the adapter's `AsyncLocalStorage` carries through a workflow step. */ +interface Frame { + key: string; + run: Run; + model?: string; +} + +interface Located { + parentKey: string; + run: Run; +} + +export interface LlamaIndexOptions { + captureMessages: boolean; + steps: boolean; + embeddings: boolean; + staleAfter: number; + reaperInterval: number; + captureLimit?: number; +} + +export function parseOptions(options: Record): LlamaIndexOptions { + return { + captureMessages: options.captureMessages !== false, + steps: options.steps !== false, + embeddings: options.embeddings === true, + staleAfter: numberOption(options.staleAfter, 600), + reaperInterval: numberOption(options.reaperInterval, 30), + captureLimit: typeof options.captureLimit === "number" ? options.captureLimit : undefined, + }; +} + +interface Bus { + on: (event: string, handler: (event: unknown) => void) => unknown; + off?: (event: string, handler: (event: unknown) => void) => unknown; +} + +/** One loaded copy of `@llamaindex/core/global` (or of the umbrella re-exporting it). */ +export interface GlobalModule { + Settings?: { callbackManager?: unknown }; + getEventCaller?: () => unknown; +} + +/** One loaded copy of `@llamaindex/core/retriever`. */ +export interface RetrieverModule { + BaseRetriever?: { prototype: object }; +} + +/** One loaded copy of `@llamaindex/workflow-core/async-context` (workflow-core ≥1.1). */ +export interface AsyncContextModule { + AsyncContext?: { Variable?: { prototype: object } }; +} + +/** One loaded copy of `@llamaindex/workflow`. */ +export interface WorkflowModule { + AgentWorkflow?: { prototype: object; name?: string }; + stopAgentEvent?: { include: (event: unknown) => boolean }; + agentToolCallEvent?: { include: (event: unknown) => boolean }; + agentToolCallResultEvent?: { include: (event: unknown) => boolean }; +} + +interface Subscribable { + subscribe: (callback: (...args: never[]) => unknown) => unknown; +} + +interface HandlerContext { + handler: (...args: unknown[]) => unknown; + [key: string]: unknown; +} + +class State { + readonly options: LlamaIndexOptions; + readonly tracker: core.RunTracker; + readonly frames = new AsyncLocalStorage(); + private readonly runs = new Map(); + private readonly leaves = new Map(); + private readonly owners = new WeakMap(); + /** Runs by the `EventCaller` of the invocation that opened them. */ + private readonly invocations = new WeakMap(); + /** Legacy task: first step id -> run. */ + private readonly tasks = new Map(); + /** Root query-engine runs, by `query-start` id. */ + private readonly queries = new Map(); + /** The retriever whose `retrieve()` is running, for the retrieval's name. */ + readonly retrievers = new AsyncLocalStorage(); + /** `EventCaller`s whose invocation we watch start and end (see `invocation`). */ + private readonly observed = new WeakSet(); + /** `EventCaller`s whose invocation has already returned or thrown. */ + private readonly returned = new WeakSet(); + /** Model leaves opened directly inside an invocation (not in a workflow step), by its `EventCaller`. */ + private readonly invocationLeaves = new WeakMap>(); + /** Errors a leaf already carries, so the run they end does not report them again. */ + private readonly carried = new WeakSet(); + /** Plain-workflow runs, by the workflow-core root handler context of their context. */ + private readonly plainRuns = new WeakMap(); + /** Step handlers of `AgentWorkflow`s, which are recorded as agent runs instead. */ + private readonly agentHandlers = new WeakSet(); + private readonly globals: GlobalModule[]; + private reaper: ReturnType | null = null; + private seq = 0; + active = true; + + constructor(options: LlamaIndexOptions, globals: GlobalModule[], frameworkPackage: string) { + this.options = options; + this.globals = globals; + this.tracker = new core.RunTracker(NAME, { + baseFields: core.frameworkFields(NAME, frameworkPackage), + fieldLimit: options.captureLimit, + }); + } + + /** The one gate every payload goes through — `captureMessages: false` drops them all. */ + capture(value: T): T | undefined { + return this.options.captureMessages ? value : undefined; + } + + private nextKey(prefix: string): string { + this.seq += 1; + return `${prefix}#${this.seq}`; + } + + // -- where does this event belong ----------------------------------------- + + private live(run: Run | undefined | null): Run | null { + return run && !run.ended ? run : null; + } + + /** The innermost run opened by an invocation on this chain — exact, per call. */ + private invocationRun(chain: object[]): Run | null { + for (const node of chain) { + const run = this.live(this.invocations.get(node)); + if (run) return run; + } + return null; + } + + /** + * Fallback for a bus without `EventCaller`s: the newest live run of an owner + * object on the caller list. `starting` is the owner of a run being opened + * now; its own runs are skipped, because without the chain they are far more + * likely concurrent siblings on a shared object than this call's parent. + */ + private ownerRun(callers: unknown[], starting?: unknown): Run | null { + for (const caller of callers) { + if (!isObject(caller) || caller === starting) continue; + const stack = this.owners.get(caller); + const run = this.live(stack?.[stack.length - 1]); + if (run) return run; + } + return null; + } + + /** + * The run an event belongs to: our step frame or LlamaIndex's invocation + * chain, whichever is deeper. `origin` defaults to the chain bound right now, + * for callers (a workflow starting) that have no event to read it from. + * `starting` is the owner of a run about to be opened (see `ownerRun`). + */ + locate(origin?: Origin, starting?: unknown): Located | null { + const from = origin ?? this.boundOrigin(); + const frame = this.frames.getStore(); + const fromFrame = frame && this.live(frame.run) ? { parentKey: frame.key, run: frame.run } : null; + // With an `EventCaller` chain, ONLY the chain decides: a run whose owner is + // on the caller list but whose invocation is not on the chain belongs to a + // different call — a concurrent request on a shared engine — never this one. + const owner = from.chain ? this.invocationRun(from.chain) : this.ownerRun(from.callers, starting); + const fromOwner = owner ? { parentKey: (owner.sub ?? owner).key, run: owner.sub ?? owner } : null; + if (fromFrame && fromOwner) return fromOwner.run.depth > fromFrame.run.depth ? fromOwner : fromFrame; + return fromFrame ?? fromOwner; + } + + private boundOrigin(): Origin { + for (const module of this.globals) { + try { + const caller = module.getEventCaller?.(); + if (!isObject(caller)) continue; + const callers = read(caller, "computedCallers"); + return { chain: invocationChain(caller), callers: Array.isArray(callers) ? callers : [] }; + } catch { + // An older build without `getEventCaller`; the frame is still consulted. + } + } + return { chain: null, callers: [] }; + } + + // -- runs --------------------------------------------------------------- + + openRun( + prefix: string, + agentId: string, + options: { + parent?: Located | null; + owner?: object | null; + caller?: object | null; + bare?: boolean; + byInvocation?: boolean; + goal?: unknown; + root?: Run | null; + fields?: Json; + }, + ): Run { + while (this.runs.size >= MAX_OPEN) { + const oldest = this.runs.values().next(); + if (oldest.done) break; + this.finishRun(oldest.value, "cancelled", undefined, "evicted"); + } + const key = this.nextKey(prefix); + const identity = this.tracker.startAgent(key, { + agentId, + parentKey: options.parent?.parentKey, + goal: this.options.captureMessages && typeof options.goal === "string" ? options.goal : undefined, + ...core.fwFields({ run_id: key, ...(options.fields ?? {}) }), + }); + const run: Run = { + key, + agentId: identity.agentId ?? agentId, + depth: identity.depth, + root: options.root ?? null, + leaves: new Set(), + usedToolIds: new Map(), + sub: null, + subSeq: 0, + owner: options.owner ?? null, + caller: options.caller ?? null, + steps: new Set(), + bare: options.bare ?? false, + byInvocation: options.byInvocation ?? false, + inFlight: 0, + ending: null, + ended: false, + lastActivity: performance.now(), + }; + this.runs.set(key, run); + if (run.owner) { + const stack = this.owners.get(run.owner) ?? []; + stack.push(run); + this.owners.set(run.owner, stack); + } + if (run.caller) this.invocations.set(run.caller, run); + return run; + } + + /** + * End a run. A success whose streamed model call is still being consumed is + * DEFERRED until that leaf closes, so the tokens are not lost; anything else + * force-closes what is open first, because an `agent_end` with an open leaf + * under it leaves the session `ongoing` forever. + */ + finishRun(run: Run, outcome: string, summary?: string, reason = "run_ended"): void { + if (run.ended) return; + if (run.sub) this.finishRun(run.sub, outcome, undefined, reason); + if (outcome === "success" && run.leaves.size > 0 && reason === "run_ended") { + run.ending = { outcome, summary }; + return; + } + // Ended BEFORE its leaves are force-closed: closing the last leaf of a bare + // run would otherwise settle it as a success in the middle of this. + run.ended = true; + for (const key of [...run.leaves]) { + const leaf = this.leaves.get(key); + if (leaf) this.closeLeaf(leaf, { closedBy: reason }); + } + this.runs.delete(run.key); + run.forget?.(); + if (run.owner) { + const stack = this.owners.get(run.owner); + const index = stack?.lastIndexOf(run) ?? -1; + if (stack && index !== -1) stack.splice(index, 1); + if (stack?.length === 0) this.owners.delete(run.owner); + } + if (run.caller && this.invocations.get(run.caller) === run) this.invocations.delete(run.caller); + if (run.root && run.root.sub === run) run.root.sub = null; + const text = summary ?? (outcome === "success" ? run.lastContent : undefined); + this.tracker.endAgent(run.key, { + outcome, + summary: this.options.captureMessages || outcome !== "success" ? text : undefined, + ...core.fwFields({ run_id: run.key }), + }); + // Every tracker link this run made goes with it — its own and any step + // still in flight. A leaked link is worse than memory: at the tracker's + // FIFO cap, the next eviction takes a LIVE run's link and its events drop. + for (const step of run.steps) this.tracker.unlink(step); + run.steps.clear(); + this.tracker.unlink(run.key); + } + + /** + * A deferred or bare run whose last leaf just closed. A bare run IS its one + * call, so it ends the way that call did: a failure, a reaped orphan + * (`cancelled` — we never learned how it ended), or a success. + */ + private settle(run: Run, outcome: string): void { + if (run.ended || run.leaves.size > 0) return; + if (run.ending) this.finishRun(run, run.ending.outcome, run.ending.summary); + else if (run.bare) this.finishRun(run, outcome, undefined, outcome === "success" ? "run_ended" : "leaf"); + } + + // -- leaves ------------------------------------------------------------- + + private openLeaf(leaf: Leaf): void { + while (this.leaves.size >= MAX_OPEN) { + const oldest = this.leaves.values().next(); + if (oldest.done) break; + this.closeLeaf(oldest.value, { closedBy: "evicted" }); + } + this.leaves.set(leaf.key, leaf); + leaf.run.leaves.add(leaf.key); + leaf.run.lastActivity = performance.now(); + } + + closeLeaf(leaf: Leaf, result: { output?: unknown; error?: string; response?: unknown; closedBy?: string }): void { + if (!this.leaves.delete(leaf.key)) return; + leaf.run.leaves.delete(leaf.key); + leaf.run.lastActivity = performance.now(); + const extras = core.fwFields({ run_id: leaf.run.key, closed_by: result.closedBy }); + if (leaf.kind === "model") { + const response = result.response; + const usage = usageOf(response); + const content = read(read(response, "message"), "content"); + if (typeof content === "string" && content !== "") leaf.run.lastContent = content; + const now = performance.now(); + this.tracker.emit("modelResponse", leaf.key, { + parentKey: leaf.parentKey, + model: leaf.model, + requestId: leaf.callId, + role: response === undefined ? undefined : "assistant", + content: this.capture(content), + stopReason: stopReasonOf(response), + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + usage: usage.usage, + error: result.error, + // Always an int: `modelResponse` does not measure its own duration. + duration_ms: core.ms(now - leaf.started), + ...extras, + ...core.fwFields({ + ttft_ms: leaf.firstChunk === undefined ? undefined : core.ms(leaf.firstChunk - leaf.started), + chunks: Array.isArray(read(response, "raw")) ? (read(response, "raw") as unknown[]).length : undefined, + }), + }); + } else { + this.tracker.emit("toolResult", leaf.key, { + parentKey: leaf.parentKey, + toolName: leaf.name, + toolCallId: leaf.callId, + output: result.error === undefined ? this.capture(result.output) : undefined, + error: result.error, + ...extras, + }); + } + this.tracker.unlink(leaf.key); + this.settle(leaf.run, result.error !== undefined ? "failed" : result.closedBy ? "cancelled" : "success"); + } + + // -- the bus -------------------------------------------------------------- + + /** True when the outermost invocation on the chain is one we can see end. */ + private observedOuter(origin: Origin): boolean { + const outer = origin.chain?.[origin.chain.length - 1]; + return outer !== undefined && this.observed.has(outer) && !this.returned.has(outer); + } + + /** Where a bus event goes; opens a root run when it belongs to nothing. */ + private placeOrOpen(event: unknown, prefix: string, agentId: () => string, origin = originOf(event)): Located { + const located = this.locate(origin); + if (located) return located; + const run = this.openRoot(origin, prefix, agentId); + return { parentKey: run.key, run }; + } + + /** + * The root run for an event nothing encloses: Python's rule, "any other + * top-level instrumented call opens the session and becomes its root agent, + * named after its class". + * + * The top-level call is the OUTERMOST invocation on the event's `EventCaller` + * chain — a `ContextChatEngine.chat`, not the retriever or the model call it + * made — so everything one chat does lands in one run instead of one session + * per retrieval and per model call. The run ends when that invocation + * returns, which only `invocation()` sees; an invocation it did not see start + * (no hook on this build, or begun before `instrument()`) falls back to a + * bare run that ends with its one leaf, as every root run used to. + */ + private openRoot(origin: Origin, prefix: string, agentId: () => string): Run { + const outer = origin.chain?.[origin.chain.length - 1]; + if (outer && this.observedOuter(origin)) { + return this.openRun(prefix, className(read(outer, "caller")) ?? agentId(), { + parent: null, + caller: outer, + byInvocation: true, + }); + } + return this.openRun(prefix, agentId(), { parent: null, bare: true }); + } + + /** + * Run one LlamaIndex invocation — the callback `withEventCaller` binds an + * `EventCaller` around — and observe how it ends. Called from the hook on + * LlamaIndex's own event-caller storage (`hookInvocations`). + * + * A native promise is REPLACED by one that settles identically, never merely + * observed: attaching a rejection handler to the caller's own promise would + * mark it handled, and a rejection the application never handled would stop + * being reported. Anything else is returned untouched and only a synchronous + * throw is seen. + */ + invocation(caller: object, fn: () => unknown): unknown { + this.observed.add(caller); + let result: unknown; + try { + result = fn(); + } catch (error) { + this.returnedFrom(caller, undefined, error, true); + throw error; + } + if (result instanceof Promise && result.constructor === Promise) { + return result.then( + (value: unknown) => { + this.returnedFrom(caller, value); + return value; + }, + (error: unknown) => { + this.returnedFrom(caller, undefined, error, true); + throw error; + }, + ); + } + if (!(isObject(result) && typeof (result as { then?: unknown }).then === "function")) { + this.returnedFrom(caller, result); + } + return result; + } + + private returnedFrom(caller: object, value: unknown, error?: unknown, failed = false): void { + core.callSafely( + () => { + if (this.active) this.invocationEnded(caller, value, error, failed); + }, + [], + `${NAME}.invocationEnded`, + ); + } + + /** + * An invocation returned or threw. A failure is the ONLY failure signal a + * model call, a query or a legacy task has — `wrapLLMEvent` has no error path + * and a failed step dispatches no `agent-end` — so it closes what that call + * left open. A success ends only a run that has no end event of its own. + */ + private invocationEnded(caller: object, value: unknown, error: unknown, failed: boolean): void { + this.returned.add(caller); + const text = failed ? errorText(error) : undefined; + if (failed) { + for (const key of this.invocationLeaves.get(caller) ?? []) { + const leaf = this.leaves.get(key); + if (!leaf) continue; + this.closeLeaf(leaf, { error: text }); + if (isObject(error)) this.carried.add(error); + } + } + this.invocationLeaves.delete(caller); + const run = this.live(this.invocations.get(caller)); + if (!run) return; + if (failed) { + for (const key of [...run.leaves]) { + const leaf = this.leaves.get(key); + if (!leaf) continue; + this.closeLeaf(leaf, { error: text }); + if (isObject(error)) this.carried.add(error); + } + // One report per failure: a leaf that already carries it is the report. + if (!(isObject(error) && this.carried.has(error))) this.reportError(run, error); + this.finishRun(run, "failed", text); + } else if (run.byInvocation) { + this.finishRun(run, "success", this.options.captureMessages ? textOf(value) : undefined); + } + } + + llmStart(event: unknown): void { + const payload = detail(event); + const id = asId(payload.id); + const frame = this.frames.getStore(); + const origin = originOf(event); + const place = this.placeOrOpen(event, "llm", () => className(callersOf(event)[0]) ?? "llm", origin); + this.closeAnsweredTools(place.run, payload.messages); + const model = modelFromCallers(event) ?? (frame && frame.run === place.run ? frame.model : undefined) ?? place.run.model; + const leaf: Leaf = { + key: `llm:${id}`, + kind: "model", + run: place.run, + parentKey: place.parentKey, + name: model ?? "llm", + callId: id, + started: performance.now(), + model, + }; + this.openLeaf(leaf); + // A model call that throws has no `llm-end`; its invocation throwing is the + // only signal. Inside a workflow step the step's failure already closes it. + const invocation = origin.chain?.[0]; + if (invocation && !frame) { + const keys = this.invocationLeaves.get(invocation) ?? new Set(); + keys.add(leaf.key); + this.invocationLeaves.set(invocation, keys); + } + this.tracker.emit("modelRequest", leaf.key, { + parentKey: leaf.parentKey, + model, + requestId: id, + messages: this.capture(messagesOf(payload.messages)), + ...core.fwFields({ run_id: place.run.key }), + }); + } + + llmStream(event: unknown): void { + const leaf = this.leaves.get(`llm:${asId(detail(event).id)}`); + if (leaf && leaf.firstChunk === undefined) leaf.firstChunk = performance.now(); + } + + llmEnd(event: unknown): void { + const payload = detail(event); + const leaf = this.leaves.get(`llm:${asId(payload.id)}`); + if (leaf) this.closeLeaf(leaf, { response: payload.response }); + } + + /** + * Close tool leaves the model has already been shown the result of. + * + * `callTool` dispatches no `llm-tool-result` when a tool throws, so without + * this a failing tool in a legacy agent stays open for the life of the run. + * The next model call carries the result as a `toolResult` message — with + * `isError` — which is the framework's own record of how the tool ended. + */ + private closeAnsweredTools(run: Run, messages: unknown): void { + if (run.leaves.size === 0 || !Array.isArray(messages)) return; + const answers = new Map(); + for (const message of messages) { + const result = read(read(message, "options"), "toolResult"); + const id = nonEmpty(read(result, "id")); + if (id && isObject(result)) answers.set(id, result); + } + for (const key of [...run.leaves]) { + const leaf = this.leaves.get(key); + if (!leaf || leaf.kind !== "tool" || !leaf.rawId) continue; + const answer = answers.get(leaf.rawId); + if (!answer) continue; + this.closeLeaf(leaf, answer.isError === true ? { error: cleanToolError(answer.result) } : { output: answer.result }); + } + } + + private toolCallId(run: Run, rawId: string | undefined): string { + if (!rawId) return randomUUID(); + const seen = run.usedToolIds.get(rawId) ?? 0; + run.usedToolIds.set(rawId, seen + 1); + // A repeat of the same provider id within a run would pair wrongly. + return seen === 0 ? rawId : `${rawId}#${seen}`; + } + + toolCall(event: unknown): void { + const call = read(detail(event), "toolCall"); + const name = nonEmpty(read(call, "name")) ?? "tool"; + const rawId = nonEmpty(read(call, "id")); + const located = this.locate(originOf(event)); + // Inside a workflow the runtime's own `agentToolCallEvent` opened this call + // already, synchronously and before the tool ran; this is the same call. + if (located && this.findTool(rawId, located.run, "workflow")) return; + const place = located ?? this.placeOrOpen(event, "tool", () => name); + this.openTool(place, name, rawId, read(call, "input"), "bus"); + } + + private openTool(place: Located, name: string, rawId: string | undefined, input: unknown, source: Leaf["source"]): void { + const leaf: Leaf = { + key: this.nextKey("tool"), + kind: "tool", + run: place.run, + parentKey: place.parentKey, + name, + callId: this.toolCallId(place.run, rawId), + rawId, + source, + started: performance.now(), + }; + this.openLeaf(leaf); + this.tracker.emit("toolUse", leaf.key, { + parentKey: leaf.parentKey, + toolName: name, + toolCallId: leaf.callId, + input: this.capture(isObject(input) && !Array.isArray(input) ? input : input === undefined ? undefined : { input }), + ...core.fwFields({ run_id: place.run.key, tool_id: rawId !== leaf.callId ? rawId : undefined }), + }); + } + + /** The open tool leaf for a provider tool call id, in the run this event belongs to first. */ + private findTool(rawId: string | undefined, run: Run | null, source?: Leaf["source"]): Leaf | undefined { + if (!rawId) return undefined; + const search = (keys: Iterable): Leaf | undefined => { + for (const key of keys) { + const leaf = this.leaves.get(key); + if (leaf?.kind === "tool" && leaf.rawId === rawId && (!source || leaf.source === source)) return leaf; + } + return undefined; + }; + if (source) return run ? search(run.leaves) : undefined; + return (run ? search(run.leaves) : undefined) ?? search(this.leaves.keys()); + } + + toolResult(event: unknown): void { + const payload = detail(event); + const leaf = this.findTool(nonEmpty(read(read(payload, "toolCall"), "id")), this.locate(originOf(event))?.run ?? null); + if (!leaf) return; + const result = read(payload, "toolResult"); + const failed = read(result, "isError") === true; + this.closeLeaf(leaf, failed ? { error: cleanToolError(read(result, "output")) } : { output: read(result, "output") }); + } + + retrieveStart(event: unknown): void { + const payload = detail(event); + const id = asId(payload.id); + // The retriever's class, as Python names it (`VectorIndexRetriever`). The + // event does not carry the retriever; the patched `retrieve()` it was + // dispatched from does, and the dispatch runs in that call's async context. + const name = className(this.retrievers.getStore()) ?? "retriever"; + const place = this.placeOrOpen(event, "retrieve", () => name); + const leaf: Leaf = { + key: `retrieve:${id}`, + kind: "retrieval", + run: place.run, + parentKey: place.parentKey, + name, + callId: id, + started: performance.now(), + }; + this.openLeaf(leaf); + this.tracker.emit("toolUse", leaf.key, { + parentKey: leaf.parentKey, + toolName: leaf.name, + toolCallId: id, + input: this.capture({ query: queryText(payload.query) }), + ...core.fwFields({ run_id: place.run.key, kind: "retrieval" }), + }); + } + + retrieveEnd(event: unknown): void { + const payload = detail(event); + const leaf = this.leaves.get(`retrieve:${asId(payload.id)}`); + if (leaf) this.closeLeaf(leaf, { output: summarizeNodes(payload.nodes) }); + } + + /** + * A query engine call is a root run when nothing encloses it — Python's + * top-level `query_engine.query()` — and nothing at all inside a run, where + * its retrievals and model calls are what is worth seeing. + */ + queryStart(event: unknown): void { + const origin = originOf(event); + const owner = origin.callers[0]; + if (this.locate(origin, owner)) return; + // Called from inside ANOTHER invocation nothing encloses — a chat engine + // whose first act is a query: that invocation is the root, and the query + // is inside it (recording nothing of its own, as inside any run). + if (origin.chain && origin.chain.length > 1 && this.observedOuter(origin)) { + this.openRoot(origin, "invocation", () => className(owner) ?? "query_engine"); + return; + } + const payload = detail(event); + const run = this.openRun(`query:${asId(payload.id)}`, className(owner) ?? "query_engine", { + owner: isObject(owner) ? owner : null, + caller: origin.chain?.[0] ?? null, + goal: queryText(payload.query), + }); + const id = asId(payload.id); + this.queries.set(id, run); + run.forget = () => this.queries.delete(id); + } + + queryEnd(event: unknown): void { + const payload = detail(event); + const id = asId(payload.id); + const run = this.queries.get(id); + if (!run) return; + this.finishRun(run, "success", this.options.captureMessages ? textOf(payload.response) : undefined); + } + + /** + * Legacy `AgentRunner` (`LLMAgent`, `OpenAIAgent`, `ReActAgent`, …). + * + * `agent-start` fires for EVERY step and `agent-end` only after the last one, + * both carrying the step. Pairing them by step id — what this adapter used to + * do — opened a span per step and closed one, so every multi-step run left + * one agent open forever and split into two sessions. A task is identified by + * its FIRST step instead, reached by walking `prevStep`. + */ + agentStart(event: unknown): void { + const step = read(detail(event), "startStep"); + if (!isObject(step)) return; + const first = firstStep(step); + const firstId = asId(read(first, "id")); + const running = this.tasks.get(firstId); + if (running) { + running.lastActivity = performance.now(); + return; + } + const origin = originOf(event); + const owner = origin.callers[0]; + let parent = this.locate(origin, owner); + if (!parent && origin.chain && origin.chain.length > 1 && this.observedOuter(origin)) { + // A legacy agent called from inside another un-run invocation nests + // under it, the way a sub-agent nests under a workflow. + const root = this.openRoot(origin, "invocation", () => "invocation"); + parent = { parentKey: root.key, run: root }; + } + const run = this.openRun(`task:${firstId}`, className(owner) ?? "AgentRunner", { + parent, + owner: isObject(owner) ? owner : null, + caller: origin.chain?.[0] ?? null, + goal: textOf((read(read(read(step, "context"), "store"), "messages") as unknown[] | undefined)?.at(-1)), + }); + run.model = nonEmpty(read(read(read(read(step, "context"), "llm"), "metadata"), "model")); + this.tasks.set(firstId, run); + run.forget = () => this.tasks.delete(firstId); + } + + agentEnd(event: unknown): void { + const step = read(detail(event), "endStep"); + if (!isObject(step)) return; + const firstId = asId(read(firstStep(step), "id")); + const run = this.tasks.get(firstId); + if (run) this.finishRun(run, "success"); + } + + // -- workflows ------------------------------------------------------------ + + /** + * Open the run for an `AgentWorkflow.runStream()` call and return the hook to + * attach to the context it is about to create. + */ + beginWorkflow( + workflow: Json, + userInput: unknown, + module: WorkflowModule, + ): { run: Run; attach: (context: unknown) => void } { + // Its steps are recorded through its context (`attachContext`), never + // again as a plain workflow's. + for (const handler of handlerNames(workflow).keys()) { + if (isObject(handler) || typeof handler === "function") this.agentHandlers.add(handler); + } + const agents = read(workflow, "agents"); + const size = agents instanceof Map ? agents.size : 1; + const rootName = nonEmpty(read(workflow, "rootAgentName")); + const agentId = size <= 1 && rootName ? rootName : (className(workflow) ?? "AgentWorkflow"); + const run = this.openRun("workflow", agentId, { + parent: this.locate(), + goal: typeof userInput === "string" ? userInput : textOf(userInput), + fields: { workflow: className(workflow), agent_name: size <= 1 ? rootName : undefined }, + }); + return { + run, + attach: (context: unknown) => { + this.attachContext(run, workflow, context, module); + }, + }; + } + + failWorkflow(run: Run | null, error: unknown): void { + if (!run) return; + // Nothing below the run saw this failure, so the run reports it — once. + this.reportError(run, error); + this.finishRun(run, "failed", errorText(error)); + } + + /** + * An `error` event, for a failure no leaf or hook already carries. The + * Python adapter's rule: a second report of one failure double-counts on the + * session's error total, so only the innermost place that saw it reports it. + */ + private reportError(run: Run, error: unknown): void { + this.tracker.emit("error", run.key, { + errorType: error instanceof Error ? error.name || "Error" : typeof error, + message: error instanceof Error ? error.message || error.name : String(error), + traceback: error instanceof Error ? error.stack : undefined, + ...core.fwFields({ run_id: run.key }), + }); + } + + private attachContext(run: Run, workflow: Json, context: unknown, module: WorkflowModule): void { + // eslint-disable-next-line @typescript-eslint/no-this-alias -- the subscriber is a named function (its name is its degradation site) + const state = this; + const callContext = read(context, "__internal__call_context") as Subscribable | undefined; + const sendEvent = read(context, "__internal__call_send_event") as Subscribable | undefined; + if (typeof sendEvent?.subscribe === "function") { + sendEvent.subscribe( + core.safe(NAME, function workflowEvent(this: void, sent: unknown): void { + state.workflowEvent(run, sent, module); + }), + ); + } + if (typeof callContext?.subscribe !== "function") { + compat.warn( + "this @llamaindex/workflow context has no __internal__call_context, so workflow steps " + + "are not recorded (the run, its model calls and its tool calls still are).", + `${NAME}:call_context`, + ); + return; + } + const names = handlerNames(workflow); + callContext.subscribe(((handlerContext: HandlerContext, next: (context: HandlerContext) => void) => { + // `next` MUST be called exactly once whatever happens here: a subscriber + // that throws before it would silently stop the customer's workflow. + let wrapped: HandlerContext = handlerContext; + try { + const original = handlerContext.handler; + if (typeof original === "function") { + const name = names.get(original) ?? nonEmpty(original.name) ?? "step"; + const inputs = handlerContext.inputs; + const input = Array.isArray(inputs) ? (inputs[0] as unknown) : undefined; + wrapped = { ...handlerContext, handler: this.wrapStep(run, workflow, name, original, input) }; + } + } catch (error) { + core.callSafely( + () => { + throw error; + }, + [], + `${NAME}.wrapStep`, + ); + } + next(wrapped); + })); + } + + workflowEvent(run: Run, sent: unknown, module: WorkflowModule): void { + // A context outlives uninstall(); its subscriptions cannot be removed. + if (!this.active) return; + if (module.stopAgentEvent?.include(sent)) { + const data = read(sent, "data"); + const result = read(data, "result"); + this.finishRun(run, "success", this.options.captureMessages ? (textOf(result) ?? textOf(read(data, "message"))) : undefined); + return; + } + if (module.agentToolCallEvent?.include(sent)) { + // The runtime announces every tool call before running it — the only + // signal on releases whose `AgentWorkflow` calls tools directly rather + // than through `callTool` (workflow 1.1.5 dispatches nothing on the bus). + const data = read(sent, "data"); + const owner = run.sub ?? run; + const frame = this.frames.getStore(); + const place = frame && frame.run === owner ? { parentKey: frame.key, run: owner } : { parentKey: owner.key, run: owner }; + const name = nonEmpty(read(data, "toolName")) ?? "tool"; + this.openTool(place, name, nonEmpty(read(data, "toolId")), read(data, "toolKwargs"), "workflow"); + return; + } + if (module.agentToolCallResultEvent?.include(sent)) { + // The runtime's own record of how a tool ended. It is the ONLY signal for + // a tool that threw: `callTool` dispatches no `llm-tool-result` then. + const data = read(sent, "data"); + const leaf = this.findTool(nonEmpty(read(data, "toolId")), run.sub ?? run); + if (!leaf) return; + const output = read(data, "toolOutput"); + if (read(output, "isError") === true) { + this.closeLeaf(leaf, { error: cleanToolError(read(output, "result")) }); + } else { + this.closeLeaf(leaf, { output: read(data, "raw") ?? read(output, "result") }); + } + } + } + + private wrapStep( + run: Run, + workflow: Json, + name: string, + original: (...args: unknown[]) => unknown, + input: unknown, + ): (...args: unknown[]) => unknown { + // eslint-disable-next-line @typescript-eslint/no-this-alias -- the wrapper needs its own `this` + const state = this; + return function failproofaiStep(this: unknown, ...args: unknown[]): unknown { + if (!state.active) return original.apply(this, args); + const step = core.callSafely(() => state.stepStart(run, workflow, name, input), [], `${NAME}.stepStart`); + if (!step) return original.apply(this, args); + const finish = (value: unknown, error?: unknown): void => { + core.callSafely(() => { + state.stepEnd(step, value, error); + }, [], `${NAME}.stepEnd`); + }; + let result: unknown; + try { + result = state.frames.run(step.frame, () => original.apply(this, args)); + } catch (error) { + finish(undefined, error); + throw error; + } + if (isObject(result) && typeof (result as { then?: unknown }).then === "function") { + return (result as unknown as PromiseLike).then( + (value) => { + finish(value); + return value; + }, + (error: unknown) => { + finish(undefined, error); + throw error; + }, + ); + } + finish(result); + return result; + }; + } + + /** + * One step handler of a plain `createWorkflow()` workflow, seen as + * workflow-core binds its handler context (`hookAsyncContext`). + * + * A plain workflow has no run boundary of its own: the application creates + * the context, sends it events and stops reading whenever it likes. So the + * run is the context's BURST of activity — it opens with the first step + * handler of a context and ends once no step of it is running and none was + * started by the last one's output (checked a macrotask later, after the + * runtime has dispatched that output). A workflow that then waits for an + * event from outside (human-in-the-loop) records the next burst as a new run. + */ + plainStep(handlerContext: Json, proceed: () => unknown): unknown { + const original = handlerContext.handler; + if (typeof original !== "function" || this.agentHandlers.has(original)) return proceed(); + const root = rootContext(handlerContext); + const inputs = handlerContext.inputs; + const input = Array.isArray(inputs) ? (inputs[0] as unknown) : undefined; + let run = this.live(this.plainRuns.get(root)); + if (!run) { + const data = read(input, "data"); + run = this.openRun("workflow", "Workflow", { + parent: this.locate(), + goal: typeof data === "string" ? data : textOf(data), + fields: { workflow: "Workflow" }, + }); + this.plainRuns.set(root, run); + } + const label = eventLabel(input); + const name = nonEmpty(read(original, "name")) ?? (label ? `handle:${label}` : "step"); + const step = this.stepStart(run, {}, name, input, true); + if (!step) return proceed(); + const owner = run; + owner.inFlight += 1; + let done = false; + const end = (value: unknown, error?: unknown): void => { + if (done) return; + done = true; + owner.inFlight -= 1; + core.callSafely( + () => { + const data = read(value, "data"); + if (error === undefined && typeof data === "string") owner.output = data; + this.stepEnd(step, value, error); + if (owner.inFlight === 0 && !owner.ended) this.whenQuiet(owner); + }, + [], + `${NAME}.plainStepEnd`, + ); + }; + // The handler object is this invocation's own, so its handler is replaced + // in place — the same thing middleware does — rather than wrapped around. + // eslint-disable-next-line @typescript-eslint/no-this-alias -- the wrapper needs its own `this` + const state = this; + handlerContext.handler = function failproofaiStep(this: unknown, ...args: unknown[]): unknown { + let result: unknown; + try { + result = state.frames.run(step.frame, () => (original as (...a: unknown[]) => unknown).apply(this, args)); + } catch (error) { + end(undefined, error); + throw error; + } + if (isObject(result) && typeof (result as { then?: unknown }).then === "function") { + return (result as unknown as PromiseLike).then( + (value) => { + end(value); + return value; + }, + (error: unknown) => { + end(undefined, error); + throw error; + }, + ); + } + end(result); + return result; + }; + return proceed(); + } + + /** + * End a plain-workflow run if it is still idle once the runtime has + * dispatched the last step's output. Two microtask hops: the step's end runs + * in a reaction to the handler's promise, the runtime's `sendEvent` in the + * reaction to the promise returned in its place — queued one hop later, and + * starting any next handler synchronously. Any later and the run would end + * after the caller that awaited it (and after an `agent()` scope around it). + */ + private whenQuiet(run: Run): void { + queueMicrotask(() => { + queueMicrotask(() => { + core.callSafely( + () => { + if (this.active && !run.ended && run.inFlight === 0) { + this.finishRun(run, "success", this.options.captureMessages ? run.output : undefined); + } + }, + [], + `${NAME}.whenQuiet`, + ); + }); + }); + } + + private stepStart(run: Run, workflow: Json, name: string, event: unknown, plain = false): Step | null { + if (run.ended) return null; + const data = read(event, "data"); + // Only an AgentWorkflow's events name the agent holding the turn; a plain + // workflow's data is the application's own, whatever its field names. + const agentName = plain + ? undefined + : (nonEmpty(read(data, "currentAgentName")) ?? nonEmpty(read(data, "agentName"))); + const owner = plain ? run : this.subAgent(run, agentName); + run.lastActivity = owner.lastActivity = performance.now(); + const key = this.nextKey(`${run.key}:step`); + this.tracker.link(key, owner.key); + owner.steps.add(key); + const agents = read(workflow, "agents"); + const agent = agents instanceof Map ? agents.get(agentName ?? read(workflow, "rootAgentName")) : undefined; + const model = nonEmpty(read(read(read(agent, "llm"), "metadata"), "model")); + if (this.options.steps) { + this.tracker.emit("hookTriggered", key, { + parentKey: owner.key, + hookName: name, + hookId: key, + triggerEvent: "workflow_step", + input: this.capture(eventData(data)), + ...core.fwFields({ run_id: run.key, step: name, input_event: eventLabel(event), agent_name: agentName }), + }); + } + return { key, name, run, owner, frame: { key, run: owner, model } }; + } + + private stepEnd(step: Step, value: unknown, error?: unknown): void { + step.run.lastActivity = step.owner.lastActivity = performance.now(); + const failed = error !== undefined; + let reported = false; + if (failed) { + // Whatever this step had open failed with it — a model call that threw + // has no `llm-end` at all. + for (const key of [...step.owner.leaves]) { + const leaf = this.leaves.get(key); + if (leaf && leaf.parentKey === step.key) { + this.closeLeaf(leaf, { error: errorText(error) }); + reported = true; + } + } + } + if (this.options.steps) { + this.tracker.emit("hookCompleted", step.key, { + parentKey: step.owner.key, + hookName: step.name, + hookId: step.key, + outcome: failed ? "failed" : "success", + output: failed ? undefined : this.capture(eventData(read(value, "data") ?? value)), + error: failed ? errorText(error) : undefined, + ...core.fwFields({ run_id: step.run.key, step: step.name, output_event: eventLabel(value) }), + }); + } + // An AgentWorkflow is a single chain of steps: a step that throws ends the + // run, and the runtime never settles `run()` to tell us so. Under + // `steps: false` there is no failed hook, so unless a leaf carried it the + // run reports it — as the Python adapter does. + if (failed) { + if (!reported && !this.options.steps) this.reportError(step.owner, error); + this.finishRun(step.run, "failed", errorText(error)); + } + // Last: the step's leaves and its hook resolve their session through it. + step.owner.steps.delete(step.key); + this.tracker.unlink(step.key); + } + + /** + * The nested agent a multi-agent step belongs to, or the run itself. + * + * The Python adapter's rule: each distinct agent name opens a nested agent + * under the workflow and a handoff closes the previous one. The name is + * sticky — tool steps carry `agentName`, not `currentAgentName`, and a step + * with neither keeps whichever agent holds the turn. + */ + private subAgent(run: Run, name: string | undefined): Run { + if (!name || name === run.agentId || (run.sub && run.sub.agentId === name)) return run.sub ?? run; + if (run.sub) this.finishRun(run.sub, "success"); + run.subSeq += 1; + const sub = this.openRun(`${run.key}:sub${run.subSeq}`, name, { + parent: { parentKey: run.key, run }, + root: run, + fields: { agent_name: name, workflow: run.agentId }, + }); + run.sub = sub; + return sub; + } + + // -- teardown ------------------------------------------------------------- + + /** + * Close what nobody is going to close. Returns how many leaves and runs. + * + * Leaves first — a model call that threw has no `llm-end` — then runs that + * have been silent for `staleAfter`: LlamaIndex.TS never signals a legacy + * task whose step threw (no `agent-end`) or a `runStream()` nobody drained, + * and a run left open is a session the dashboard shows as `ongoing` forever. + * Python reaps only leaves because its span handler sees every drop; here + * the run-level signal genuinely does not exist. + */ + sweep(now = performance.now()): number { + const cutoff = now - this.options.staleAfter * 1000; + let closed = 0; + for (const leaf of [...this.leaves.values()]) { + if (leaf.started > cutoff) continue; + this.closeLeaf(leaf, { closedBy: "stale" }); + closed += 1; + } + for (const run of [...this.runs.values()].reverse()) { + if (run.ended || run.root !== null || run.leaves.size > 0 || run.lastActivity > cutoff) continue; + this.finishRun(run, "cancelled", undefined, "stale"); + closed += 1; + } + return closed; + } + + startReaper(): void { + if (this.options.reaperInterval <= 0 || this.reaper !== null) return; + this.reaper = setInterval(() => { + core.callSafely(() => this.sweep(), [], `${NAME}.reaper`); + }, this.options.reaperInterval * 1000); + this.reaper.unref(); + } + + /** What is still held, for tests: every per-run table, and the tracker. */ + residue(): { runs: number; leaves: number; tasks: number; queries: number; tracker: core.RunTracker } { + return { + runs: this.runs.size, + leaves: this.leaves.size, + tasks: this.tasks.size, + queries: this.queries.size, + tracker: this.tracker, + }; + } + + shutdown(): void { + this.active = false; + if (this.reaper !== null) clearInterval(this.reaper); + this.reaper = null; + // Newest first, so a sub-agent closes before the workflow that opened it. + for (const run of [...this.runs.values()].reverse()) this.finishRun(run, "cancelled", undefined, "uninstrument"); + this.leaves.clear(); + this.tasks.clear(); + this.queries.clear(); + this.tracker.reset(); + } +} + +interface Step { + key: string; + name: string; + run: Run; + owner: Run; + frame: Frame; +} + +function firstStep(step: Json): Json { + let current = step; + const seen = new Set(); + for (;;) { + const previous = read(current, "prevStep"); + if (!isObject(previous) || seen.has(previous)) return current; + seen.add(previous); + current = previous; + } +} + +/** A workflow-core handler context's root: one per `createContext()`, so one per context. */ +function rootContext(handlerContext: Json): object { + const root = read(handlerContext, "root"); + if (isObject(root)) return root; + let current: Json = handlerContext; + const seen = new Set(); + for (;;) { + const previous = read(current, "prev"); + if (!isObject(previous) || seen.has(previous)) return current; + seen.add(previous); + current = previous; + } +} + +/** Step handler functions are instance arrow fields, so they are named by the field. */ +function handlerNames(workflow: Json): Map { + const names = new Map(); + try { + for (const key of Object.keys(workflow)) { + const value = read(workflow, key); + if (typeof value === "function") names.set(value, key); + } + } catch { + // A frozen or exotic object; unnamed steps fall back to the function name. + } + return names; +} + +/** A workflow event's label: workflow-core tags each event with `Symbol.toStringTag`. */ +function eventLabel(event: unknown): string | undefined { + if (!isObject(event)) return undefined; + try { + const tag = /^\[object (.+)\]$/.exec(Object.prototype.toString.call(event))?.[1]; + return tag !== undefined && tag !== "Object" && !tag.startsWith("WorkflowEvent") ? tag : undefined; + } catch { + return undefined; + } +} + +/** A workflow event's payload, minus the run's whole state object. */ +function eventData(data: unknown): unknown { + if (!isObject(data) || Array.isArray(data)) return data; + const rest = { ...data }; + delete rest.state; + return rest; +} + +/** + * `AgentWorkflow` stores a thrown tool as `Error: ${new Error(String(output))}`, + * where `output` is already `prettifyError`'s rendering — three nested + * prefixes for one failure. Keep the innermost, as `: `. + * + * `prettifyError` has two spellings: `Error: ` on older releases and + * `Error(): ` on llamaindex 0.12, which recorded verbatim read + * "Error: Error(Error): unknown region: latam". + */ +function cleanToolError(result: unknown): string { + let text = typeof result === "string" ? result : String(result); + while (/^Error: (\w*Error: |Error\(\w*\): )/.test(text)) text = text.slice("Error: ".length); + const named = /^Error\((\w*)\): ([\s\S]*)$/.exec(text); + if (named) text = `${named[1] || "Error"}: ${named[2]}`; + return text; +} + +// --------------------------------------------------------------------------- +// Attachment +// --------------------------------------------------------------------------- + +interface Installed { + state: State; + unsubscribe: Array<() => void>; + patcher: core.Patcher; +} + +let installed: Installed | null = null; + +const BUS_EVENTS: Array<[string, keyof State]> = [ + ["llm-start", "llmStart"], + ["llm-stream", "llmStream"], + ["llm-end", "llmEnd"], + ["llm-tool-call", "toolCall"], + ["llm-tool-result", "toolResult"], + ["retrieve-start", "retrieveStart"], + ["retrieve-end", "retrieveEnd"], + ["query-start", "queryStart"], + ["query-end", "queryEnd"], + ["agent-start", "agentStart"], + ["agent-end", "agentEnd"], +]; + +/** + * Subscribe to every copy of the bus and patch every copy of `AgentWorkflow`. + * + * Separate from `install()` so a test can hand it stand-ins for the framework + * modules; `install()` is only about FINDING the right copies. + * + * THROWS while an earlier attach is still installed, before touching anything. + * Overwriting it would orphan that install's bus subscriptions and prototype + * patch — `uninstall()` only reaches the latest — so they would record for the + * life of the process, every event twice. A throw rather than a no-op because a + * no-op would hand back a handle for an install that did not happen, with + * options that were never applied. `instrument()` never reaches this: it skips + * an adapter that is already active. + * + * @internal Not part of the public API. + */ +export function attach( + rawOptions: Record, + modules: { + globals: GlobalModule[]; + workflows: WorkflowModule[]; + retrievers?: RetrieverModule[]; + asyncContexts?: AsyncContextModule[]; + frameworkPackage?: string; + }, +): { sweep: (now?: number) => number; residue: () => ReturnType } { + if (installed !== null) { + throw new Error( + "the llamaindex adapter is already installed; uninstrument(\"llamaindex\") (or " + + "adapter.uninstall()) before attaching again.", + ); + } + const options = parseOptions(rawOptions); + const state = new State(options, modules.globals, modules.frameworkPackage ?? PACKAGE); + const current: Installed = { state, unsubscribe: [], patcher: new core.Patcher() }; + installed = current; + + const buses = new Set(); + for (const module of modules.globals) { + const bus = module.Settings?.callbackManager as Bus | undefined; + if (bus && typeof bus.on === "function") buses.add(bus); + } + if (buses.size === 0) { + throw new Error( + "Settings.callbackManager is missing or has no `on` — this build of LlamaIndex does not " + + "expose the callback bus this adapter subscribes to.", + ); + } + for (const bus of buses) { + if (typeof bus.off !== "function") { + compat.warn( + "this build of LlamaIndex has no `callbackManager.off`, so uninstrument() cannot detach " + + "the handlers. They stay subscribed for the life of the process and emit nothing.", + `${NAME}:off`, + ); + } + for (const [event, method] of BUS_EVENTS) { + // Named per event BEFORE `safe()` reads the name: the name is the + // degradation site, and one shared site would let a handler that keeps + // failing on, say, `retrieve-end` switch off `llm-start` with it. + const named = { + [method](raw: unknown): void { + if (!state.active) return; + (state[method] as (event: unknown) => void).call(state, raw); + }, + }[method]!; + const handler = core.safe(NAME, named); + bus.on(event, handler); + current.unsubscribe.push(() => { + bus.off?.(event, handler); + }); + } + } + if (options.embeddings) { + logger.debug( + "llamaindex: embeddings=true has nothing to record — LlamaIndex.TS dispatches no embedding events.", + ); + } + + const storages = new Set>(); + for (const module of modules.globals) { + const storage = eventCallerStorage(module); + if (storage) storages.add(storage); + } + for (const storage of storages) hookInvocations(current, storage); + compat.probe(NAME, "EventCaller storage", () => storages.size > 0 || modules.globals.every((m) => !m.getEventCaller)); + for (const module of modules.retrievers ?? []) { + patchRetriever(current, module); + } + for (const module of modules.workflows) { + patchWorkflow(current, module); + } + for (const module of modules.asyncContexts ?? []) { + hookAsyncContext(current, module); + } + state.startReaper(); + logger.debug(`llamaindex adapter subscribed on ${buses.size} bus(es), ${current.patcher.size} patch(es)`); + return { sweep: (now?: number) => state.sweep(now), residue: () => state.residue() }; +} + +/** + * LlamaIndex's own `AsyncLocalStorage` of `EventCaller`s — module-private in + * `@llamaindex/core/global`, so found by watching which storage one call of + * the exported `getEventCaller()` reads. The prototype is swapped back before + * this returns: the window is one synchronous call, with no other code in it. + * + * @internal Exported for the unit tests. + */ +export function eventCallerStorage(module: GlobalModule): AsyncLocalStorage | null { + const getEventCaller = module.getEventCaller; + if (typeof getEventCaller !== "function") return null; + const proto = AsyncLocalStorage.prototype; + // eslint-disable-next-line @typescript-eslint/unbound-method -- restored as the same unbound function + const original = proto.getStore; + let found: unknown = null; + proto.getStore = function getStore(this: AsyncLocalStorage): unknown { + // eslint-disable-next-line @typescript-eslint/no-this-alias -- recording WHICH storage is read is the point + found ??= this; + return original.call(this); + }; + try { + getEventCaller(); + } catch { + // An exotic build; no storage, so no invocation boundaries. + } finally { + proto.getStore = original; + } + return found instanceof AsyncLocalStorage ? (found as AsyncLocalStorage) : null; +} + +/** + * See every LlamaIndex invocation start and end: `withEventCaller` runs each + * `@wrapEventCaller` method (a chat engine's `chat`, a query engine's `query`, + * a provider's `chat`, `AgentRunner.chat`) as `storage.run(new EventCaller(...), fn)`. + * An own `run` on that ONE storage object wraps `fn`; no other storage in the + * process is touched, and uninstall restores it. + * + * This is the run boundary the callback bus lacks: without it a chat engine + * has no start or end event at all, so its retrieval and its model call were + * two unrelated root runs in two sessions. + */ +function hookInvocations(current: Installed, storage: AsyncLocalStorage): void { + const state = current.state; + // Whatever `run` this storage has now (normally the prototype's), called with the storage as `this`. + const original = Reflect.get(storage, "run") as (this: unknown, ...args: unknown[]) => unknown; + const replacement = function run(this: unknown, store: unknown, callback: unknown, ...args: unknown[]): unknown { + if (!state.active || typeof callback !== "function" || !isObject(store) || !("caller" in store)) { + return original.call(this, store, callback, ...args); + } + const fn = callback as (...a: unknown[]) => unknown; + return original.call( + this, + store, + function invocation(this: unknown, ...inner: unknown[]): unknown { + return state.invocation(store, () => fn.apply(this, inner)); + }, + ...args, + ); + }; + current.patcher.patch(storage, "run", replacement); +} + +/** + * `BaseRetriever.prototype.retrieve` — a plain prototype method, so this also + * covers retrievers built before `instrument()` — binds the retriever for the + * `retrieve-start` it dispatches, which carries only the query. + */ +function patchRetriever(current: Installed, module: RetrieverModule): void { + const proto = module.BaseRetriever?.prototype as Record | undefined; + const ok = compat.probe(NAME, "BaseRetriever.retrieve", () => typeof proto?.retrieve === "function"); + if (!ok || !proto) return; + const original = proto.retrieve as (...args: unknown[]) => unknown; + const state = current.state; + const replacement = function retrieve(this: object, ...args: unknown[]): unknown { + if (!state.active || !isObject(this)) return original.apply(this, args); + return state.retrievers.run(this, () => original.apply(this, args)); + }; + current.patcher.patch(proto, "retrieve", replacement); +} + +/** + * Plain `createWorkflow()` workflows. workflow-core runs every step handler as + * `handlerContextAsyncLocalStorage.run(handlerContext, …)`, and from 1.1 that + * storage is an `AsyncContext.Variable` — a class exported from + * `@llamaindex/workflow-core/async-context`, so its prototype `run` sees every + * handler of every context, including workflows built before `instrument()`. + * (Before 1.1, and on `@llama-flow/core`, it is a closure: nothing to hook.) + * Only a value shaped like a handler context is acted on; every other use of + * the class passes straight through. + */ +function hookAsyncContext(current: Installed, module: AsyncContextModule): void { + const proto = module.AsyncContext?.Variable?.prototype as Record | undefined; + if (typeof proto?.run !== "function") { + logger.debug("llamaindex: this workflow-core has no AsyncContext.Variable; plain workflows are not runs."); + return; + } + const original = proto.run as (value: unknown, fn: () => unknown) => unknown; + const state = current.state; + const replacement = function run(this: unknown, value: unknown, fn: () => unknown): unknown { + if (!state.active || !isHandlerContext(value)) return original.call(this, value, fn); + let proceeded = false; + const proceed = (): unknown => { + proceeded = true; + return original.call(this, value, fn); + }; + try { + return state.plainStep(value, proceed); + } catch (error) { + // A failure of ours before the handler ran must not stop the workflow; + // one from the handler (after `proceed`) is the application's own. + if (proceeded) throw error; + core.callSafely( + () => { + throw error; + }, + [], + `${NAME}.plainStep`, + ); + return original.call(this, value, fn); + } + }; + current.patcher.patch(proto, "run", replacement); +} + +/** workflow-core's per-invocation handler context, by shape. */ +function isHandlerContext(value: unknown): value is Json { + return ( + isObject(value) && + typeof read(value, "handler") === "function" && + Array.isArray(read(value, "inputs")) && + read(value, "next") instanceof Set && + "prev" in value + ); +} + +function patchWorkflow(current: Installed, module: WorkflowModule): void { + const proto = module.AgentWorkflow?.prototype as Record | undefined; + const ok = compat.probe(NAME, "AgentWorkflow.runStream", () => typeof proto?.runStream === "function"); + if (!ok || !proto) return; + const original = proto.runStream as (...args: unknown[]) => unknown; + const state = current.state; + const replacement = function runStream(this: Json, ...args: unknown[]): unknown { + if (!state.active) return original.apply(this, args); + const wf = read(this, "workflow") as Json | undefined; + const createContext = read(wf, "createContext"); + const own = wf ? Object.getOwnPropertyDescriptor(wf, "createContext") : undefined; + // Without the context there is no end signal, so an opened run would stay + // open until the reaper: record the calls inside it as loose runs instead. + const observable = compat.probe( + NAME, + "AgentWorkflow.workflow.createContext", + () => typeof createContext === "function" && own?.writable === true, + ); + if (!observable) return original.apply(this, args); + let attach: ((context: unknown) => void) | undefined; + let run: Run | null = null; + core.callSafely( + () => { + ({ run, attach } = state.beginWorkflow(this, args[0], module)); + }, + [], + `${NAME}.beginWorkflow`, + ); + const intercept = attach !== undefined; + if (intercept) { + // `runStream` creates the context and sends the start event in one + // synchronous call, and the first step runs inside that send. So the + // subscriptions go on the context the moment it exists, for exactly the + // duration of this call. + wf!.createContext = function createContextOnce(this: unknown, ...inner: unknown[]): unknown { + const context = (createContext as (...a: unknown[]) => unknown).apply(this, inner); + core.callSafely(() => attach!(context), [], `${NAME}.attachContext`); + return context; + }; + } + try { + return original.apply(this, args); + } catch (error) { + core.callSafely(() => state.failWorkflow(run, error), [], `${NAME}.failWorkflow`); + throw error; + } finally { + if (intercept && own) Object.defineProperty(wf!, "createContext", own); + } + }; + current.patcher.patch(proto, "runStream", replacement); +} + +// --------------------------------------------------------------------------- +// The adapter +// --------------------------------------------------------------------------- + +async function loadCopies(specifier: string): Promise { + try { + return await compat.requireModuleCopies(specifier, INSTALL); + } catch { + return null; + } +} + +export const adapter: Adapter = { + name: NAME, + + async install(options: Record = {}): Promise { + compat.checkVersion(NAME, PACKAGE, { + minimum: MIN_VERSION, + below: BELOW_VERSION, + reason: "the first release whose agent() runs on the @llamaindex/workflow 1.1 runtime", + }); + + // `@llamaindex/core/global` is where the bus singleton lives, and it is the + // one module every LlamaIndex install has — an app on `@llamaindex/core` + + // `@llamaindex/workflow` alone never installs the umbrella. The umbrella + // re-exports the same `Settings`, so it is only the fallback for a layout + // (pnpm, strict) where the app cannot resolve the scoped package itself. + let globals = (await loadCopies("@llamaindex/core/global")) as GlobalModule[] | null; + let frameworkPackage = compat.versionString(PACKAGE) !== null ? PACKAGE : CORE_PACKAGE; + let retrievers = (await loadCopies("@llamaindex/core/retriever")) as RetrieverModule[] | null; + if (!globals?.some((module) => module.Settings)) { + globals = (await compat.requireModuleCopies(PACKAGE, INSTALL)) as GlobalModule[]; + frameworkPackage = PACKAGE; + // The umbrella re-exports `BaseRetriever` from the same copy of core. + retrievers = globals as RetrieverModule[]; + } + + // The workflow package is optional: a legacy-agent or query-engine app does + // not have it, and that is not a reason to record nothing. + let workflows: WorkflowModule[] = []; + if (compat.versionString(WORKFLOW_PACKAGE) !== null) { + compat.checkVersion(NAME, WORKFLOW_PACKAGE, { + minimum: WORKFLOW_MIN, + below: WORKFLOW_BELOW, + reason: "1.1 moved agent workflows onto @llamaindex/workflow-core", + }); + workflows = ((await loadCopies(WORKFLOW_PACKAGE)) ?? []) as WorkflowModule[]; + } + // Plain workflows: workflow-core ≥1.1 only (see `hookAsyncContext`). Resolved + // from the application, so a layout that does not hoist it (pnpm) leaves + // plain workflows unrecorded as runs — their model calls still are. + const asyncContexts = ((await loadCopies(ASYNC_CONTEXT_MODULE)) ?? []) as AsyncContextModule[]; + attach(options, { globals, workflows, retrievers: retrievers ?? [], asyncContexts, frameworkPackage }); + }, + + uninstall(): void { + const current = installed; + installed = null; + if (current === null) return; + for (const unsubscribe of current.unsubscribe) { + try { + unsubscribe(); + } catch { + // Detaching is best-effort; `state.active = false` below is what + // actually stops events being recorded. + } + } + current.patcher.restoreAll(); + core.callSafely(() => { + current.state.shutdown(); + }, [], `${NAME}.shutdown`); + }, +}; diff --git a/sdk/typescript/src/integrations/mastra.ts b/sdk/typescript/src/integrations/mastra.ts new file mode 100644 index 000000000..cd037a659 --- /dev/null +++ b/sdk/typescript/src/integrations/mastra.ts @@ -0,0 +1,1802 @@ +/** + * Mastra (`@mastra/core`). + * + * ## The mapping + * + * Mastra has no Python counterpart, so this is derived from the rule every + * adapter follows — "a framework construct becomes an agent if and only if it + * owns an LLM decision loop and has its own goal; everything else with a start + * and an end becomes the closest kind of leaf" — with CrewAI (agents calling + * agents) and LlamaIndex (workflow steps) as the analogues: + * + * | Mastra | FailproofAI | + * |---|---| + * | `Agent.generate()` / `.stream()` (and the legacy / VNext variants) | `agent_start` / `agent_end`, `agent_id` = the agent's `name` (its `id` if unnamed), never a UUID | + * | a sub-agent another agent delegates to (`agents: {…}`) | nested `agent_start` / `agent_end`, `parent_id` = the caller, inside the caller's `agent-` tool call | + * | each LLM step of the loop | `model_request` / `model_response`, paired on `request_id`, with the model id, integer tokens, a string `stop_reason` and `duration_ms` | + * | a tool the model calls | `tool_use` / `tool_result` carrying the MODEL's `toolCallId`, attributed to the agent whose loop called it | + * | a workflow run (`run.start()`, `.stream()`, `.resume()`) | `agent_start` / `agent_end`, `agent_id` = the workflow id | + * | a workflow step (incl. every branch taken, parallel step and loop iteration) | `hook_triggered` / `hook_completed`, `trigger_event="workflow_step"` | + * | a nested workflow, or an agent used as a step | a nested agent under the workflow, inside that step's hook | + * | a step's `suspend()` … `run.resume({ step, resumeData })` | `human_wait` + `agent_pause` … `agent_resume` + `human_input` on the SAME workflow span, which stays open while suspended; ids `:` | + * | `agent.network()` | ONE agent (`fw_method="network"`); the router Mastra builds per decision (`routing-agent`) is that agent's own model steps, and the agent it delegates to is nested under it | + * | a run an input or output processor blocks (`abort()` / tripwire) | `agent_end` `outcome="rejected"` | + * | Mastra's own machinery — the agentic loop and a network are built from internal workflows | nothing | + * + * **Sessions.** A root run (nothing enclosing it, no `failproofai.session()` / + * `agent()` scope) takes the conversation id Mastra gives it, the rule every + * adapter follows ("a conversation spanning runs is one session where the + * framework provides a conversation id"): an agent run's memory thread + * (`memory: { thread }`, 0.x `threadId`), so every turn of a thread is one + * session; a workflow run's run id, so a suspended run and its resume — in + * this process or another — are one session. Otherwise the session is new. + * An enclosing scope always wins, and a nested run is in its parent's. + * + * A streamed call finalises when Mastra finishes the stream — which it does as + * the caller consumes it — not when `stream()` returns: the model events carry + * the real token counts and stop reason, and `agent_end` is not emitted while + * the loop is still running. A failure is recorded once, on the event it + * happened in (the `model_response`, the `tool_result`, the `hook_completed`), + * and closes the enclosing agent `failed`; there is no separate `error` event + * for each layer the exception unwound through. + * + * Token counts are the provider's own, read off each step. A STREAMED step + * from an OpenAI-compatible endpoint carries none unless the request asked for + * them (`stream_options.include_usage`), and the model Mastra's router builds + * for a `{ id, url }` / custom provider does not ask — Mastra itself then + * reports zero usage — so such a step is recorded without tokens. Nothing a + * call passes turns it on (`providerOptions` cannot: the provider overwrites + * `stream_options`); a model built with usage on — `@ai-sdk/openai`, or + * `createOpenAICompatible({ includeUsage: true })` — is recorded with them. + * + * ## Why prototype patching, and where + * + * Mastra 1.x has a real tracing extension point — an `ObservabilityExporter` + * receiving `agent_run` / `model_step` / `tool_call` / `workflow_step` spans — + * but it only exists for agents and workflows registered on a `Mastra` instance + * configured with `@mastra/observability`. A bare `new Agent(...)` gets a no-op + * tracer and emits nothing to hook, and that is the most common way Mastra is + * used outside its dev server. It is also absent from 0.x in this shape. So the + * adapter patches the few PROTOTYPE methods every run goes through, on both + * lines alike: + * + * * `Agent.prototype.generate` / `.stream` (+ variants) and `.network` — the + * agent span. + * * `Agent.prototype.__runInputProcessors` — where an input processor's + * `abort()` lands. A blocked `stream()` calls none of its callbacks, so this + * is what ends it (1.x also exposes the stream's finish; 0.x does not). + * * `Agent.prototype.resolveModelConfig` — every model the agent will call + * passes through here, per run. The resolved model is handed back behind a + * proxy whose `doGenerate` / `doStream` are observed: that is the AI SDK + * `LanguageModel` contract (V1–V3), which Mastra itself calls once per LLM + * step, so each step is one pair with the provider's own usage and finish + * reason — rather than one collapsed pair read off the final result. + * * `Agent.prototype.convertTools` — the one step that turns every kind of tool + * (assigned, memory, toolset, client, sub-agent, workflow-as-tool) into what + * the loop executes. Wrapping its OUTPUT covers tools created before + * `instrument()` ran, which patching the `createTool` export never could: an + * ES-module namespace is read-only, and a Mastra tool's `execute` lives on + * the instance. + * * `Run.prototype._start` / `_resume` (+ `_restart`, `_timeTravel`) and + * `DefaultExecutionEngine.prototype.executeStep` — the workflow span and its + * steps. Runs Mastra marks internal (`isInternalWorkflow`, or a + * `tracingPolicy.internal` with the WORKFLOW bit) are skipped, which is what + * keeps the agentic loop's own `execution-workflow` / `agentic-loop` out. + * + * Every copy of `@mastra/core` the application loads is patched — see + * `compat.requireModuleCopies()`: the ES-module and CommonJS builds are two + * unrelated sets of classes. + * + * ## How events find their agent + * + * An adapter-private `AsyncLocalStorage` carries the Mastra run that is + * executing (`Frame`). It never binds FailproofAI identity — it only answers + * "which of OUR runs is this?", and every event is still emitted through the + * `RunTracker` with an explicit parent key. The model proxy and the tool + * wrappers capture the frame when the run builds them, so an event keeps its + * agent even when Mastra later executes it from a stream the caller is + * pulling on a different async path. A tool call runs its body inside a + * `tool` frame of the calling agent, which is what nests a sub-agent under the + * agent that delegated to it — and, inside a workflow step, what attributes a + * tool to the agent that called it rather than to the workflow. + */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; + +import { current as currentIdentity } from "../context.js"; +import { agent as agentScope } from "../scopes.js"; +import * as compat from "./compat.js"; +import * as core from "./core.js"; +import type { Adapter } from "./core.js"; + +const NAME = "mastra"; +const PACKAGE = "@mastra/core"; +const INSTALL = "npm install @mastra/core"; + +/** + * The oldest release the patch points above all exist in with this shape: + * `resolveModelConfig` and `Run._start` both arrived in the 0.20 line, when + * `generate` / `stream` became the loop that calls a `LanguageModelV2` once per + * step. Earlier releases route model calls through the AI SDK's own + * `generateText` and run workflows through a different `Run`, so nothing here + * would see a model step or a workflow step. + */ +const MINIMUM = "0.20.0"; + +/* + * ## Lifetime + * + * `instrument()` hands Mastra objects that outlive it: a model behind a proxy, + * tools behind wrappers, a stream the caller is still reading. Restoring the + * prototypes does not reach any of them, so every recording path checks that + * the tracker it began on is still LIVE (`live()`), and nothing recreates a + * tracker behind `uninstrument()`'s back: + * + * * `tracker` exists exactly while `enabled` — created by `install()`, dropped + * by `uninstall()`. A proxy, wrapper, span or frame remembers the tracker it + * was built under; once that tracker is gone it is a pass-through for good, + * even after a later `instrument()` creates a new one. + * * `uninstall()` turns `enabled` off FIRST, then closes what is still open — + * model steps, tool calls, workflow steps (marked `fw_incomplete`), then + * agents `cancelled` — so the trace ends where the recording did instead of + * leaving spans `ongoing` forever. + * * Open spans are tracked only so teardown can close them. Every end path + * removes its own entry; a stream nobody ever consumes has no end path, so + * the registries are bounded like the tracker's own run table. + * * `wrapTool()` works without `instrument()` — it is the call-site helper for + * bundled applications — so a call with no live installation records on a + * `standalone` tracker, which is self-contained (its run opens and closes in + * the one call) and never touched by `uninstall()`. + */ + +let enabled = false; +let tracker: core.RunTracker | null = null; +let standalone: core.RunTracker | null = null; +let patcher: core.Patcher | null = null; + +function newTracker(options: Record = {}): core.RunTracker { + return new core.RunTracker(NAME, { + baseFields: core.frameworkFields(NAME, PACKAGE), + fieldLimit: typeof options.captureLimit === "number" ? options.captureLimit : undefined, + }); +} + +/** May something begun on `t` still record? */ +function live(t: core.RunTracker | null | undefined): t is core.RunTracker { + return t !== null && t !== undefined && ((enabled && t === tracker) || t === standalone); +} + +/** Same bound as `RunTracker`'s own run table. */ +const MAX_OPEN = 10_000; + +/** A span that is open until it is `closed`, on the tracker it began on. */ +interface Span { + tracker: core.RunTracker; + closed: boolean; +} + +/** Open spans, oldest first, forgetting the oldest past `MAX_OPEN`. */ +class OpenSpans { + private readonly items = new Set(); + + add(item: T): void { + while (this.items.size >= MAX_OPEN) { + const oldest = this.items.values().next(); + if (oldest.done) break; + this.items.delete(oldest.value); + } + this.items.add(item); + } + + /** End `item`: true when it was still open on a live tracker. */ + close(item: T): boolean { + this.items.delete(item); + if (item.closed) return false; + item.closed = true; + return live(item.tracker); + } + + /** Remove and return everything still open, newest first. */ + drain(): T[] { + const all = [...this.items].reverse().filter((item) => !item.closed); + this.items.clear(); + for (const item of all) item.closed = true; + return all; + } + + get size(): number { + return this.items.size; + } +} + +/** The Mastra run currently executing, as far as this adapter is concerned. */ +interface Frame { + /** The tracker the run was begun on; a frame from a dead one records nothing. */ + tracker: core.RunTracker; + /** Our `RunTracker` key for the agent or workflow run. */ + key: string; + /** The `Agent` or workflow `Run` instance that owns the run. */ + owner: object; + /** `tool` while a tool the agent called is executing. */ + kind: "agent" | "workflow" | "tool"; + /** Set on the frame of an `agent.network()` run, whose router it absorbs. */ + network?: boolean; + /** Ends the agent run this frame is, for an end no Mastra callback reports. */ + close?: (outcome: string) => void; +} + +const frames = new AsyncLocalStorage(); + +/** Mastra workflow run id -> our key, so a step can find its workflow span. */ +const workflowRuns = new Map(); + +const WRAPPED = Symbol.for("failproofai.wrapped"); + +/** + * Our own model proxies and agent-tool wrappers, to what they wrap. A reused + * agent can hand a later run the proxy or tool an earlier run built; that one + * is bound to the earlier run (or a dead installation), so it is unwrapped and + * observed afresh rather than skipped as "already wrapped". + */ +const modelProxies = new WeakMap(); +const toolWrappers = new WeakMap unknown>(); + +function markWrapped(wrapper: T, original: unknown): T { + (wrapper as Record)[WRAPPED] = original; + return wrapper; +} + +// --------------------------------------------------------------------------- +// Small readers. Each takes whatever Mastra or a provider handed us and +// returns a value the schema accepts, or undefined — never throws. +// --------------------------------------------------------------------------- + +function errorOf(error: unknown): { type: string; message: string } { + if (error instanceof Error) return { type: error.name || "Error", message: error.message }; + if (typeof error === "object" && error !== null) { + const value = error as { name?: unknown; message?: unknown }; + if (typeof value.message === "string") { + return { type: typeof value.name === "string" ? value.name : "Error", message: value.message }; + } + } + return { type: typeof error, message: String(error) }; +} + +const describeError = (error: unknown): string => { + const detail = errorOf(error); + return `${detail.type}: ${detail.message}`; +}; + +function asInt(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.round(value) + : undefined; +} + +/** + * A token count in any of the provider shapes: a number (V1 `promptTokens`, + * V2 `inputTokens`) or V3's `{ total, … }` breakdown. + */ +function tokenCount(value: unknown): number | undefined { + if (typeof value === "object" && value !== null) return asInt((value as { total?: unknown }).total); + return asInt(value); +} + +function usageOf(usage: unknown): { inputTokens?: number; outputTokens?: number } { + if (typeof usage !== "object" || usage === null) return {}; + const value = usage as Record; + return { + inputTokens: tokenCount(value.inputTokens ?? value.promptTokens), + outputTokens: tokenCount(value.outputTokens ?? value.completionTokens), + }; +} + +/** V1/V2 finish reasons are strings; V3's are `{ unified, raw }`. */ +function finishReasonOf(value: unknown): string | undefined { + if (typeof value === "string" && value) return value; + if (typeof value === "object" && value !== null) { + const { unified, raw } = value as { unified?: unknown; raw?: unknown }; + if (typeof unified === "string" && unified) return unified; + if (typeof raw === "string" && raw) return raw; + } + return undefined; +} + +function agentLabel(instance: unknown): { label: string; rawId?: string } { + const value = instance as { name?: unknown; id?: unknown } | undefined; + const raw = typeof value?.name === "string" && value.name ? value.name : value?.id; + const label = core.normalizeAgentId(raw, "mastra-agent"); + const rawId = typeof value?.id === "string" && value.id !== label ? value.id : undefined; + return { label, rawId }; +} + +/** The run's goal: the prompt, when the caller handed one over as text. */ +function goalOf(input: unknown): string | undefined { + if (typeof input === "string") return input; + if (!Array.isArray(input)) return undefined; + for (let i = input.length - 1; i >= 0; i -= 1) { + const item = input[i] as unknown; + if (typeof item === "string") return item; + const message = item as { role?: unknown; content?: unknown } | null; + if (message?.role === "user" && typeof message.content === "string") return message.content; + } + return undefined; +} + +/** One `LanguageModel` prompt part, reduced to what a reader needs. */ +function partToWire(part: unknown): unknown { + const value = (part ?? {}) as Record; + switch (value.type) { + case "text": + return { type: "text", text: value.text }; + case "tool-call": + return { type: "tool_call", id: value.toolCallId, name: value.toolName, input: value.input ?? value.args }; + case "tool-result": + return { type: "tool_result", id: value.toolCallId, name: value.toolName, output: value.output ?? value.result }; + default: + return { type: value.type }; + } +} + +function contentToWire(content: unknown): unknown { + if (!Array.isArray(content)) return content; + if (content.every((part) => (part as { type?: unknown } | null)?.type === "text")) { + return content.map((part) => (part as { text?: unknown }).text).join(""); + } + return content.map(partToWire); +} + +/** + * A `LanguageModel` call's prompt as `{messages, system}`. + * + * The provider-level prompt is what the model actually receives — the agent's + * instructions as a system message, memory, the tool results of earlier steps + * — so it is recorded rather than what the caller passed to `generate()`. + */ +function promptOf(options: unknown): { + messages?: Array>; + system?: unknown; + tools?: Array>; +} { + const value = (options ?? {}) as { prompt?: unknown; tools?: unknown; mode?: { tools?: unknown } }; + const out: ReturnType = {}; + if (Array.isArray(value.prompt)) { + const system: unknown[] = []; + const messages: Array> = []; + for (const item of value.prompt) { + const message = (item ?? {}) as { role?: unknown; content?: unknown }; + if (message.role === "system") system.push(message.content); + else messages.push({ role: message.role, content: contentToWire(message.content) }); + } + out.messages = messages; + if (system.length > 0) out.system = system.length === 1 ? system[0] : system; + } + const tools = Array.isArray(value.tools) ? value.tools : value.mode?.tools; + if (Array.isArray(tools) && tools.length > 0) { + out.tools = tools.map((tool) => { + const entry = (tool ?? {}) as { name?: unknown; description?: unknown }; + return { name: entry.name, description: entry.description }; + }); + } + return out; +} + +// --------------------------------------------------------------------------- +// Model steps +// --------------------------------------------------------------------------- + +interface ModelCall extends Span { + requestId: string; + runKey: string; + model?: string; + started: number; +} + +const openModels = new OpenSpans(); + +interface ModelOutcome { + model?: string; + finishReason?: string; + usage?: unknown; + text?: string; + toolCalls?: Array>; + error?: unknown; +} + +function modelIdOf(model: object): string | undefined { + const id = (model as { modelId?: unknown }).modelId; + return typeof id === "string" && id ? id : undefined; +} + +function beginModelCall(frame: Frame, model: object, options: unknown): ModelCall | undefined { + const t = frame.tracker; + if (!live(t)) return undefined; + const runKey = frame.key; + const requestId = randomUUID(); + const modelId = modelIdOf(model); + const provider = (model as { provider?: unknown }).provider; + t.emit("modelRequest", requestId, { + parentKey: runKey, + model: modelId, + ...promptOf(options), + requestId, + ...core.fwFields({ provider: typeof provider === "string" ? provider : undefined }), + }); + const call: ModelCall = { tracker: t, closed: false, requestId, runKey, model: modelId, started: Date.now() }; + openModels.add(call); + return call; +} + +function endModelCall(call: ModelCall, outcome: ModelOutcome): void { + // Already closed — by teardown, when its installation is gone. + if (!openModels.close(call)) return; + const usage = usageOf(outcome.usage); + call.tracker.emit("modelResponse", call.requestId, { + parentKey: call.runKey, + model: outcome.model ?? call.model, + stopReason: outcome.finishReason ?? (outcome.error !== undefined ? "error" : undefined), + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + content: outcome.text ? outcome.text : undefined, + role: "assistant", + requestId: call.requestId, + error: outcome.error === undefined ? undefined : describeError(outcome.error), + ...core.fwFields({ + duration_ms: core.ms(Date.now() - call.started), + tool_calls: outcome.toolCalls?.length ? outcome.toolCalls : undefined, + }), + }); +} + +/** What a `doGenerate` result says, in any of the V1–V3 shapes. */ +function generateOutcome(result: unknown): ModelOutcome { + const value = (result ?? {}) as { + content?: unknown; + text?: unknown; + toolCalls?: unknown; + finishReason?: unknown; + usage?: unknown; + response?: { modelId?: unknown }; + }; + let text = typeof value.text === "string" ? value.text : ""; + const toolCalls: Array> = []; + if (Array.isArray(value.content)) { + for (const part of value.content) { + const entry = (part ?? {}) as Record; + if (entry.type === "text" && typeof entry.text === "string") text += entry.text; + if (entry.type === "tool-call") { + toolCalls.push({ id: entry.toolCallId, name: entry.toolName, input: entry.input ?? entry.args }); + } + } + } + if (Array.isArray(value.toolCalls)) { + for (const call of value.toolCalls) { + const entry = (call ?? {}) as Record; + toolCalls.push({ id: entry.toolCallId, name: entry.toolName, input: entry.args ?? entry.input }); + } + } + return { + model: typeof value.response?.modelId === "string" ? value.response.modelId : undefined, + finishReason: finishReasonOf(value.finishReason), + usage: value.usage, + text, + toolCalls, + }; +} + +/** + * Fold one `doStream` part into the step's outcome. The stream is the only + * place a streamed step's usage and finish reason exist, so they are read as + * they pass rather than from Mastra's aggregate afterwards. + */ +function foldStreamPart(outcome: ModelOutcome, part: unknown): void { + const value = (part ?? {}) as Record; + switch (value.type) { + case "text-delta": { + const delta = value.delta ?? value.textDelta; + if (typeof delta === "string") outcome.text = (outcome.text ?? "") + delta; + break; + } + case "tool-call": + (outcome.toolCalls ??= []).push({ + id: value.toolCallId, + name: value.toolName, + input: value.input ?? value.args, + }); + break; + case "response-metadata": + if (typeof value.modelId === "string" && value.modelId) outcome.model = value.modelId; + break; + case "finish": + outcome.finishReason = finishReasonOf(value.finishReason); + outcome.usage = value.usage; + break; + case "error": + outcome.error = value.error; + break; + default: + break; + } +} + +type ModelMethod = (options: unknown) => PromiseLike; + +/** + * One observed `doGenerate` / `doStream` call: a step. + * + * The result decides how the step ends, not the method name. A provider's + * `doGenerate` answers with the finished content, but the model Mastra 0.x + * resolves is its own V5 wrapper, whose `doGenerate` ALSO answers with a + * `stream` — reading the result as a finished generation there records a step + * with no tokens and no stop reason. So a result carrying a readable stream is + * observed as one, whichever method produced it. + */ +function observedCall(target: object, original: ModelMethod, frame: Frame, name: string): ModelMethod { + const observed = async function (options: unknown): Promise { + // Built under an installation that is gone: the model, untouched. + if (!live(frame.tracker)) return original.call(target, options); + const call = core.callSafely(beginModelCall, [frame, target, options], `${NAME}.model`); + let result: unknown; + try { + result = await original.call(target, options); + } catch (error) { + if (call) core.callSafely(endModelCall, [call, { error }], `${NAME}.model`); + throw error; + } + if (!call) return result; + const stream = (result as { stream?: unknown } | null)?.stream; + if (typeof (stream as ReadableStream | undefined)?.getReader !== "function") { + core.callSafely( + (value: unknown) => endModelCall(call, generateOutcome(value)), + [result], + `${NAME}.model`, + ); + return result; + } + const outcome: ModelOutcome = {}; + return { + ...(result as object), + // Pull-based, so a consumer that cancels still closes the step (see + // core.observeStream). A cancel ends it with the cancel reason as its error. + stream: core.observeStream( + stream as ReadableStream, + (part) => foldStreamPart(outcome, part), + (error) => endModelCall(call, error === undefined ? outcome : { ...outcome, error: outcome.error ?? error }), + `${NAME}.stream`, + ), + }; + }; + Object.defineProperty(observed, "name", { value: name, configurable: true }); + return observed; +} + +/** + * Hand a resolved model back behind a proxy that observes its two call + * methods and is otherwise the model. + * + * Every other method is bound to the real model, and every property read with + * the real model as receiver: a provider class that keeps state in `#private` + * fields throws when one of its methods runs with a proxy as `this`. + */ +function observeModel(model: unknown, frame: Frame): unknown { + if (typeof model !== "object" || model === null) return model; + // One of ours from an earlier run: observe the model it wraps, for THIS run. + const target = modelProxies.get(model) ?? model; + const doGenerate = (target as { doGenerate?: unknown }).doGenerate; + const doStream = (target as { doStream?: unknown }).doStream; + if (typeof doGenerate !== "function" && typeof doStream !== "function") return model; + if ((target as Record)[WRAPPED] !== undefined) return model; + + const cache = new Map(); + const proxy = new Proxy(target, { + get(object, property) { + if (property === WRAPPED) return target; + const value: unknown = Reflect.get(object, property); + if (typeof value !== "function" || property === "constructor") return value; + let bound = cache.get(property); + if (bound === undefined || (bound as { [WRAPPED]?: unknown })[WRAPPED] !== value) { + const method = value as ModelMethod; + bound = + property === "doGenerate" || property === "doStream" + ? observedCall(object, method, frame, property) + : (method as (...args: unknown[]) => unknown).bind(object); + markWrapped(bound as object, value); + cache.set(property, bound); + } + return bound; + }, + }); + modelProxies.set(proxy, target); + return proxy; +} + +// --------------------------------------------------------------------------- +// Tools +// --------------------------------------------------------------------------- + +/** + * A tool call that returned a failure instead of throwing it. + * + * Mastra 0.x catches a tool's exception and hands the model a `MastraError` + * (1.x re-throws it); both lines return a `{ error: true, message }` object + * for input that fails validation. Either way the call failed, and a + * `tool_result` recording it as output would say otherwise. + */ +function failureOf(value: unknown): unknown { + if (value instanceof Error) return value; + if (typeof value === "object" && value !== null) { + const flagged = value as { error?: unknown; message?: unknown }; + if (flagged.error === true && typeof flagged.message === "string") return flagged; + } + return undefined; +} + +function inputRecord(input: unknown): Record | undefined { + if (input === undefined || input === null) return undefined; + if (typeof input === "object" && !Array.isArray(input)) return input as Record; + return { input }; +} + +interface ToolCall extends Span { + key: string; + toolName: string; + toolCallId: string; + parentKey?: string; + /** Set when the call opened its own root run, which it must close. */ + ownRun?: string; +} + +const openTools = new OpenSpans(); + +function beginToolCall( + t: core.RunTracker, + toolName: string, + toolCallId: string, + input: unknown, + parentKey: string | undefined, +): ToolCall | undefined { + if (!live(t)) return undefined; + let ownRun: string | undefined; + if (parentKey === undefined && currentIdentity().sessionId === null) { + // Nothing is running and no scope is open: a tool called on its own is its + // own run, exactly as a bare LangChain tool is, rather than an event + // dropped for want of a session. + ownRun = randomUUID(); + t.startAgent(ownRun, { agentId: core.normalizeAgentId(toolName, "tool"), ...core.fwFields({ kind: "tool" }) }); + parentKey = ownRun; + } + const key = `${parentKey ?? "ambient"}:tool:${toolCallId}:${randomUUID()}`; + t.emit("toolUse", key, { parentKey, toolName, toolCallId, input: inputRecord(input) }); + const call: ToolCall = { tracker: t, closed: false, key, toolName, toolCallId, parentKey, ownRun }; + openTools.add(call); + return call; +} + +function endToolCall(call: ToolCall, output: unknown, error?: unknown): void { + // Already closed — by teardown, when its installation is gone. + if (!openTools.close(call)) return; + const t = call.tracker; + const failure = error ?? failureOf(output); + t.emit("toolResult", call.key, { + parentKey: call.parentKey, + toolName: call.toolName, + toolCallId: call.toolCallId, + output: failure === undefined ? output : undefined, + error: failure === undefined ? undefined : describeError(failure), + }); + if (call.ownRun !== undefined) { + t.endAgent(call.ownRun, { outcome: failure === undefined ? "success" : "failed" }); + } +} + +/** + * Run `execute` bracketed by a tool span; the one `try` only re-throws. + * `frame` is what the tool's own body runs inside, when there is one. + */ +function runTool( + begin: () => ToolCall | undefined, + execute: () => unknown, + frame: Frame | undefined, +): unknown { + const call = core.callSafely(begin, [], `${NAME}.tool`); + if (!call) return execute(); + const end = (output: unknown, error?: unknown): void => { + core.callSafely(endToolCall, [call, output, error], `${NAME}.tool`); + }; + let result: unknown; + try { + result = frame ? frames.run(frame, execute) : execute(); + } catch (error) { + end(undefined, error); + throw error; + } + if (typeof (result as PromiseLike | null)?.then === "function") { + return (result as PromiseLike).then( + (value) => { + end(value); + return value; + }, + (error: unknown) => { + end(undefined, error); + throw error; + }, + ); + } + end(result); + return result; +} + +/** A copy of `tool` with a new `execute`, keeping its prototype and fields. */ +function withExecute(tool: T, execute: unknown): T { + return Object.assign(Object.create(Object.getPrototypeOf(tool) as object | null) as T, tool, { execute }); +} + +/** + * Wrap the tools one agent run is about to execute. + * + * These are the loop's converted tools, which all share the AI SDK signature + * `execute(input, { toolCallId, … })` whatever kind of tool they came from, on + * both major lines — so the input is the validated arguments and the id is the + * model's own. + */ +function observeTools(tools: unknown, runFrame: Frame, agent: object): unknown { + if (typeof tools !== "object" || tools === null) return tools; + const out: Record = { ...(tools as Record) }; + const frame: Frame = { tracker: runFrame.tracker, key: runFrame.key, owner: agent, kind: "tool" }; + for (const [name, tool] of Object.entries(out)) { + const execute = (tool as { execute?: unknown } | null)?.execute; + if (typeof tool !== "object" || tool === null || typeof execute !== "function") continue; + // One of ours from an earlier run: wrap what it wraps, for THIS run. + const ours = toolWrappers.get(execute); + if (ours === undefined && (execute as unknown as Record)[WRAPPED] !== undefined) continue; + const original = ours ?? (execute as (...args: unknown[]) => unknown); + const wrapped = function failproofaiToolExecute(this: unknown, ...args: unknown[]): unknown { + // Built under an installation that is gone: the tool, untouched. + if (!live(frame.tracker)) return original.apply(this, args); + const options = args[1] as { toolCallId?: unknown } | undefined; + const toolCallId = typeof options?.toolCallId === "string" ? options.toolCallId : randomUUID(); + return runTool( + () => beginToolCall(frame.tracker, name, toolCallId, args[0], frame.key), + () => original.apply(this, args), + frame, + ); + }; + toolWrappers.set(wrapped, original); + out[name] = withExecute(tool, markWrapped(wrapped, original)); + } + return out; +} + +/** + * The input and call id of a direct `tool.execute(...)` call, in either + * major's signature: 1.x `execute(input, { agent: { toolCallId } })`, 0.x + * `execute({ context: input, runtimeContext, … }, { toolCallId })`. + */ +function toolCallArgs(args: unknown[]): { input: unknown; toolCallId?: string } { + const [first, second] = args as [unknown, Record | undefined]; + const pick = (...values: unknown[]): string | undefined => + values.find((value): value is string => typeof value === "string" && value !== ""); + const legacy = + typeof first === "object" && + first !== null && + "context" in first && + ("runtimeContext" in first || second === undefined || typeof second.toolCallId === "string"); + if (legacy) { + const envelope = first as { context?: unknown; toolCallId?: unknown }; + return { input: envelope.context, toolCallId: pick(second?.toolCallId, envelope.toolCallId) }; + } + const agentContext = second?.agent as { toolCallId?: unknown } | undefined; + return { input: first, toolCallId: pick(agentContext?.toolCallId, second?.toolCallId) }; +} + +/** + * Wrap one Mastra tool's `execute` so a DIRECT call is recorded. + * + * A tool an agent executes is already recorded by `instrument()` — including + * tools built before it ran — so this is for the calls no agent makes: a tool + * invoked from your own code or a workflow step. Inside a Mastra run or a + * `failproofai` scope it records under that; with nothing open it records the + * call as its own run, named after the tool. A wrapped tool handed to an agent + * is not recorded twice. + */ +export function wrapTool unknown }>( + tool: T, +): T { + const execute = tool.execute; + if (typeof execute !== "function" || core.isWrapped(execute)) return tool; + const toolName = typeof tool.id === "string" && tool.id ? tool.id : "tool"; + const original = execute as unknown as (...args: unknown[]) => unknown; + + const wrapped = function failproofaiToolExecute(this: unknown, ...args: unknown[]): unknown { + const frame = frames.getStore(); + // The agent's own tool wrapper is already recording this very call; and + // inside a run whose installation is gone, nothing records. + if (frame?.kind === "tool" || (frame !== undefined && !live(frame.tracker))) { + return original.apply(this, args); + } + return runTool( + () => { + const { input, toolCallId } = toolCallArgs(args); + const t = frame?.tracker ?? (enabled && tracker !== null ? tracker : (standalone ??= newTracker())); + return beginToolCall(t, toolName, toolCallId ?? randomUUID(), input, frame?.key); + }, + () => original.apply(this, args), + undefined, + ); + }; + // A new object rather than a mutation: a Mastra tool may be frozen, and + // assigning to a frozen object fails silently in sloppy mode. The prototype + // is kept so it is still a `Tool` to Mastra's own checks. + return withExecute(tool, markWrapped(wrapped, original)); +} + +// --------------------------------------------------------------------------- +// Agents +// --------------------------------------------------------------------------- + +/** + * How an agent method is called: `generate` resolves with the finished run, + * `stream` resolves as soon as the loop starts and finishes as the caller + * consumes it, and `network` resolves with a stream that runs on its own. + */ +type AgentCallMode = "generate" | "stream" | "network"; + +interface AgentRun { + tracker: core.RunTracker; + key: string; + frame: Frame; + args: unknown[]; + started: number; + ended: boolean; + /** + * Set when this call is recorded as part of an enclosing span rather than as + * a span of its own — a network's router — so it never emits `agent_end`. + */ + borrowed?: boolean; +} + +function endAgentRun(run: AgentRun, outcome: string): void { + if (run.ended) return; + run.ended = true; + if (run.borrowed) return; + // Closed `cancelled` already, by the teardown that made it not live. + if (!live(run.tracker)) return; + run.tracker.endAgent(run.key, { + outcome, + ...core.fwFields({ duration_ms: core.ms(Date.now() - run.started) }), + }); +} + +/** How a finished `generate()` / `onFinish` result ended. */ +function outcomeOf(result: unknown): string { + const value = (result ?? {}) as { error?: unknown; finishReason?: unknown; tripwire?: unknown }; + if (value.tripwire) return "rejected"; + if (value.error || value.finishReason === "error") return "failed"; + return "success"; +} + +/** + * Compose our end-of-stream callbacks with the caller's. + * + * Mastra calls `onFinish` / `onError` / `onAbort` when the stream actually + * finishes — as it is consumed — which is the moment the run ends. Reading one + * of the output's promise getters instead would START consumption: Mastra + * drains the stream into a buffer the first time one is touched, so observing + * the run would change what it does. + */ +function withStreamCallbacks(options: unknown, run: AgentRun): Record { + const base = (typeof options === "object" && options !== null ? options : {}) as Record; + const chain = + (name: string, ours: (...args: unknown[]) => void) => + (...args: unknown[]): unknown => { + core.callSafely(ours, args, `${NAME}.${name}`); + const theirs = base[name]; + return typeof theirs === "function" ? (theirs as (...a: unknown[]) => unknown).apply(base, args) : undefined; + }; + return { + ...base, + onFinish: chain("onFinish", (result) => endAgentRun(run, outcomeOf(result))), + onError: chain("onError", () => endAgentRun(run, "failed")), + onAbort: chain("onAbort", () => endAgentRun(run, "cancelled")), + }; +} + +/** + * Close a streamed run that no callback will close. + * + * A stream a processor blocks — an input processor's `abort()`, or one on the + * output stream — ends with a `tripwire` chunk and calls none of `onFinish` / + * `onError` / `onAbort` (1.x and 0.x alike), so without this its agent stayed + * open forever. The output's `_waitUntilFinished()` settles when the stream + * does, without starting consumption the way its promise getters would; a run + * the callbacks already closed makes this a no-op. + */ +function closeWhenFinished(run: AgentRun, output: unknown): void { + const wait = (output as { _waitUntilFinished?: unknown } | null)?._waitUntilFinished; + if (typeof wait !== "function") return; + const waiting = (wait as () => unknown).call(output); + const finish = (): void => { + endAgentRun(run, (output as { tripwire?: unknown }).tripwire ? "rejected" : "success"); + }; + Promise.resolve(waiting).then( + () => core.callSafely(finish, [], `${NAME}.stream`), + () => undefined, + ); +} + +/** + * Close a network run when its stream does. + * + * `network()` resolves with a stream that runs by itself — its workflow starts + * as the stream is constructed and does not wait on the reader — and whose + * `status` getter settles once it has finished, without consuming anything. + */ +function closeNetworkWhenFinished(run: AgentRun, stream: unknown): void { + const status = (stream as { status?: unknown } | null)?.status; + if (typeof (status as PromiseLike | undefined)?.then !== "function") { + endAgentRun(run, "success"); + return; + } + const site = `${NAME}.network`; + Promise.resolve(status).then( + (value) => core.callSafely(endAgentRun, [run, workflowOutcome(value)], site), + () => core.callSafely(endAgentRun, [run, "failed"], site), + ); +} + +const nonEmpty = (value: unknown): string | undefined => + typeof value === "string" && value !== "" ? value : undefined; + +/** + * The conversation a call belongs to: its memory thread and resource, in the + * 1.x shape (`memory: { thread, resource }`, the thread a string or `{ id }`) + * or 0.x's older top-level `threadId` / `resourceId`. + */ +function conversationOf(options: unknown): { thread?: string; resource?: string } { + const value = (typeof options === "object" && options !== null ? options : {}) as { + threadId?: unknown; + resourceId?: unknown; + memory?: unknown; + }; + const memory = (typeof value.memory === "object" && value.memory !== null ? value.memory : {}) as { + thread?: unknown; + resource?: unknown; + }; + const threadObject = (typeof memory.thread === "object" && memory.thread !== null ? memory.thread : {}) as { + id?: unknown; + }; + return { + thread: nonEmpty(memory.thread) ?? nonEmpty(threadObject.id) ?? nonEmpty(value.threadId), + resource: nonEmpty(memory.resource) ?? nonEmpty(value.resourceId), + }; +} + +/** + * The agent an `agent.network()` builds for itself to route with. + * + * Every routing decision and completion check of a network is a fresh + * `routing-agent` running on the network agent's own model. It IS the + * network's decision loop, so its model calls are recorded as the network + * agent's own steps rather than as one more agent per decision. + */ +function isRoutingAgent(agent: object): boolean { + const { id, name } = agent as { id?: unknown; name?: unknown }; + return id === "routing-agent" || name === "routing-agent"; +} + +/** + * Whether a run is a root nothing has put in a session yet — then the + * conversation id the framework gives it is its session. An enclosing + * `failproofai.session()` / `agent()` scope always wins. + */ +function isUnscopedRoot(parent: Frame | undefined): boolean { + return parent === undefined && currentIdentity().sessionId === null; +} + +function beginAgentRun(agent: object, method: string, args: unknown[], mode: AgentCallMode): AgentRun | undefined { + const t = tracker; + if (!enabled || t === null) return undefined; + const parent = frames.getStore(); + // Inside a run whose installation is gone: that whole tree is unrecorded. + if (parent !== undefined && !live(parent.tracker)) return undefined; + // Re-entry from inside the same run — 0.x's `generate` is `stream` under the + // hood — is the same agent span, not a nested one. + if (parent?.kind === "agent" && parent.owner === agent) return undefined; + if (parent?.network === true && isRoutingAgent(agent)) { + // The network's own router: recorded on the network's span, not its own. + return { + tracker: t, + key: parent.key, + frame: { tracker: t, key: parent.key, owner: agent, kind: "agent" }, + args, + started: Date.now(), + ended: false, + borrowed: true, + }; + } + const key = randomUUID(); + const { label, rawId } = agentLabel(agent); + const { thread, resource } = conversationOf(args[1]); + t.startAgent(key, { + agentId: label, + parentKey: parent?.key, + // A memory thread is a conversation that spans runs: the session, for a + // run nothing else has put in one. + sessionId: isUnscopedRoot(parent) ? thread : undefined, + goal: goalOf(args[0]), + ...core.fwFields({ + agent_id: rawId, + method, + streaming: mode === "stream" || undefined, + thread_id: thread, + resource_id: resource, + }), + }); + const run: AgentRun = { + tracker: t, + key, + frame: { tracker: t, key, owner: agent, kind: "agent", network: mode === "network" || undefined }, + args, + started: Date.now(), + ended: false, + }; + run.frame.close = (outcome: string): void => endAgentRun(run, outcome); + if (mode === "stream") { + const next = [...args]; + while (next.length < 2) next.push(undefined); + next[1] = withStreamCallbacks(args[1], run); + run.args = next; + } + return run; +} + +function patchAgentMethod(prototype: object, method: string, mode: AgentCallMode): boolean { + const original = (prototype as Record)[method]; + if (typeof original !== "function" || core.isWrapped(original)) return false; + const fn = original as (...args: unknown[]) => unknown; + const site = `${NAME}.${method}`; + + const wrapper = function failproofaiAgentCall(this: object, ...args: unknown[]): unknown { + const run = core.callSafely(beginAgentRun, [this, method, args, mode], site); + if (!run) return fn.apply(this, args); + const settle = (value: unknown): void => { + if (run.borrowed) return; + // A stream ends through its callbacks — `stream()` returning only means + // the loop has started — or, when none will fire, as it finishes. + if (mode === "stream") core.callSafely(closeWhenFinished, [run, value], site); + else if (mode === "network") core.callSafely(closeNetworkWhenFinished, [run, value], site); + else core.callSafely(endAgentRun, [run, outcomeOf(value)], site); + }; + let result: unknown; + try { + result = frames.run(run.frame, () => fn.apply(this, run.args)); + } catch (error) { + core.callSafely(endAgentRun, [run, "failed"], site); + throw error; + } + if (typeof (result as PromiseLike | null)?.then === "function") { + return (result as PromiseLike).then( + (value) => { + settle(value); + return value; + }, + (error: unknown) => { + core.callSafely(endAgentRun, [run, "failed"], site); + throw error; + }, + ); + } + settle(result); + return result; + }; + Object.defineProperty(wrapper, "name", { value: fn.name, configurable: true }); + return patcher!.patch(prototype, method, markWrapped(wrapper, original)); +} + +/** + * End a run the moment an input processor blocks its prompt. + * + * `__runInputProcessors` is where every input processor's `abort()` lands, and + * its result says so (`tripwire` in 1.x, `tripwireTriggered` in 0.x). A + * blocked `generate()` resolves with the tripwire and ends there anyway, but a + * blocked `stream()` calls none of its callbacks — and 0.x's output offers no + * other end signal — so without this its agent stayed open for good. Nothing + * the run would have done follows a blocked prompt, so it ends here. + */ +function patchInputProcessors(prototype: object): void { + const method = "__runInputProcessors"; + const original = (prototype as Record)[method]; + if (typeof original !== "function" || core.isWrapped(original)) return; + const fn = original as (...args: unknown[]) => unknown; + const site = `${NAME}.inputProcessors`; + const blocked = (value: unknown): boolean => { + const result = (value ?? {}) as { tripwire?: unknown; tripwireTriggered?: unknown }; + return result.tripwireTriggered === true || (result.tripwire !== undefined && result.tripwire !== null && result.tripwire !== false); + }; + const wrapper = function failproofaiInputProcessors(this: object, ...args: unknown[]): unknown { + const frame = frames.getStore(); + const result = fn.apply(this, args); + const close = frame?.kind === "agent" && frame.owner === this ? frame.close : undefined; + if (close === undefined) return result; + const check = (value: unknown): void => { + if (blocked(value)) close("rejected"); + }; + if (typeof (result as PromiseLike | null)?.then === "function") { + return (result as PromiseLike).then((value) => { + core.callSafely(check, [value], site); + return value; + }); + } + core.callSafely(check, [result], site); + return result; + }; + Object.defineProperty(wrapper, "name", { value: fn.name, configurable: true }); + patcher!.patch(prototype, method, markWrapped(wrapper, original)); +} + +/** + * Patch a method that BUILDS something the run will use later, so the result + * is observed with the run that built it. `observe` sees only calls made by + * the agent's own run; any other caller gets Mastra's value untouched. + */ +function patchBuilder( + prototype: object, + method: string, + observe: (value: unknown, frame: Frame, agent: object) => unknown, +): boolean { + const original = (prototype as Record)[method]; + if (typeof original !== "function" || core.isWrapped(original)) return false; + const fn = original as (...args: unknown[]) => unknown; + const site = `${NAME}.${method}`; + const wrapper = function failproofaiBuilder(this: object, ...args: unknown[]): unknown { + const frame = frames.getStore(); + const result = fn.apply(this, args); + if (frame?.kind !== "agent" || frame.owner !== this || !live(frame.tracker)) return result; + const apply = (value: unknown): unknown => { + const observed = core.callSafely(observe, [value, frame, this], site); + return observed === undefined ? value : observed; + }; + return typeof (result as PromiseLike | null)?.then === "function" + ? (result as PromiseLike).then(apply) + : apply(result); + }; + Object.defineProperty(wrapper, "name", { value: fn.name, configurable: true }); + return patcher!.patch(prototype, method, markWrapped(wrapper, original)); +} + +// --------------------------------------------------------------------------- +// Workflows +// --------------------------------------------------------------------------- + +interface WorkflowRunLike { + workflowId?: unknown; + runId?: unknown; + isInternalWorkflow?: unknown; + tracingPolicy?: { internal?: unknown }; +} + +/** `InternalSpans.WORKFLOW` — Mastra's own bit for "this workflow is plumbing". */ +const INTERNAL_WORKFLOW = 1; + +function isInternalRun(run: WorkflowRunLike): boolean { + if (run.isInternalWorkflow === true) return true; + const internal = run.tracingPolicy?.internal; + return typeof internal === "number" && (internal & INTERNAL_WORKFLOW) !== 0; +} + +/** A workflow's result status as an `agent_end` outcome. */ +function workflowOutcome(status: unknown): string { + switch (status) { + case "success": + return "success"; + case "failed": + return "failed"; + case "canceled": + case "cancelled": + return "cancelled"; + case "tripwire": + return "rejected"; + default: + return typeof status === "string" && status ? status : "success"; + } +} + +interface WorkflowSpan { + tracker: core.RunTracker; + key: string; + frame: Frame; + runId?: string; + started: number; +} + +/** + * The workflows `agent.network()` is built from. 1.x marks them internal + * (`tracingPolicy.internal`); 0.x marks them nothing, so inside a network run + * they are recognised by id — anywhere else a workflow of that name is the + * user's and recorded as usual. + */ +const NETWORK_WORKFLOWS: ReadonlySet = new Set([ + "agent-loop-main-workflow", + "Agent-Network-Outer-Workflow", + "iteration-with-validation", +]); + +// --------------------------------------------------------------------------- +// Suspend / resume (human in the loop) +// --------------------------------------------------------------------------- + +/** + * A workflow run that suspended and waits to be resumed. + * + * A suspended run deliberately keeps its agent open, as a LangGraph interrupt + * does: closing it would close the pause with it, zeroing the one interval + * that measures how long the human took. The resume continues the SAME span. + * A resume that lands in another process — the ordinary deployment shape, one + * worker suspends and whichever picks up the approval resumes from storage — + * finds nothing here, starts a new span in the same session (a root workflow's + * session is its run id) and closes the pause by its deterministic id. + */ +interface PausedRun { + tracker: core.RunTracker; + key: string; + started: number; + /** Open pause id -> the step path it waits on. */ + pauses: Map; +} + +/** Mastra workflow run id -> its paused span. */ +const pausedRuns = new Map(); + +function rememberPaused(runId: string, paused: PausedRun): void { + pausedRuns.delete(runId); + while (pausedRuns.size >= MAX_OPEN) { + const oldest = pausedRuns.entries().next(); + if (oldest.done) break; + pausedRuns.delete(oldest.value[0]); + // Never closed from here: the resume may still come, in any process. + oldest.value[1].tracker.forget(oldest.value[1].key); + } + pausedRuns.set(runId, paused); +} + +/** A suspend payload or resume answer as the text a human read or wrote. */ +function humanText(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === "string") return value; + if (typeof value === "object" && !Array.isArray(value)) { + const { prompt, question, message } = value as Record; + const text = prompt ?? question ?? message; + if (typeof text === "string") return text; + } + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +/** The steps a `suspended` result waits on: their path, and what each asked. */ +function suspendedStepsOf(result: unknown): Array<{ path: string; payload: unknown }> { + const value = (result ?? {}) as { + suspended?: unknown; + steps?: Record; + suspendPayload?: Record; + }; + const payloadOf = (id: string): unknown => value.steps?.[id]?.suspendPayload ?? value.suspendPayload?.[id]; + const out: Array<{ path: string; payload: unknown }> = []; + if (Array.isArray(value.suspended)) { + for (const entry of value.suspended) { + const ids = (Array.isArray(entry) ? entry : [entry]).filter( + (id): id is string => typeof id === "string" && id !== "", + ); + if (ids.length > 0) out.push({ path: ids.join("."), payload: payloadOf(ids[0]!) }); + } + } + if (out.length === 0 && typeof value.steps === "object" && value.steps !== null) { + for (const [id, step] of Object.entries(value.steps)) { + if (step?.status === "suspended") out.push({ path: id, payload: step.suspendPayload }); + } + } + return out; +} + +/** + * The step a `run.resume({ step, resumeData })` resumes, as a path: a step id, + * a `Step`, or an array of either for a step inside a nested workflow. + */ +function resumeTargetOf(params: unknown): { path?: string; answer: unknown } { + const value = (params ?? {}) as { step?: unknown; resumeData?: unknown }; + const ids = (Array.isArray(value.step) ? value.step : [value.step]) + .map((step: unknown) => (typeof step === "string" ? step : (step as { id?: unknown } | null)?.id)) + .filter((id): id is string => typeof id === "string" && id !== ""); + return { path: ids.length > 0 ? ids.join(".") : undefined, answer: value.resumeData }; +} + +const pauseIdOf = (runId: string, path: string): string => `${runId}:${path}`; + +/** `human_wait` + `agent_pause` for each step newly suspended, in that order. */ +function suspendRun(span: WorkflowSpan, runId: string, result: unknown): void { + const existing = pausedRuns.get(runId); + const paused: PausedRun = + existing !== undefined && existing.key === span.key + ? existing + : { tracker: span.tracker, key: span.key, started: span.started, pauses: new Map() }; + for (const { path, payload } of suspendedStepsOf(result)) { + const pauseId = pauseIdOf(runId, path); + if (paused.pauses.has(pauseId)) continue; + paused.pauses.set(pauseId, path); + const marker = core.fwFields({ step_id: path, workflow_run_id: runId }); + span.tracker.emit("humanWait", span.key, { + inputId: pauseId, + prompt: humanText(payload), + reason: "mastra_suspend", + ...marker, + }); + span.tracker.emit("agentPause", span.key, { pauseId, reason: "mastra_suspend", ...marker }); + } + rememberPaused(runId, paused); +} + +/** + * `agent_resume` + `human_input` for the pauses a resume answers: the step it + * names (and anything nested under it), or every open pause when it names + * none. With no pause open in this process, the one the named step must have + * opened elsewhere — the same id, since it is derived from the run and step. + */ +function resumeRun(span: WorkflowSpan, runId: string, params: unknown, paused: PausedRun | undefined): void { + const { path, answer } = resumeTargetOf(params); + const response = humanText(answer); + const emit = (pauseId: string, stepPath: string, elsewhere: boolean): void => { + const marker = core.fwFields({ + step_id: stepPath, + workflow_run_id: runId, + resumed_elsewhere: elsewhere || undefined, + }); + span.tracker.emit("agentResume", span.key, { pauseId, reason: "mastra_resume", ...marker }); + span.tracker.emit("humanInput", span.key, { inputId: pauseId, response, ...marker }); + }; + if (paused === undefined) { + if (path !== undefined) emit(pauseIdOf(runId, path), path, true); + return; + } + for (const [pauseId, stepPath] of [...paused.pauses]) { + if (path !== undefined && stepPath !== path && !stepPath.startsWith(`${path}.`)) continue; + paused.pauses.delete(pauseId); + emit(pauseId, stepPath, false); + } +} + +function patchRunMethod(prototype: object, method: string): boolean { + const original = (prototype as Record)[method]; + if (typeof original !== "function" || core.isWrapped(original)) return false; + const fn = original as (...args: unknown[]) => unknown; + const site = `${NAME}.workflow`; + const resuming = method === "_resume"; + + const begin = (run: WorkflowRunLike & object, args: unknown[]): WorkflowSpan | undefined => { + const t = tracker; + if (!enabled || t === null || isInternalRun(run)) return undefined; + const parent = frames.getStore(); + if (parent !== undefined && !live(parent.tracker)) return undefined; + if (parent?.kind === "workflow" && parent.owner === run) return undefined; + // A network's own machinery (0.x marks it nothing): its agents land on + // the network's span. + if (parent?.network === true && NETWORK_WORKFLOWS.has(run.workflowId)) return undefined; + const runId = typeof run.runId === "string" ? run.runId : undefined; + + const paused = resuming && runId !== undefined ? pausedRuns.get(runId) : undefined; + if (paused !== undefined && paused.tracker === t && t.isOpen(paused.key)) { + // Resuming a run this process suspended: the same span, continued. + const span: WorkflowSpan = { + tracker: t, + key: paused.key, + frame: { tracker: t, key: paused.key, owner: run, kind: "workflow" }, + runId, + started: paused.started, + }; + workflowRuns.set(runId!, paused.key); + resumeRun(span, runId!, args[0], paused); + return span; + } + + const key = randomUUID(); + t.startAgent(key, { + agentId: core.normalizeAgentId(run.workflowId, "workflow"), + parentKey: parent?.key, + // A root run's session is its run id — the one thing its resume, in + // this process or another, is sure to share with it. + sessionId: isUnscopedRoot(parent) ? runId : undefined, + ...core.fwFields({ kind: "workflow", workflow_run_id: runId, method: method.replace(/^_/, "") }), + }); + const span: WorkflowSpan = { + tracker: t, + key, + frame: { tracker: t, key, owner: run, kind: "workflow" }, + runId, + started: Date.now(), + }; + if (runId !== undefined) { + workflowRuns.set(runId, key); + if (resuming) resumeRun(span, runId, args[0], undefined); + } + return span; + }; + const end = (span: WorkflowSpan, value: unknown, failed = false): void => { + if (span.runId !== undefined && workflowRuns.get(span.runId) === span.key) workflowRuns.delete(span.runId); + // Closed `cancelled` already, by the teardown that made it not live. + if (!live(span.tracker)) return; + const status = failed ? "failed" : (value as { status?: unknown } | null)?.status; + if (status === "suspended" && span.runId !== undefined) { + suspendRun(span, span.runId, value); + return; + } + if (span.runId !== undefined && pausedRuns.get(span.runId)?.key === span.key) pausedRuns.delete(span.runId); + span.tracker.endAgent(span.key, { + outcome: workflowOutcome(status), + ...core.fwFields({ duration_ms: core.ms(Date.now() - span.started) }), + }); + }; + + const wrapper = function failproofaiWorkflowRun(this: WorkflowRunLike & object, ...args: unknown[]): unknown { + const span = core.callSafely(begin, [this, args], site); + if (!span) return fn.apply(this, args); + let result: unknown; + try { + result = frames.run(span.frame, () => fn.apply(this, args)); + } catch (error) { + core.callSafely(end, [span, undefined, true], site); + throw error; + } + if (typeof (result as PromiseLike | null)?.then === "function") { + return (result as PromiseLike).then( + (value) => { + core.callSafely(end, [span, value], site); + return value; + }, + (error: unknown) => { + core.callSafely(end, [span, undefined, true], site); + throw error; + }, + ); + } + core.callSafely(end, [span, result], site); + return result; + }; + Object.defineProperty(wrapper, "name", { value: fn.name, configurable: true }); + return patcher!.patch(prototype, method, markWrapped(wrapper, original)); +} + +let stepSequence = 0; + +interface StepSpan extends Span { + runKey: string; + hookName: string; + hookId: string; +} + +const openSteps = new OpenSpans(); + +function beginStep(params: unknown): StepSpan | undefined { + const t = tracker; + if (!enabled || t === null) return undefined; + const value = (params ?? {}) as { + step?: { id?: unknown }; + runId?: unknown; + executionContext?: { runId?: unknown }; + prevOutput?: unknown; + }; + const runId = value.executionContext?.runId ?? value.runId; + const runKey = typeof runId === "string" ? workflowRuns.get(runId) : undefined; + const hookName = value.step?.id; + if (runKey === undefined || typeof hookName !== "string" || !hookName) return undefined; + // Only a step of the recorded run ITSELF. An agent that runs inside it + // executes its own loop as internal workflows, and one called with the + // run's id — as 0.x's network calls its sub-agents — hands those internal + // steps the same run id; they execute in the agent's frame, not the run's. + const frame = frames.getStore(); + if (frame?.kind !== "workflow" || frame.key !== runKey) return undefined; + stepSequence += 1; + const hookId = `${runKey}:${hookName}:${stepSequence}`; + t.emit("hookTriggered", runKey, { + hookName, + hookId, + triggerEvent: "workflow_step", + input: value.prevOutput, + }); + const span: StepSpan = { tracker: t, closed: false, runKey, hookName, hookId }; + openSteps.add(span); + return span; +} + +interface StepResult { + status?: unknown; + output?: unknown; + error?: unknown; +} + +/** + * The step result inside what `executeStep` resolved with: `{ result: {…} }` + * from 0.24 on, the bare `{ status, output, error }` before that. + */ +function stepResultOf(value: unknown): StepResult { + if (typeof value !== "object" || value === null) return {}; + const wrapped = (value as { result?: unknown }).result; + if (typeof wrapped === "object" && wrapped !== null && "status" in wrapped) return wrapped; + return value; +} + +/** + * A failed step does not throw out of `executeStep` — Mastra catches it and + * resolves with `status: "failed"` — so the failure is read off the result. + * 0.x hands the error over as a rendered string with its stack attached; only + * the first line is the error. + */ +function stepOutcome(value: unknown, thrown?: unknown): { outcome: string; output?: unknown; error?: string } { + const result = stepResultOf(value); + const error = thrown ?? (result.status === "failed" ? (result.error ?? "step failed") : undefined); + if (error !== undefined) { + return { outcome: "failed", error: typeof error === "string" ? error.split("\n")[0] : describeError(error) }; + } + return { + outcome: typeof result.status === "string" && result.status ? workflowOutcome(result.status) : "success", + output: result.output, + }; +} + +function endStep(span: StepSpan, value: unknown, thrown?: unknown): void { + // Already closed — by teardown, when its installation is gone. + if (!openSteps.close(span)) return; + span.tracker.emit("hookCompleted", span.runKey, { + hookName: span.hookName, + hookId: span.hookId, + ...stepOutcome(value, thrown), + }); +} + +function patchExecuteStep(prototype: object): boolean { + const original = (prototype as Record).executeStep; + if (typeof original !== "function" || core.isWrapped(original)) return false; + const fn = original as (...args: unknown[]) => unknown; + const site = `${NAME}.step`; + const wrapper = function failproofaiStep(this: unknown, ...args: unknown[]): unknown { + const span = core.callSafely(beginStep, [args[0]], site); + if (!span) return fn.apply(this, args); + let result: unknown; + try { + result = fn.apply(this, args); + } catch (error) { + core.callSafely(endStep, [span, undefined, error], site); + throw error; + } + if (typeof (result as PromiseLike | null)?.then === "function") { + return (result as PromiseLike).then( + (value) => { + core.callSafely(endStep, [span, value], site); + return value; + }, + (error: unknown) => { + core.callSafely(endStep, [span, undefined, error], site); + throw error; + }, + ); + } + core.callSafely(endStep, [span, result], site); + return result; + }; + Object.defineProperty(wrapper, "name", { value: fn.name, configurable: true }); + return patcher!.patch(prototype, "executeStep", markWrapped(wrapper, original)); +} + +/** + * Run `body` inside an agent span named for a workflow. + * + * Mastra workflow runs are recorded automatically once `instrument("mastra")` + * has run — the run as an agent, its steps as hooks — so this is not needed + * for them. It stays for grouping work that is not a Mastra workflow (or runs + * where the adapter is not installed) under one named span. + */ +export function workflow(workflowName: string, body: () => T): T { + return agentScope(core.normalizeAgentId(workflowName, "workflow"), { fw_kind: "workflow" }, body); +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +type ClassLike = { prototype: object }; + +const AGENT_METHODS = [ + ["generate", "generate"], + ["stream", "stream"], + ["generateVNext", "generate"], + ["streamVNext", "stream"], + ["generateLegacy", "generate"], + ["streamLegacy", "stream"], + ["network", "network"], +] as const; + +function installAgent(Agent: ClassLike): number { + let patched = 0; + for (const [method, mode] of AGENT_METHODS) { + if (patchAgentMethod(Agent.prototype, method, mode)) patched += 1; + } + const proto = Agent.prototype as Record; + if (compat.probe(NAME, "Agent.resolveModelConfig", () => typeof proto.resolveModelConfig === "function")) { + patchBuilder(Agent.prototype, "resolveModelConfig", (model, frame) => observeModel(model, frame)); + } + if (compat.probe(NAME, "Agent.convertTools", () => typeof proto.convertTools === "function")) { + patchBuilder(Agent.prototype, "convertTools", (tools, frame, agent) => observeTools(tools, frame, agent)); + } + // Optional, and silent when absent: without it a blocked stream is closed + // by its finish (1.x) or at teardown (0.x), rather than never recorded. + patchInputProcessors(Agent.prototype); + return patched; +} + +function installWorkflows(module: unknown): void { + const { Run, DefaultExecutionEngine } = (module ?? {}) as { + Run?: ClassLike; + DefaultExecutionEngine?: ClassLike; + }; + if (compat.probe(NAME, "workflows Run._start", () => typeof (Run?.prototype as Record | undefined)?._start === "function")) { + for (const method of ["_start", "_resume", "_restart", "_timeTravel"]) patchRunMethod(Run!.prototype, method); + } + if ( + compat.probe( + NAME, + "workflows DefaultExecutionEngine.executeStep", + () => typeof (DefaultExecutionEngine?.prototype as Record | undefined)?.executeStep === "function", + ) + ) { + patchExecuteStep(DefaultExecutionEngine!.prototype); + } +} + +export const adapter: Adapter = { + name: NAME, + + async install(options: Record = {}): Promise { + compat.checkVersion(NAME, PACKAGE, { + minimum: MINIMUM, + below: "2.0.0", + reason: "model steps are observed at resolveModelConfig and workflow runs at Run._start", + }); + const agentCopies = await compat.requireModuleCopies("@mastra/core/agent", INSTALL); + // Workflows are optional: an app with no workflows is ordinary, and a + // workflow module that fails to load must not cost the agent patches below. + const workflowCopies = await compat + .requireModuleCopies("@mastra/core/workflows", INSTALL) + .catch(() => [] as unknown[]); + + // A fresh tracker per installation, never one left over: whatever an + // earlier installation built stays bound to ITS tracker, which is dead. + teardown(); + tracker = newTracker(options); + patcher = new core.Patcher(); + + let patched = 0; + for (const copy of agentCopies) { + const Agent = (copy as { Agent?: unknown }).Agent; + if (typeof Agent === "function") patched += installAgent(Agent); + } + if (patched === 0) { + teardown(); + throw new Error( + "could not patch any Agent generate/stream method — this build of @mastra/core " + + "exposes none of them under a writable name.", + ); + } + for (const copy of workflowCopies) installWorkflows(copy); + enabled = true; + }, + + uninstall(): void { + teardown(); + }, +}; + +/** + * Stop recording and close what is still open, on the tracker it opened on. + * The switch goes FIRST: nothing may begin a span while the rest are closed. + */ +function teardown(): void { + enabled = false; + patcher?.restoreAll(); + patcher = null; + workflowRuns.clear(); + pausedRuns.clear(); + const t = tracker; + tracker = null; + const models = openModels.drain(); + const tools = openTools.drain(); + const steps = openSteps.drain(); + if (t === null) return; + const site = `${NAME}.uninstall`; + const incomplete = core.fwFields({ incomplete: true }); + // Innermost first — a step's model calls and tools before the step — and + // every leaf before the agents that own it. + for (const call of models) { + if (call.tracker !== t) continue; + core.callSafely( + () => + t.emit("modelResponse", call.requestId, { + parentKey: call.runKey, + model: call.model, + stopReason: "incomplete", // the LangChain adapter's word for a model call cut off mid-flight + requestId: call.requestId, + ...core.fwFields({ duration_ms: core.ms(Date.now() - call.started), incomplete: true }), + }), + [], + site, + ); + } + for (const call of tools) { + if (call.tracker !== t) continue; + core.callSafely( + () => + t.emit("toolResult", call.key, { + parentKey: call.parentKey, + toolName: call.toolName, + toolCallId: call.toolCallId, + ...incomplete, + }), + [], + site, + ); + } + for (const span of steps) { + if (span.tracker !== t) continue; + core.callSafely( + () => + t.emit("hookCompleted", span.runKey, { + hookName: span.hookName, + hookId: span.hookId, + outcome: "cancelled", + ...incomplete, + }), + [], + site, + ); + } + core.callSafely(() => t.closeOpenAgents("cancelled"), [], site); + t.reset(); +} + +/** + * The pure readers above and two lifecycle probes, for this package's own + * unit tests. + * + * @internal Not part of the public API — `stripInternal` drops it from the + * published declarations: their shapes follow Mastra's and the providers' internals, and + * change whenever those do. Nothing exported references it. + */ +export const _internals = { + /** Whether an installation is live. */ + isEnabled: (): boolean => enabled, + /** How many suspended workflow runs are waiting on a resume. */ + pausedRuns: (): number => pausedRuns.size, + /** How many spans of each kind are open, for the bookkeeping tests. */ + openSpans: (): Record => ({ + models: openModels.size, + tools: openTools.size, + steps: openSteps.size, + workflowRuns: workflowRuns.size, + agents: tracker?.openAgents().length ?? 0, + }), + usageOf, + finishReasonOf, + promptOf, + generateOutcome, + foldStreamPart, + toolCallArgs, + isInternalRun, + workflowOutcome, + stepResultOf, + stepOutcome, + conversationOf, + isRoutingAgent, + suspendedStepsOf, + resumeTargetOf, + humanText, +}; diff --git a/sdk/typescript/src/logger.ts b/sdk/typescript/src/logger.ts new file mode 100644 index 000000000..d1445416d --- /dev/null +++ b/sdk/typescript/src/logger.ts @@ -0,0 +1,98 @@ +/** + * The one logging surface for this SDK. + * + * Python's SDK reaches for `logging.getLogger(__name__)`, which gives the host + * application a named, level-filtered, individually-silenceable channel for + * free. Node has no stdlib equivalent, and this package is contractually + * zero-dependency, so it cannot pull in `pino` or `debug` — a telemetry library + * that installs into other people's agent processes must not hand them a + * dependency they did not choose. + * + * So: a minimal level-filtered logger over `console`, with a `setLogger()` + * escape hatch so a host that already has structured logging can route ours + * into it rather than having two formats interleaved on stderr. + * + * DEFAULT LEVEL IS `warn`, deliberately. Every `logger.warning(...)` in the + * ported modules marks a condition that silently costs the caller telemetry — + * a full queue, a dropped event, an unresolvable session. Defaulting to + * `error` would hide exactly the messages that exist because the failure is + * otherwise invisible; defaulting to `info` would put startup chatter into + * somebody's agent output. + */ + +export type LogLevel = "debug" | "info" | "warn" | "error" | "silent"; + +export interface Logger { + debug(message: string, ...args: unknown[]): void; + info(message: string, ...args: unknown[]): void; + warn(message: string, ...args: unknown[]): void; + error(message: string, ...args: unknown[]): void; +} + +const ORDER: Record = { + debug: 10, + info: 20, + warn: 30, + error: 40, + silent: 100, +}; + +function envLevel(): LogLevel { + const raw = (process.env.FAILPROOFAI_SDK_LOG_LEVEL ?? "").trim().toLowerCase(); + return raw in ORDER ? (raw as LogLevel) : "warn"; +} + +let level: LogLevel = envLevel(); +let sink: Logger | null = null; + +/** Route this SDK's log lines into the host's own logger. `null` restores the default. */ +export function setLogger(next: Logger | null): void { + sink = next; +} + +/** Override the level. Also readable from `FAILPROOFAI_SDK_LOG_LEVEL`. */ +export function setLogLevel(next: LogLevel): void { + level = next; +} + +/** Re-read `FAILPROOFAI_SDK_LOG_LEVEL`. Used by tests that mutate the environment. */ +export function resetLogLevel(): void { + level = envLevel(); +} + +function emit( + method: "debug" | "info" | "warn" | "error", + message: string, + args: unknown[], +): void { + if (ORDER[method] < ORDER[level]) return; + // A logger that throws is a logger that takes the host agent down over a + // diagnostic. Nothing in this file may do that. + try { + if (sink) { + sink[method](message, ...args); + return; + } + // `console.warn`/`console.error` go to stderr, which is where a library's + // diagnostics belong: stdout is frequently the agent's own protocol channel + // (an MCP server speaks JSON-RPC over it) and writing there corrupts it. + const line = `[failproofai-sdk] ${message}`; + if (method === "warn") console.warn(line, ...args); + else console.error(line, ...args); + } catch { + /* empty */ + } +} + +export const logger: Logger = { + debug: (message, ...args) => emit("debug", message, args), + info: (message, ...args) => emit("info", message, args), + warn: (message, ...args) => emit("warn", message, args), + error: (message, ...args) => emit("error", message, args), +}; + +/** `logger.error` plus the error's stack, mirroring Python's `logger.exception`. */ +export function logException(message: string, error: unknown): void { + const detail = error instanceof Error ? (error.stack ?? error.message) : String(error); + emit("error", `${message}: ${detail}`, []); +} diff --git a/sdk/typescript/src/next.ts b/sdk/typescript/src/next.ts new file mode 100644 index 000000000..c88eaa015 --- /dev/null +++ b/sdk/typescript/src/next.ts @@ -0,0 +1,115 @@ +/** + * Next.js support: `withFailproofai(nextConfig)`. + * + * ## Why this exists + * + * `next build` BUNDLES a server's dependencies into its own output by default, + * and none of LangChain, Mastra or LlamaIndex is on Next's built-in external + * list. `instrument()` attaches to the framework copy in `node_modules`; a + * bundled app runs a different copy, so the adapter reports success and + * records nothing. (The Vercel AI SDK is unaffected: ai 7 reads its telemetry + * integrations from a global, and `telemetry()` is passed at the call site.) + * + * The cure Next documents for exactly this is `serverExternalPackages`: a + * package listed there is loaded from `node_modules` at run time, so the copy + * `instrument()` patched is the copy the routes run. This wrapper adds the + * packages the adapters need, keeps everything the app already lists, and + * records what it added so `instrument()` can tell a configured app from an + * unconfigured one (see `nextExternalsGap` in `integrations/index.ts`). + * + * // next.config.ts + * import { withFailproofai } from "@failproofai/sdk/next"; + * export default withFailproofai({ ...yourConfig }); + * + * `@failproofai/sdk` itself is on the list for a second reason: bundled, the + * SDK would be compiled separately into `instrumentation.ts` and into each + * route, so `instrument()` would configure one copy while the routes record + * through another. + * + * This module runs where `next.config` runs — Node, at build and at server + * start — and imports nothing, so it is safe to load from any config file. + */ + +/** + * Packages a Next.js server must load from `node_modules` for `instrument()` to + * reach them. Listing one the app does not install is harmless: Next only + * consults the list for imports that actually occur. + */ +export const NEXT_EXTERNAL_PACKAGES: readonly string[] = [ + "@failproofai/sdk", + // LangChain / LangGraph: every run goes through @langchain/core's callback + // manager, which is what the adapter patches. Listing core is sufficient — + // a bundled `@langchain/langgraph` importing it still gets the external copy. + "@langchain/core", + "@langchain/langgraph", + "langchain", + // Mastra: Agent and the workflow engine are patched in @mastra/core. + "@mastra/core", + // LlamaIndex: the callback bus lives in @llamaindex/core, and the agent + // workflow runtime that is patched lives in @llamaindex/workflow. + "llamaindex", + "@llamaindex/core", + "@llamaindex/workflow", + "@llamaindex/workflow-core", +]; + +/** + * The environment variable `withFailproofai` sets when Next evaluates the + * config: the comma-separated external list, so `instrument()` in the same + * server process knows the app was configured. Set it yourself (to the + * packages you listed, or to `1`) if you configure `serverExternalPackages` by + * hand and want to silence the warning. + */ +export const NEXT_EXTERNALS_ENV = "FAILPROOFAI_NEXT_EXTERNALS"; + +type NextConfigObject = Record & { + serverExternalPackages?: string[]; + transpilePackages?: string[]; +}; + +/** What `withFailproofai` guarantees about the config it returns. */ +export interface WithFailproofaiExternals { + serverExternalPackages: string[]; +} + +/** + * Add the packages `instrument()` needs to `serverExternalPackages`. + * + * Accepts every form a `next.config` may export: an object, or a function of + * `(phase, context)` returning one (sync or async). The result has the same + * shape as the input. Packages the app lists in `transpilePackages` are left + * out — Next rejects a package that is both — and `instrument()` will warn for + * the framework instead, which is the honest outcome. + */ +export function withFailproofai( + config: (...args: A) => R, +): (...args: A) => R extends Promise ? Promise : R & WithFailproofaiExternals; +export function withFailproofai(config?: C): C & WithFailproofaiExternals; +export function withFailproofai(config?: unknown): unknown { + if (typeof config === "function") { + const wrapped = (...args: unknown[]): unknown => { + const produced = (config as (...a: unknown[]) => unknown)(...args); + return produced instanceof Promise + ? produced.then((value) => apply(value as NextConfigObject)) + : apply(produced as NextConfigObject); + }; + return wrapped; + } + return apply((config ?? {}) as NextConfigObject); +} + +function apply(config: NextConfigObject): NextConfigObject { + const existing = Array.isArray(config.serverExternalPackages) ? config.serverExternalPackages : []; + const transpiled = new Set(Array.isArray(config.transpilePackages) ? config.transpilePackages : []); + const merged = [...new Set([...existing, ...NEXT_EXTERNAL_PACKAGES.filter((p) => !transpiled.has(p))])]; + try { + // Next evaluates the config inside the server process for `next start` and + // `next dev`, before `instrumentation.ts` runs — so this reaches + // `instrument()`. A standalone build does not evaluate it; there + // `instrument()` reads the resolved config Next stores instead. + process.env[NEXT_EXTERNALS_ENV] = merged.join(","); + } catch { + // An environment that forbids writing env vars loses only the marker. + } + return { ...config, serverExternalPackages: merged }; +} diff --git a/sdk/typescript/src/node-require.ts b/sdk/typescript/src/node-require.ts new file mode 100644 index 000000000..d7f24899d --- /dev/null +++ b/sdk/typescript/src/node-require.ts @@ -0,0 +1,446 @@ +/** + * A CommonJS `require` this package can use from either module system. + * + * The adapters need three things Node only offers through `require`: reading a + * framework's `package.json` for its version, asking whether a package + * resolves at all, and inspecting `require.cache`. Reaching for them the + * obvious way — `createRequire(import.meta.url)` — compiles under ESM and is a + * syntax error under CommonJS, so a package that ships both builds cannot use + * it without a bundler shim. This module has none. + * + * The anchor is the **consuming application**, not this file: the frameworks + * we detect are the application's dependencies, and under pnpm or a workspace + * this package may sit somewhere that cannot see them at all. Where the + * application IS has two answers and neither is always right — see `anchors()`. + */ + +import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, isAbsolute, join } from "node:path"; + +// `createRequire` wants a file path to resolve relative to; the file need not +// exist, only its directory is used. +const requireAt = (dir: string): NodeJS.Require => createRequire(join(dir, "__failproofai_anchor__.js")); +const appRequire = requireAt(process.cwd()); + +/** + * The directories to resolve the application's dependencies from, best first. + * + * The working directory alone was the first answer, and it is wrong twice: a + * service started with no working directory set runs from `/`, where nothing + * resolves and `instrument()` reports it found no framework; and in a monorepo + * whose root hoists a different `@langchain/core` than the app's own nested + * one, it resolved the ROOT copy — patched a class the app never loads, and + * reported success. + * + * The entry script's directory is the better anchor whenever the entry is the + * application's own code. It is not when the entry is a launcher living in + * `node_modules` (`next start`, a process manager's wrapper): resolving from + * there finds whatever the launcher's package sits beside, so the working + * directory keeps precedence in that case. Both are always tried; which one is + * asked FIRST only matters when they disagree. + */ +function anchors(): string[] { + const cwd = process.cwd(); + const entry = process.argv[1]; + const dirs: string[] = []; + if (typeof entry === "string" && entry !== "" && isAbsolute(entry)) { + const dir = dirname(entry); + const launcher = dir.split(/[\\/]/).includes("node_modules"); + if (launcher) dirs.push(cwd, dir); + else dirs.push(dir, cwd); + } else { + dirs.push(cwd); + } + return [...new Set(dirs)]; +} + +export const nodeRequire: NodeJS.Require = appRequire; + +/** Every `require.cache` key currently loaded, for "is this already imported". */ +export function loadedModulePaths(): string[] { + try { + return Object.keys(appRequire.cache); + } catch { + return []; + } +} + +/** The resolved filename for `specifier`, or null when it cannot be found. */ +export function resolveFrom(specifier: string): string | null { + const found: string[] = []; + for (const dir of anchors()) { + try { + const path = requireAt(dir).resolve(specifier); + if (!found.includes(path)) found.push(path); + } catch { + // Either the package is absent from here or its `exports` map does not + // expose this subpath. Both are ordinary; try the next anchor. + } + } + // A copy something has already `require`d is the one in use, whichever + // anchor found it. + return found.find(isRequired) ?? found[0] ?? null; +} + +/** + * Whether the process entry point is a CommonJS module. + * + * `require.main` is the entry module under CommonJS and `undefined` when the + * entry is an ES module — and the entry's module system decides which copy of + * a dual-published framework the application's own imports reach. That is the + * copy an adapter has to patch; see `resolveCopies`. + */ +export function entryIsCommonJs(): boolean { + try { + return isCommonJsMain(appRequire.main); + } catch { + return false; + } +} + +/** + * Whether the application's own imports of a dual-published framework reach + * its CommonJS copy — the copy an adapter must patch. See `requireModuleCopies`. + * + * Usually that is the entry point's module system. The exception is a server + * whose entry is a CommonJS LAUNCHER that loads the application's code another + * way. Next.js is the one that matters: `next start` is a CommonJS script, but + * every package the server loads from `node_modules` at run time (anything in + * `serverExternalPackages`) is loaded with `import()` — by Turbopack and by + * webpack alike — so the application runs the ES-module copies. Reading the + * launcher as "a CommonJS app" patched the CommonJS copies instead, and + * `instrument()` in `instrumentation.ts` reported success for LangChain, Mastra + * and LlamaIndex while recording nothing. + * + * Next sets `NEXT_RUNTIME` in its server processes (and inlines it into the + * bundles it builds), before `instrumentation.ts` runs. + */ +export function appImportsReachCommonJs(): boolean { + if (isNextServer()) return false; + return entryIsCommonJs(); +} + +/** Running inside a Next.js server process (Node or Edge runtime). */ +export function isNextServer(): boolean { + try { + const runtime = process.env.NEXT_RUNTIME; + return typeof runtime === "string" && runtime !== ""; + } catch { + return false; + } +} + +/** + * Whether a `require.main` value names a CommonJS entry module. + * + * Node answers `undefined` for an ES-module entry; Deno answers `null`. A bare + * `!== undefined` read Deno's `null` as "CommonJS", so an ES-module Deno app + * had its frameworks' CommonJS copies patched while its own imports reached the + * untouched ES-module copies: `instrument()` returned success and nothing was + * recorded. Only an actual module object counts. + */ +export function isCommonJsMain(main: unknown): boolean { + return typeof main === "object" && main !== null; +} + +/** Whether `path` is in the CommonJS module cache, i.e. has been `require`d. */ +export function isRequired(path: string): boolean { + try { + return Object.prototype.hasOwnProperty.call(appRequire.cache, path); + } catch { + return false; + } +} + +/** + * The file an ES-module `import(specifier)` from the application would load, + * or null when it cannot be worked out. + * + * A dual-published framework (`@langchain/core`, `@mastra/core`, `llamaindex`, + * …) ships two copies of every module — `exports["./x"].import` and + * `exports["./x"].require` — and they are two different objects in memory. + * `resolveFrom` can only ever name the `require` one, because `createRequire` + * resolves with CommonJS conditions. Patching that copy in an ES-module + * application is a silent no-op: `instrument()` reports success, the app's own + * imports reach the untouched ESM copy, and no event is ever recorded. + * + * `import.meta.resolve` would answer this directly, but it does not exist in + * the CommonJS build and cannot take a parent URL without a flag — and the + * parent has to be the APPLICATION, not this file (see the module comment). So + * this reads the package's `exports` map itself, with Node's ESM conditions + * (`node`, `import`, `default`). It only has to handle what packages actually + * publish: condition objects, arrays, and single-`*` subpath patterns. + */ +export function resolveEsm(specifier: string): string | null { + const { name, subpath } = splitSpecifier(specifier); + if (name === null) return null; + const root = packageRoot(name, specifier); + if (root === null) return null; + let manifest: { exports?: unknown; main?: unknown }; + try { + manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as typeof manifest; + } catch { + return null; + } + if (manifest.exports === undefined) { + // No exports map: Node's ESM loader reads `main` exactly as `require` + // does, so there is only one copy and CommonJS resolution already names it. + return null; + } + const target = matchExports(manifest.exports, subpath, ESM_CONDITIONS); + if (target === null || !target.startsWith("./")) return null; + const file = join(root, target); + return existsSync(file) ? file : null; +} + +const ESM_CONDITIONS: readonly string[] = ["node", "import", "default"]; +const CJS_CONDITIONS: readonly string[] = ["node", "require", "default"]; + +/** + * The file `subpath` of the package installed at `root` resolves to, for an + * `import` or a `require` — or null. `root` is a package directory found on + * disk rather than by resolution (see `nestedCopies`), so there is no + * specifier to hand Node's resolver; this reads the package's own `exports`. + */ +export function resolveExportsAt(root: string, subpath: string, kind: "import" | "require"): string | null { + let manifest: { exports?: unknown; main?: unknown }; + try { + manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as typeof manifest; + } catch { + return null; + } + let target: string | null; + if (manifest.exports === undefined) { + if (subpath !== ".") return null; + target = typeof manifest.main === "string" ? manifest.main : "index.js"; + if (!target.startsWith("./")) target = `./${target}`; + } else { + target = matchExports(manifest.exports, subpath, kind === "import" ? ESM_CONDITIONS : CJS_CONDITIONS); + } + if (target === null || !target.startsWith("./")) return null; + const file = join(root, target); + return existsSync(file) ? file : null; +} + +/** Bounds on `nestedCopies`' walk: it runs once, at `instrument()`, over a tree nobody sized. */ +const NESTED_MAX_DEPTH = 4; +const NESTED_MAX_DIRS = 20_000; + +/** + * Every OTHER installed copy of package `name` the application can end up + * running: the real directories of the copies nested BENEATH some dependency + * (`node_modules//node_modules/`, at any depth up to a bound), in + * every `node_modules` on the application's resolution chain — plus, under + * pnpm, every version in the virtual store (`node_modules/.pnpm/@`). + * + * A copy exists there because a dependency could not share the application's + * — it pinned a different version as a hard dependency — and everything that + * dependency builds is built on it. `resolveFrom` can never name one: Node's + * resolution from the application stops at the application's own copy. + * + * Deliberately NOT included: a `` sitting directly in some ancestor + * `node_modules`. The nearest one is the application's own copy, and a farther + * one is a monorepo root's hoist the application's imports never reach. + */ +export function nestedCopies(name: string): string[] { + const found = new Map(); + const seen = new Set(); + let budget = NESTED_MAX_DIRS; + const real = (path: string): string | null => { + try { + return realpathSync(path); + } catch { + return null; + } + }; + const list = (dir: string): string[] => { + try { + return readdirSync(dir); + } catch { + return []; + } + }; + const candidate = (dir: string): void => { + const path = real(dir); + if (path !== null && manifestNames(join(path, "package.json"), name)) found.set(path, true); + }; + const scan = (modules: string, depth: number): void => { + const key = real(modules); + if (key === null || seen.has(key)) return; + seen.add(key); + for (const entry of list(modules)) { + if (budget <= 0) return; + if (entry === ".pnpm") { + // pnpm's virtual store: one directory per installed version. + const prefix = `${name.replace("/", "+")}@`; + for (const version of list(join(modules, entry))) { + if (version.startsWith(prefix)) candidate(join(modules, entry, version, "node_modules", name)); + } + continue; + } + if (entry.startsWith(".")) continue; + const packages = entry.startsWith("@") ? list(join(modules, entry)).map((sub) => join(modules, entry, sub)) : [join(modules, entry)]; + for (const pkg of packages) { + if (budget-- <= 0) return; + // The top level of a scanned `node_modules` is never a candidate. + if (depth > 0 && pkg === join(modules, name)) { + candidate(pkg); + continue; + } + if (depth < NESTED_MAX_DEPTH) { + const inner = join(pkg, "node_modules"); + if (existsSync(inner)) scan(inner, depth + 1); + } + } + } + }; + for (const anchor of anchors()) { + let dir = anchor; + for (;;) { + const modules = join(dir, "node_modules"); + if (existsSync(modules)) scan(modules, 0); + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } + return [...found.keys()]; +} + +function splitSpecifier(specifier: string): { name: string | null; subpath: string } { + const parts = specifier.split("/"); + const scoped = specifier.startsWith("@"); + if (scoped && parts.length < 2) return { name: null, subpath: "." }; + const name = scoped ? `${parts[0]}/${parts[1]}` : parts[0]!; + const rest = parts.slice(scoped ? 2 : 1); + return { name, subpath: rest.length === 0 ? "." : `./${rest.join("/")}` }; +} + +/** + * The directory holding `name`'s `package.json`, as the application sees it. + * + * CommonJS resolution of the package (or of the requested subpath) is used + * first, because that is precisely the install the application would load — + * under pnpm, a workspace, or nested `node_modules` alike — and then walked up + * to the manifest that names it. Only when CommonJS cannot resolve it at all + * (an ESM-only package whose `exports` has no `require` condition) does this + * fall back to walking `node_modules` up from each of `anchors()`. + */ +function packageRoot(name: string, specifier: string): string | null { + for (const candidate of [specifier, name, `${name}/package.json`]) { + const entry = resolveFrom(candidate); + if (entry === null) continue; + let dir = dirname(entry); + for (;;) { + if (manifestNames(join(dir, "package.json"), name)) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } + for (const anchor of anchors()) { + let dir = anchor; + for (;;) { + const candidate = join(dir, "node_modules", name); + if (manifestNames(join(candidate, "package.json"), name)) return candidate; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } + return null; +} + +function manifestNames(path: string, name: string): boolean { + try { + return (JSON.parse(readFileSync(path, "utf8")) as { name?: unknown }).name === name; + } catch { + return false; + } +} + +/** Node's `PACKAGE_EXPORTS_RESOLVE`, reduced to what packages publish. */ +function matchExports(exports: unknown, subpath: string, conditions: readonly string[]): string | null { + const isSubpathMap = + typeof exports === "object" && + exports !== null && + !Array.isArray(exports) && + Object.keys(exports).some((key) => key.startsWith(".")); + if (!isSubpathMap) { + return subpath === "." ? resolveTarget(exports, conditions, null) : null; + } + const map = exports as Record; + if (Object.prototype.hasOwnProperty.call(map, subpath) && !subpath.includes("*")) { + return resolveTarget(map[subpath], conditions, null); + } + // Longest matching `prefix*suffix` pattern wins, as in Node. + let best: { key: string; match: string } | null = null; + for (const key of Object.keys(map)) { + const star = key.indexOf("*"); + if (star === -1 || key.indexOf("*", star + 1) !== -1) continue; + const prefix = key.slice(0, star); + const suffix = key.slice(star + 1); + if ( + subpath.startsWith(prefix) && + subpath.endsWith(suffix) && + subpath.length >= key.length && + (best === null || prefix.length > best.key.indexOf("*")) + ) { + best = { key, match: subpath.slice(prefix.length, subpath.length - suffix.length) }; + } + } + return best === null ? null : resolveTarget(map[best.key], conditions, best.match); +} + +function resolveTarget(target: unknown, conditions: readonly string[], match: string | null): string | null { + if (typeof target === "string") { + return match === null ? target : target.replaceAll("*", match); + } + if (Array.isArray(target)) { + for (const entry of target) { + const resolved = resolveTarget(entry, conditions, match); + if (resolved !== null) return resolved; + } + return null; + } + if (typeof target === "object" && target !== null) { + for (const [key, value] of Object.entries(target)) { + if (key !== "default" && !conditions.includes(key)) continue; + const resolved = resolveTarget(value, conditions, match); + if (resolved !== null) return resolved; + } + } + return null; +} + +/** `require(specifier)`, or null on any failure. Never throws. */ +export function tryRequire(specifier: string): T | null { + try { + return appRequire(specifier) as T; + } catch { + return null; + } +} + +/** + * A real dynamic `import()`, in both halves of the dual build. + * + * TypeScript downlevels `import(x)` to `require(x)` when it emits CommonJS, + * which is exactly wrong for the packages this SDK reaches for: the Vercel AI + * SDK is ESM-only, so the CommonJS build would report the framework as "not + * importable" for the users most likely to have it. Building the import through + * the `Function` constructor hides it from that transform, so both builds + * perform a genuine dynamic import and both can load an ESM-only framework. + * + * The `Function` here takes no caller-controlled input: the body is this fixed + * literal and the specifier arrives as an argument. + */ +// eslint-disable-next-line @typescript-eslint/no-implied-eval -- fixed literal body, no caller input; see above +const dynamicImport = new Function("specifier", "return import(specifier);") as ( + specifier: string, +) => Promise; + +export async function importModule(specifier: string): Promise { + return await dynamicImport(specifier); +} diff --git a/sdk/typescript/src/redact.ts b/sdk/typescript/src/redact.ts new file mode 100644 index 000000000..9838e885c --- /dev/null +++ b/sdk/typescript/src/redact.ts @@ -0,0 +1,305 @@ +/** + * Deterministic credential scrubbing for SDK spool files. + * + * This mirrors the daemon's minimal redaction boundary — and, line for line, + * `failproofai_sdk/_redact.py`. The SDK applies it before bytes reach disk; the + * daemon applies it again before upload so batches written by older SDKs + * receive the same protection. + * + * The scan is index-based rather than regex-based on purpose: several of the + * rules (the JWT segment walk, the assignment's backwards name scan, the bearer + * token's byte budget) are not regular, and a half-regex/half-manual + * implementation is exactly how the two sides of a redactor drift apart. + */ + +import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, join } from "node:path"; + +const PREFIX_RULES: ReadonlyArray = [ + ["sk-ant-api", 16, "anthropic-key"], + ["sk-ant-", 16, "anthropic-key"], + ["sk-proj-", 16, "openai-key"], + ["sk-", 16, "api-key"], + ["ghp_", 20, "github-token"], + ["gho_", 20, "github-token"], + ["ghu_", 20, "github-token"], + ["ghs_", 20, "github-token"], + ["ghr_", 20, "github-token"], + ["github_pat_", 20, "github-token"], + ["sb_secret_", 16, "supabase-key"], + ["sbp_", 20, "supabase-key"], + ["xoxb-", 16, "slack-token"], + ["xoxp-", 16, "slack-token"], + ["AKIA", 16, "aws-access-key-id"], + ["ASIA", 16, "aws-access-key-id"], +]; + +const STRONG_SECRET_NAMES = ["secret", "password", "passwd", "credential"] as const; +const WEAK_SECRET_NAMES = ["key", "token"] as const; +const MIN_ASSIGNMENT_VALUE = 12; + +type Match = readonly [length: number, label: string]; + +function isAsciiAlnum(code: number): boolean { + return ( + (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122) + ); +} + +function isTokenChar(value: string, index: number): boolean { + const code = value.charCodeAt(index); + return isAsciiAlnum(code) || code === 0x5f /* _ */ || code === 0x2d /* - */; +} + +/** + * UTF-8 byte length of the code unit at `index`. + * + * Summed across a string this is the exact UTF-8 byte count, including for + * astral characters: each half of a surrogate pair reports 2, and 2 + 2 is the + * 4 bytes such a code point actually occupies. That is what lets the byte + * budgets below scan by code unit without ever decoding the string. + */ +function utf8LenAt(value: string, index: number): number { + const code = value.charCodeAt(index); + if (code < 0x80) return 1; + if (code < 0x800) return 2; + return code >= 0xd800 && code <= 0xdfff ? 2 : 3; +} + +export function utf8Length(value: string): number { + let total = 0; + for (let i = 0; i < value.length; i += 1) total += utf8LenAt(value, i); + return total; +} + +// Python's `str.isspace()`: the ASCII whitespace set plus the file/group/record/ +// unit separators, plus Unicode separators. `\s` covers the Unicode half; the +// C1 separators are spelled out because `\s` does not include them. +function isSpaceChar(char: string): boolean { + return /\s/.test(char) || (char >= "\u001c" && char <= "\u001f"); +} + +function isUpperChar(char: string): boolean { + return char.toLowerCase() !== char && char.toUpperCase() === char; +} + +function atBoundary(value: string, start: number): boolean { + return start === 0 || !isTokenChar(value, start - 1); +} + +function matchPrefix(value: string, start: number): Match | null { + if (!atBoundary(value, start)) return null; + for (const [prefix, minimum, label] of PREFIX_RULES) { + if (!value.startsWith(prefix, start)) continue; + let end = start + prefix.length; + while (end < value.length && isTokenChar(value, end)) end += 1; + if (end - start - prefix.length >= minimum) return [end - start, label]; + } + return null; +} + +function matchJwt(value: string, start: number): Match | null { + if (!atBoundary(value, start) || !value.startsWith("eyJ", start)) return null; + let end = start; + let segments = 0; + while (segments < 3) { + const segmentStart = end; + while (end < value.length) { + const code = value.charCodeAt(end); + if ( + !( + isAsciiAlnum(code) || + code === 0x2d /* - */ || + code === 0x5f /* _ */ || + code === 0x3d /* = */ + ) + ) { + break; + } + end += 1; + } + if (end === segmentStart) break; + segments += 1; + if (segments < 3 && end < value.length && value[end] === ".") { + end += 1; + } else if (segments < 3) { + break; + } + } + const length = end - start; + return segments === 3 && length >= 40 ? [length, "jwt"] : null; +} + +function matchBearer(value: string, start: number): Match | null { + if (value.slice(start, start + 7).toLowerCase() !== "bearer ") return null; + let end = start + 7; + let tokenBytes = 0; + while (end < value.length) { + const char = value[end]!; + if (isSpaceChar(char) || char === '"' || char === "'") break; + tokenBytes += utf8LenAt(value, end); + end += 1; + } + return tokenBytes >= 8 ? [end - start, "bearer-token"] : null; +} + +function isSecretName(name: string): boolean { + const raw = name.replace(/^-+/, "").replace(/-+$/, ""); + const lowered = raw.toLowerCase(); + const compound = + raw.includes("_") || + raw.includes("-") || + Array.from(raw.slice(1)).some((char) => isUpperChar(char)); + return ( + STRONG_SECRET_NAMES.some((part) => lowered.endsWith(part)) || + (compound && WEAK_SECRET_NAMES.some((part) => lowered.endsWith(part))) + ); +} + +const OPAQUE_VALUE_PREFIXES = ["{", "$", "<", "(", "`", "[redacted:"] as const; + +function isLiteralSecret(value: string): boolean { + return ( + utf8Length(value) >= MIN_ASSIGNMENT_VALUE && + !OPAQUE_VALUE_PREFIXES.some((prefix) => value.startsWith(prefix)) + ); +} + +function matchAssignment(value: string, start: number): Match | null { + if (start === 0) return null; + if (value[start - 1] === "=" && (value[start] === '"' || value[start] === "'")) return null; + + const previous = value[start - 1]!; + const quote = previous === '"' || previous === "'" ? previous : null; + const equals = quote ? start - 2 : start - 1; + if (equals < 0 || value[equals] !== "=") return null; + + let nameStart = equals; + while (nameStart > 0) { + if (!isTokenChar(value, nameStart - 1)) break; + nameStart -= 1; + } + if (nameStart === equals) return null; + if ( + !isSecretName(value.slice(nameStart, equals)) || + ["{", "$", "<", "(", "`"].some((prefix) => value.startsWith(prefix, start)) + ) { + return null; + } + + let end = start; + let valueBytes = 0; + while (end < value.length) { + const char = value[end]!; + if ( + (quote && char === quote) || + (!quote && (isSpaceChar(char) || char === ";" || char === "&" || char === '"' || char === "'")) + ) { + break; + } + valueBytes += utf8LenAt(value, end); + end += 1; + } + return valueBytes >= MIN_ASSIGNMENT_VALUE ? [end - start, "secret-assignment"] : null; +} + +/** The minimally redacted string and the replacement count. */ +export function scrubString(value: string): [string, number] { + const out: string[] = []; + let cursor = 0; + let copiedThrough = 0; + let hits = 0; + while (cursor < value.length) { + const match = + matchPrefix(value, cursor) ?? + matchJwt(value, cursor) ?? + matchBearer(value, cursor) ?? + matchAssignment(value, cursor); + if (match === null) { + cursor += 1; + continue; + } + const [length, label] = match; + out.push(value.slice(copiedThrough, cursor)); + out.push(`[redacted:${label}]`); + cursor += length; + copiedThrough = cursor; + hits += 1; + } + if (hits === 0) return [value, 0]; + out.push(value.slice(copiedThrough)); + return [out.join(""), hits]; +} + +/** Read the daemon's redaction switch, defaulting safely to minimal. */ +export function redactionEnabled(baseDir: string): boolean { + const configuredHome = process.env.FAILPROOFAI_HOME; + let configPath: string; + if (basename(baseDir) === "custom-agents") { + configPath = join(dirname(baseDir), "config.json"); + } else if (configuredHome) { + configPath = join(configuredHome, "config.json"); + } else { + configPath = join(homedir(), ".failproofai", "config.json"); + } + try { + const config: unknown = JSON.parse(readFileSync(configPath, "utf-8")); + if (typeof config !== "object" || config === null || Array.isArray(config)) return true; + const collector = (config as Record).collector; + if (typeof collector !== "object" || collector === null || Array.isArray(collector)) { + return true; + } + return (collector as Record).redact !== "off"; + } catch { + // A missing, unreadable or malformed config means we do not know the + // operator's intent — so redact. Failing open here would mean one typo in + // `config.json` silently ships credentials. + return true; + } +} + +/** Redact credential-shaped keys and string values in a valid JSON event line. */ +export function redactJsonLine(encoded: string): string { + const event: unknown = JSON.parse(encoded); + let hits = 0; + + const scrub = (value: unknown, fieldName?: string): unknown => { + if (typeof value === "string") { + const [scrubbed, count] = scrubString(value); + hits += count; + if (count === 0 && typeof fieldName === "string" && isSecretName(fieldName)) { + if (isLiteralSecret(scrubbed)) { + hits += 1; + return "[redacted:secret-assignment]"; + } + } + return scrubbed; + } + if (Array.isArray(value)) { + // Elements are more values for the same field, so its name still applies. + return value.map((item) => scrub(item, fieldName)); + } + if (typeof value === "object" && value !== null) { + const result: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + const [redactedKey, count] = scrubString(key); + hits += count; + let uniqueKey = redactedKey; + let suffix = 2; + while (Object.prototype.hasOwnProperty.call(result, uniqueKey)) { + uniqueKey = `${redactedKey}#${suffix}`; + suffix += 1; + } + result[uniqueKey] = scrub(item, key); + } + return result; + } + return value; + }; + + const redacted = scrub(event); + return hits ? JSON.stringify(redacted) : encoded; +} + +export { isSecretName, isLiteralSecret }; diff --git a/sdk/typescript/src/resolver.ts b/sdk/typescript/src/resolver.ts new file mode 100644 index 000000000..879450304 --- /dev/null +++ b/sdk/typescript/src/resolver.ts @@ -0,0 +1,120 @@ +import { homedir } from "node:os"; +import { isAbsolute, join, resolve, sep } from "node:path"; + +import { shared } from "./shared.js"; + +/** + * Where this SDK writes its event spool. + * + * Resolution order, most explicit first: + * + * 1. `setBaseDir()` — a caller said so outright + * 2. `~/.failproofai/custom-agents` — always + * + * There is no environment variable that redirects the spool off the umbrella, + * and that is the point. `FAILPROOFAI_HOME` (honoured by + * `failproofaiCustomAgentsDir()` below) MOVES the umbrella; it cannot take you + * outside it, because the `custom-agents` segment is appended unconditionally. + * Wherever the home is, the spool is inside it. + * + * ## Why `$AGENTEYE_HOME` does not resolve here + * + * In the Python SDK it used to sit at step 2 and win over the default, which + * made it possible to aim the SDK at `~/.agenteye` — or anywhere else — from + * the environment. That is a redirect with no confirmation and no error: + * batches land in a directory, something may or may not read it, and an unread + * spool is indistinguishable from an idle one. An operator who exports it for + * the OTHER component that reads it (the older `agenteye-collector`) moved the + * SDK's spool as a side effect they never asked for. This SDK never had it. + * + * ## Nothing is stranded by this + * + * `failproofaid`, the daemon this SDK ships beside, watches BOTH + * `~/.failproofai/custom-agents/events` AND `~/.agenteye/events` + * (`crates/fpai-collect/src/config.rs`, `spool_dirs`). Batches sitting under + * the legacy root still drain. + * + * ## The one case that needs a deliberate choice + * + * A host running the older `agenteye-collector` and nothing else. That + * collector never learned the umbrella, so it does not watch where this SDK + * writes. The supported bridges, in order of preference: + * + * * run `failproofaid` instead — it watches both roots; or + * * point the collector AT the SDK with + * `AGENTEYE_HOME=~/.failproofai/custom-agents`; or + * * `configure({ baseDir: "~/.agenteye" })` in the application, which is + * explicit, visible at the call site, and cannot happen by inheriting + * somebody else's environment. + * + * `test/spool-contract.test.ts` reads the Rust, the TypeScript in `src/hooks/` + * and the Python resolver and fails if any of them drifts from this. + */ +export function getBaseDir(): string { + const baseDir = shared().baseDir; + if (baseDir !== null) return baseDir; + return failproofaiCustomAgentsDir(); +} + +/** + * `~/.failproofai/custom-agents`, honouring `$FAILPROOFAI_HOME`. + * + * Mirrors `customAgentsDir()` in `src/hooks/fp-home.ts`, + * `custom_agents_events_dir()` in `crates/fpai-collect/src/config.rs` and + * `failproofai_custom_agents_dir()` in the Python SDK. All four must agree; a + * divergence would mean this SDK writes somewhere the daemon never reads, with + * NO error on either side. + * + * Returns a path unconditionally and never checks whether it exists. The caller + * creates it: `writer`'s file write already does a recursive `mkdir` on the + * directory it is about to write into. An existence check here is what made an + * earlier opt-in dead — a spool root that must pre-exist can never be the place + * a first batch is written. + */ +export function failproofaiCustomAgentsDir(): string { + const fpHome = process.env.FAILPROOFAI_HOME; + const base = fpHome ? expandUser(fpHome) : join(homedir(), ".failproofai"); + return join(base, "custom-agents"); +} + +/** + * `~/.agenteye` — the root the SDK family wrote to before the default moved. + * + * Not part of resolution, and no environment variable can put it back — see + * `getBaseDir`. Kept as a named constant because the migration notes and the + * tests still refer to it, and because `failproofaid` goes on watching + * `~/.agenteye/events`. Spelling it in four places is how the sides drift apart. + */ +export function legacyAgenteyeDir(): string { + return join(homedir(), ".agenteye"); +} + +/** + * Expand a leading `~` against the current user's home. + * + * Load-bearing, not a nicety: the migration bridge this module itself + * prescribes — `configure({ baseDir: "~/.agenteye" })`, listed in `getBaseDir`'s + * docs and in the README as the explicit, visible-at-the-call-site option — is + * a RELATIVE path whose first segment is the literal character `~`. Without + * this, the writer's recursive `mkdir` cheerfully creates a `~` directory under + * the process's cwd and spools into it: nothing on the machine watches that + * path, so 100% of the telemetry is lost, which is the precise "an unread spool + * is indistinguishable from an idle one" failure the prose above is written to + * prevent. + */ +export function expandUser(path: string): string { + if (path === "~") return homedir(); + if (path.startsWith(`~${sep}`) || path.startsWith("~/")) { + return join(homedir(), path.slice(2)); + } + return path; +} + +export function setBaseDir(path: string | null | undefined): void { + if (path === null || path === undefined) { + shared().baseDir = null; + return; + } + const expanded = expandUser(path); + shared().baseDir = isAbsolute(expanded) ? expanded : resolve(expanded); +} diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts new file mode 100644 index 000000000..b4df86de9 --- /dev/null +++ b/sdk/typescript/src/runtime.ts @@ -0,0 +1,29 @@ +/** + * The process-wide writer and event namespace. + * + * These live in a leaf module so that `scopes.ts` and the framework adapters + * can reach the namespace without importing the package index, which would be + * a circular import. + * + * Constructing `EventWriter` starts the flush interval and registers the + * module-level exit flush, and that happens at `import "@failproofai/sdk"` + * time — the index imports this module, so the timing matches the Python SDK's. + * The interval is `unref`'d, so importing this package never keeps a process + * alive on its own. + * + * Reach the namespace through the `runtime` object rather than a direct named + * import, so a test can swap in a recording namespace and the scopes and + * adapters pick it up. A `let` export would work under ESM live bindings and + * silently not under the CJS build, which is exactly the kind of difference + * that shows up only in somebody else's project. + */ + +import { EventNamespace } from "./events.js"; +import { EventWriter } from "./writer.js"; + +const writer = new EventWriter(); + +export const runtime: { writer: EventWriter; event: EventNamespace } = { + writer, + event: new EventNamespace(writer), +}; diff --git a/sdk/typescript/src/schema.ts b/sdk/typescript/src/schema.ts new file mode 100644 index 000000000..5a564d2fe --- /dev/null +++ b/sdk/typescript/src/schema.ts @@ -0,0 +1,410 @@ +/** + * One builder per event type. `toDict()` IS the wire format. + * + * Frozen byte-for-byte by `test/wire-format.test.ts`, and kept in lockstep with + * `failproofai_sdk/_schema.py`: the two SDKs write into the same spool + * directories and the same ingest endpoint, so a field that differs between + * them is a field the dashboard renders for one language and not the other. + * + * KEY ORDER IS PART OF THE FORMAT. JavaScript preserves insertion order for + * string keys, so the order below is the order on the wire, and it matches + * Python's `_build`: the base identity block, then `environment`, then the + * declared optionals that were supplied, then the caller's extras. + */ + +import { getEnvironment } from "./environment.js"; + +export type JsonValue = unknown; +export type ExtraFields = Record; + +/** + * Build an ordered event object, omitting absent optionals, then merge extras. + * + * `extra` is merged LAST and verbatim — which is why `events.ts` refuses a + * reserved name and why `integrations/core.ts` namespaces everything `fw_*`. + * An extra called `tool_name` would otherwise overwrite the declared one and + * silently change a promoted column. + */ +function build( + base: Record, + specifics: ReadonlyArray, + extra: ExtraFields, +): Record { + const result: Record = { ...base }; + result.environment = getEnvironment(); + for (const [key, value] of specifics) { + if (value !== undefined && value !== null) result[key] = value; + } + return Object.assign(result, extra); +} + +interface Identity { + timestamp: string; + sessionId: string; + agentId: string; +} + +function identityBlock( + { timestamp, sessionId, agentId }: Identity, + type: string, + required: Record = {}, +): Record { + return { timestamp, session_id: sessionId, agent_id: agentId, type, ...required }; +} + +export interface ToolUseEvent extends Identity { + toolName: string; + toolCallId: string; + input?: Record | null; + extraFields?: ExtraFields; +} + +export function toolUseEvent(event: ToolUseEvent): Record { + return build( + identityBlock(event, "tool_use", { + tool_name: event.toolName, + tool_call_id: event.toolCallId, + }), + [["input", event.input]], + event.extraFields ?? {}, + ); +} + +export interface ToolResultEvent extends Identity { + toolName: string; + toolCallId: string; + output?: unknown; + error?: string | null; + durationMs?: number | null; + extraFields?: ExtraFields; +} + +export function toolResultEvent(event: ToolResultEvent): Record { + return build( + identityBlock(event, "tool_result", { + tool_name: event.toolName, + tool_call_id: event.toolCallId, + }), + [ + ["output", event.output], + ["error", event.error], + ["duration_ms", event.durationMs], + ], + event.extraFields ?? {}, + ); +} + +export interface ModelRequestEvent extends Identity { + model?: string | null; + messages?: Array> | null; + system?: unknown; + tools?: Array> | null; + /** + * Pairs this request with its response. Appended LAST in the ordered list so + * an event that omits it serialises byte-for-byte as before — + * `wire-format.test.ts` freezes those bytes, and the dedup key hashes them. + */ + requestId?: string | null; + extraFields?: ExtraFields; +} + +export function modelRequestEvent(event: ModelRequestEvent): Record { + return build( + identityBlock(event, "model_request"), + [ + ["model", event.model], + ["messages", event.messages], + ["system", event.system], + ["tools", event.tools], + ["request_id", event.requestId], + ], + event.extraFields ?? {}, + ); +} + +export interface ModelResponseEvent extends Identity { + model?: string | null; + stopReason?: string | null; + inputTokens?: number | null; + outputTokens?: number | null; + content?: unknown; + role?: string | null; + /** The `requestId` of the `model_request` this answers. See above. */ + requestId?: string | null; + extraFields?: ExtraFields; +} + +export function modelResponseEvent(event: ModelResponseEvent): Record { + return build( + identityBlock(event, "model_response"), + [ + ["model", event.model], + ["stop_reason", event.stopReason], + ["input_tokens", event.inputTokens], + ["output_tokens", event.outputTokens], + ["content", event.content], + ["role", event.role], + ["request_id", event.requestId], + ], + event.extraFields ?? {}, + ); +} + +export interface AgentStartEvent extends Identity { + goal?: string | null; + parentId?: string | null; + extraFields?: ExtraFields; +} + +export function agentStartEvent(event: AgentStartEvent): Record { + return build( + identityBlock(event, "agent_start"), + [ + ["goal", event.goal], + ["parent_id", event.parentId], + ], + event.extraFields ?? {}, + ); +} + +export interface AgentEndEvent extends Identity { + outcome?: string | null; + summary?: string | null; + extraFields?: ExtraFields; +} + +export function agentEndEvent(event: AgentEndEvent): Record { + return build( + identityBlock(event, "agent_end"), + [ + ["outcome", event.outcome], + ["summary", event.summary], + ], + event.extraFields ?? {}, + ); +} + +export interface AgentPauseEvent extends Identity { + pauseId: string; + reason?: string | null; + userId?: string | null; + extraFields?: ExtraFields; +} + +export function agentPauseEvent(event: AgentPauseEvent): Record { + return build( + identityBlock(event, "agent_pause", { pause_id: event.pauseId }), + [ + ["reason", event.reason], + ["user_id", event.userId], + ], + event.extraFields ?? {}, + ); +} + +export interface AgentResumeEvent extends Identity { + pauseId: string; + durationMs?: number | null; + reason?: string | null; + userId?: string | null; + extraFields?: ExtraFields; +} + +export function agentResumeEvent(event: AgentResumeEvent): Record { + return build( + identityBlock(event, "agent_resume", { pause_id: event.pauseId }), + [ + ["duration_ms", event.durationMs], + ["reason", event.reason], + ["user_id", event.userId], + ], + event.extraFields ?? {}, + ); +} + +export interface HookTriggeredEvent extends Identity { + hookName: string; + hookId: string; + triggerEvent?: string | null; + input?: unknown; + extraFields?: ExtraFields; +} + +export function hookTriggeredEvent(event: HookTriggeredEvent): Record { + return build( + identityBlock(event, "hook_triggered", { + hook_name: event.hookName, + hook_id: event.hookId, + }), + [ + ["trigger_event", event.triggerEvent], + ["input", event.input], + ], + event.extraFields ?? {}, + ); +} + +export interface HookCompletedEvent extends Identity { + hookName: string; + hookId: string; + outcome?: string | null; + output?: unknown; + error?: string | null; + durationMs?: number | null; + extraFields?: ExtraFields; +} + +export function hookCompletedEvent(event: HookCompletedEvent): Record { + return build( + identityBlock(event, "hook_completed", { + hook_name: event.hookName, + hook_id: event.hookId, + }), + [ + ["outcome", event.outcome], + ["output", event.output], + ["error", event.error], + ["duration_ms", event.durationMs], + ], + event.extraFields ?? {}, + ); +} + +export interface ErrorEvent extends Identity { + errorType: string; + message: string; + traceback?: string | null; + extraFields?: ExtraFields; +} + +export function errorEvent(event: ErrorEvent): Record { + return build( + identityBlock(event, "error", { + error_type: event.errorType, + message: event.message, + }), + [["traceback", event.traceback]], + event.extraFields ?? {}, + ); +} + +export interface HumanWaitEvent extends Identity { + inputId: string; + prompt?: string | null; + options?: string[] | null; + reason?: string | null; + extraFields?: ExtraFields; +} + +export function humanWaitEvent(event: HumanWaitEvent): Record { + return build( + identityBlock(event, "human_wait", { input_id: event.inputId }), + [ + ["prompt", event.prompt], + ["options", event.options], + ["reason", event.reason], + ], + event.extraFields ?? {}, + ); +} + +export interface HumanInputEvent extends Identity { + inputId: string; + response?: string | null; + durationMs?: number | null; + extraFields?: ExtraFields; +} + +export function humanInputEvent(event: HumanInputEvent): Record { + return build( + identityBlock(event, "human_input", { input_id: event.inputId }), + [ + ["response", event.response], + ["duration_ms", event.durationMs], + ], + event.extraFields ?? {}, + ); +} + +export interface HumanPauseEvent extends Identity { + reason?: string | null; + userId?: string | null; + extraFields?: ExtraFields; +} + +export function humanPauseEvent(event: HumanPauseEvent): Record { + return build( + identityBlock(event, "human_pause"), + [ + ["reason", event.reason], + ["user_id", event.userId], + ], + event.extraFields ?? {}, + ); +} + +export interface HumanInterruptEvent extends Identity { + reason?: string | null; + userId?: string | null; + atStep?: string | null; + extraFields?: ExtraFields; +} + +export function humanInterruptEvent(event: HumanInterruptEvent): Record { + return build( + identityBlock(event, "human_interrupt"), + [ + ["reason", event.reason], + ["user_id", event.userId], + ["at_step", event.atStep], + ], + event.extraFields ?? {}, + ); +} + +/** + * Every field name any event builder above declares, plus the wire names of the + * identity block. `integrations/core.ts` derives its forbidden-extras set from + * this so that adding a field here cannot leave a stale copy there. + */ +export const DECLARED_FIELD_NAMES: ReadonlySet = new Set([ + "timestamp", + "session_id", + "agent_id", + "type", + "environment", + "tool_name", + "tool_call_id", + "input", + "output", + "error", + "duration_ms", + "model", + "messages", + "system", + "tools", + "request_id", + "stop_reason", + "input_tokens", + "output_tokens", + "content", + "role", + "goal", + "parent_id", + "outcome", + "summary", + "pause_id", + "reason", + "user_id", + "hook_name", + "hook_id", + "trigger_event", + "error_type", + "message", + "traceback", + "input_id", + "prompt", + "options", + "response", + "at_step", +]); diff --git a/sdk/typescript/src/scopes.ts b/sdk/typescript/src/scopes.ts new file mode 100644 index 000000000..96fd9d16d --- /dev/null +++ b/sdk/typescript/src/scopes.ts @@ -0,0 +1,701 @@ +/** + * Scopes: `session()`, `agent()`, `toolCall()`. + * + * These are the ergonomic surface over `failproofai.event.*`. Each binds run + * identity onto the `AsyncLocalStorage` in `context.ts` so that everything + * emitted inside — including code that has never heard of the SDK's identity + * options — lands on the right session and agent. + * + * ## Two forms, and why + * + * **Callback (preferred).** `await agent("planner", fn)` runs `fn` inside + * `AsyncLocalStorage.run()`. Nothing to unwind: the binding exists for exactly + * the async subtree the callback creates and disappears with it, so the + * entire class of "a scope was entered here and exited over there" bugs is + * unreachable. Use this unless you cannot. + * + * **`using` (escape hatch).** `using span = agent.open("planner")` binds with + * `enterWith` and unwinds in `[Symbol.dispose]`. Needed when the work is not a + * single function — a scope opened in a constructor and closed in a teardown, + * a block that straddles an existing control structure. It carries the same + * hazard Python's context managers do: a scope opened in one async context and + * disposed in another leaves its frame bound where it was set. `context.ts` + * warns once when it detects that. + * + * Both forms emit byte-identical events; the only difference is who unwinds. + */ + +import { randomUUID } from "node:crypto"; + +import * as context from "./context.js"; +import { nowMicros } from "./clock.js"; +import { ProcessExit, onProcessExit } from "./exit.js"; +import type { Identity, Store } from "./context.js"; +import { runtime } from "./runtime.js"; + +// `Symbol.dispose` is only defined from Node 20.5 / V8 11.7. Defining it here +// means `using` works on every runtime this package supports, and assigning it +// when it already exists is a no-op rather than a conflict. +interface SymbolConstructorWithDispose { + dispose?: symbol; + asyncDispose?: symbol; +} +const symbolShim = Symbol as unknown as SymbolConstructorWithDispose; +symbolShim.dispose ??= Symbol.for("nodejs.dispose"); +symbolShim.asyncDispose ??= Symbol.for("nodejs.asyncDispose"); + +/** + * `AUTO` — infer the enclosing agent from the context stack (the default); + * `null` — force a root span, emitting no `parentId` at all; + * a string — use this id verbatim. + * + * `parentId` has three states and `null` is a meaningful one of them, so the + * default cannot be `null`. + */ +export const AUTO = Symbol.for("failproofai.AUTO"); +export type ParentId = string | null | typeof AUTO; + +/** + * True for a cancellation rather than a failure. + * + * A cancellation must not emit an `error` event, or every cancelled run + * pollutes the Errors surface. JavaScript has no `CancelledError`; the + * equivalent is the `AbortError` an `AbortSignal` produces, which both + * `DOMException` and Node's own APIs raise under that name (`ABORT_ERR` is the + * `code` Node sets on its own variant). + */ +export function isCancellation(error: unknown): boolean { + if (typeof error !== "object" || error === null) return false; + const named = error as { name?: unknown; code?: unknown }; + return named.name === "AbortError" || named.code === "ABORT_ERR"; +} + +function errorName(error: unknown): string { + if (error instanceof Error) { + // Several SDKs subclass Error without setting `name` — openai's + // `BadRequestError` reports `name === "Error"` — so the class, which is + // what anyone reading the Errors surface is looking for, would be lost. + const own = error.name || "Error"; + const ctor = (error as { constructor?: { name?: unknown } }).constructor?.name; + if (own === "Error" && typeof ctor === "string" && ctor !== "" && ctor !== "Error") return ctor; + return own; + } + return typeof error; +} + +// --------------------------------------------------------------------------- +// what is still open when the process exits +// --------------------------------------------------------------------------- + +/** One open scope; `close` emits its closing event with a `ProcessExit`. */ +interface OpenScope { + closed: boolean; + opened: number; + close(exitCode: number): void; +} + +const openScopes = new Set(); + +/** + * Track a scope until it settles. Returns a guard the settle path calls: it + * answers false when the exit closer already ended the scope, so the event is + * never emitted twice. + */ +function trackOpen(close: (exitCode: number) => void): { settle(): boolean } { + const entry: OpenScope = { closed: false, opened: nowMicros(), close }; + openScopes.add(entry); + return { + settle(): boolean { + openScopes.delete(entry); + if (entry.closed) return false; + entry.closed = true; + return true; + }, + }; +} + +onProcessExit(() => + [...openScopes].map((entry) => ({ + opened: entry.opened, + close(exitCode: number): void { + openScopes.delete(entry); + if (entry.closed) return; + entry.closed = true; + entry.close(exitCode); + }, + })), +); + +function errorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} + +function describe(error: unknown): string { + const text = errorMessage(error); + const name = errorName(error); + return text ? `${name}: ${text}` : name; +} + +function stackOf(error: unknown): string | undefined { + return error instanceof Error ? (error.stack ?? undefined) : undefined; +} + +function isThenable(value: unknown): value is PromiseLike { + return ( + typeof value === "object" && + value !== null && + typeof (value as PromiseLike).then === "function" + ); +} + +/** + * Run `body`, then `settle`, preserving whether `body` was sync or async. + * + * A scope whose callback is synchronous must stay synchronous: wrapping every + * body in a promise would make `session(() => 1)` return a `Promise`, + * which breaks the one case where a caller genuinely cannot await — a + * constructor, a synchronous framework hook, an `EventEmitter` listener. + */ +function settleWith( + body: () => T, + onSuccess: (value: Awaited) => void, + onFailure: (error: unknown) => void, +): T { + let result: T; + try { + result = body(); + } catch (error) { + onFailure(error); + throw error; + } + if (isThenable(result)) { + return (result as PromiseLike>).then( + (value) => { + onSuccess(value); + return value; + }, + (error: unknown) => { + onFailure(error); + throw error; + }, + ) as T; + } + onSuccess(result as Awaited); + return result; +} + +// --------------------------------------------------------------------------- +// session +// --------------------------------------------------------------------------- + +export interface SessionOptions { + sessionId?: string; + agentId?: string; +} + +/** The handle `session.open()` returns. Disposing it unwinds the binding. */ +export class SessionScope { + readonly id: string; + private readonly previous: Store; + private readonly entered: Store; + private readonly agentId: string | undefined; + private disposed = false; + + constructor(options: SessionOptions = {}) { + const store = context.snapshot(); + this.id = options.sessionId ?? store.sessionId ?? randomUUID().replace(/-/g, ""); + let next = context.withSessionBound(store, this.id); + this.agentId = options.agentId; + if (options.agentId !== undefined) next = context.withAgentPushed(next, options.agentId); + this.entered = next; + this.previous = context.enterWith(next); + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + // Unwind unconditionally: a scope that leaks an agent frame misattributes + // every later event in the process. When the current store is not the one + // we entered, something below us bound its own and never unwound, or we are + // being disposed from a different async context — repair by value rather + // than stamping the caller's context with a store from somewhere else. + const now = context.snapshot(); + if (now === this.entered) { + context.enterWith(this.previous); + return; + } + context.noteCrossContextExit(); + let repaired = now; + if (this.agentId !== undefined) repaired = context.withAgentDiscarded(repaired, this.agentId); + context.enterWith({ sessionId: this.previous.sessionId, agentStack: repaired.agentStack }); + } + + [Symbol.dispose](): void { + this.dispose(); + } +} + +export interface SessionFn { + /** Bind a session for the duration of `body`. Emits no events — identity only. */ + (body: (sessionId: string) => T): T; + (options: SessionOptions, body: (sessionId: string) => T): T; + /** The `using` form: `using s = failproofai.session.open()`. */ + open(options?: SessionOptions): SessionScope; +} + +/** + * Bind a session id (and optionally an agent id) for the enclosing work. + * + * await failproofai.session(async (sid) => { + * failproofai.event.agentStart({ agentId: "main", goal: "..." }); + * }); + * + * Emits **no events** — it is identity only. `agent()` is what brackets a run + * with `agent_start`/`agent_end`. + * + * An omitted `sessionId` reuses an already-bound session if there is one, and + * otherwise generates one. That inheritance is what lets a nested scope stay + * inside one run instead of splitting it into two sessions. + */ +const sessionImpl = (( + first: SessionOptions | ((sessionId: string) => T), + second?: (sessionId: string) => T, +): T => { + const options = typeof first === "function" ? {} : first; + const body = typeof first === "function" ? first : second!; + const store = context.snapshot(); + const id = options.sessionId ?? store.sessionId ?? randomUUID().replace(/-/g, ""); + let next = context.withSessionBound(store, id); + if (options.agentId !== undefined) next = context.withAgentPushed(next, options.agentId); + return context.runWith(next, () => body(id)); +}) as SessionFn; + +sessionImpl.open = (options: SessionOptions = {}): SessionScope => new SessionScope(options); + +export const session: SessionFn = sessionImpl; + +// --------------------------------------------------------------------------- +// agent +// --------------------------------------------------------------------------- + +export interface AgentOptions { + sessionId?: string; + goal?: string; + parentId?: ParentId; + /** The `outcome` on `agent_end` when the block completes normally. */ + outcome?: string; + summary?: string; + /** Any other key is attached to `agent_start` only. */ + [field: string]: unknown; +} + +function resolveAgentEntry( + agentId: string, + options: AgentOptions, +): { sid: string; parent: string | null | undefined; next: Store } { + const store = context.snapshot(); + const sid = options.sessionId ?? store.sessionId ?? randomUUID().replace(/-/g, ""); + + let parent: string | null | undefined; + if (options.parentId === undefined || options.parentId === AUTO) { + // Only inherit within the SAME session. An explicit `sessionId` that + // differs from the ambient one is the documented way to start a NEW run, + // and the enclosing agent does not exist in it — the span tree is keyed by + // session, so the child would render as a root with a dangling parent, or + // get grafted onto whatever agent in its own session happened to share the + // id. The ordinary long-lived-server shape reaches it directly: + // + // await agent("server", { sessionId: "boot" }, async () => { + // for (const rid of requests) { + // await agent("handler", { sessionId: rid }, ...); // parent="server" + // } + // }); + // + // A caller who genuinely wants a cross-session link can still pass + // `parentId` explicitly. + const currentAgent = + store.agentStack.length > 0 ? store.agentStack[store.agentStack.length - 1]! : null; + parent = sid === store.sessionId ? currentAgent : null; + } else { + parent = options.parentId; + } + + const next = context.withAgentPushed(context.withSessionBound(store, sid), agentId); + return { sid, parent, next }; +} + +function agentStartFields(options: AgentOptions): Record { + const { sessionId, goal, parentId, outcome, summary, ...fields } = options; + void sessionId; + void goal; + void parentId; + void outcome; + void summary; + return fields; +} + +function emitAgentEnd( + sid: string, + agentId: string, + options: AgentOptions, + error: unknown, + hadError: boolean, +): void { + let outcome: string; + if (!hadError) { + outcome = options.outcome ?? "success"; + } else if (isCancellation(error)) { + outcome = "cancelled"; + } else { + outcome = "failed"; + // `error` strictly BEFORE `agent_end`, because the dashboard closes the + // agent span at `agent_end` and anything after it is attributed to nothing. + runtime.event.error({ + sessionId: sid, + agentId, + errorType: errorName(error), + message: errorMessage(error), + // A ProcessExit's stack is the SDK's own exit path — noise, not a cause. + traceback: error instanceof ProcessExit ? undefined : stackOf(error), + }); + } + // The literal is `"failed"`, never `"failure"` — only + // `error|failed|timeout|rejected` count as a failure server-side. + runtime.event.agentEnd({ + sessionId: sid, + agentId, + outcome, + summary: options.summary, + }); +} + +/** The handle `agent.open()` returns. Disposing it emits `agent_end`. */ +export class AgentScope { + readonly agentId: string; + readonly sessionId: string; + readonly identity: Identity; + private readonly options: AgentOptions; + private readonly previous: Store; + private readonly entered: Store; + private disposed = false; + /** Set by `fail()` so a `using` block can still record a failure. */ + private failure: { error: unknown } | null = null; + + constructor(agentId = "main", options: AgentOptions = {}) { + this.agentId = agentId; + this.options = options; + const { sid, parent, next } = resolveAgentEntry(agentId, options); + this.sessionId = sid; + this.entered = next; + this.previous = context.enterWith(next); + try { + runtime.event.agentStart({ + sessionId: sid, + agentId, + goal: options.goal, + parentId: parent, + ...agentStartFields(options), + }); + } catch (error) { + // A rejected `agent_start` (a reserved extra, say) must not leave a + // half-entered scope behind: the disposer never runs if the constructor + // throws. + this.unwind(); + throw error; + } + this.identity = context.current(); + this.open = trackOpen((exitCode) => + emitAgentEnd( + this.sessionId, + this.agentId, + this.options, + new ProcessExit(exitCode, `agent ${JSON.stringify(this.agentId)}`), + true, + ), + ); + } + + private readonly open: { settle(): boolean }; + + /** + * Record that this span failed. `using` has no exception channel to a + * disposer, so a caller who catches inside the block tells us here. + */ + fail(error: unknown): void { + this.failure = { error }; + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + try { + if (!this.open.settle()) return; + emitAgentEnd( + this.sessionId, + this.agentId, + this.options, + this.failure?.error, + this.failure !== null, + ); + } finally { + // Unwound in a `finally` so the stack is intact even if emission itself + // blew up. A leaked frame is worse than a lost event. + this.unwind(); + } + } + + private unwind(): void { + const now = context.snapshot(); + if (now === this.entered) { + context.enterWith(this.previous); + return; + } + context.noteCrossContextExit(); + context.enterWith({ + sessionId: this.previous.sessionId, + agentStack: context.withAgentDiscarded(now, this.agentId).agentStack, + }); + } + + [Symbol.dispose](): void { + this.dispose(); + } +} + +export interface AgentFn { + (agentId: string, body: (identity: Identity) => T): T; + (agentId: string, options: AgentOptions, body: (identity: Identity) => T): T; + /** The `using` form: `using span = failproofai.agent.open("planner")`. */ + open(agentId?: string, options?: AgentOptions): AgentScope; +} + +/** + * Bracket a run (or a sub-run) with `agent_start` / `agent_end`. + * + * await failproofai.agent("planner", { goal: question }, async () => { ... }); + * + * Keep `agentId` low-cardinality (a node/role name, never a UUID): it is a + * `LowCardinality(String)` column and the primary facet across every session. + * + * Extra keys are attached to `agent_start` only; `agent_end` carries `outcome` + * and `summary`. + * + * Exit semantics, which are the whole point: + * + * | thrown | events | outcome | + * |-------------------------|-------------------|---------------| + * | nothing | `agent_end` | `options.outcome` | + * | any error | `error`, then end | `"failed"` | + * | an `AbortError` | `agent_end` only | `"cancelled"` | + * + * The error is always re-thrown. + */ +const agentImpl = (( + agentId: string, + second: AgentOptions | ((identity: Identity) => T), + third?: (identity: Identity) => T, +): T => { + const options = typeof second === "function" ? {} : second; + const body = typeof second === "function" ? second : third!; + const { sid, parent, next } = resolveAgentEntry(agentId, options); + + return context.runWith(next, () => { + runtime.event.agentStart({ + sessionId: sid, + agentId, + goal: options.goal, + parentId: parent, + ...agentStartFields(options), + }); + const open = trackOpen((exitCode) => + emitAgentEnd(sid, agentId, options, new ProcessExit(exitCode, `agent ${JSON.stringify(agentId)}`), true), + ); + return settleWith( + () => body(context.current()), + () => { + if (open.settle()) emitAgentEnd(sid, agentId, options, undefined, false); + }, + (error) => { + if (open.settle()) emitAgentEnd(sid, agentId, options, error, true); + }, + ); + }); +}) as AgentFn; + +agentImpl.open = (agentId = "main", options: AgentOptions = {}): AgentScope => + new AgentScope(agentId, options); + +export const agent: AgentFn = agentImpl; + +// --------------------------------------------------------------------------- +// toolCall +// --------------------------------------------------------------------------- + +/** The handle a `toolCall` body receives. Set `.output`; read `.id`. */ +export class ToolCall { + readonly id: string; + /** + * What the tool produced. When the body returns a value and this was never + * set, the returned value is recorded instead — so the common + * `await toolCall("search", { input }, () => search(q))` needs no assignment. + */ + output: unknown = undefined; + private assigned = false; + + constructor(toolCallId: string) { + this.id = toolCallId; + // A plain field would make "never set" and "set to undefined" + // indistinguishable, and the difference decides whether the body's return + // value is used. + let stored: unknown; + Object.defineProperty(this, "output", { + get: () => stored, + set: (value: unknown) => { + stored = value; + this.assigned = true; + }, + enumerable: true, + configurable: true, + }); + } + + /** True once `.output` has been assigned, whatever it was assigned to. */ + get outputAssigned(): boolean { + return this.assigned; + } +} + +export interface ToolCallOptions { + toolCallId?: string; + input?: Record | null; + /** Any other key is attached to `tool_use` only. */ + [field: string]: unknown; +} + +function toolUseFields(options: ToolCallOptions): Record { + const { toolCallId, input, ...fields } = options; + void toolCallId; + void input; + return fields; +} + +/** The handle `toolCall.open()` returns. Disposing it emits `tool_result`. */ +export class ToolCallScope { + readonly call: ToolCall; + private readonly toolName: string; + private readonly sid: string | null; + private readonly aid: string; + private disposed = false; + private failure: { error: unknown } | null = null; + + constructor(toolName: string, options: ToolCallOptions = {}) { + this.toolName = toolName; + // Resolve once, at entry: a tool that opens its own scope inside must not + // make the closing `tool_result` land on a different agent id. + this.sid = context.sessionId(); + this.aid = context.agentId(); + this.call = new ToolCall(options.toolCallId ?? randomUUID().replace(/-/g, "")); + runtime.event.toolUse({ + sessionId: this.sid, + agentId: this.aid, + toolName, + toolCallId: this.call.id, + input: options.input, + ...toolUseFields(options), + }); + } + + fail(error: unknown): void { + this.failure = { error }; + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + // A tool still open at exit is closed by the event namespace (exit.ts), + // which sees every tool_use; its tool_result there makes this a no-op. + const failed = this.failure !== null && !isCancellation(this.failure.error); + runtime.event.toolResult({ + sessionId: this.sid, + agentId: this.aid, + toolName: this.toolName, + toolCallId: this.call.id, + output: this.call.output, + error: failed ? describe(this.failure!.error) : undefined, + }); + } + + [Symbol.dispose](): void { + this.dispose(); + } +} + +export interface ToolCallFn { + (toolName: string, body: (call: ToolCall) => T): T; + (toolName: string, options: ToolCallOptions, body: (call: ToolCall) => T): T; + /** The `using` form: `using t = failproofai.toolCall.open("web_search")`. */ + open(toolName: string, options?: ToolCallOptions): ToolCallScope; +} + +/** + * Bracket a tool invocation with `tool_use` / `tool_result`. + * + * const hits = await failproofai.toolCall("web_search", { input: { q } }, () => search(q)); + * + * `toolCallId` defaults to a fresh uuid. Identity comes from the enclosing + * scope; if nothing is bound, the underlying `event.toolUse()` throws the usual + * `TypeError` naming the fix. + * + * The body's resolved value is recorded as `output` unless the handle's + * `.output` was assigned, in which case that wins. + * + * On failure this emits `tool_result({ error: "TypeName: msg" })` and **no + * `error` event**. A tool failure the agent loop catches is not a run-level + * error, and one that propagates is reported exactly once, by the enclosing + * `agent()`. Cancellation closes the leaf with no `error` string at all, for + * the same reason `agent()` does not mark it failed. + */ +const toolCallImpl = (( + toolName: string, + second: ToolCallOptions | ((call: ToolCall) => T), + third?: (call: ToolCall) => T, +): T => { + const options = typeof second === "function" ? {} : second; + const body = typeof second === "function" ? second : third!; + + const sid = context.sessionId(); + const aid = context.agentId(); + const call = new ToolCall(options.toolCallId ?? randomUUID().replace(/-/g, "")); + + runtime.event.toolUse({ + sessionId: sid, + agentId: aid, + toolName, + toolCallId: call.id, + input: options.input, + ...toolUseFields(options), + }); + + const finish = (output: unknown, error: unknown, failed: boolean): void => { + runtime.event.toolResult({ + sessionId: sid, + agentId: aid, + toolName, + toolCallId: call.id, + output, + error: failed && !isCancellation(error) ? describe(error) : undefined, + }); + }; + + return settleWith( + () => body(call), + (value) => finish(call.outputAssigned ? call.output : value, undefined, false), + (error) => finish(call.outputAssigned ? call.output : undefined, error, true), + ); +}) as ToolCallFn; + +toolCallImpl.open = (toolName: string, options: ToolCallOptions = {}): ToolCallScope => + new ToolCallScope(toolName, options); + +export const toolCall: ToolCallFn = toolCallImpl; diff --git a/sdk/typescript/src/shared.ts b/sdk/typescript/src/shared.ts new file mode 100644 index 000000000..3e5f6312f --- /dev/null +++ b/sdk/typescript/src/shared.ts @@ -0,0 +1,33 @@ +/** + * Settings every copy of this package in one process must agree on. + * + * A process routinely holds more than one copy: the ESM and CommonJS builds + * (the dual-package case), or — on Next.js without `withFailproofai` — a copy + * bundled into each route beside the one `instrumentation.ts` imported from + * `node_modules`. Module-level state is per copy, so `configure({ environment: + * "production" })` in `instrumentation.ts` never reached the route's copy, and + * everything the route emitted went out labelled `dev` — recorded, shipped, + * and in the wrong bucket, with nothing said. + * + * `environment` and `baseDir` decide what an event is labelled and where it is + * written, so they live here, keyed through the process-wide `Symbol.for` + * registry every copy can reach. `flushInterval` is timing only and stays with + * each copy's writer. + */ + +const KEY = Symbol.for("@failproofai/sdk.settings"); + +export interface SharedSettings { + environment: string | null; + baseDir: string | null; +} + +export function shared(): SharedSettings { + const holder = globalThis as unknown as Record; + let found = holder[KEY]; + if (found === undefined) { + found = { environment: null, baseDir: null }; + holder[KEY] = found; + } + return found; +} diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts new file mode 100644 index 000000000..4f7457bd2 --- /dev/null +++ b/sdk/typescript/src/version.ts @@ -0,0 +1,5 @@ +// The version this package publishes under. Mirrors `failproofai_sdk/_version.py` +// in the Python SDK, but the two version INDEPENDENTLY — npm and PyPI have +// different pre-release grammars and different release cadences, and nothing +// downstream compares them. See CHANGELOG.md for this package's own history. +export const VERSION = "0.0.1-beta.0"; diff --git a/sdk/typescript/src/writer.ts b/sdk/typescript/src/writer.ts new file mode 100644 index 000000000..78042045a --- /dev/null +++ b/sdk/typescript/src/writer.ts @@ -0,0 +1,934 @@ +import { + closeSync, + fsyncSync, + mkdirSync, + openSync, + renameSync, + rmSync, + writeSync, + constants as fsConstants, +} from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import { join } from "node:path"; + +import { runExitClosers } from "./exit.js"; +import { logException, logger } from "./logger.js"; +import { redactJsonLine, redactionEnabled } from "./redact.js"; +import { getBaseDir } from "./resolver.js"; + +/** + * Per-process batch counter, so two batches written inside the same millisecond + * cannot land on the same filename. + * + * The timestamp alone is not enough, and the way it fails is invisible. Two + * batches in the same millisecond produce the same stem, and the second rename + * overwrites the first — no exception, no log line, no trace that the events + * had ever existed. Three routine situations hit it: + * + * * the exit flush racing the interval's own final cycle, which is exactly + * when the last events of a run are written; + * * any caller invoking `flushNow()` from more than one place; + * * several agent processes sharing one spool root — the normal deployment. + * Nothing in the stem identifies the writer, so unrelated processes would + * silently overwrite each other's batches. + * + * Hence the pid as well as the counter: the counter fixes the in-process race + * and the pid fixes the cross-process one. The daemons require only that a + * batch file end in `.jsonl` and not `.tmp` (`crates/fpai-collect/src/spool.rs`), + * so the rest of the stem is ours to make unique. + */ +let batchSeq = 0; + +/** + * Hard cap on the in-memory queue, matching `events.ts`'s `PENDING_CAP`. + * + * `submit()` is called from the caller's own agent loop and must never block or + * throw, so it cannot apply backpressure — which leaves an unbounded queue as + * the only other option, and that is a memory leak wearing a different hat. Any + * condition that stops the spool draining (a full disk, a read-only mount) then + * converts a telemetry outage into an OOM kill of the host agent. Losing the + * oldest events is the better failure: it is bounded, it is logged, and the + * events most worth having are the recent ones. + * + * At the default 500 ms interval a process would have to emit 20,000 + * events/second to reach it. Anything that does hit this cap is not a busy + * agent, it is a spool that has stopped. + * + * A COUNT alone is not the bound this comment claims, because it says nothing + * about how big an event is. `integrations/core.ts` budgets 128 KiB of `fw_*` + * extras per event on top of the declared fields, so 10,000 of them is ~1.3 GB. + * So the queue is bounded by BYTES as well, below. + */ +const QUEUE_CAP = 10_000; + +/** + * The ceiling, enforced against MEASURED bytes rather than an estimate. Chosen + * to sit well under the memory a small container is given (512 MB is the common + * floor), because the whole point is that a spool which has stopped draining + * must not take the customer's agent down with it. + */ +const QUEUE_BYTE_CAP = 64 * 1024 * 1024; + +/** + * Per-STRING cap inside one event, mirroring `MAX_FIELD_BYTES` in + * `crates/fpai-collect/src/spool.rs`. The Rust spool writer has always enforced + * this; an SDK publishing into the same directories must too. + */ +const MAX_FIELD_BYTES = 1024 * 1024; + +/** + * Roll a batch file once it reaches this, mirroring `DEFAULT_MAX_BATCH_BYTES` + * in `spool.rs` and staying under the uploader's `DEFAULT_MAX_UPLOAD_BYTES`. + * + * `uploader.rs` documents the invariant this restores: "A single line longer + * than max is emitted alone rather than dropped: the spool writer already + * guarantees no such line exists." `splitLines` can only split on newlines — so + * one oversized event would be POSTed whole, rejected, and the WHOLE spool file + * (every unrelated event batched with it) parked, retried three times and + * poisoned. Never delivered, and nothing in the host process would ever learn. + */ +const MAX_BATCH_BYTES = 8 * 1024 * 1024; + +/** + * An encoded event above this is over-large on its own and gets its fields + * capped. Sits below `MAX_BATCH_BYTES` so a capped event still leaves room for + * the batch framing around it. + */ +const MAX_EVENT_BYTES = 4 * 1024 * 1024; + +/** + * How deep `sanitize` will walk before giving up on a branch. Guards the + * fallback path against a stack overflow, which would defeat the point of + * having a fallback at all. + */ +const MAX_SANITIZE_DEPTH = 50; + +/** + * How `JSON.stringify` writes a lone surrogate. + * + * Unlike Python's `ensure_ascii=True`, JavaScript emits non-ASCII characters + * literally, so this substring can only appear because a real lone surrogate + * was escaped (ES2019 well-formed `JSON.stringify`) or because the payload's + * own text contained the literal characters `\ud8…`. The second is rare enough + * to be worth the certainty; it costs one re-encode and changes nothing. + * + * The lead nibble matters. `\ud` alone also matches U+D000–U+D7FF, which is + * most of the Hangul syllable block. Real surrogates are U+D800–U+DFFF, whose + * escapes all begin `\ud8`, `\ud9`, `\uda`…`\udf`. + */ +const SURROGATE_ESCAPES: readonly string[] = [..."89abcdefABCDEF"].map((c) => `\\ud${c}`); + +const FIELD_TRUNCATION_MARKER = "…[truncated]"; +const CYCLE_MARKER = ""; +const DEPTH_MARKER = ""; + +/** + * Every live writer. Both the exit flush and `flushAllNow()` iterate this + * rather than binding to one instance. + * + * Node has no `fork()` hazard to guard here the way the Python SDK does: a + * `worker_threads` Worker and a `cluster` child each load this module afresh + * and get their own writer with its own timer, so there is no inherited queue + * with no thread to drain it. + */ +const liveWriters = new Set(); + +/** + * A flush interval the timer can actually run on. + * + * Rejected at the boundary, where a caller still has a stack trace pointing at + * their own `configure()` call: + * + * -1 -> a timer that fires immediately and forever + * NaN -> Node coerces to 1 ms; a busy loop rewriting the spool + * Inf -> Node coerces to 1 ms; the same busy loop + * 0 -> waits not at all; pins a core and rewrites the spool as fast as + * the disk allows + * + * Every one of those is a telemetry library becoming the reason the host agent + * is slow, and none of them throws on its own. + */ +export function validatedInterval(flushInterval: number): number { + const interval = Number(flushInterval); + if (!Number.isFinite(interval) || interval <= 0) { + throw new Error( + `flushInterval must be a finite number greater than zero, got ${String(flushInterval)}`, + ); + } + return interval; +} + +/** + * Roughly how many bytes `value` will occupy once encoded. + * + * Walks NODES, not characters: `.length` on a string is O(1), so an ordinary + * event costs microseconds even though it may carry megabytes of text. That is + * what makes it affordable on `submit`, which runs on the caller's agent loop. + * + * Deliberately approximate — it ignores JSON punctuation and escaping — because + * it backs a backstop against unbounded growth, not an exact quota. + */ +export function approxSize(value: unknown, depth = 0): number { + if (depth > MAX_SANITIZE_DEPTH) return 16; + if (value === null || value === undefined) return 8; + const kind = typeof value; + if (kind === "boolean" || kind === "number" || kind === "bigint") return 8; + if (kind === "string") return (value as string).length; + // NOTHING here may throw. `submit` runs on the caller's agent loop, so an + // exception escaping this function is a telemetry call taking down the host + // agent — the one failure mode this whole module is written to avoid. A + // getter on the caller's object can throw anything at all. + try { + if (ArrayBuffer.isView(value)) return (value).byteLength; + if (Array.isArray(value)) { + let total = 0; + for (const item of value) total += approxSize(item, depth + 1); + return total; + } + if (value instanceof Map) { + let total = 0; + for (const [key, item] of value) { + total += (typeof key === "string" ? key.length : 16) + approxSize(item, depth + 1); + } + return total; + } + if (value instanceof Set) { + let total = 0; + for (const item of value) total += approxSize(item, depth + 1); + return total; + } + if (kind === "object") { + let total = 0; + for (const [key, item] of Object.entries(value as Record)) { + total += key.length + approxSize(item, depth + 1); + } + return total; + } + } catch { + return 16; + } + return 16; +} + +/** Drop a trailing high surrogate so a slice never invents a lone one. */ +function sliceWithoutSplitting(value: string, end: number): string { + if (end <= 0) return ""; + const code = value.charCodeAt(end - 1); + const safeEnd = code >= 0xd800 && code <= 0xdbff ? end - 1 : end; + return value.slice(0, safeEnd); +} + +/** + * Truncate every string in `value` to `limit`, marking what was cut. + * + * Mirrors `truncate_strings` in `crates/fpai-collect/src/spool.rs`, which has + * always enforced this on the Rust side of the same spool. + */ +export function capFields(value: unknown, limit: number, depth = 0): unknown { + if (depth > MAX_SANITIZE_DEPTH) return value; + // Same rule as `approxSize`: never throw. This runs from `encodeEntry`, whose + // contract is that ONE bad event is dropped alone rather than taking the + // batch beside it down. + try { + if (typeof value === "string" && value.length > limit) { + return sliceWithoutSplitting(value, limit) + FIELD_TRUNCATION_MARKER; + } + if (Array.isArray(value)) return value.map((item) => capFields(item, limit, depth + 1)); + if (value !== null && typeof value === "object" && !(value instanceof Date)) { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = capFields(item, limit, depth + 1); + } + return out; + } + } catch { + return value; + } + return value; +} + +/** + * A string with any lone surrogate made inert. + * + * `\ud800`-`\udfff` outside a valid pair is what a byte sequence that is not + * valid UTF-8 becomes when it is decoded leniently — a filesystem path, a + * truncated tool output. `JSON.stringify` escapes them happily, so nothing + * fails locally, and then the SERVER skips the whole event: verified against a + * live ingest, `{"accepted":0,"skipped":1}` at 200 OK. Replacing each with its + * visible escape keeps the byte in the payload instead of dropping it. + */ +export function scrubSurrogates(value: string): string { + // Matches a high surrogate not followed by a low one, or a low surrogate not + // preceded by a high one. + return value.replace( + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`, + ); +} + +/** + * Rewrite one payload into something `JSON.stringify` can definitely encode. + * + * Only ever reached from `encodeEntry`'s fallback, so it may be slow; it must + * not be lossy in the ordinary case, and it must not throw. + * + * `seen` tracks the objects on the CURRENT PATH, not every object visited. A + * payload that mentions the same object twice as siblings is a DAG, not a + * cycle, and JSON encodes it fine — flagging it would corrupt a perfectly good + * event. + */ +export function sanitize(value: unknown, seen: Set, depth = 0): unknown { + if (depth > MAX_SANITIZE_DEPTH) return DEPTH_MARKER; + if (value === undefined || value === null) return null; + const kind = typeof value; + // NaN / Infinity / -Infinity. `JSON.stringify` already writes these as `null`, + // but making it explicit here means the sanitized copy and the strict copy + // agree rather than differing by an encoder default. + if (kind === "number") return Number.isFinite(value) ? value : null; + if (kind === "bigint") return (value as bigint).toString(); + if (kind === "boolean") return value; + if (kind === "string") return scrubSurrogates(value as string); + if (kind === "function" || kind === "symbol") return null; + + const object = value; + if (seen.has(object)) return CYCLE_MARKER; + + try { + if (object instanceof Date) { + return Number.isNaN(object.getTime()) ? null : object.toISOString(); + } + if (ArrayBuffer.isView(object)) return null; + + seen.add(object); + try { + if (Array.isArray(object)) { + return object.map((item) => sanitize(item, seen, depth + 1)); + } + if (object instanceof Set) { + return [...object].map((item) => sanitize(item, seen, depth + 1)); + } + const entries = + object instanceof Map + ? [...object.entries()].map(([k, v]) => [String(k), v] as const) + : Object.entries(object as Record); + const out: Record = {}; + for (const [key, item] of entries) { + // Keys go through the SAME surrogate scrub as values. A filesystem + // path, the realistic source, is most naturally a KEY + // (`{path: contents}`). An unscrubbed key reaches the wire as a JSON + // lone-surrogate escape, ingest answers 200 `{"accepted":0,"skipped":1}`, + // and the uploader parks that batch and poisons it after three retries + // — so one bad key loses every event batched with it. + out[scrubSurrogates(key)] = sanitize(item, seen, depth + 1); + } + return out; + } finally { + seen.delete(object); + } + } catch { + seen.delete(object); + return null; + } +} + +/** + * The replacer for the FAST path. Handles the values `JSON.stringify` cannot, + * and nothing else — this is the encoder that must stay byte-identical for + * every payload that was already valid JSON. + * + * Mirrors Python's `default=str`: the values that have an obvious textual form + * get one, and everything else falls through to the sanitized rebuild below. + * Functions and symbols are DROPPED rather than stringified — Python's + * `default=str` would put a lambda's source location on the wire, which is + * worse than an absent key. + */ +function replacer(this: unknown, _key: string, value: unknown): unknown { + if (typeof value === "bigint") return value.toString(); + if (value instanceof Map) return Object.fromEntries(value); + if (value instanceof Set) return [...value]; + if (value instanceof Error) { + return { name: value.name, message: value.message, stack: value.stack }; + } + return value; +} + +/** + * One event as a JSON line, or null if it cannot be encoded at all. + * + * THE POINT IS ISOLATION. A single `JSON.stringify` over the whole batch means + * one unencodable payload takes every event beside it down: the flush restores + * the batch, the next interval retries the identical batch, and the spool never + * advances again. An object holding a back-reference — an ordinary thing to + * hand a telemetry call, and the shape of every framework's run context — + * permanently ends recording for the process. + * + * So: try strict first (the fast path, byte-identical to the plain encoder for + * every payload that was already valid JSON), fall back to a sanitised copy, + * and only then give up on that ONE event. + */ +export function encodeEntry(entry: Record): string | null { + let encoded: string | null = null; + // Not a narrow catch on TypeError. A `toJSON()` or a getter runs the CALLER'S + // code, which can throw anything at all — a RuntimeError out of a lazy ORM + // attribute, an error out of a property that touches the network. Those would + // propagate out of the batch write and put the whole batch back on the queue + // to be retried identically forever: the exact wedge this function exists to + // prevent, reached through a different error type. + try { + encoded = JSON.stringify(entry, replacer) ?? null; + } catch { + encoded = null; + } + + if (encoded !== null && !SURROGATE_ESCAPES.some((escape) => encoded.includes(escape))) { + return capEncoded(entry, encoded); + } + + try { + const sanitized = sanitize(entry, new Set()) as Record; + const text = JSON.stringify(sanitized, replacer); + if (text === undefined) return null; + return capEncoded(sanitized, text); + } catch (error) { + // Nothing left to try. Losing this event is the correct outcome; losing the + // batch around it is not. + logException( + `could not serialize an event (type=${String(entry?.type)}); dropping it`, + error, + ); + return null; + } +} + +/** + * Bound ONE event, re-encoding only when it is actually over-large. + * + * Why it has to happen at all: `uploader.rs` states the invariant it relies on + * — "A single line longer than max is emitted alone rather than dropped: the + * spool writer already guarantees no such line exists." The Rust spool writer + * does guarantee it (`truncate_strings` at `MAX_FIELD_BYTES`). An SDK writer + * publishing into the same directories must too — or one + * `toolResult({ output: })` is written as a single line, POSTed + * whole because `splitLines` can only split on newlines, rejected, and the + * ENTIRE spool file parked, retried three times and poisoned. Every unrelated + * event batched alongside it goes too, and nothing in the host process ever + * learns. + */ +function capEncoded(entry: Record, encoded: string): string { + // `encoded.length` is UTF-16 units, not bytes, so the cheap check has to be + // the byte count — `Buffer.byteLength` is a native scan and costs far less + // than the walk it guards. + if (Buffer.byteLength(encoded, "utf8") <= MAX_EVENT_BYTES) return encoded; + const capped = capFields(entry, MAX_FIELD_BYTES); + let recoded: string; + try { + const text = JSON.stringify(capped, replacer); + if (text === undefined) return encoded; + recoded = text; + } catch { + return encoded; + } + logger.warn( + `truncated an oversized event (type=${String(entry?.type)}) from ` + + `${Buffer.byteLength(encoded, "utf8")} to ${Buffer.byteLength(recoded, "utf8")} bytes; ` + + `fields above ${MAX_FIELD_BYTES} bytes were cut so the batch stays deliverable`, + ); + return recoded; +} + +interface EncodedLine { + readonly text: string; + readonly bytes: number; +} + +/** Split encoded lines into batches that each stay under `limit` bytes. */ +export function roll(lines: readonly EncodedLine[], limit: number): EncodedLine[][] { + const chunks: EncodedLine[][] = []; + let chunk: EncodedLine[] = []; + let size = 0; + for (const line of lines) { + const cost = line.bytes + 1; // the newline this line will be joined with + if (chunk.length > 0 && size + cost > limit) { + chunks.push(chunk); + chunk = []; + size = 0; + } + chunk.push(line); + size += cost; + } + if (chunk.length > 0) chunks.push(chunk); + return chunks; +} + +function batchStem(): string { + // `2026-09-23T12-34-56-789Z`: the timestamp orders batches for a human + // reading the directory, and the pid+counter suffix is what makes the name + // unique. See `batchSeq` above. + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + batchSeq += 1; + return `event-${stamp}-${process.pid}-${batchSeq - 1}`; +} + +interface InFlight { + readonly chunks: EncodedLine[][]; + /** Incremented in the SAME synchronous tick as the rename that made a chunk durable. */ + done: number; +} + +export class EventWriter { + private queue: Record[] = []; + /** + * Encoded lines from a failed write, retried ahead of the queue. + * + * A plain "put the entries back" would re-encode them and, when a batch had + * already rolled into several files and only a later one failed, rewrite the + * files that succeeded — duplicating every event in them. Holding the + * ENCODED lines instead means a partial failure retries exactly what did not + * land. + */ + private pending: EncodedLine[] = []; + private flushInterval: number; + private dropped = 0; + private queuedBytes = 0; + private timer: ReturnType | null = null; + private flushing: Promise | null = null; + private inFlight: InFlight | null = null; + private closed = false; + + constructor(flushInterval = 0.5) { + this.flushInterval = validatedInterval(flushInterval); + this.startTimer(); + liveWriters.add(this); + ensureExitHook(); + } + + private startTimer(): void { + if (this.timer !== null) clearInterval(this.timer); + this.timer = setInterval(() => { + void this.flush().catch((error: unknown) => { + // Recording must never die permanently because one flush hit a + // transient filesystem error. `flush` restores the encoded lines before + // rejecting, so the next interval retries them. + logException("event flush failed; buffered events will be retried", error); + }); + }, this.flushInterval * 1000); + // The single most important line in this file for not being noticed: an + // un-unref'd interval keeps the Node event loop alive forever, so a script + // that merely imports this SDK never exits. `unref` makes the timer stop + // counting as a reason to stay running, exactly like the daemon thread the + // Python SDK uses. + this.timer.unref?.(); + } + + submit(entry: Record): void { + if (this.closed) return; + const size = approxSize(entry); + // BOTH bounds, and the byte one against measured sizes. A count alone is + // not a memory bound (the adapters budget 128 KiB of extras per event, so + // 10,000 of them is ~1.3 GB), and an average-based byte bound is not one + // either until the average has been learned. + while ( + this.queue.length > 0 && + (this.queue.length >= QUEUE_CAP || this.queuedBytes + size > QUEUE_BYTE_CAP) + ) { + const evicted = this.queue.shift(); + if (evicted === undefined) break; + this.queuedBytes = Math.max(0, this.queuedBytes - approxSize(evicted)); + this.dropped += 1; + // Powers of ten, so a stuck spool says so without becoming the thing that + // fills the disk it is complaining about. + if (this.dropped === 1 || this.dropped % 1000 === 0) { + logger.warn( + `event queue is full (${this.queue.length} events, ${this.queuedBytes} bytes); ` + + `discarding oldest. ${this.dropped} dropped so far — the spool is not draining.`, + ); + } + } + this.queue.push(entry); + this.queuedBytes += size; + } + + setFlushInterval(interval: number): void { + // Validate first, assign second: a rejected value must leave the writer + // running on the interval it already had, not on a half-applied one. + this.flushInterval = validatedInterval(interval); + // Restart so the new interval applies from now, not from the end of a cycle + // that may be an hour long. + if (!this.closed) this.startTimer(); + } + + getFlushInterval(): number { + return this.flushInterval; + } + + /** Drain and write any buffered entries immediately. */ + async flushNow(): Promise { + // Twice, on purpose. `flush()` hands back a flush that is ALREADY running + // if there is one, and that flush drained the queue before any event + // emitted since — so awaiting it alone resolves with those events still in + // memory. The second call either drains what is left or joins a newer + // flush that started after this call, which drained it too. Either way, + // every event submitted before `flushNow()` was called is on disk when it + // resolves; with nothing queued the second pass returns immediately. + await this.flush(); + await this.flush(); + } + + /** + * The synchronous flush, for a signal handler or `process.on("exit")`. + * + * `exit` handlers may not await, so the periodic path's async writes are no + * use there. This one blocks, which is exactly right at the end of a process + * and exactly wrong in a steady-state agent loop. + */ + flushSync(): void { + const carried = this.takeInFlightRemainder(); + const lines = [...carried, ...this.drainToLines()]; + if (lines.length === 0) return; + for (const chunk of roll(lines, MAX_BATCH_BYTES)) { + try { + this.writeOneFileSync(chunk); + } catch (error) { + logException("final flush failed; buffered events were lost", error); + return; + } + } + } + + /** + * Chunks an in-flight async write had not yet renamed. + * + * A chunk whose `renameSync` already ran is durable and complete on disk even + * though the promise that would have recorded it never resolved, so taking it + * again here would publish a byte-identical duplicate. `done` is incremented + * in the same synchronous tick as that rename precisely so this check has no + * window to be wrong in. + */ + private takeInFlightRemainder(): EncodedLine[] { + const active = this.inFlight; + this.inFlight = null; + if (active === null) return []; + return active.chunks.slice(active.done).flat(); + } + + private drainToLines(): EncodedLine[] { + const entries = this.queue; + this.queue = []; + this.queuedBytes = 0; + const carried = this.pending; + this.pending = []; + return [...carried, ...this.encode(entries)]; + } + + /** + * Encode BEFORE touching the filesystem. An unencodable event is a permanent + * condition — retrying it produces the identical failure — so it is dropped + * here, while a filesystem error is raised from the write and the whole batch + * goes back to be retried. + */ + private encode(entries: readonly Record[]): EncodedLine[] { + const lines: EncodedLine[] = []; + let dropped = 0; + const redact = redactionEnabled(getBaseDir()); + for (const entry of entries) { + let encoded = encodeEntry(entry); + if (encoded === null) { + dropped += 1; + continue; + } + if (redact) { + try { + encoded = redactJsonLine(encoded); + } catch (error) { + // A redactor that throws must not publish the unredacted line it was + // handed. Dropping the event is the only safe answer: the whole + // reason this runs is that the line may carry a credential. + logException("redaction failed; dropping the event rather than publishing it", error); + dropped += 1; + continue; + } + } + lines.push({ text: encoded, bytes: Buffer.byteLength(encoded, "utf8") }); + } + if (dropped > 0) { + logger.error( + `dropped ${dropped} unpublishable event(s) from a batch of ${entries.length}; ` + + "the rest of the batch was published", + ); + } + return lines; + } + + /** + * Serialised, so a batch is never drained by two callers at once and — the + * case that actually bites — so a `flushNow()` from a signal handler waits + * for an in-flight write instead of racing it. + */ + private flush(): Promise { + if (this.flushing !== null) return this.flushing; + const run = this.flushOnce().finally(() => { + this.flushing = null; + }); + this.flushing = run; + return run; + } + + private async flushOnce(): Promise { + const lines = this.drainToLines(); + if (lines.length === 0) return; + const chunks = roll(lines, MAX_BATCH_BYTES); + const active: InFlight = { chunks, done: 0 }; + this.inFlight = active; + try { + for (const chunk of chunks) { + await this.writeOneFile(chunk, active); + } + } catch (error) { + // Only what did NOT land goes back, at the FRONT — the events that + // already reached disk must not be published twice. + const remainder = active.chunks.slice(active.done).flat(); + this.pending = [...remainder, ...this.pending]; + this.queuedBytes += remainder.reduce((total, line) => total + line.bytes, 0); + this.boundPending(); + throw error; + } finally { + if (this.inFlight === active) this.inFlight = null; + } + } + + /** + * The retry buffer obeys the same ceilings as the queue. A spool that has + * stopped draining must not turn into unbounded growth just because the + * events are already encoded. + */ + private boundPending(): void { + while ( + this.pending.length > 0 && + (this.pending.length > QUEUE_CAP || this.queuedBytes > QUEUE_BYTE_CAP) + ) { + const evicted = this.pending.shift(); + if (evicted === undefined) break; + this.queuedBytes = Math.max(0, this.queuedBytes - evicted.bytes); + this.dropped += 1; + } + } + + private eventsDir(): string { + return join(getBaseDir(), "events"); + } + + private content(lines: readonly EncodedLine[]): Buffer { + return Buffer.from(`${lines.map((line) => line.text).join("\n")}\n`, "utf8"); + } + + private async writeOneFile(lines: readonly EncodedLine[], active?: InFlight): Promise { + const dir = this.eventsDir(); + // 0700, and the batch below 0600. These files are not metadata: they carry + // goals, prompt text, tool arguments and tool output straight from the host + // agent. Under the ordinary umask 022 they would land 0644 inside 0755 + // directories, so on any shared host — a build box, a bastion, a container + // with several service accounts — every other local user could read every + // agent transcript this SDK spools, for the whole flush+upload window and + // forever if no daemon is running. The daemon reads these as the SAME user, + // so tightening them costs no delivery. + await mkdir(dir, { recursive: true, mode: 0o700 }); + + const stem = batchStem(); + const tmpPath = join(dir, `${stem}.tmp`); + const finalPath = join(dir, `${stem}.jsonl`); + const body = this.content(lines); + + try { + // O_EXCL + an explicit 0600. The mode is applied at CREATE, so the payload + // is never briefly world-readable the way a follow-up chmod would leave it. + const handle = await open( + tmpPath, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + 0o600, + ); + try { + await handle.write(body); + // fsync BEFORE the rename. A rename is atomic with respect to readers, + // but atomic is not durable: it orders nothing against the page cache, + // so a power loss or kernel crash can leave a correctly-named, + // zero-length or truncated `.jsonl`. The collector reads whatever is + // there, POSTs it, and then DELETES the file (`remove_file` in + // `crates/fpai-collect/src/uploader.rs`) — so the loss is permanent and + // silent, and an empty batch is accepted with a 200. This repo's own + // Rust spool writer calls `sync_all()` here for exactly this reason. + await handle.sync(); + } finally { + await handle.close(); + } + + // SYNCHRONOUS rename, deliberately, in an otherwise-async write. It is a + // metadata operation measured in microseconds, and doing it synchronously + // is what lets `done` be incremented in the same tick — so `flushSync` + // can never see a chunk as un-written after it has become durable, and + // never publishes a duplicate at exit. + renameSync(tmpPath, finalPath); + if (active) active.done += 1; + } catch (error) { + // Clean up the partial file on ANY failure. Each flush picks a fresh + // stem, so without this a persistent fault — a full disk, a read-only + // mount, a cross-device rename — strands one `.tmp` per flush cycle: + // roughly 170,000 files a day at the default interval, on the very disk + // that is already the problem. The watcher ignores them by extension, so + // nothing else would ever notice or collect them. + // + // The batch itself is NOT lost by this: the caller returns the lines to + // the retry buffer and the next cycle rewrites them under a new name. + rmSync(tmpPath, { force: true }); + throw error; + } + + this.syncDirectory(dir); + } + + private writeOneFileSync(lines: readonly EncodedLine[]): void { + const dir = this.eventsDir(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + + const stem = batchStem(); + const tmpPath = join(dir, `${stem}.tmp`); + const finalPath = join(dir, `${stem}.jsonl`); + const body = this.content(lines); + + try { + const fd = openSync( + tmpPath, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, + 0o600, + ); + try { + writeSync(fd, body); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(tmpPath, finalPath); + } catch (error) { + rmSync(tmpPath, { force: true }); + throw error; + } + + this.syncDirectory(dir); + } + + /** + * fsync the DIRECTORY, or the rename itself can be lost while the file's + * contents survive — leaving the batch on disk under its `.tmp` name, which + * the watcher ignores by design. + * + * Best-effort: opening a directory for fsync is POSIX behaviour, and + * platforms that refuse it (Windows) still get the content fsync above, which + * is the half that prevents a truncated delivery. + */ + private syncDirectory(dir: string): void { + let fd: number; + try { + fd = openSync(dir, fsConstants.O_RDONLY); + } catch { + return; + } + try { + fsyncSync(fd); + } catch { + /* platform dependent */ + } finally { + closeSync(fd); + } + } + + /** Stop the timer and drop this writer from the exit hook. Tests only. */ + close(): void { + this.closed = true; + if (this.timer !== null) clearInterval(this.timer); + this.timer = null; + liveWriters.delete(this); + releaseExitHook(); + } + + /** Diagnostics for tests and for anyone debugging a spool that is not draining. */ + stats(): { queued: number; queuedBytes: number; dropped: number; pending: number } { + return { + queued: this.queue.length, + queuedBytes: this.queuedBytes, + dropped: this.dropped, + pending: this.pending.length, + }; + } +} + +/** Final synchronous flush for every live writer. */ +export function flushAllSync(): void { + for (const writer of [...liveWriters]) { + try { + writer.flushSync(); + } catch (error) { + logException("final flush failed; buffered events were lost", error); + } + } +} + +/** Await a flush of every live writer. Exposed as `failproofai.flush()`. */ +export async function flushAllNow(): Promise { + await Promise.all( + [...liveWriters].map(async (writer) => { + try { + await writer.flushNow(); + } catch (error) { + logException("flush failed", error); + } + }), + ); +} + +/** + * Register the exit flush, once, and only once there is something to flush. + * + * `exit` is the last point at which anything in this process runs, and it runs + * on a normal return, on an unhandled rejection that terminates, and on an + * explicit `process.exit()`. It may not await, which is why `flushSync` exists. + * + * Registered LAZILY and released when the last writer goes, for two reasons. + * It keeps a process that merely imports this package from carrying a hook it + * will never use — and, more practically, it stops the listener count growing + * when this module is evaluated more than once in one process, which happens + * whenever both halves of the dual build are loaded (a CommonJS app with an ESM + * dependency that also uses this SDK) and which Node reports as a + * `MaxListenersExceededWarning` in the host's own output. + * + * Exceptions are swallowed inside `flushAllSync` on purpose: an uncaught one + * here prints a full stack into the host agent's stderr during shutdown, where + * it reads as a crash in the application rather than a telemetry flush that + * failed. + * + * SIGINT/SIGTERM are NOT installed. Registering a signal handler CHANGES the + * process's behaviour — Node's default is to terminate, and a listener + * suppresses that — so a telemetry library that installed one would silently + * stop Ctrl-C from working. The README documents the two-line handler instead. + */ +let exitHookInstalled = false; + +/** + * Close what the process is abandoning (`exit.ts`), THEN flush — so the + * `agent_end` / `tool_result` those closers emit are in the final batch rather + * than queued behind a flush that already ran. + */ +function onExit(exitCode: number): void { + runExitClosers(typeof exitCode === "number" ? exitCode : 0); + flushAllSync(); +} + +function ensureExitHook(): void { + if (exitHookInstalled) return; + exitHookInstalled = true; + process.on("exit", onExit); +} + +function releaseExitHook(): void { + if (!exitHookInstalled || liveWriters.size > 0) return; + exitHookInstalled = false; + process.removeListener("exit", onExit); +} diff --git a/sdk/typescript/test/adapters.test.ts b/sdk/typescript/test/adapters.test.ts new file mode 100644 index 000000000..0773edd98 --- /dev/null +++ b/sdk/typescript/test/adapters.test.ts @@ -0,0 +1,397 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import * as core from "../src/integrations/core.js"; +import { session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The adapters' translation tables, exercised against the shapes the real + * frameworks emit. + * + * These are the parts that can silently stop recording: a framework renames a + * callback, moves its token counts, or changes where the model id lives, and + * the adapter installs cleanly and produces nothing. Nothing else in the suite + * would notice. + */ + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); +}); +afterEach(async () => { + await spool.cleanup(); + setLogger(null); +}); + +describe("the Vercel AI SDK tracer", () => { + it("turns the SDK's own spans into an agent span, a model pair and a tool pair", async () => { + const { telemetry } = await import("../src/integrations/ai.js"); + const { tracer } = telemetry({ functionId: "answer-question" }); + + await session({ sessionId: "s1" }, async () => { + // Exactly the shape `generateText` produces: a root span, a doGenerate + // span inside it, and a toolCall span beside that. Each callback ends + // its own span, as the SDK's `recordSpan` does — OpenTelemetry's + // `startActiveSpan` never ends one for you. + await tracer.startActiveSpan( + "ai.generateText", + { attributes: { "ai.operationId": "ai.generateText", "ai.telemetry.functionId": "answer-question" } }, + async (root: { setAttribute: (k: string, v: unknown) => unknown; end: () => void }) => { + await tracer.startActiveSpan( + "ai.generateText.doGenerate", + { + attributes: { + "ai.operationId": "ai.generateText.doGenerate", + "ai.model.id": "gpt-4o", + "ai.model.provider": "openai", + "ai.prompt.messages": JSON.stringify([{ role: "user", content: "hi" }]), + }, + }, + (span: { setAttributes: (a: Record) => unknown; end: () => void }) => { + span.setAttributes({ + "ai.response.text": "hello", + "ai.response.finishReason": "stop", + "ai.usage.inputTokens": 11, + "ai.usage.outputTokens": 22, + }); + span.end(); + }, + ); + await tracer.startActiveSpan( + "ai.toolCall", + { + attributes: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": "get_weather", + "ai.toolCall.id": "call-1", + "ai.toolCall.args": JSON.stringify({ city: "Faro" }), + }, + }, + (span: { setAttribute: (k: string, v: unknown) => unknown; end: () => void }) => { + span.setAttribute("ai.toolCall.result", JSON.stringify({ celsius: 21 })); + span.end(); + }, + ); + root.setAttribute("ai.response.text", "hello"); + root.end(); + }, + ); + }); + + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual([ + "agent_start", + "model_request", + "model_response", + "tool_use", + "tool_result", + "agent_end", + ]); + + const request = events.find((event) => event.type === "model_request")!; + expect(request.model).toBe("gpt-4o"); + expect(request.messages).toEqual([{ role: "user", content: "hi" }]); + + const response = events.find((event) => event.type === "model_response")!; + expect(response.input_tokens).toBe(11); + expect(response.output_tokens).toBe(22); + expect(response.stop_reason).toBe("stop"); + // The pair is correlated, which is what lets the dashboard join them. + expect(response.request_id).toBe(request.request_id); + + const toolUse = events.find((event) => event.type === "tool_use")!; + expect(toolUse.tool_name).toBe("get_weather"); + expect(toolUse.tool_call_id).toBe("call-1"); + expect(toolUse.input).toEqual({ city: "Faro" }); + expect(events.find((event) => event.type === "tool_result")!.output).toEqual({ celsius: 21 }); + + // `functionId` names the agent span, and `agent_id` is a low-cardinality + // facet, so it must be the label and not the span id. + expect(events[0]!.agent_id).toBe("answer-question"); + expect(events.at(-1)!.outcome).toBe("success"); + }); + + it("reads v4's token attribute names as well as v5's", async () => { + const { tracer } = await import("../src/integrations/ai.js"); + const t = tracer(); + await session({ sessionId: "s1" }, () => { + t.startActiveSpan( + "ai.generateText.doGenerate", + { + attributes: { + "ai.operationId": "ai.generateText.doGenerate", + "ai.usage.promptTokens": 5, + "ai.usage.completionTokens": 7, + }, + }, + (span: { end: () => void }) => span.end(), + ); + }); + const response = (await flushed(spool)).find((event) => event.type === "model_response")!; + expect([response.input_tokens, response.output_tokens]).toEqual([5, 7]); + }); + + it("records a thrown error as a failed span", async () => { + const { tracer } = await import("../src/integrations/ai.js"); + const t = tracer(); + await session({ sessionId: "s1" }, () => { + expect(() => + t.startActiveSpan( + "ai.generateText", + { attributes: { "ai.operationId": "ai.generateText" } }, + (span: { recordException: (e: unknown) => void; setStatus: (s: { code: number }) => unknown; end: () => void }) => { + // What the SDK's `recordSpan` does with an error it catches. + const error = new Error("provider is down"); + span.recordException(error); + span.setStatus({ code: 2 }); + span.end(); + throw error; + }, + ), + ).toThrow("provider is down"); + }); + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual(["agent_start", "error", "agent_end"]); + expect(events.at(-1)!.outcome).toBe("failed"); + }); +}); + +describe("the AI SDK middleware", () => { + it("records a model call and its usage", async () => { + const { middleware } = await import("../src/integrations/ai.js"); + const mw = middleware() as { + wrapGenerate: (arg: { + doGenerate: () => Promise; + params: Record; + model?: { modelId?: string; provider?: string }; + }) => Promise; + }; + + await session({ sessionId: "s1" }, async () => { + await mw.wrapGenerate({ + params: { prompt: [{ role: "user", content: "hi" }] }, + model: { modelId: "claude-opus-5", provider: "anthropic" }, + doGenerate: async () => ({ + content: [{ type: "text", text: "hello" }], + finishReason: "stop", + usage: { inputTokens: 3, outputTokens: 4 }, + }), + }); + }); + + const events = await flushed(spool); + // A bare session has no agent to attribute the call to, so the call is a + // run of its own, named after the model (see the adapter's mapping). + expect(events.map((event) => event.type)).toEqual([ + "agent_start", + "model_request", + "model_response", + "agent_end", + ]); + expect(events[0]!.agent_id).toBe("claude-opus-5"); + expect(events[1]!.model).toBe("claude-opus-5"); + expect(events[2]!.input_tokens).toBe(3); + expect(events[2]!.request_id).toBe(events[1]!.request_id); + }); + + it("emits the response only when a STREAM finishes, so usage is not lost", async () => { + const { middleware } = await import("../src/integrations/ai.js"); + const mw = middleware() as { + wrapStream: (arg: { + doStream: () => Promise; + params: Record; + model?: { modelId?: string }; + }) => Promise<{ stream: ReadableStream }>; + }; + + const result = await session({ sessionId: "s1" }, async () => + mw.wrapStream({ + params: { prompt: [{ role: "user", content: "hi" }] }, + model: { modelId: "claude-opus-5" }, + doStream: async () => ({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: "text-delta", delta: "hel" }); + controller.enqueue({ type: "text-delta", delta: "lo" }); + controller.enqueue({ + type: "finish", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 2 }, + }); + controller.close(); + }, + }), + }), + }), + ); + + // Before the consumer reads it, only the request exists: a stream's usage + // and finish reason live in its FINAL part. + expect((await flushed(spool)).map((event) => event.type)).toEqual(["agent_start", "model_request"]); + + const chunks: unknown[] = []; + for await (const chunk of result.stream as unknown as AsyncIterable) chunks.push(chunk); + // Every chunk still reaches the caller untouched. + expect(chunks).toHaveLength(3); + + const events = await flushed(spool); + const response = events.find((event) => event.type === "model_response")!; + expect(response.content).toBe("hello"); + expect(response.output_tokens).toBe(2); + expect(response.stop_reason).toBe("stop"); + }); +}); + +describe("the LangChain adapter", () => { + /** The `CallbackManager` shape `@langchain/core` exposes, minus the rest of it. */ + function fakeLangChain() { + class CallbackManager { + handlers: unknown[] = []; + addHandler(handler: unknown): void { + this.handlers.push(handler); + } + static configure(): CallbackManager | undefined { + return undefined; + } + } + return { CallbackManager }; + } + + it("translates the handler surface into events", async () => { + // The adapter's `install()` imports `@langchain/core`, which is not a + // dependency here. Its HANDLER is the translation table, and that is what + // is worth testing — so drive it through the same tracker the adapter uses. + const tracker = new core.RunTracker("langchain", { + baseFields: core.frameworkFields("langchain"), + }); + + tracker.startAgent("chain-1", { agentId: "agent", sessionId: "s1", fw_run_type: "chain" }); + tracker.emit("modelRequest", "llm-1", { + parentKey: "chain-1", + model: "gpt-4o", + messages: [{ role: "user", content: "hi" }], + requestId: "llm-1", + }); + tracker.emit("modelResponse", "llm-1", { + parentKey: "chain-1", + inputTokens: 9, + outputTokens: 8, + content: "hello", + requestId: "llm-1", + }); + tracker.emit("toolUse", "tool-1", { + parentKey: "chain-1", + toolName: "search", + toolCallId: "tool-1", + input: { input: "kites" }, + }); + tracker.emit("toolResult", "tool-1", { + parentKey: "chain-1", + toolName: "search", + toolCallId: "tool-1", + output: "results", + }); + tracker.endAgent("chain-1", { outcome: "success" }); + + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual([ + "agent_start", + "model_request", + "model_response", + "tool_use", + "tool_result", + "agent_end", + ]); + // Every event carries the framework label, which is how a mixed process is + // read back apart. + expect(events.every((event) => event.framework === "langchain")).toBe(true); + expect(events.every((event) => event.session_id === "s1")).toBe(true); + }); + + it("attaches itself to a manager `configure` built, idempotently", () => { + const { CallbackManager } = fakeLangChain(); + const handler = { name: "failproofai" }; + const manager = new CallbackManager(); + // The adapter's `attach` is idempotent by identity: LangChain calls + // `configure` per invocation and a child manager may inherit the parent's + // handler list, so a blind add would emit each event once per attachment. + const attach = (target: InstanceType): void => { + if (target.handlers.includes(handler)) return; + target.addHandler(handler); + }; + attach(manager); + attach(manager); + expect(manager.handlers).toHaveLength(1); + }); + + it("hands out a working handler without instrument()", async () => { + // The patch-free path: `callbacks: [langchainHandler()]` with no + // `instrument()` call anywhere. It used to throw here; see + // test/langchain.test.ts for what it records. + const { adapter, langchainHandler } = await import("../src/integrations/langchain.js"); + try { + expect(typeof langchainHandler().handleChainStart).toBe("function"); + } finally { + adapter.uninstall(); + } + }); +}); + +describe("the Mastra helpers", () => { + it("wraps a tool's execute without mutating the original", async () => { + const { wrapTool } = await import("../src/integrations/mastra.js"); + const original = vi.fn(async () => ({ celsius: 21 })); + const tool = { id: "get_weather", execute: original }; + const wrapped = wrapTool(tool); + + // A Mastra tool may be frozen, so wrapping returns a new object rather + // than assigning into theirs. + expect(wrapped).not.toBe(tool); + expect(tool.execute).toBe(original); + + await session({ sessionId: "s1" }, async () => { + await (wrapped.execute as (context: unknown) => Promise)({ + toolCallId: "call-1", + context: { city: "Faro" }, + }); + }); + + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual(["tool_use", "tool_result"]); + expect(events[0]!.tool_name).toBe("get_weather"); + expect(events[0]!.input).toEqual({ city: "Faro" }); + expect(events[1]!.output).toEqual({ celsius: 21 }); + }); + + it("records a tool failure on the leaf and re-throws", async () => { + const { wrapTool } = await import("../src/integrations/mastra.js"); + const wrapped = wrapTool({ + id: "flaky", + execute: async () => { + throw new RangeError("out of range"); + }, + }); + await session({ sessionId: "s1" }, async () => { + await expect( + (wrapped.execute as (context: unknown) => Promise)({ toolCallId: "c1" }), + ).rejects.toThrow("out of range"); + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect(result.error).toBe("RangeError: out of range"); + }); + + it("brackets a workflow run with an agent span", async () => { + const { workflow } = await import("../src/integrations/mastra.js"); + await session({ sessionId: "s1" }, async () => { + await workflow("nightly-report", async () => undefined); + }); + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual(["agent_start", "agent_end"]); + expect(events[0]!.agent_id).toBe("nightly-report"); + expect(events[0]!.fw_kind).toBe("workflow"); + }); +}); diff --git a/sdk/typescript/test/ai.test.ts b/sdk/typescript/test/ai.test.ts new file mode 100644 index 000000000..2e81962fa --- /dev/null +++ b/sdk/typescript/test/ai.test.ts @@ -0,0 +1,1076 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import * as core from "../src/integrations/core.js"; +import { + FailproofSpan, + _internals, + adapter, + integration, + middleware, + responseContent, + stopReasonOf, + toolCallsOf, + tracer, + usageTokens, +} from "../src/integrations/ai.js"; +import { setLogger } from "../src/logger.js"; +import { runtime } from "../src/runtime.js"; +import { agent, session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The Vercel AI SDK adapter's logic, against the shapes each `ai` major hands + * it. The real-framework proof is `integration/ai.test.ts`; this file pins the + * translation rules it depends on, without installing four copies of `ai`. + */ + +let spool: Spool; +const warnings: string[] = []; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); + warnings.length = 0; + setLogger({ + debug: () => undefined, + info: () => undefined, + warn: (message: string) => warnings.push(message), + error: (message: string) => warnings.push(message), + }); +}); +afterEach(async () => { + adapter.uninstall(); + await spool.cleanup(); + setLogger(null); +}); + +const types = (events: Array>) => events.map((e) => e.type); +const count = (events: Array>, type: string) => events.filter((e) => e.type === type).length; + +interface Span { + setAttributes: (a: Record) => unknown; + recordException: (e: unknown) => void; + setStatus: (s: { code: number; message?: string }) => unknown; + end: () => void; +} + +describe("reading every major's spelling", () => { + it("reads tokens from v4, v5, v6+ and v7-normalised usage", () => { + expect(usageTokens({ promptTokens: 5, completionTokens: 7 })).toEqual({ inputTokens: 5, outputTokens: 7 }); + expect(usageTokens({ inputTokens: 11, outputTokens: 7, totalTokens: 18 })).toEqual({ inputTokens: 11, outputTokens: 7 }); + expect( + usageTokens({ inputTokens: { total: 23, noCache: 23 }, outputTokens: { total: 9, text: 9 } }), + ).toEqual({ inputTokens: 23, outputTokens: 9 }); + expect(usageTokens({ tokens: 4 })).toEqual({ inputTokens: 4, outputTokens: undefined }); + expect(usageTokens(undefined)).toEqual({}); + // A count that is not a count is dropped, never shipped as NaN. + expect(usageTokens({ inputTokens: { total: undefined }, outputTokens: "x" })).toEqual({ + inputTokens: undefined, + outputTokens: undefined, + }); + }); + + it("reduces every finish-reason shape to a string", () => { + expect(stopReasonOf("stop")).toBe("stop"); + expect(stopReasonOf({ unified: "tool-calls", raw: "tool_use" })).toBe("tool-calls"); + expect(stopReasonOf(["length"])).toBe("length"); + expect(stopReasonOf("")).toBeUndefined(); + expect(stopReasonOf({})).toBeUndefined(); + }); + + it("uses the tool calls as content when a step produced no text", () => { + const call = { type: "tool-call", toolCallId: "c1", toolName: "weather", input: '{"city":"Paris"}' }; + expect(responseContent({ content: [call] })).toEqual([ + { toolCallId: "c1", toolName: "weather", input: { city: "Paris" } }, + ]); + expect(responseContent({ content: [{ type: "text", text: "hi" }, call] })).toBe("hi"); + // v4: `text` is "" on a tool-call step — "" is not content. + expect( + responseContent({ text: "", toolCalls: [{ toolCallId: "c1", toolName: "weather", args: "{}" }] }), + ).toEqual([{ toolCallId: "c1", toolName: "weather", input: {} }]); + expect(responseContent({ text: "" })).toBeUndefined(); + }); +}); + +describe("the tracer (ai v4–v6)", () => { + it("never ends a span for the caller: a stream span stays open until the SDK ends it", async () => { + const t = tracer(); + let stream: Span | undefined; + await session({ sessionId: "s1" }, async () => { + // `streamText` opens its root span with endWhenDone: false and returns + // before a single token has been produced. + await t.startActiveSpan( + "ai.streamText", + { attributes: { "ai.operationId": "ai.streamText", "ai.telemetry.functionId": "streamer" } }, + async (span: Span) => { + stream = span; + }, + ); + }); + expect(types(await flushed(spool))).toEqual(["agent_start"]); + stream!.end(); + expect(types(await flushed(spool))).toEqual(["agent_start", "agent_end"]); + }); + + it("stamps the model request when the step starts, before the tool it causes", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => + t.startActiveSpan("ai.streamText", { attributes: { "ai.operationId": "ai.streamText" } }, async (root: Span) => { + let step: Span | undefined; + t.startActiveSpan( + "ai.streamText.doStream", + { attributes: { "ai.operationId": "ai.streamText.doStream", "ai.model.id": "m" } }, + (span: Span) => { + step = span; + }, + ); + // The tool runs while the model's stream is still open. + t.startActiveSpan( + "ai.toolCall", + { attributes: { "ai.operationId": "ai.toolCall", "ai.toolCall.name": "weather", "ai.toolCall.id": "c1" } }, + (span: Span) => span.end(), + ); + step!.setAttributes({ + "ai.response.finishReason": "tool-calls", + "ai.response.text": "", + "ai.response.toolCalls": JSON.stringify([{ toolCallId: "c1", toolName: "weather", input: "{}" }]), + }); + step!.end(); + root.end(); + }), + ); + const events = await flushed(spool); + expect(types(events)).toEqual([ + "agent_start", + "model_request", + "tool_use", + "tool_result", + "model_response", + "agent_end", + ]); + expect(events.every((e) => e.agent_id === "ai.streamText")).toBe(true); + const response = events.find((e) => e.type === "model_response")!; + expect(response.stop_reason).toBe("tool-calls"); + // The call's input is parsed, as on every other path — not a JSON string in JSON. + expect(response.content).toEqual([{ toolCallId: "c1", toolName: "weather", input: {} }]); + }); + + it("records a failed model step once, on its model_response", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, async () => { + await expect( + t.startActiveSpan("ai.generateText", { attributes: { "ai.operationId": "ai.generateText" } }, async (root: Span) => { + try { + await t.startActiveSpan( + "ai.generateText.doGenerate", + { attributes: { "ai.operationId": "ai.generateText.doGenerate", "ai.model.id": "m" } }, + async (span: Span) => { + const error = new Error("model exploded"); + span.recordException(error); + span.setStatus({ code: 2, message: error.message }); + span.end(); + throw error; + }, + ); + } catch (error) { + root.recordException(error); + root.setStatus({ code: 2 }); + root.end(); + throw error; + } + }), + ).rejects.toThrow("model exploded"); + }); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + const response = events.find((e) => e.type === "model_response")!; + expect(response.error).toBe("Error: model exploded"); + expect(response.stop_reason).toBe("error"); + expect(events.at(-1)!.outcome).toBe("failed"); + }); + + it("nests an operation under an enclosing agent() scope", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => + agent("planner", () => + t.startActiveSpan("ai.generateText", { attributes: { "ai.operationId": "ai.generateText" } }, (span: Span) => + span.end(), + ), + ), + ); + const events = await flushed(spool); + const start = events.find((e) => e.type === "agent_start" && e.agent_id === "ai.generateText")!; + expect(start.parent_id).toBe("planner"); + expect(start.session_id).toBe("s1"); + }); +}); + +describe("the telemetry integration (ai v7)", () => { + const usage = (i: number, o: number) => ({ inputTokens: i, outputTokens: o, totalTokens: i + o }); + + it("turns v7's lifecycle callbacks into one agent, model pairs and tool pairs", async () => { + await session({ sessionId: "s1" }, async () => { + const callId = "call-abc"; + const base = { callId, functionId: "weather-agent", provider: "p", modelId: "m" }; + integration.onStart({ ...base, operationId: "ai.generateText" }); + integration.onLanguageModelCallStart({ ...base, messages: [{ role: "user", content: "hi" }] }); + await integration.executeLanguageModelCall({ callId, execute: async () => "ok" }); + integration.onLanguageModelCallEnd({ + ...base, + finishReason: "tool-calls", + usage: usage(11, 7), + content: [{ type: "tool-call", toolCallId: "tc1", toolName: "weather", input: { city: "Paris" } }], + performance: { responseTimeMs: 12.6 }, + }); + const toolCall = { type: "tool-call", toolCallId: "tc1", toolName: "weather", input: { city: "Paris" } }; + integration.onToolExecutionStart({ ...base, toolCall }); + integration.onToolExecutionEnd({ ...base, toolCall, toolOutput: { type: "tool-result", output: { c: 20 } } }); + integration.onLanguageModelCallStart({ ...base, messages: [] }); + integration.onLanguageModelCallEnd({ ...base, finishReason: "stop", usage: usage(23, 9), content: [{ type: "text", text: "sunny" }] }); + integration.onEnd({ ...base, finishReason: "stop" }); + }); + const events = await flushed(spool); + expect(types(events)).toEqual([ + "agent_start", + "model_request", + "model_response", + "tool_use", + "tool_result", + "model_request", + "model_response", + "agent_end", + ]); + expect(events.every((e) => e.agent_id === "weather-agent" && e.session_id === "s1")).toBe(true); + const responses = events.filter((e) => e.type === "model_response"); + expect(responses.map((e) => [e.input_tokens, e.output_tokens, e.stop_reason])).toEqual([ + [11, 7, "tool-calls"], + [23, 9, "stop"], + ]); + expect(responses[0]!.duration_ms).toBe(13); + const requests = events.filter((e) => e.type === "model_request"); + expect(responses.map((e) => e.request_id)).toEqual(requests.map((e) => e.request_id)); + expect(events.find((e) => e.type === "tool_use")!.tool_call_id).toBe("tc1"); + expect(events.find((e) => e.type === "tool_result")!.output).toEqual({ c: 20 }); + expect(events.at(-1)!.outcome).toBe("success"); + }); + + it("ignores ai v6's integration events, which carry no callId (v6 records through the tracer)", async () => { + await session({ sessionId: "s1" }, () => { + integration.onStart({ functionId: "x", model: { modelId: "m" } }); + integration.onToolExecutionStart({ toolCall: { toolCallId: "t", toolName: "w" } }); + integration.onEnd({ finishReason: "stop" }); + }); + expect(await flushed(spool)).toEqual([]); + }); + + it("records a failing provider call once, on its model_response", async () => { + await session({ sessionId: "s1" }, async () => { + const base = { callId: "c2", functionId: "f", modelId: "m" }; + integration.onStart({ ...base, operationId: "ai.generateText" }); + integration.onLanguageModelCallStart(base); + await expect( + integration.executeLanguageModelCall({ + callId: "c2", + execute: async () => { + throw new Error("model exploded"); + }, + }), + ).rejects.toThrow("model exploded"); + integration.onError({ callId: "c2", error: new Error("model exploded") }); + }); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[2]!.error).toBe("Error: model exploded"); + expect(events[2]!.stop_reason).toBe("error"); + expect(events[3]!.outcome).toBe("failed"); + }); + + it("emits a single error when the operation itself failed and no leaf carried it", async () => { + await session({ sessionId: "s1" }, () => { + integration.onStart({ callId: "c3", operationId: "ai.streamObject", functionId: "extract" }); + integration.onEnd({ callId: "c3", error: new Error("schema mismatch") }); + }); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "error", "agent_end"]); + expect(events[2]!.outcome).toBe("failed"); + }); + + it("records a failing tool on its tool_result", async () => { + await session({ sessionId: "s1" }, () => { + const toolCall = { toolCallId: "t1", toolName: "weather", input: {} }; + integration.onStart({ callId: "c4", operationId: "ai.generateText", functionId: "f" }); + integration.onToolExecutionStart({ callId: "c4", toolCall }); + integration.onToolExecutionEnd({ callId: "c4", toolCall, toolOutput: { type: "tool-error", error: new Error("down") } }); + integration.onEnd({ callId: "c4" }); + }); + const events = await flushed(spool); + expect(events.find((e) => e.type === "tool_result")!.error).toBe("Error: down"); + expect(types(events)).not.toContain("error"); + }); + + it("nests an operation started inside a tool under the operation that called it", async () => { + await session({ sessionId: "s1" }, async () => { + integration.onStart({ callId: "outer", operationId: "ai.generateText", functionId: "supervisor" }); + await integration.executeTool({ + callId: "outer", + execute: async () => { + integration.onStart({ callId: "inner", operationId: "ai.generateText", functionId: "researcher" }); + integration.onEnd({ callId: "inner" }); + }, + }); + integration.onEnd({ callId: "outer" }); + }); + const events = await flushed(spool); + const inner = events.find((e) => e.type === "agent_start" && e.agent_id === "researcher")!; + expect(inner.parent_id).toBe("supervisor"); + }); + + it("is registered process-wide by instrument() exactly once, and removed by uninstrument()", async () => { + const g = globalThis as { AI_SDK_TELEMETRY_INTEGRATIONS?: unknown[] }; + const before = g.AI_SDK_TELEMETRY_INTEGRATIONS; + try { + g.AI_SDK_TELEMETRY_INTEGRATIONS = [{ someone: "else" }]; + await adapter.install({}); + await adapter.install({}); + expect(g.AI_SDK_TELEMETRY_INTEGRATIONS.filter((i) => i === integration)).toHaveLength(1); + adapter.uninstall(); + expect(g.AI_SDK_TELEMETRY_INTEGRATIONS).toEqual([{ someone: "else" }]); + } finally { + g.AI_SDK_TELEMETRY_INTEGRATIONS = before; + } + }); +}); + +describe("the middleware", () => { + const generate = (result: Record) => ({ + params: { prompt: [{ role: "user", content: "hi" }] }, + model: { modelId: "gpt-x", provider: "p" }, + doGenerate: async () => result, + }); + + it("makes a call outside any scope its own run, in a session of its own", async () => { + await middleware().wrapGenerate(generate({ content: [{ type: "text", text: "hi" }], finishReason: "stop" })); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events.every((e) => e.agent_id === "gpt-x")).toBe(true); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + expect(warnings).toEqual([]); + }); + + it("records a call inside agent() as a step of that agent", async () => { + await session({ sessionId: "s1" }, () => + agent("planner", () => + middleware().wrapGenerate( + generate({ + content: [{ type: "text", text: "ok" }], + finishReason: { unified: "stop", raw: "end_turn" }, + usage: { inputTokens: { total: 3 }, outputTokens: { total: 4 } }, + }), + ), + ), + ); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events.every((e) => e.agent_id === "planner")).toBe(true); + const response = events[2]!; + expect([response.input_tokens, response.output_tokens, response.stop_reason]).toEqual([3, 4, "stop"]); + }); + + it("records a failing model on its model_response, and rethrows", async () => { + await expect( + middleware().wrapGenerate({ + params: {}, + model: { modelId: "gpt-x" }, + doGenerate: async () => { + throw new Error("rate limited"); + }, + }), + ).rejects.toThrow("rate limited"); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[2]!.error).toBe("Error: rate limited"); + expect(events[3]!.outcome).toBe("failed"); + }); + + it("defers to the tracer inside a model span, so each call is recorded once", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => + t.startActiveSpan( + "ai.generateText.doGenerate", + { attributes: { "ai.operationId": "ai.generateText.doGenerate" } }, + async (span: Span) => { + await middleware().wrapGenerate(generate({ text: "x" })); + span.end(); + }, + ), + ); + const events = await flushed(spool); + expect(types(events)).toEqual(["model_request", "model_response"]); + }); + + it("defers to the v7 integration inside executeLanguageModelCall", async () => { + await integration.executeLanguageModelCall({ + execute: () => middleware().wrapGenerate(generate({ text: "x" })), + }); + expect(await flushed(spool)).toEqual([]); + }); + + it("reads a v4 stream's textDelta parts", async () => { + const result = await middleware().wrapStream({ + params: {}, + model: { modelId: "gpt-x" }, + doStream: async () => ({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: "text-delta", textDelta: "he" }); + controller.enqueue({ type: "text-delta", textDelta: "y" }); + controller.enqueue({ type: "finish", finishReason: "stop", usage: { promptTokens: 1, completionTokens: 2 } }); + controller.close(); + }, + }), + }), + }); + for await (const _ of result.stream as unknown as AsyncIterable) void _; + const response = (await flushed(spool)).find((e) => e.type === "model_response")!; + expect([response.content, response.input_tokens, response.output_tokens, response.stop_reason]).toEqual([ + "hey", + 1, + 2, + "stop", + ]); + }); +}); + +describe("the middleware on a stream that does not finish", () => { + const endless = (onCancel: (reason: unknown) => void) => + new ReadableStream({ + pull(controller) { + controller.enqueue({ type: "text-delta", delta: "x" }); + }, + cancel(reason) { + onCancel(reason); + }, + }); + + it("closes a streamed call the consumer cancels, as cancelled, and cancels the provider stream", async () => { + let sourceCancelled: unknown; + const result = await middleware().wrapStream({ + params: {}, + model: { modelId: "gpt-x" }, + doStream: async () => ({ stream: endless((reason) => (sourceCancelled = reason)) }), + }); + const reader = (result.stream as ReadableStream).getReader(); + await reader.read(); + await reader.read(); + await reader.cancel("client disconnected"); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + const response = events[2]!; + expect(response.stop_reason).toBe("cancelled"); + expect(response.error).toBeUndefined(); + expect(response.content).toMatch(/^x+$/); + expect(response.request_id).toBe(events[1]!.request_id); + expect(events[3]!.outcome).toBe("cancelled"); + expect(sourceCancelled).toBe("client disconnected"); + expect(count(events, "error")).toBe(0); + }); + + it("closes a streamed call whose stream errors, with the error, and ends its run failed", async () => { + const result = await middleware().wrapStream({ + params: {}, + model: { modelId: "gpt-x" }, + doStream: async () => ({ + stream: new ReadableStream({ + async pull(controller) { + controller.enqueue({ type: "text-delta", delta: "y" }); + await Promise.resolve(); + controller.error(new Error("ECONNRESET")); + }, + }), + }), + }); + const reader = (result.stream as ReadableStream).getReader(); + await expect( + (async () => { + while (!(await reader.read()).done) { + /* drain */ + } + })(), + ).rejects.toThrow("ECONNRESET"); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[2]!.stop_reason).toBe("error"); + expect(events[2]!.error).toBe("Error: ECONNRESET"); + expect(events[3]!.outcome).toBe("failed"); + // Recorded once, on the model_response — not again as an `error` event. + expect(count(events, "error")).toBe(0); + }); + + it("closes a cancelled call inside agent() without ending the enclosing agent", async () => { + await session({ sessionId: "s1" }, () => + agent("planner", async () => { + const result = await middleware().wrapStream({ + params: {}, + model: { modelId: "gpt-x" }, + doStream: async () => ({ stream: endless(() => undefined) }), + }); + const reader = (result.stream as ReadableStream).getReader(); + await reader.read(); + await reader.cancel(); + }), + ); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events.every((e) => e.agent_id === "planner")).toBe(true); + expect(events[2]!.stop_reason).toBe("cancelled"); + // The only agent_end is the planner's own, from its scope. + expect(events[3]!.outcome).toBe("success"); + }); +}); + +describe("instrument('ai') and the process-wide OpenTelemetry slot", () => { + /** + * A throwaway application with `ai` at `version` and a stand-in for + * `@opentelemetry/api` that keeps the real one's global-registration rule: + * the first provider wins and every later `setGlobalTracerProvider` returns + * false. (The real API against real `ai` releases is proven in + * `integration/ai.test.ts`.) + */ + const apps: string[] = []; + const makeApp = (version: string): string => { + const app = mkdtempSync(join(tmpdir(), "failproofai-ai-otel-")); + apps.push(app); + const write = (path: string, text: string): void => { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, text); + }; + write(join(app, "node_modules", "ai", "package.json"), JSON.stringify({ name: "ai", version, main: "index.js" })); + write(join(app, "node_modules", "ai", "index.js"), "module.exports = {};\n"); + write( + join(app, "node_modules", "@opentelemetry", "api", "package.json"), + JSON.stringify({ name: "@opentelemetry/api", version: "1.9.0", main: "index.js" }), + ); + write( + join(app, "node_modules", "@opentelemetry", "api", "index.js"), + [ + "let delegate = null;", + "class NoopTracerProvider { getTracer() { return {}; } }", + "class ProxyTracerProvider { getDelegate() { return delegate ?? new NoopTracerProvider(); } }", + "const proxy = new ProxyTracerProvider();", + "module.exports = {", + " ProxyTracerProvider,", + " trace: {", + " setGlobalTracerProvider(p) { if (delegate) return false; delegate = p; return true; },", + " getTracerProvider() { return proxy; },", + " disable() { delegate = null; },", + " },", + "};", + ].join("\n"), + ); + return app; + }; + interface FakeOtel { + trace: { + setGlobalTracerProvider(p: unknown): boolean; + getTracerProvider(): { getDelegate(): unknown }; + }; + } + const otelOf = (app: string): FakeOtel => createRequire(join(app, "main.js"))("@opentelemetry/api") as FakeOtel; + const inApp = async (app: string, fn: () => unknown): Promise => { + const cwd = process.cwd(); + process.chdir(app); + try { + await fn(); + } finally { + process.chdir(cwd); + } + }; + afterEach(() => { + for (const app of apps.splice(0)) rmSync(app, { recursive: true, force: true }); + }); + + it.each(["4.3.19", "5.0.0", "6.0.288"])( + "on ai %s does not take the global slot by default, so the customer's own provider still registers", + async (version) => { + const app = makeApp(version); + await inApp(app, () => adapter.install({})); + const theirs = { getTracer: () => ({ theirs: true }) }; + expect(otelOf(app).trace.setGlobalTracerProvider(theirs)).toBe(true); + expect(otelOf(app).trace.getTracerProvider().getDelegate()).toBe(theirs); + // …and says, once, what instrument("ai") does and does not cover here. + await inApp(app, () => adapter.install({})); + const advice = warnings.filter((w) => w.includes("registerGlobalTracer")); + expect(advice).toHaveLength(1); + expect(advice[0]).toContain("telemetry()"); + // …naming an import that exists. The root package has no `ai` export, so + // `failproofai.ai.telemetry()` sent the reader to `undefined`. + expect(advice[0]).toContain('from "@failproofai/sdk/ai"'); + expect(advice[0]).not.toContain("failproofai.ai."); + }, + ); + + it("registers the global tracer on ai 4–6 only when asked to", async () => { + const app = makeApp("6.0.288"); + await inApp(app, () => adapter.install({ registerGlobalTracer: true })); + const delegate = otelOf(app).trace.getTracerProvider().getDelegate() as { getTracer?: () => unknown }; + expect((delegate.getTracer?.() as object | undefined)?.constructor.name).toBe("FailproofTracer"); + expect(warnings.filter((w) => w.includes("registerGlobalTracer"))).toEqual([]); + // uninstrument() gives the slot back. + adapter.uninstall(); + expect(otelOf(app).trace.setGlobalTracerProvider({ getTracer: () => ({}) })).toBe(true); + }); + + it("stays quiet on ai 4–6 when told registerGlobalTracer: false", async () => { + const app = makeApp("5.0.0"); + await inApp(app, () => adapter.install({ registerGlobalTracer: false })); + expect(warnings).toEqual([]); + expect(otelOf(app).trace.setGlobalTracerProvider({ getTracer: () => ({}) })).toBe(true); + }); + + it("on ai 7 neither touches OpenTelemetry nor warns", async () => { + const app = makeApp("7.0.111"); + await inApp(app, () => adapter.install({})); + expect(warnings).toEqual([]); + expect(otelOf(app).trace.setGlobalTracerProvider({ getTracer: () => ({}) })).toBe(true); + }); +}); + +describe("bookkeeping", () => { + interface TrackerMaps { + links: Map; + runs: Map; + } + const maps = (): TrackerMaps => _internals.tracker() as unknown as TrackerMaps; + + it("leaves no residue after 20k completed operations, and never evicts a live run's links", async () => { + const captured: Array> = []; + const original = runtime.event; + runtime.event = new Proxy( + {}, + { + get: (_, method) => (options: Record) => + captured.push({ method: String(method), ...options }), + }, + ) as typeof runtime.event; + try { + const t = tracer(); + const mw = middleware(); + + // A run that stays open throughout: a root operation with an + // intermediate span under it, whose child only arrives at the very end. + let liveRoot: FailproofSpan | undefined; + let liveStep: FailproofSpan | undefined; + await session({ sessionId: "live" }, () => + t.startActiveSpan( + "ai.generateText", + { attributes: { "ai.operationId": "ai.generateText", "ai.telemetry.functionId": "live-agent" } }, + (root) => { + liveRoot = root; + t.startActiveSpan("ai.step", (step) => { + liveStep = step; + }); + }, + ), + ); + + const stream = (mode: "finish" | "cancel" | "error") => async () => ({ + stream: new ReadableStream({ + pull(controller) { + controller.enqueue({ type: "text-delta", delta: "x" }); + if (mode === "finish") { + controller.enqueue({ type: "finish", finishReason: "stop" }); + controller.close(); + } else if (mode === "error") { + controller.error(new Error("reset")); + } + }, + }), + }); + const drain = async (s: unknown, cancel: boolean): Promise => { + const reader = (s as ReadableStream).getReader(); + try { + if (cancel) { + await reader.read(); + await reader.cancel(); + return; + } + while (!(await reader.read()).done) { + /* drain */ + } + } catch { + // the errored stream + } + }; + + const N = 20_000; + for (let i = 0; i < N; i += 1) { + // ai v4–v6: the tracer, every span kind. + t.startActiveSpan("ai.generateText", { attributes: { "ai.operationId": "ai.generateText" } }, (root) => { + t.startActiveSpan( + "ai.generateText.doGenerate", + { attributes: { "ai.operationId": "ai.generateText.doGenerate" } }, + (span) => span.end(), + ); + t.startActiveSpan( + "ai.toolCall", + { attributes: { "ai.operationId": "ai.toolCall", "ai.toolCall.id": `t${String(i)}` } }, + (span) => span.end(), + ); + t.startActiveSpan("ai.other", (span) => span.end()); + if (i % 2 === 0) root.recordException(new Error("x")); + root.end(); + }); + // ai v7: the integration — success, error and abort paths. + const callId = `c${String(i)}`; + const toolCall = { toolCallId: `tc${String(i)}`, toolName: "w", input: {} }; + integration.onStart({ callId, operationId: "ai.generateText", functionId: "f" }); + integration.onLanguageModelCallStart({ callId }); + integration.onLanguageModelCallEnd({ callId, finishReason: "tool-calls" }); + integration.onToolExecutionStart({ callId, toolCall }); + if (i % 3 === 0) { + integration.onToolExecutionEnd({ callId, toolCall, toolOutput: { type: "tool-result", output: 1 } }); + integration.onEnd({ callId }); + } else if (i % 3 === 1) { + integration.onLanguageModelCallStart({ callId }); + integration.onError({ callId, error: new Error("boom") }); + } else { + integration.onAbort({ callId }); + } + // The middleware: generate, and a stream that finishes, is cancelled, or errors. + await mw.wrapGenerate({ params: {}, model: { modelId: "m" }, doGenerate: async () => ({ text: "x" }) }); + const mode = (["finish", "cancel", "error"] as const)[i % 3]!; + const result = await mw.wrapStream({ params: {}, model: { modelId: "m" }, doStream: stream(mode) }); + await drain(result.stream, mode === "cancel"); + } + + // Only the live run is left: its agent, and its intermediate span's link. + expect(maps().runs.size).toBe(1); + expect(maps().links.size).toBe(1); + expect(_internals.openCalls()).toBe(0); + + // That link survived 20k runs through the table: the live run's late + // child still resolves to it. + captured.length = 0; + const step = new FailproofSpan("ai.generateText.doGenerate", liveStep, { + "ai.operationId": "ai.generateText.doGenerate", + }); + step.end(); + liveStep!.end(); + liveRoot!.end(); + expect(captured.map((e) => [e.method, e.agentId, e.sessionId])).toEqual([ + ["modelRequest", "live-agent", "live"], + ["modelResponse", "live-agent", "live"], + ["agentEnd", "live-agent", "live"], + ]); + expect(maps().runs.size).toBe(0); + expect(maps().links.size).toBe(0); + } finally { + runtime.event = original; + } + }, 120_000); +}); + +describe("tool calls in a model_response", () => { + it("reads every major's ai.response.toolCalls into one shape, input parsed", () => { + // v4: toolCallType + args as a JSON string. + expect(toolCallsOf(JSON.stringify([{ toolCallType: "function", toolCallId: "a", toolName: "w", args: '{"city":"Paris"}' }]))).toEqual([ + { toolCallId: "a", toolName: "w", input: { city: "Paris" } }, + ]); + // v5/v6 generateText: input as a JSON string. + expect(toolCallsOf(JSON.stringify([{ toolCallId: "b", toolName: "w", input: '{"city":"Rome"}' }]))).toEqual([ + { toolCallId: "b", toolName: "w", input: { city: "Rome" } }, + ]); + // v5/v6 streamText: a typed part with input already an object. + expect(toolCallsOf(JSON.stringify([{ type: "tool-call", toolCallId: "c", toolName: "w", input: { city: "Oslo" } }]))).toEqual([ + { toolCallId: "c", toolName: "w", input: { city: "Oslo" } }, + ]); + expect(toolCallsOf(undefined)).toBeUndefined(); + expect(toolCallsOf("[]")).toBeUndefined(); + expect(toolCallsOf("not json")).toBeUndefined(); + }); +}); + +describe("the tracer when a stream does not finish (ai v4–v6)", () => { + type Tracer = ReturnType; + const streamRoot = (t: Tracer, functionId: string, body: (root: Span) => void): void => { + t.startActiveSpan( + "ai.streamText", + { attributes: { "ai.operationId": "ai.streamText", "ai.telemetry.functionId": functionId } }, + (root: Span) => body(root), + ); + }; + const openStep = (t: Tracer): Span => { + let step: Span | undefined; + t.startActiveSpan( + "ai.streamText.doStream", + { attributes: { "ai.operationId": "ai.streamText.doStream", "ai.model.id": "m" } }, + (span: Span) => { + step = span; + }, + ); + return step!; + }; + + it("closes the model call an aborted stream left open, and ends the agent cancelled", async () => { + const t = tracer(); + let step: Span | undefined; + await session({ sessionId: "s1" }, () => { + streamRoot(t, "counter", (root) => { + step = openStep(t); + // v5/v6 on abort: the root ends from the result stream's flush; the + // model step's span is never ended. + root.end(); + }); + }); + // A late end of the abandoned step is not news. + step!.setAttributes({ "ai.response.finishReason": "stop" }); + step!.end(); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + const response = events.find((e) => e.type === "model_response")!; + expect(response.stop_reason).toBe("cancelled"); + expect(response.model).toBe("m"); + expect(response.request_id).toBe(events.find((e) => e.type === "model_request")!.request_id); + expect(events.at(-1)!.outcome).toBe("cancelled"); + expect(count(events, "error")).toBe(0); + expect(_internals.tracker()!.stats()).toEqual({ runs: 0, links: 0 }); + }); + + it("closes a tool a cut-off operation left running, as cancelled", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => { + streamRoot(t, "agent", (root) => { + t.startActiveSpan( + "ai.toolCall", + { attributes: { "ai.operationId": "ai.toolCall", "ai.toolCall.name": "weather", "ai.toolCall.id": "tc1" } }, + () => undefined, + ); + root.end(); + }); + }); + const events = await flushed(spool); + const result = events.find((e) => e.type === "tool_result")!; + expect(result.tool_call_id).toBe("tc1"); + expect(result.error).toMatch(/cancelled/); + expect(events.at(-1)!.outcome).toBe("cancelled"); + }); + + it("ends a streamText that finished no step as cancelled, and a completed one as success", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => { + streamRoot(t, "aborted-between-steps", (root) => root.end()); + streamRoot(t, "finished", (root) => { + root.setAttributes({ "ai.response.finishReason": "stop" }); + root.end(); + }); + t.startActiveSpan("ai.generateText", { attributes: { "ai.operationId": "ai.generateText" } }, (root: Span) => root.end()); + }); + const ends = (await flushed(spool)).filter((e) => e.type === "agent_end"); + expect(ends.map((e) => [e.agent_id, e.outcome])).toEqual([ + ["aborted-between-steps", "cancelled"], + ["finished", "success"], + ["ai.generateText", "success"], + ]); + }); + + it("ends an operation nothing will ever end — its root span collected — as cancelled and abandoned", async () => { + const { setFlagsFromString } = await import("node:v8"); + const { runInNewContext } = await import("node:vm"); + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + + // A client that disconnected: the SDK's result stream never flushes, so + // neither span is ended, and every reference to them is then dropped. + const start = (): void => { + const t = tracer(); + streamRoot(t, "disconnected", () => { + openStep(t); + }); + }; + await session({ sessionId: "s1" }, () => { + start(); + }); + for (let i = 0; i < 20 && _internals.tracker()!.stats().runs > 0; i += 1) { + gc(); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[2]!.stop_reason).toBe("cancelled"); + expect(events.at(-1)!.outcome).toBe("cancelled"); + expect(events.at(-1)!.fw_abandoned).toBe(true); + expect(events.every((e) => e.session_id === "s1" && e.agent_id === "disconnected")).toBe(true); + expect(_internals.tracker()!.stats()).toEqual({ runs: 0, links: 0 }); + }); +}); + +describe("embeddings", () => { + type Tracer = ReturnType; + const embedSpans = (t: Tracer, operation = "ai.embed", functionId?: string): void => { + t.startActiveSpan( + operation, + { attributes: { "ai.operationId": operation, ...(functionId ? { "ai.telemetry.functionId": functionId } : {}) } }, + (root: Span) => { + t.startActiveSpan( + `${operation}.doEmbed`, + { attributes: { "ai.operationId": `${operation}.doEmbed`, "ai.model.id": "embedder" } }, + (span: Span) => { + span.setAttributes({ "ai.usage.tokens": 3 }); + span.end(); + }, + ); + root.end(); + }, + ); + }; + const lines = (events: Array>) => events.map((e) => `${String(e.agent_id)} ${String(e.type)}`); + + it("tracer: a bare embed() is its own run, named by functionId", async () => { + embedSpans(tracer(), "ai.embed", "indexer"); + const events = await flushed(spool); + expect(lines(events)).toEqual([ + "indexer agent_start", + "indexer model_request", + "indexer model_response", + "indexer agent_end", + ]); + expect(events[2]!.input_tokens).toBe(3); + }); + + it("tracer: an embed() inside agent() is a model call of that agent, not a nested agent", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => + agent("rag", () => { + embedSpans(t); + embedSpans(t, "ai.embedMany"); + }), + ); + const events = await flushed(spool); + expect(lines(events)).toEqual([ + "rag agent_start", + "rag model_request", + "rag model_response", + "rag model_request", + "rag model_response", + "rag agent_end", + ]); + expect(_internals.tracker()!.stats()).toEqual({ runs: 0, links: 0 }); + }); + + it("tracer: an embed() inside a tool is a model call of the operation that ran the tool", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => { + t.startActiveSpan( + "ai.generateText", + { attributes: { "ai.operationId": "ai.generateText", "ai.telemetry.functionId": "weather-agent" } }, + (root: Span) => { + t.startActiveSpan( + "ai.toolCall", + { attributes: { "ai.operationId": "ai.toolCall", "ai.toolCall.name": "lookup", "ai.toolCall.id": "tc1" } }, + (tool: Span) => { + embedSpans(t); + tool.end(); + }, + ); + root.end(); + }, + ); + }); + const events = await flushed(spool); + expect(lines(events)).toEqual([ + "weather-agent agent_start", + "weather-agent tool_use", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent tool_result", + "weather-agent agent_end", + ]); + expect(events.at(-1)!.outcome).toBe("success"); + }); + + it("tracer: a failing enclosed embed() reports its error under the enclosing agent", async () => { + const t = tracer(); + await session({ sessionId: "s1" }, () => + agent("rag", () => { + t.startActiveSpan("ai.embed", { attributes: { "ai.operationId": "ai.embed" } }, (root: Span) => { + root.recordException(new Error("too many values")); + root.end(); + }); + }), + ); + expect(lines(await flushed(spool))).toEqual(["rag agent_start", "rag error", "rag agent_end"]); + }); + + it("v7: bare, inside agent(), and inside a tool — the same three answers", async () => { + const embed = (callId: string, functionId?: string): void => { + integration.onStart({ callId, operationId: "ai.embed", ...(functionId ? { functionId } : {}) }); + integration.onEmbedStart({ callId, modelId: "embedder", values: ["a"] }); + integration.onEmbedEnd({ callId, modelId: "embedder", usage: { tokens: 3 } }); + integration.onEnd({ callId }); + }; + embed("bare", "indexer"); + await session({ sessionId: "s1" }, async () => { + await agent("rag", () => { + embed("scoped"); + }); + integration.onStart({ callId: "outer", operationId: "ai.generateText", functionId: "weather-agent" }); + await integration.executeTool({ + callId: "outer", + execute: async () => { + embed("in-tool"); + }, + }); + integration.onEnd({ callId: "outer" }); + }); + const events = await flushed(spool); + expect(lines(events)).toEqual([ + "indexer agent_start", + "indexer model_request", + "indexer model_response", + "indexer agent_end", + "rag agent_start", + "rag model_request", + "rag model_response", + "rag agent_end", + "weather-agent agent_start", + "weather-agent model_request", + "weather-agent model_response", + "weather-agent agent_end", + ]); + expect(events.filter((e) => e.type === "model_response").map((e) => e.input_tokens)).toEqual([3, 3, 3]); + expect(_internals.openCalls()).toBe(0); + expect(_internals.tracker()!.stats()).toEqual({ runs: 0, links: 0 }); + }); +}); + +describe("the telemetry integration on abort and failure (ai v7)", () => { + it("closes the interrupted model call as cancelled, keeping its model", async () => { + await session({ sessionId: "s1" }, () => { + integration.onStart({ callId: "a1", operationId: "ai.streamText", functionId: "counter" }); + integration.onLanguageModelCallStart({ callId: "a1", modelId: "m" }); + integration.onAbort({ callId: "a1" }); + }); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[2]!.stop_reason).toBe("cancelled"); + expect(events[2]!.model).toBe("m"); + expect(events[3]!.outcome).toBe("cancelled"); + }); + + it("keeps the model on a call closed by a failure", async () => { + await session({ sessionId: "s1" }, () => { + integration.onStart({ callId: "e1", operationId: "ai.generateText", functionId: "f" }); + integration.onLanguageModelCallStart({ callId: "e1", modelId: "m" }); + integration.onError({ callId: "e1", error: new Error("boom") }); + }); + const response = (await flushed(spool)).find((e) => e.type === "model_response")!; + expect(response.stop_reason).toBe("error"); + expect(response.model).toBe("m"); + }); +}); diff --git a/sdk/typescript/test/copies.test.ts b/sdk/typescript/test/copies.test.ts new file mode 100644 index 000000000..fcb9bb5da --- /dev/null +++ b/sdk/typescript/test/copies.test.ts @@ -0,0 +1,204 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { resolveEsm } from "../src/node-require.js"; + +/** + * Which copy of a dual-published framework an adapter patches. + * + * The first release resolved every framework with `createRequire`, which can + * only name the CommonJS build — so in an ES-module application, the default + * for a new TypeScript project, `instrument()` patched a copy nothing used and + * recorded nothing. These tests build a fake dual package on disk and run + * `requireModuleCopies` from REAL entry points of each module system, because + * "which module system is the entry" is a property of the process and cannot be + * faked from inside a test runner. + */ + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +let app: string; + +function write(path: string, text: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text); +} + +/** A copy that says which build it is and records that it was loaded. */ +const copy = (marker: string, esm: boolean): string => + esm + ? `globalThis.__loaded = [...(globalThis.__loaded ?? []), "${marker}"];\nexport const marker = "${marker}";\nexport class Thing {}\n` + : `globalThis.__loaded = [...(globalThis.__loaded ?? []), "${marker}"];\nexports.marker = "${marker}";\nexports.Thing = class Thing {};\n`; + +beforeAll(() => { + app = mkdtempSync(join(tmpdir(), "failproofai-copies-")); + const modules = join(app, "node_modules"); + + // A dual package with a root entry, a subpath and a pattern, conditions + // nested the way real frameworks publish them (`@langchain/core` does + // exactly `{ import: { types, default }, require: { types, default } }`). + write( + join(modules, "dualfw", "package.json"), + JSON.stringify({ + name: "dualfw", + version: "1.2.3", + exports: { + ".": { import: { types: "./x.d.ts", default: "./esm/index.js" }, require: "./cjs/index.cjs" }, + "./callbacks/manager": { import: "./esm/manager.js", require: "./cjs/manager.cjs" }, + "./tools/*": { node: { import: "./esm/tools/*.js", require: "./cjs/tools/*.cjs" } }, + }, + }), + ); + write(join(modules, "dualfw", "esm", "index.js"), copy("esm", true)); + write(join(modules, "dualfw", "cjs", "index.cjs"), copy("cjs", false)); + write(join(modules, "dualfw", "esm", "manager.js"), copy("esm-manager", true)); + write(join(modules, "dualfw", "cjs", "manager.cjs"), copy("cjs-manager", false)); + write(join(modules, "dualfw", "esm", "tools", "search.js"), copy("esm-search", true)); + write(join(modules, "dualfw", "cjs", "tools", "search.cjs"), copy("cjs-search", false)); + + // ESM-only: no `require` condition, so CommonJS resolution cannot find it. + write( + join(modules, "@scope", "esmonly", "package.json"), + JSON.stringify({ name: "@scope/esmonly", exports: { ".": { import: "./index.js" } } }), + ); + write(join(modules, "@scope", "esmonly", "index.js"), copy("esmonly", true)); + + // No exports map: Node reads `main` for both, so there is one copy. + write(join(modules, "plainfw", "package.json"), JSON.stringify({ name: "plainfw", main: "main.js" })); + write(join(modules, "plainfw", "main.js"), copy("plain", false)); +}); + +afterAll(() => { + rmSync(app, { recursive: true, force: true }); +}); + +describe("resolveEsm", () => { + const inApp = (fn: () => T): T => { + const cwd = process.cwd(); + process.chdir(app); + try { + return fn(); + } finally { + process.chdir(cwd); + } + }; + + it("names the import build of a dual package, root and subpath", () => { + inApp(() => { + expect(resolveEsm("dualfw")).toBe(join(app, "node_modules", "dualfw", "esm", "index.js")); + expect(resolveEsm("dualfw/callbacks/manager")).toBe( + join(app, "node_modules", "dualfw", "esm", "manager.js"), + ); + }); + }); + + it("expands a subpath pattern through nested conditions", () => { + inApp(() => { + expect(resolveEsm("dualfw/tools/search")).toBe( + join(app, "node_modules", "dualfw", "esm", "tools", "search.js"), + ); + }); + }); + + it("finds an ESM-only package that CommonJS resolution cannot", () => { + inApp(() => { + expect(resolveEsm("@scope/esmonly")).toBe(join(app, "node_modules", "@scope", "esmonly", "index.js")); + }); + }); + + it("returns null where there is no separate ESM copy or no such module", () => { + inApp(() => { + expect(resolveEsm("plainfw")).toBeNull(); + expect(resolveEsm("dualfw/not-exported")).toBeNull(); + expect(resolveEsm("not-installed-anywhere")).toBeNull(); + }); + }); +}); + +describe("requireModuleCopies", () => { + /** Run `body` in a fresh process whose ENTRY is an ES module or CommonJS. */ + const run = (entry: "esm" | "cjs", body: string, where: { dir?: string; cwd?: string } = {}): string[] => { + const compat = + entry === "esm" + ? pathToFileURL(join(root, "dist", "esm", "integrations", "compat.js")).href + : join(root, "dist", "cjs", "integrations", "compat.js"); + const file = join(where.dir ?? app, entry === "esm" ? "main.mjs" : "main.cjs"); + const header = + entry === "esm" + ? `import { createRequire } from "node:module";\nimport * as compat from ${JSON.stringify(compat)};\nconst require = createRequire(import.meta.url);\n` + : `const compat = require(${JSON.stringify(compat)});\n`; + write( + file, + `${header}(async () => {\n${body}\nconsole.log(JSON.stringify({ result, loaded: globalThis.__loaded ?? [] }));\n})().catch((e) => { console.log(JSON.stringify({ error: String(e.message) })); });\n`, + ); + const child = spawnSync(process.execPath, [file], { cwd: where.cwd ?? app, encoding: "utf8" }); + expect(child.stderr).toBe(""); + const parsed = JSON.parse(child.stdout.trim()) as { result?: string[]; loaded?: string[]; error?: string }; + if (parsed.error !== undefined) return [`error: ${parsed.error}`]; + return [...parsed.result!, "|", ...parsed.loaded!]; + }; + + const markers = 'const result = (await compat.requireModuleCopies(SPEC, "npm i x")).map((m) => m.marker);'; + + it("patches the ES-module copy for an ES-module app, and loads nothing else", () => { + expect(run("esm", markers.replace("SPEC", '"dualfw"'))).toEqual(["esm", "|", "esm"]); + }); + + it("also patches the CommonJS copy when something already required it", () => { + const body = `require("dualfw");\n${markers.replace("SPEC", '"dualfw"')}`; + expect(run("esm", body)).toEqual(["esm", "cjs", "|", "cjs", "esm"]); + }); + + it("patches the CommonJS copy for a CommonJS app, without loading the ESM build", () => { + expect(run("cjs", markers.replace("SPEC", '"dualfw"'))).toEqual(["cjs", "|", "cjs"]); + expect(run("cjs", markers.replace("SPEC", '"dualfw/callbacks/manager"'))).toEqual([ + "cjs-manager", + "|", + "cjs-manager", + ]); + }); + + it("uses the one copy an ESM-only or exports-less package has", () => { + expect(run("esm", markers.replace("SPEC", '"@scope/esmonly"'))).toEqual(["esmonly", "|", "esmonly"]); + expect(run("cjs", markers.replace("SPEC", '"@scope/esmonly"'))).toEqual(["esmonly", "|", "esmonly"]); + expect(run("esm", markers.replace("SPEC", '"plainfw"'))).toEqual(["plain", "|", "plain"]); + }); + + it("finds the framework from the entry script when the service runs from /", () => { + // systemd with no WorkingDirectory=, a container with WORKDIR unset. + expect(run("esm", markers.replace("SPEC", '"dualfw"'), { cwd: "/" })).toEqual(["esm", "|", "esm"]); + expect(run("cjs", markers.replace("SPEC", '"dualfw"'), { cwd: "/" })).toEqual(["cjs", "|", "cjs"]); + }); + + it("patches the app's own nested copy, not the one a monorepo root hoists", () => { + // root/node_modules/dualfw is the hoisted copy; root/apps/web has its own. + const web = join(app, "apps", "web"); + const nested = join(web, "node_modules", "dualfw"); + write( + join(nested, "package.json"), + JSON.stringify({ name: "dualfw", exports: { ".": { import: "./esm.js", require: "./cjs.cjs" } } }), + ); + write(join(nested, "esm.js"), copy("nested-esm", true)); + write(join(nested, "cjs.cjs"), copy("nested-cjs", false)); + // Started from the monorepo root, as a root-level script runner does. + expect(run("esm", markers.replace("SPEC", '"dualfw"'), { dir: web, cwd: app })).toEqual([ + "nested-esm", + "|", + "nested-esm", + ]); + expect(run("cjs", markers.replace("SPEC", '"dualfw"'), { dir: web, cwd: app })).toEqual([ + "nested-cjs", + "|", + "nested-cjs", + ]); + }); + + it("throws with the install command when the framework is absent", () => { + const [line] = run("esm", markers.replace("SPEC", '"not-installed-anywhere"')); + expect(line).toMatch(/not importable\. Install it with: {2}npm i x/); + }); +}); diff --git a/sdk/typescript/test/edge.test.ts b/sdk/typescript/test/edge.test.ts new file mode 100644 index 000000000..cbb27b4ec --- /dev/null +++ b/sdk/typescript/test/edge.test.ts @@ -0,0 +1,183 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import * as edge from "../src/edge/index.js"; +import { EDGE_NOTICE, resetNotice } from "../src/edge/notice.js"; + +/** + * The no-op build an Edge / Worker runtime or a browser bundle gets through the + * `edge-light` / `workerd` / `worker` / `browser` export conditions. + * + * Found by `integration/nextjs.test.ts`: a Next.js route with + * `export const runtime = "edge"` importing the SDK failed `next build` + * outright — "Native module not found: node:fs" — because the real entry + * statically imports Node builtins and an ES module cannot catch that. + */ + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const edgeDir = join(root, "src", "edge"); + +afterEach(() => { + resetNotice(); + vi.restoreAllMocks(); +}); + +describe("the edge build's imports", () => { + it("imports no Node builtin and nothing from the Node build but VERSION and types", () => { + for (const file of readdirSync(edgeDir).filter((name) => name.endsWith(".ts"))) { + const source = readFileSync(join(edgeDir, file), "utf8"); + const imports = [...source.matchAll(/^import\s+(type\s+)?[^;]*?from\s+"([^"]+)";/gms)].map((m) => ({ + typeOnly: m[1] !== undefined, + from: m[2]!, + })); + for (const { typeOnly, from } of imports) { + if (typeOnly) continue; + const allowed = from.startsWith("./") || from === "../version.js"; + expect(allowed, `${file} imports ${from} at run time`).toBe(true); + } + expect(source, file).not.toMatch(/\bprocess\./); + expect(source, file).not.toMatch(/require\(/); + } + }); +}); + +/** + * Exports of the Node build that are internals (test helpers, parsing + * functions) rather than API; an edge module need not mirror them. Anything + * NOT listed here must exist in the edge build, or an Edge import of it is a + * link error — the crash this build exists to prevent. + */ +const INTERNAL: Record = { + index: [], + ai: ["toolCallsOf", "_internals", "usageTokens", "stopReasonOf", "responseContent"], + langchain: [ + "ABANDONED_ROOT_GRACE_MS", + "PAUSED_SESSION_TTL_MS", + "_stats", + "captureLimitOf", + "interruptIdOf", + "isCancellation", + "isControlFlow", + "nodeOf", + "normalizeMessages", + "resetOrphanWarning", + "promptOf", + "readOptions", + "summarizeDocuments", + "toolOutput", + "usageOf", + ], + mastra: ["_internals"], + llamaindex: ["BELOW_VERSION", "MIN_VERSION", "eventCallerStorage", "parseOptions", "summarizeNodes", "usageOf"], +}; + +const PAIRS: Array<[string, () => Promise>, () => Promise>]> = [ + ["index", () => import("../src/index.js"), () => import("../src/edge/index.js")], + ["ai", () => import("../src/integrations/ai.js"), () => import("../src/edge/ai.js")], + ["langchain", () => import("../src/integrations/langchain.js"), () => import("../src/edge/langchain.js")], + ["mastra", () => import("../src/integrations/mastra.js"), () => import("../src/edge/mastra.js")], + ["llamaindex", () => import("../src/integrations/llamaindex.js"), () => import("../src/edge/llamaindex.js")], +]; + +describe.each(PAIRS)("the edge %s module", (name, loadReal, loadEdge) => { + it("exports every public name the Node build does, as the same kind of value", async () => { + const real = await loadReal(); + const noop = await loadEdge(); + const missing: string[] = []; + for (const key of Object.keys(real)) { + if (INTERNAL[name]!.includes(key)) continue; + if (!(key in noop)) missing.push(key); + else expect(typeof noop[key], `${name}.${key}`).toBe(typeof real[key]); + } + expect(missing).toEqual([]); + }); + + it("is what every edge export condition selects", () => { + const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { + exports: Record>; + }; + const entry = manifest.exports[name === "index" ? "." : `./${name}`]!; + const conditions = Object.keys(entry); + // Ahead of import/require: a bundler takes the FIRST condition it matches. + expect(conditions.slice(0, 4)).toEqual(["edge-light", "workerd", "worker", "browser"]); + for (const condition of conditions.slice(0, 4)) { + expect(entry[condition]).toEqual({ + import: { types: expect.any(String), default: `./dist/esm/edge/${name}.js` }, + require: { types: expect.any(String), default: `./dist/cjs/edge/${name}.js` }, + }); + // The REAL declarations, so code that compiles for Node compiles here. + const real = name === "index" ? "index" : `integrations/${name}`; + expect((entry[condition] as { import: { types: string } }).import.types).toBe(`./dist/esm/${real}.d.ts`); + } + }); +}); + +describe("the edge build's behaviour", () => { + it("runs scope bodies and returns their values, recording nothing", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const out = await edge.session({ sessionId: "s-1" }, (sessionId) => + edge.agent("planner", { goal: "g" }, (identity) => + edge.toolCall("search", { toolCallId: "call-1", input: { q: 1 } }, (call) => { + call.output = 42; + return { sessionId, agent: identity.agentId, call: call.id, assigned: call.outputAssigned }; + }), + ), + ); + expect(out).toEqual({ sessionId: "s-1", agent: "planner", call: "call-1", assigned: true }); + expect(edge.session(() => "no options")).toBe("no options"); + expect(await edge.agent("a", async () => 7)).toBe(7); + + const scope = edge.agent.open("planner"); + expect(scope.agentId).toBe("planner"); + scope.dispose(); + edge.session.open().dispose(); + edge.toolCall.open("t").dispose(); + + for (const method of Object.keys(edge.event)) { + expect(() => (edge.event as unknown as Record void>)[method]!({})).not.toThrow(); + } + expect(Object.keys(edge.event)).toHaveLength(15); + await expect(edge.flush()).resolves.toBeUndefined(); + expect(await edge.instrument()).toEqual([]); + expect(await (edge.instrument as (name: string) => Promise)("langchain")).toEqual([]); + expect(edge.uninstrument()).toEqual([]); + expect(edge.current().sessionId).toBeNull(); + expect(edge.AUTO).toBe(Symbol.for("failproofai.AUTO")); + + // Said once, on first use — and it says nothing is recorded. + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]![0]).toBe(`[failproofai-sdk] ${EDGE_NOTICE}`); + expect(EDGE_NOTICE).toContain("NOTHING is recorded"); + }); + + it("is silent when merely imported", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await import("../src/edge/ai.js"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("hands the frameworks values they accept and that record nothing", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const ai = await import("../src/edge/ai.js"); + expect(ai.telemetry({ functionId: "f" })).toEqual({ isEnabled: false, functionId: "f" }); + const model = { modelId: "m" }; + expect(await ai.wrapModel(model)).toBe(model); + const mw = ai.middleware(); + expect(await mw.wrapGenerate({ doGenerate: async () => "generated" })).toBe("generated"); + expect(await mw.wrapStream({ doStream: async () => "streamed" })).toBe("streamed"); + const tool = { execute: () => 1 }; + expect(ai.wrapTool("t", tool)).toBe(tool); + expect(ai.tracer().startActiveSpan("x", (span: { isRecording(): boolean }) => span.isRecording())).toBe(false); + + const langchain = await import("../src/edge/langchain.js"); + expect(typeof langchain.langchainHandler()).toBe("object"); + const mastra = await import("../src/edge/mastra.js"); + expect(mastra.workflow("w", () => "ran")).toBe("ran"); + expect(mastra.wrapTool(tool)).toBe(tool); + const llamaindex = await import("../src/edge/llamaindex.js"); + expect(typeof llamaindex.attach()).toBe("function"); + }); +}); diff --git a/sdk/typescript/test/evaluator-client.test.ts b/sdk/typescript/test/evaluator-client.test.ts new file mode 100644 index 000000000..16f86c48e --- /dev/null +++ b/sdk/typescript/test/evaluator-client.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it, vi } from "vitest"; + +import { EvaluatorAPIError, EvaluatorClient } from "../src/evaluator/client.js"; +import { PROTOCOL_VERSION } from "../src/evaluator/protocol.js"; + +/** + * Four properties in here are security or correctness, not style: + * redirects are refused, every URL is pinned to the configured origin, + * responses are bounded WHILE they are read, and `claim` is never retried. + */ + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function makeClient(options: Partial[0]> = {}) { + const fetchImpl = vi.fn(async () => + jsonResponse({ + protocol_version: PROTOCOL_VERSION, + evaluator_instance_id: "i1", + evaluator_kind: "customer", + heartbeat_interval_seconds: 30, + lease_duration_seconds: 120, + poll_interval_seconds: 10, + claim_limit: 4, + disabled_definitions: [], + }), + ); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "token", + fetchImpl: fetchImpl, + sleep: async () => undefined, + ...options, + }); + return { client, fetchImpl }; +} + +describe("construction", () => { + it("refuses plaintext http to a non-loopback host", () => { + expect(() => new EvaluatorClient({ baseUrl: "http://api.example.test", credential: "t" })).toThrow( + /must use https unless it targets loopback/, + ); + }); + + it("allows http to loopback, where there is no network to eavesdrop", () => { + for (const host of ["http://localhost:8020", "http://127.0.0.1:8020", "http://[::1]:8020"]) { + expect(() => new EvaluatorClient({ baseUrl: host, credential: "t" })).not.toThrow(); + } + }); + + it("refuses a credential that would let a header be injected", () => { + expect(() => + new EvaluatorClient({ baseUrl: "https://a.test", credential: "tok\r\nX-Evil: 1" }), + ).toThrow(/control characters/); + expect(() => new EvaluatorClient({ baseUrl: "https://a.test", credential: " " })).toThrow( + /must not be empty/, + ); + }); + + it("refuses a non-absolute base url", () => { + expect(() => new EvaluatorClient({ baseUrl: "/v1", credential: "t" })).toThrow( + /absolute http\(s\) URL/, + ); + }); +}); + +describe("requests", () => { + it("sends the bearer credential and refuses redirects", async () => { + const { client, fetchImpl } = makeClient(); + await client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }); + const [, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; + expect((init.headers as Record).Authorization).toBe("Bearer token"); + // Following a redirect would carry the Authorization header to wherever it + // points, which turns a misconfigured server into credential exfiltration. + expect(init.redirect).toBe("error"); + }); + + it("refuses a server-supplied URL outside the configured origin", async () => { + const { client } = makeClient(); + await expect( + client.transcript( + { + assignmentId: "a1", + leaseGeneration: 1, + leaseExpiresAt: "", + sessionId: "s", + sessionRevisionId: "r", + agentId: "main", + environment: "dev", + triggerReason: "x", + eventCount: 0, + transcriptUrl: "https://attacker.test/transcript", + definitionsUrl: "", + }, + "w1", + ), + ).rejects.toThrow(/outside the configured API origin/); + }); + + it("retries a retryable status and stops at the cap", async () => { + const fetchImpl = vi.fn(async () => jsonResponse({}, 503)); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + maxRetries: 2, + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }), + ).rejects.toThrow(EvaluatorAPIError); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it("does not retry a non-retryable status", async () => { + const fetchImpl = vi.fn(async () => + jsonResponse( + { + protocol_version: PROTOCOL_VERSION, + error: { code: "invalid_credentials", message: "nope", retryable: false, request_id: "r" }, + }, + 401, + ), + ); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }), + ).rejects.toMatchObject({ code: "invalid_credentials", retryable: false }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("NEVER retries a claim, because a lost response may already have leased work", async () => { + const fetchImpl = vi.fn(async () => jsonResponse({}, 503)); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + maxRetries: 5, + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.claim({ workerId: "w1", catalogRevision: "sha256:0", capacity: 1 }), + ).rejects.toThrow(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("reports a non-JSON body as an invalid response rather than crashing", async () => { + const fetchImpl = vi.fn(async () => new Response("gateway", { status: 200 })); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }), + ).rejects.toMatchObject({ code: "invalid_response" }); + }); + + it("stops reading a body past the limit instead of buffering it whole", async () => { + const huge = "x".repeat(4 * 1024 * 1024); + const fetchImpl = vi.fn(async () => jsonResponse({ padding: huge })); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }), + ).rejects.toMatchObject({ code: "response_too_large" }); + }); + + it("classifies a transport failure as retryable", async () => { + const fetchImpl = vi.fn(async () => { + throw new Error("ECONNREFUSED"); + }); + const client = new EvaluatorClient({ + baseUrl: "https://api.example.test", + credential: "t", + maxRetries: 0, + fetchImpl: fetchImpl, + sleep: async () => undefined, + }); + await expect( + client.register({ + workerId: "w1", + sdkVersion: "0", + catalogRevision: "sha256:0", + maxConcurrency: 1, + definitions: [], + }), + ).rejects.toMatchObject({ code: "transport_error", retryable: true }); + }); +}); diff --git a/sdk/typescript/test/evaluator-protocol.test.ts b/sdk/typescript/test/evaluator-protocol.test.ts new file mode 100644 index 000000000..882537a2b --- /dev/null +++ b/sdk/typescript/test/evaluator-protocol.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "vitest"; + +import { + Assertion, + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, +} from "../src/evaluator/authoring.js"; +import { + ExecutionMode, + PROTOCOL_VERSION, + ProtocolError, + ResultKind, + UnsupportedProtocolVersion, + assignmentDefinitionFromWire, + assignmentFromWire, + claimResponseFromWire, + planResponseFromWire, + registerResponseFromWire, + sessionTranscriptFromWire, +} from "../src/evaluator/protocol.js"; +import { transcriptWire } from "./helpers.js"; + +/** + * Every `fromWire` is a VALIDATOR, not a cast. The server is across a network + * boundary: a field that arrives as a number where a string was promised has to + * fail here, naming the field — not three frames later as an undefined property + * nobody can explain. + */ + +describe("protocol version", () => { + it("refuses a version it does not implement", () => { + expect(() => + registerResponseFromWire({ protocol_version: "3", evaluator_instance_id: "x" }), + ).toThrow(UnsupportedProtocolVersion); + }); + + it("accepts its own", () => { + expect(PROTOCOL_VERSION).toBe("2"); + }); +}); + +describe("wire validation", () => { + const assignment = { + assignment_id: "a1", + lease_generation: 1, + lease_expires_at: "2026-01-01T00:02:00Z", + session_id: "s1", + session_revision_id: "r1", + agent_id: "main", + environment: "dev", + trigger_reason: "session_end", + event_count: 3, + transcript_url: "/v1/evaluator/assignments/a1/transcript", + }; + + it("reads a well-formed assignment", () => { + expect(assignmentFromWire(assignment)).toMatchObject({ assignmentId: "a1", eventCount: 3 }); + }); + + it("names the offending field", () => { + expect(() => assignmentFromWire({ ...assignment, session_id: 7 })).toThrow( + /session_id must be a string/, + ); + expect(() => assignmentFromWire({ ...assignment, lease_generation: 0 })).toThrow( + /lease_generation must be greater than zero/, + ); + expect(() => assignmentFromWire({ ...assignment, event_count: -1 })).toThrow( + /event_count must not be negative/, + ); + }); + + it("requires execution_mode explicitly rather than defaulting to local", () => { + // Coercing a missing value to `local` silently runs a server-authored + // definition down the customer-local path, or the reverse. + const definition = { + eval_key: "k", + display_name: "K", + eval_version: "1", + result_kind: "score", + labels: [], + }; + expect(() => assignmentDefinitionFromWire(definition)).toThrow(ProtocolError); + expect( + assignmentDefinitionFromWire({ ...definition, execution_mode: "local" }).executionMode, + ).toBe(ExecutionMode.LOCAL); + // `python` is the wire spelling of "the server authored this, sandbox it". + expect( + assignmentDefinitionFromWire({ ...definition, execution_mode: "python" }).executionMode, + ).toBe(ExecutionMode.SANDBOX); + }); + + it("refuses a transcript whose event_count disagrees with its events", () => { + const wire = transcriptWire([{ type: "tool_use" }]); + wire.event_count = 5; + expect(() => sessionTranscriptFromWire(wire)).toThrow(/event_count is 5/); + }); + + it("refuses a transcript schema version it does not implement", () => { + const wire = transcriptWire(); + wire.schema_version = "9"; + expect(() => sessionTranscriptFromWire(wire)).toThrow(/unsupported transcript schema version/); + }); + + it("round-trips a transcript through toWire", () => { + const wire = transcriptWire([{ type: "tool_use", payload: { a: 1 } }]); + expect(sessionTranscriptFromWire(sessionTranscriptFromWire(wire).toWire()).events).toHaveLength(1); + }); + + it("refuses a non-boolean idempotent_replay", () => { + expect(() => + planResponseFromWire({ + protocol_version: "2", + assignment_id: "a1", + assignment_status: "planned", + runs: [], + idempotent_replay: "yes", + }), + ).toThrow(/idempotent_replay must be a boolean/); + }); + + it("refuses a non-array assignments list", () => { + expect(() => claimResponseFromWire({ protocol_version: "2", assignments: {} })).toThrow( + /assignments must be an array/, + ); + }); +}); + +describe("transcript helpers", () => { + const session = sessionTranscriptFromWire( + transcriptWire([{ type: "tool_use" }, { type: "tool_result" }, { type: "tool_result" }]), + ); + + it("filters and counts by event type", () => { + expect(session.count("tool_result")).toBe(2); + expect(session.eventsOfType("tool_use")).toHaveLength(1); + expect(session.count("absent")).toBe(0); + }); +}); + +describe("authoring bounds", () => { + it("keeps a score inside 0..1", () => { + expect(() => new Score(1.5)).toThrow(/between 0 and 1/); + expect(() => new Score(Number.NaN)).toThrow(/must be finite/); + expect(new Score(0.5).unit).toBe("ratio"); + }); + + it("refuses control characters the server would 422 on", () => { + // A non-retryable 422 loses a SUCCESSFUL evaluation and dead-letters its + // assignment, so this has to fail at the line that wrote it. + expect(() => new EvalResult({ reasoning: "line\u0000break" })).toThrow( + /must not contain control characters/, + ); + // Real multi-line reasoning is fine. + expect(() => new EvalResult({ reasoning: "line\nbreak\ttab" })).not.toThrow(); + }); + + it("normalizes and de-duplicates labels", () => { + expect(new EvalResult({ score: new Score(1), labels: ["b", "a"] }).labels).toEqual(["a", "b"]); + expect(() => new EvalResult({ labels: ["a", "a"] })).toThrow(/must be unique/); + }); + + it("requires at least one result", () => { + expect(() => new EvalResult({}).resultItems("k")).toThrow(/must contain a score, metric/); + }); + + it("attaches the eval's reasoning to the PRIMARY result of a non-score eval", () => { + const items = new EvalResult({ + metrics: { latency: 12, other: 3 }, + reasoning: "because", + }).resultItems("latency"); + expect(items.find((item) => item.resultKey === "latency")!.reasoning).toBe("because"); + expect(items.find((item) => item.resultKey === "other")!.reasoning).toBeNull(); + }); + + it("coerces a bare number or boolean into a Metric or an Assertion", () => { + const items = new EvalResult({ metrics: { a: 1 }, assertions: { b: true } }).resultItems("a"); + expect(items.map((item) => item.resultKind)).toEqual([ResultKind.METRIC, ResultKind.ASSERTION]); + expect(new Metric(1).unit).toBe(""); + expect(new Assertion(true).passed).toBe(true); + }); + + it("validates a condition's reason code as a key", () => { + expect(() => new ConditionResult(false, "Not A Key")).toThrow(/must match/); + expect(new ConditionResult(false).reasonCode).toBe("condition_false"); + }); +}); + +describe("Evaluator registry", () => { + it("validates the eval key shape", () => { + const app = new Evaluator({ name: "n", version: "1" }); + expect(() => app.eval("Bad-Key", { version: "1" }, () => new EvalResult())).toThrow(/must match/); + }); + + it("refuses a duplicate key", () => { + const app = new Evaluator({ name: "n", version: "1" }); + app.eval("a", { version: "1" }, () => new EvalResult()); + expect(() => app.eval("a", { version: "2" }, () => new EvalResult())).toThrow(/duplicate/); + }); + + it("derives a display name from the key", () => { + const app = new Evaluator({ name: "n", version: "1" }); + app.eval("tool_success_rate", { version: "1" }, () => new EvalResult()); + expect(app.definition("tool_success_rate").displayName).toBe("Tool success rate"); + }); + + it("produces a stable catalog revision that changes with the catalog", () => { + const build = (version: string): string => { + const app = new Evaluator({ name: "n", version: "1" }); + app.eval("b", { version }, () => new EvalResult()); + app.eval("a", { version: "1" }, () => new EvalResult()); + return app.catalogRevision; + }; + expect(build("1")).toBe(build("1")); + expect(build("1")).not.toBe(build("2")); + expect(build("1")).toMatch(/^sha256:[0-9a-f]{64}$/); + // Sorted, so registration order cannot change the hash. + const app = new Evaluator({ name: "n", version: "1" }); + app.eval("z", { version: "1" }, () => new EvalResult()); + app.eval("a", { version: "1" }, () => new EvalResult()); + expect(app.catalog().map((item) => item.evalKey)).toEqual(["a", "z"]); + }); +}); diff --git a/sdk/typescript/test/events.test.ts b/sdk/typescript/test/events.test.ts new file mode 100644 index 000000000..a0ae674c2 --- /dev/null +++ b/sdk/typescript/test/events.test.ts @@ -0,0 +1,193 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { EventNamespace } from "../src/events.js"; +import { agent, session } from "../src/scopes.js"; +import { runtime } from "../src/runtime.js"; +import { setLogger } from "../src/logger.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); +}); + +afterEach(async () => { + await spool.cleanup(); + setLogger(null); +}); + +const event = (): EventNamespace => runtime.event; + +describe("identity resolution", () => { + it("resolves sessionId and agentId from the enclosing scope", async () => { + await session({ sessionId: "s1" }, async () => { + await agent("planner", async () => { + event().toolUse({ toolName: "t", toolCallId: "c1" }); + }); + }); + const events = await flushed(spool); + const toolUse = events.find((e) => e.type === "tool_use")!; + expect(toolUse.session_id).toBe("s1"); + expect(toolUse.agent_id).toBe("planner"); + }); + + it("falls back to 'main' for agentId but never invents a sessionId", async () => { + await session({ sessionId: "s1" }, () => { + event().toolUse({ toolName: "t", toolCallId: "c1" }); + }); + expect((await flushed(spool))[0]!.agent_id).toBe("main"); + }); + + it("throws, naming the fix, when nothing is bound", () => { + expect(() => event().toolUse({ toolName: "t", toolCallId: "c1" })).toThrow( + /sessionId is required and nothing is bound/, + ); + }); + + it("refuses an empty sessionId, which the server would accept and silently merge", () => { + expect(() => event().toolUse({ sessionId: " ", toolName: "t", toolCallId: "c" })).toThrow( + /must not be empty/, + ); + }); + + it("refuses a non-string sessionId, which the server skips at 200 OK", () => { + expect(() => + event().toolUse({ sessionId: 42 as unknown as string, toolName: "t", toolCallId: "c" }), + ).toThrow(TypeError); + }); +}); + +describe("reserved and promoted fields", () => { + it("refuses a reserved name as a custom field", () => { + expect(() => + event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c", type: "nope" }), + ).toThrow(/Reserved field names/); + }); + + it("reports the reserved-field fault before the identity fault", () => { + // Both are true here. The reserved name reads identically from anywhere, so + // it is the more useful of the two to hear. + expect(() => event().toolUse({ toolName: "t", toolCallId: "c", timestamp: "x" })).toThrow( + /Reserved field names/, + ); + }); + + it("drops a nullish promoted extra with a warning rather than NULLing the column", async () => { + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + event().agentStart({ sessionId: "s", model: null }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("model was passed as null")); + const [emitted] = await flushed(spool); + expect("model" in emitted!).toBe(false); + }); + + it("refuses a non-integer token count, which the server stores as NULL", () => { + expect(() => event().modelResponse({ sessionId: "s", inputTokens: 1.5 })).toThrow(TypeError); + expect(() => event().modelResponse({ sessionId: "s", outputTokens: -1 })).toThrow(RangeError); + expect(() => event().modelResponse({ sessionId: "s", inputTokens: 2 ** 32 })).toThrow(RangeError); + }); + + it("refuses a non-string promoted extra", () => { + expect(() => + event().agentStart({ sessionId: "s", tool_name: 7 as unknown as string }), + ).toThrow(TypeError); + }); +}); + +describe("duration pairing", () => { + it("computes duration_ms from the matching start event", async () => { + event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c1" }); + await new Promise((resolve) => setTimeout(resolve, 12)); + event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c1" }); + const result = (await flushed(spool)).find((e) => e.type === "tool_result")!; + expect(typeof result.duration_ms).toBe("number"); + expect(result.duration_ms as number).toBeGreaterThanOrEqual(10); + }); + + it("refuses a caller-supplied duration_ms on every paired event", () => { + const cases: Array<() => void> = [ + () => event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c", duration_ms: 1 }), + () => event().agentResume({ sessionId: "s", pauseId: "p", duration_ms: 1 }), + () => event().hookCompleted({ sessionId: "s", hookName: "h", hookId: "h", duration_ms: 1 }), + () => event().humanInput({ sessionId: "s", inputId: "i", duration_ms: 1 }), + ]; + for (const run of cases) expect(run).toThrow(/auto-computed/); + }); + + it("keys the pair by kind AND session, so a shared step id cannot cross-pair", async () => { + // A tool call and a hook sharing an id is not exotic — both are frequently + // the harness's own step id. + event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "step-1" }); + event().hookCompleted({ sessionId: "s", hookName: "h", hookId: "step-1" }); + event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "step-1" }); + + const events = await flushed(spool); + const hook = events.find((e) => e.type === "hook_completed")!; + const tool = events.find((e) => e.type === "tool_result")!; + // The hook never opened, so it has no duration; the tool's survived. + expect("duration_ms" in hook).toBe(false); + expect(typeof tool.duration_ms).toBe("number"); + }); + + it("does not key the pair by agent, so a tool opened and closed under different agents still pairs", async () => { + event().toolUse({ sessionId: "s", agentId: "planner", toolName: "t", toolCallId: "c1" }); + event().toolResult({ sessionId: "s", agentId: "worker", toolName: "t", toolCallId: "c1" }); + const result = (await flushed(spool)).find((e) => e.type === "tool_result")!; + expect(typeof result.duration_ms).toBe("number"); + }); + + it("separates identical step ids in different sessions", async () => { + event().toolUse({ sessionId: "a", toolName: "t", toolCallId: "step" }); + event().toolUse({ sessionId: "b", toolName: "t", toolCallId: "step" }); + event().toolResult({ sessionId: "a", toolName: "t", toolCallId: "step" }); + const result = (await flushed(spool)).find((e) => e.type === "tool_result")!; + expect(result.session_id).toBe("a"); + expect(typeof result.duration_ms).toBe("number"); + }); +}); + +describe("timestamps", () => { + it("formats with six fractional digits, matching the Python SDK and the ingest parser", async () => { + event().agentStart({ sessionId: "s" }); + const [emitted] = await flushed(spool); + expect(emitted!.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/); + }); + + it("orders a burst inside one millisecond: every timestamp strictly after the last", async () => { + // A fast agent emits model_response, tool_use, tool_result and the next + // model_request inside one millisecond. With the last three digits always + // 000 they all carried the same timestamp, and the dashboard — which sorts + // on it — showed tool_result before its tool_use. + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(new Date("2026-09-23T15:55:44.134Z")); + event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c1" }); + event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c1" }); + event().modelRequest({ sessionId: "s" }); + event().agentEnd({ sessionId: "s" }); + } finally { + vi.useRealTimers(); + } + const stamps = (await flushed(spool)).map((e) => e.timestamp as string); + expect(stamps).toHaveLength(4); + for (const stamp of stamps) expect(stamp.startsWith("2026-09-23T15:55:44.134")).toBe(true); + expect([...stamps].sort()).toEqual(stamps); + expect(new Set(stamps).size).toBe(4); + }); + + it("re-anchors to the wall clock when it steps back, rather than freezing", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(new Date("2026-09-23T16:00:00.000Z")); + event().agentStart({ sessionId: "s" }); + vi.setSystemTime(new Date("2026-09-23T15:00:00.000Z")); // NTP stepped back an hour + event().agentEnd({ sessionId: "s" }); + } finally { + vi.useRealTimers(); + } + const [, end] = await flushed(spool); + expect((end!.timestamp as string).startsWith("2026-09-23T15:00:00.000")).toBe(true); + }); +}); diff --git a/sdk/typescript/test/expression.test.ts b/sdk/typescript/test/expression.test.ts new file mode 100644 index 000000000..f3311b70a --- /dev/null +++ b/sdk/typescript/test/expression.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "vitest"; + +import { + EvaluationBudgetExceeded, + MAX_POW_EXPONENT, + UnsafeEvaluatorSource, + compileExpression, +} from "../src/evaluator/expression.js"; +import { sessionTranscriptFromWire } from "../src/evaluator/protocol.js"; +import { transcriptWire } from "./helpers.js"; + +/** + * The language is the correctness boundary for server-authored source — the + * `worker_threads` sandbox around it bounds RESOURCES, not reach. So the cases + * that matter most here are the ones that must be refused, and every one of + * them is a real escape in a naive `eval`- or `vm`-based design. + */ + +const session = sessionTranscriptFromWire( + transcriptWire([ + { type: "tool_use", payload: { tool_name: "search" } }, + { type: "tool_result", payload: { error: "timeout" } }, + { type: "tool_result", payload: { output: "ok" } }, + ]), +); + +const GLOBALS = ["session", "Math", "Object", "Array", "Number", "JSON"] as const; + +function run(source: string): unknown { + const compiled = compileExpression(source, { + fieldName: "evaluator_source", + maximumBytes: 128 * 1024, + globalNames: GLOBALS, + }); + return compiled({ session }); +} + +function compileOnly(source: string): void { + compileExpression(source, { + fieldName: "evaluator_source", + maximumBytes: 128 * 1024, + globalNames: GLOBALS, + }); +} + +describe("what the language can express", () => { + it("reads the transcript surface", () => { + expect(run("session.eventCount")).toBe(3); + expect(run("session.count('tool_result')")).toBe(2); + expect(run("session.eventsOfType('tool_use').length")).toBe(1); + expect(run("session.agentId")).toBe("main"); + }); + + it("reads arbitrary payload keys, which no static allowlist could enumerate", () => { + expect(run("session.eventsOfType('tool_result')[0].payload.error")).toBe("timeout"); + expect(run("session.eventsOfType('tool_result')[1].payload['output']")).toBe("ok"); + }); + + it("supports `!= null`, the idiom for an optional payload field", () => { + // Mapping `!=` to `!==` would silently stop this matching a missing key, + // so an evaluation over a payload that omits the field would report the + // opposite of the truth. + expect( + run("session.eventsOfType('tool_result').filter(e => e.payload.error != null).length"), + ).toBe(1); + expect(run("session.eventsOfType('tool_result')[1].payload.error == null")).toBe(true); + }); + + it("supports arrow functions, arithmetic, comparison and the conditional operator", () => { + expect(run("[1, 2, 3].map(x => x * 2).filter(x => x > 2)")).toEqual([4, 6]); + expect(run("session.eventCount > 0 ? 'busy' : 'idle'")).toBe("busy"); + expect(run("Math.round(1 / 3 * 100) / 100")).toBe(0.33); + expect(run("[1, 2, 3].reduce((a, b) => a + b, 0)")).toBe(6); + }); + + it("supports template strings and object literals", () => { + expect(run("`saw ${session.eventCount} events`")).toBe("saw 3 events"); + expect(run("({ total: session.eventCount, ok: true })")).toEqual({ total: 3, ok: true }); + }); + + it("supports the namespace globals it advertises", () => { + expect(run("Object.keys({ a: 1, b: 2 })")).toEqual(["a", "b"]); + expect(run("Array.isArray(session.events)")).toBe(true); + expect(run("Number.isInteger(session.eventCount)")).toBe(true); + expect(run("JSON.stringify({ a: 1 })")).toBe('{"a":1}'); + }); +}); + +describe("escapes that must be refused", () => { + const rejected: Array<[string, string]> = [ + ["a function constructor", "(x => x).constructor"], + ["a literal computed constructor", "session['constructor']"], + ["prototype", "session.prototype"], + ["__proto__", "session.__proto__"], + ["call", "session.count.call"], + ["bind", "session.count.bind"], + ["toString, which runs arbitrary code", "session.toString"], + ["a reference to process", "process.env"], + ["a reference to globalThis", "globalThis"], + ["require", "require('fs')"], + ["import", "import('fs')"], + ["new", "new Object()"], + ["a function expression", "function () { return 1 }"], + ["assignment", "session.eventCount = 5"], + ["a statement", "if (true) 1"], + ["a block-bodied arrow", "(x) => { return x }"], + ["an unbound identifier", "somethingElse"], + ["a private name", "_secret"], + ["a regular expression literal", "/abc/.test('abc')"], + ["a comment, which the grammar does not accept", "1 // trailing"], + ["a compile-time exponent bomb", "10 ** 99"], + ["a generator", "function* () { yield 1 }"], + ["a BigInt literal", "1n"], + ["optional chaining onto a forbidden key", "session?.constructor"], + ]; + + for (const [label, source] of rejected) { + it(`refuses ${label}`, () => { + expect(() => compileOnly(source)).toThrow(UnsafeEvaluatorSource); + }); + } + + it("refuses a bare method reference, which is only useful for smuggling one out", () => { + expect(() => compileOnly("session.count")).toThrow(/only to call it/); + expect(() => compileOnly("[].map")).toThrow(/only to call it/); + }); + + it("names the exponent ceiling in the message", () => { + expect(() => compileOnly("2 ** 100")).toThrow( + new RegExp(`integer constant in 0\\.\\.${MAX_POW_EXPONENT}`), + ); + }); + + it("refuses a RUNTIME-computed forbidden key, which no source check can see", () => { + // This is why the boundary is `readProperty` and not the parser. + expect(() => run("session[['con','structor'].join('')]")).toThrow(UnsafeEvaluatorSource); + expect(() => run("session['con' + 'structor']")).toThrow(UnsafeEvaluatorSource); + }); + + it("refuses a non-allowlisted method on a built-in prototype", () => { + expect(() => run("[1, 2].sort()")).toThrow(UnsafeEvaluatorSource); + expect(() => run("'abc'.matchAll('a')")).toThrow(UnsafeEvaluatorSource); + }); + + it("refuses a namespace member outside its set", () => { + expect(() => run("Object.getPrototypeOf({})")).toThrow(UnsafeEvaluatorSource); + expect(() => run("Math.random()")).toThrow(UnsafeEvaluatorSource); + }); + + it("does not walk a prototype for a plain object's missing key", () => { + expect(run("({}).somethingAbsent")).toBeUndefined(); + }); +}); + +describe("budgets", () => { + it("bounds unbounded self-application", () => { + // `(f => f(f))(f => f(f))` is reachable with nothing but arrow functions, + // and runs forever. + expect(() => run("(f => f(f))(f => f(f))")).toThrow(EvaluationBudgetExceeded); + }); + + it("bounds a string bomb", () => { + expect(() => run("'x'.repeat(64) .repeat(64) .repeat(64) .repeat(64)")).toThrow( + EvaluationBudgetExceeded, + ); + }); + + it("refuses source above the byte ceiling", () => { + expect(() => + compileExpression(`'${"x".repeat(200)}'`, { + fieldName: "evaluator_source", + maximumBytes: 100, + globalNames: GLOBALS, + }), + ).toThrow(/exceeds 100 bytes/); + }); + + it("refuses empty source", () => { + expect(() => compileOnly(" ")).toThrow(/must not be empty/); + }); +}); diff --git a/sdk/typescript/test/global-setup.ts b/sdk/typescript/test/global-setup.ts new file mode 100644 index 000000000..7780cc51f --- /dev/null +++ b/sdk/typescript/test/global-setup.ts @@ -0,0 +1,26 @@ +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Point the evaluator sandbox at this checkout's built worker. + * + * The tests run against `src/` through Vitest's TypeScript loader, but a + * `worker_threads` Worker is a real Node process entry and cannot load a `.ts` + * file. In an installed package `source.ts` resolves `@failproofai/sdk/sandbox-worker` + * through the exports map; here there is no installed package, so the env + * override — the same one the code documents for bundled consumers — points at + * `dist/`. `npm test` builds first, which is why this can assert rather than + * skip: a sandbox test that silently did not run is worse than no sandbox test. + */ +export default function setup(): void { + const root = dirname(dirname(fileURLToPath(import.meta.url))); + const worker = resolve(root, "dist/cjs/evaluator/sandbox-worker.js"); + if (!existsSync(worker)) { + throw new Error( + `the evaluator sandbox worker is not built at ${worker}. Run \`npm run build\` first ` + + "(`npm test` does it for you).", + ); + } + process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = worker; +} diff --git a/sdk/typescript/test/helpers.ts b/sdk/typescript/test/helpers.ts new file mode 100644 index 000000000..925b2a94d --- /dev/null +++ b/sdk/typescript/test/helpers.ts @@ -0,0 +1,130 @@ +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { setBaseDir } from "../src/resolver.js"; +import { runtime } from "../src/runtime.js"; + +export interface Spool { + dir: string; + /** Every event written so far, oldest batch first. */ + events(): Array>; + /** The raw lines, for tests that care about bytes rather than values. */ + lines(): string[]; + files(): string[]; + /** + * Drain the writer into THIS spool, then remove it. + * + * Draining first is not tidiness. The writer is process-wide and its queue + * outlives a test, so a test that left events buffered would have them + * flushed by the NEXT test — into that test's assertions, and, in the window + * where no base directory is set, into the developer's real + * `~/.failproofai/custom-agents`, where a running daemon would collect them. + */ + cleanup(): Promise; +} + +/** + * A throwaway spool root with the writer pointed at it. + * + * Every test that emits anything needs this: without it they would write into + * the developer's real `~/.failproofai/custom-agents`, where a daemon would + * happily collect the fixtures and ship them. + */ +export function useSpool(): Spool { + const dir = mkdtempSync(join(tmpdir(), "failproofai-sdk-test-")); + setBaseDir(dir); + const eventsDir = join(dir, "events"); + return { + dir, + files(): string[] { + try { + return readdirSync(eventsDir) + .filter((name) => name.endsWith(".jsonl")) + .sort(); + } catch { + return []; + } + }, + lines(): string[] { + return this.files().flatMap((name) => + readFileSync(join(eventsDir, name), "utf8").split("\n").filter(Boolean), + ); + }, + events(): Array> { + return this.lines().map((line) => JSON.parse(line) as Record); + }, + async cleanup(): Promise { + await runtime.writer.flushNow(); + setBaseDir(null); + rmSync(dir, { recursive: true, force: true }); + }, + }; +} + +/** Flush the process-wide writer and read back what landed. */ +export async function flushed(spool: Spool): Promise>> { + await runtime.writer.flushNow(); + return spool.events(); +} + +/** The built ESM entry, as a URL a child process can import. */ +export function indexUrl(): string { + const root = dirname(dirname(fileURLToPath(import.meta.url))); + const entry = join(root, "dist/esm/index.js"); + if (!existsSync(entry)) { + throw new Error(`dist/esm/index.js is not built. Run \`npm run build\` (\`npm test\` does it).`); + } + return pathToFileURL(entry).href; +} + +/** + * Run `source` in a FRESH Node process and collect its output. + * + * Some of this SDK's guarantees are about process lifetime — the flush interval + * must not keep the loop alive, and the exit hook must write what is buffered. + * Neither can be observed from inside a test runner, which holds the loop open + * by itself and never exits. + */ +export async function runNode( + source: string, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--input-type=module", "--eval", source], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + child.on("error", reject); + child.on("close", (code) => { + resolve({ code, stdout, stderr }); + }); + }); +} + +/** A minimal transcript for the evaluator tests. */ +export function transcriptWire( + events: Array<{ type: string; payload?: Record }> = [], +): Record { + return { + schema_version: "2", + assignment_id: "assignment-1", + session_id: "session-1", + session_revision_id: "revision-1", + agent_id: "main", + environment: "dev", + started_at: "2026-01-01T00:00:00.000000Z", + ended_at: "2026-01-01T00:05:00.000000Z", + event_count: events.length, + events: events.map((event, index) => ({ + id: `e${index}`, + ts: "2026-01-01T00:00:01.000000Z", + event_type: event.type, + payload: event.payload ?? {}, + })), + }; +} diff --git a/sdk/typescript/test/integrations.test.ts b/sdk/typescript/test/integrations.test.ts new file mode 100644 index 000000000..91c502932 --- /dev/null +++ b/sdk/typescript/test/integrations.test.ts @@ -0,0 +1,369 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { agent, session } from "../src/scopes.js"; +import { setLogger } from "../src/logger.js"; +import * as core from "../src/integrations/core.js"; +import { parseVersion } from "../src/integrations/compat.js"; +import { activeFrameworks, available, instrument, uninstrument } from "../src/integrations/index.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); + core.setStrict(false); + core.resetFailures(); +}); +afterEach(async () => { + uninstrument(); + await spool.cleanup(); + core.setStrict(null); + setLogger(null); +}); + +describe("failure policy", () => { + it("swallows a throwing hook and degrades the call site after repeated failures", () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + const boom = (): never => { + throw new Error("hook is broken"); + }; + const wrapped = core.safe("test", boom); + for (let i = 0; i < 3; i += 1) expect(() => wrapped()).not.toThrow(); + // A broken adapter costs one log line, not 40% of the process. + expect(core.isDegraded("test.boom")).toBe(true); + }); + + it("catches a REJECTED PROMISE, not just a synchronous throw", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + // Half of every framework's callback surface is async. A bare try/catch + // sees nothing when the body rejects; the rejection lands as an unhandled + // rejection, which Node terminates the process for. + const wrapped = core.safe("test", async () => { + throw new Error("async hook is broken"); + }); + await expect(wrapped()).resolves.toBeUndefined(); + }); + + it("re-throws under FAILPROOFAI_SDK_STRICT", () => { + core.setStrict(true); + const wrapped = core.safe("test", () => { + throw new Error("visible"); + }); + expect(() => wrapped()).toThrow("visible"); + }); +}); + +describe("wrapCallable", () => { + it("returns the original value and re-throws the original error, identity intact", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + const failure = new Error("from the framework"); + const throwing = core.wrapCallable( + () => { + throw failure; + }, + { + before: () => { + throw new Error("before is broken"); + }, + after: () => { + throw new Error("after is broken"); + }, + onError: () => { + throw new Error("onError is broken"); + }, + }, + ); + // Every one of our hooks throws, and the caller still sees exactly theirs. + expect(() => throwing()).toThrow(failure); + + const resolving = core.wrapCallable(async () => "value", { + after: () => { + throw new Error("after is broken"); + }, + }); + await expect(resolving()).resolves.toBe("value"); + }); + + it("settles our hooks on the PROMISE, not on the call", async () => { + const seen: string[] = []; + const wrapped = core.wrapCallable(async () => "v", { + after: () => seen.push("after"), + }); + const pending = wrapped(); + expect(seen).toEqual([]); + await pending; + expect(seen).toEqual(["after"]); + }); +}); + +describe("Patcher", () => { + it("restores exactly what it replaced", () => { + const target = { method: (): string => "original" }; + const original = target.method; + const patcher = new core.Patcher(); + patcher.patch(target, "method", () => "patched"); + expect(target.method()).toBe("patched"); + patcher.restoreAll(); + expect(target.method).toBe(original); + }); + + it("leaves a third party's patch alone rather than deleting it", () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + const target = { method: (): string => "original" }; + const patcher = new core.Patcher(); + patcher.patch(target, "method", () => "ours"); + const theirs = (): string => "theirs"; + target.method = theirs; + patcher.restoreAll(); + // Restoring here would delete their patch. Two instrumentation libraries + // un-patching each other is how both silently stop recording. + expect(target.method).toBe(theirs); + }); + + it("reports rather than pretends when a target refuses assignment", () => { + const frozen = Object.freeze({ method: (): string => "original" }); + const patcher = new core.Patcher(); + expect(patcher.patch(frozen, "method", () => "ours")).toBe(false); + expect(patcher.size).toBe(0); + }); +}); + +describe("payload discipline", () => { + it("truncates a long string and says so", () => { + const out = core.payload({ fw_prompt: "x".repeat(20_000) }); + expect((out.fw_prompt as string).length).toBeLessThanOrEqual(core.FIELD_LIMIT); + expect(out.fw_truncated).toBe(true); + }); + + it("keeps the SMALL fields when the budget runs out, not the first ones", () => { + // The adapters put the big payload before the metadata, so insertion order + // would drop `fw_run_id` — the field that says which run the payload + // belongs to — and keep the oversized blob. + const out = core.payload({ + fw_inputs: "x".repeat(core.EVENT_BUDGET * 2), + fw_run_id: "run-1", + fw_node: "retrieve", + }); + expect(out.fw_run_id).toBe("run-1"); + expect(out.fw_node).toBe("retrieve"); + expect(out.fw_truncated).toBe(true); + }); + + it("caps list and depth growth", () => { + const deep = core.truncate({ a: { b: { c: { d: { e: { f: { g: 1 } } } } } } }); + expect(JSON.stringify(deep)).not.toContain('"g"'); + const wide = core.truncate(Array.from({ length: 300 }, (_, i) => i)) as unknown[]; + expect(wide.length).toBeLessThanOrEqual(101); + expect(String(wide.at(-1))).toContain("more items truncated"); + }); + + it("renders a class instance as data rather than as a repr string", () => { + class Weather { + constructor( + readonly city: string, + readonly celsius: number, + ) {} + } + expect(core.truncate(new Weather("Faro", 21))).toEqual({ city: "Faro", celsius: 21 }); + }); + + it("does not run a throwing getter into the caller's face", () => { + const hostile = { + get boom(): never { + throw new Error("lazy attribute"); + }, + }; + expect(() => core.truncate(hostile)).not.toThrow(); + }); +}); + +describe("extras namespacing", () => { + it("prefixes fw_ and drops nullish values", () => { + expect(core.fwFields({ run_id: "r", node: "n", tags: null, missing: undefined })).toEqual({ + fw_run_id: "r", + fw_node: "n", + }); + }); + + it("leaves the deliberate top-level names alone", () => { + expect(core.fwFields({ duration_ms: 12, usage: { a: 1 }, request_id: "r" })).toEqual({ + duration_ms: 12, + usage: { a: 1 }, + request_id: "r", + }); + }); + + it("refuses an extra that would overwrite a declared field", () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + // The schema merges extras LAST, so this would silently change a promoted + // column and every test would still pass. + expect(core.guardExtras({ tool_name: "hijacked", fw_ok: 1 })).toEqual({ fw_ok: 1 }); + core.setStrict(true); + expect(() => core.guardExtras({ outcome: "hijacked" })).toThrow(/would overwrite/); + }); +}); + +describe("agent id normalization", () => { + it("replaces a bare id, which would poison a LowCardinality facet", () => { + expect(core.normalizeAgentId("3f9a1c2b-0000-4000-8000-000000000000")).toBe("main"); + expect(core.normalizeAgentId("a".repeat(32))).toBe("main"); + }); + + it("strips a per-run id from a readable prefix rather than discarding the name", () => { + expect(core.normalizeAgentId("agent-3f9a1c2b-0000-4000-8000-000000000000")).toBe("agent"); + expect(core.normalizeAgentId("crew_" + "a".repeat(32))).toBe("crew"); + }); + + it("leaves an ordinary name untouched, separators and all", () => { + expect(core.normalizeAgentId("node_a_b")).toBe("node_a_b"); + expect(core.normalizeAgentId("agent-v2")).toBe("agent-v2"); + expect(core.normalizeAgentId("step-3")).toBe("step-3"); + }); + + it("bounds the length and falls back on nothing", () => { + expect(core.normalizeAgentId("x".repeat(200)).length).toBe(64); + expect(core.normalizeAgentId(null)).toBe("main"); + expect(core.normalizeAgentId(" ")).toBe("main"); + }); +}); + +describe("RunTracker", () => { + it("emits against a framework's own run ids", async () => { + const tracker = new core.RunTracker("test"); + tracker.startAgent("run-1", { agentId: "planner", sessionId: "s1", goal: "g" }); + tracker.emit("toolUse", "tool-1", { + parentKey: "run-1", + toolName: "search", + toolCallId: "tool-1", + }); + tracker.endAgent("run-1", { outcome: "success" }); + + const events = await flushed(spool); + expect(events.map((event) => event.type)).toEqual(["agent_start", "tool_use", "agent_end"]); + expect(events.every((event) => event.session_id === "s1")).toBe(true); + expect(events[1]!.agent_id).toBe("planner"); + }); + + it("walks the parent chain more than one hop", async () => { + const tracker = new core.RunTracker("test"); + tracker.startAgent("root", { agentId: "planner", sessionId: "s1" }); + // An intermediate framework run is a link, not a span. + tracker.link("middle", "root"); + tracker.emit("toolUse", "leaf", { parentKey: "middle", toolName: "t", toolCallId: "leaf" }); + const toolUse = (await flushed(spool)).find((event) => event.type === "tool_use")!; + expect(toolUse.agent_id).toBe("planner"); + }); + + it("joins a hand-written scope, producing ONE tree rather than two", async () => { + const tracker = new core.RunTracker("test"); + await agent("planner", { sessionId: "s1" }, () => { + tracker.startAgent("run-1", { agentId: "retriever" }); + }); + const starts = (await flushed(spool)).filter((event) => event.type === "agent_start"); + const nested = starts.find((event) => event.agent_id === "retriever")!; + expect(nested.session_id).toBe("s1"); + expect(nested.parent_id).toBe("planner"); + }); + + it("does NOT claim a parent inside a bare session, which would render as a forever-ongoing span", async () => { + const tracker = new core.RunTracker("test"); + await session({ sessionId: "s1" }, () => { + tracker.startAgent("run-1", { agentId: "retriever" }); + }); + const start = (await flushed(spool)).find((event) => event.type === "agent_start")!; + expect(start.parent_id).toBeUndefined(); + }); + + it("drops an unresolvable event with ONE warning rather than inventing a session", async () => { + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + const tracker = new core.RunTracker("test"); + tracker.emit("toolUse", "orphan", { toolName: "t", toolCallId: "orphan" }); + tracker.emit("toolUse", "orphan2", { toolName: "t", toolCallId: "orphan2" }); + // A synthesized session id splits one run into many. + expect(await flushed(spool)).toEqual([]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("closes what it opened, so a dead session does not render as ongoing forever", async () => { + const tracker = new core.RunTracker("test"); + tracker.startAgent("a", { agentId: "one", sessionId: "s1" }); + tracker.startAgent("b", { agentId: "two", sessionId: "s1" }); + tracker.closeOpenAgents(); + const ends = (await flushed(spool)).filter((event) => event.type === "agent_end"); + expect(ends.map((event) => event.agent_id)).toEqual(["two", "one"]); + expect(ends.every((event) => event.outcome === "cancelled")).toBe(true); + }); + + it("evicts oldest-first rather than growing unboundedly", () => { + const tracker = new core.RunTracker("test", { maxOpen: 4 }); + for (let i = 0; i < 20; i += 1) { + tracker.startAgent(`run-${i}`, { agentId: "a", sessionId: "s1" }); + } + expect(tracker.openAgents().length).toBeLessThanOrEqual(4); + }); + + it("truncates the declared fields as well as the extras, under ONE truncation flag", async () => { + const tracker = new core.RunTracker("test", { fieldLimit: 64 }); + tracker.startAgent("run-1", { agentId: "a", sessionId: "s1" }); + tracker.emit("toolResult", "t1", { + parentKey: "run-1", + toolName: "search", + toolCallId: "t1", + output: "y".repeat(500), + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect((result.output as string).length).toBeLessThanOrEqual(64 + "…[truncated]".length); + // `fw_truncated` has to mean "this event lost data" — `output` is cut on + // essentially every real tool loop, and it is the field an operator filters + // on to find where. + expect(result.fw_truncated).toBe(true); + }); +}); + +describe("the registry", () => { + it("lists the frameworks it can instrument", () => { + expect(available()).toEqual(["ai", "langchain", "llamaindex", "mastra"]); + }); + + it("throws on an unknown name, listing the valid ones", async () => { + // A typo that silently records nothing is the worst outcome available. + await expect(instrument("langchian")).rejects.toThrow(/unknown framework/); + }); + + it("accepts the spellings people actually type", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + // LangGraph is served by the LangChain adapter; neither is installed here, + // so the install fails and is skipped — which is itself the contract. + await expect(instrument("langgraph")).resolves.toEqual([]); + expect(activeFrameworks()).toEqual([]); + }); + + it("does not throw when a framework is absent — it logs and moves on", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + await expect(instrument("mastra")).resolves.toEqual([]); + }); + + it("uninstrumenting an unknown name is a no-op, not a throw", () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + expect(uninstrument("nonsense")).toEqual([]); + }); +}); + +describe("version parsing", () => { + it("reads the leading numeric components only", () => { + expect(parseVersion("1.5.2")).toEqual([1, 5, 2]); + expect(parseVersion("2.0.0-beta.1")).toEqual([2, 0, 0]); + expect(parseVersion("0.14.23+build")).toEqual([0, 14, 23]); + expect(parseVersion("next")).toEqual([]); + }); +}); + +describe("duration rounding", () => { + it("returns a whole non-negative integer, because the server drops anything else", () => { + expect(core.ms(12.7)).toBe(13); + expect(core.ms(-5)).toBe(0); + expect(Number.isInteger(core.ms(0.4))).toBe(true); + }); +}); diff --git a/sdk/typescript/test/langchain-copies.test.ts b/sdk/typescript/test/langchain-copies.test.ts new file mode 100644 index 000000000..22cb70397 --- /dev/null +++ b/sdk/typescript/test/langchain-copies.test.ts @@ -0,0 +1,204 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { nestedCopies, resolveExportsAt } from "../src/node-require.js"; + +/** + * A SECOND `@langchain/core`, nested under a dependency that pinned its own. + * + * A provider declaring `@langchain/core` as a hard dependency on a range the + * application's copy does not satisfy gets its own copy at + * `node_modules//node_modules/@langchain/core`, and everything it + * exports is built on it. `instrument()` resolved `@langchain/core` from the + * application and patched that copy only — so a run the provider's classes + * started as a ROOT went through the nested copy's untouched `configure` and was + * recorded nowhere (reproduced end to end in + * `integration/fixtures/langchain-dup-core`). These tests pin the two halves of + * the fix with fake copies on disk: finding every nested copy, and patching the + * build of each that the application's module system will load. + */ + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +let app: string; +let modules: string; + +function write(path: string, text: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text); +} + +/** + * A fake `@langchain/core` whose `CallbackManager.configure` records, per copy + * and per build, whether it was handed the failproofai handler. + */ +function fakeCore(dir: string, marker: string, version: string): void { + write( + join(dir, "package.json"), + JSON.stringify({ + name: "@langchain/core", + version, + exports: { + "./callbacks/manager": { + import: { types: "./x.d.ts", default: "./esm/manager.js" }, + require: { types: "./x.d.cts", default: "./cjs/manager.cjs" }, + }, + "./package.json": "./package.json", + }, + }), + ); + const body = (build: string) => + `class CallbackManager {\n` + + ` static configure(inheritable) {\n` + + ` const ours = Array.isArray(inheritable) && inheritable.some((h) => h && h.name === "failproofai");\n` + + ` globalThis.__seen = [...(globalThis.__seen ?? []), "${marker}-${build}:" + ours];\n` + + ` return undefined;\n` + + ` }\n` + + `}\n` + + `globalThis.__loaded = [...(globalThis.__loaded ?? []), "${marker}-${build}"];\n`; + write(join(dir, "esm", "manager.js"), `${body("esm")}export { CallbackManager };\n`); + write(join(dir, "cjs", "manager.cjs"), `${body("cjs")}exports.CallbackManager = CallbackManager;\n`); +} + +beforeAll(() => { + app = realpathSync(mkdtempSync(join(tmpdir(), "failproofai-lc-copies-"))); + modules = join(app, "node_modules"); + // The application's own copy. + fakeCore(join(modules, "@langchain", "core"), "app", "1.2.12"); + // A provider that pinned 0.3 — the layout this is about. + fakeCore(join(modules, "lc-provider", "node_modules", "@langchain", "core"), "provider", "0.3.80"); + write(join(modules, "lc-provider", "package.json"), JSON.stringify({ name: "lc-provider" })); + // Nested two levels down, under a scoped package. + fakeCore( + join(modules, "@acme", "agents", "node_modules", "tools", "node_modules", "@langchain", "core"), + "deep", + "1.1.0", + ); + // Outside the declared range: found, but never loaded or patched. + fakeCore(join(modules, "ancient", "node_modules", "@langchain", "core"), "ancient", "0.1.52"); + // pnpm's virtual store, with the provider's copy symlinked beside it the way + // pnpm lays a dependency out — ONE copy, however many links point at it. + fakeCore(join(modules, ".pnpm", "@langchain+core@1.0.4", "node_modules", "@langchain", "core"), "pnpm", "1.0.4"); + mkdirSync(join(modules, ".pnpm", "prov@1.0.0", "node_modules", "@langchain"), { recursive: true }); + symlinkSync( + join(modules, ".pnpm", "@langchain+core@1.0.4", "node_modules", "@langchain", "core"), + join(modules, ".pnpm", "prov@1.0.0", "node_modules", "@langchain", "core"), + ); +}); + +afterAll(() => { + rmSync(app, { recursive: true, force: true }); +}); + +const inApp = (fn: () => T): T => { + const cwd = process.cwd(); + process.chdir(app); + try { + return fn(); + } finally { + process.chdir(cwd); + } +}; + +describe("nestedCopies", () => { + it("finds every copy nested under a dependency, and the pnpm store's, but never the app's own", () => { + const found = inApp(() => nestedCopies("@langchain/core")).sort(); + expect(found).toEqual( + [ + join(modules, ".pnpm", "@langchain+core@1.0.4", "node_modules", "@langchain", "core"), + join(modules, "@acme", "agents", "node_modules", "tools", "node_modules", "@langchain", "core"), + join(modules, "ancient", "node_modules", "@langchain", "core"), + join(modules, "lc-provider", "node_modules", "@langchain", "core"), + ].sort(), + ); + }); + + it("finds nothing where nothing is nested", () => { + expect(inApp(() => nestedCopies("not-installed-anywhere"))).toEqual([]); + }); +}); + +describe("resolveExportsAt", () => { + it("names each build of a subpath, through nested conditions", () => { + const provider = join(modules, "lc-provider", "node_modules", "@langchain", "core"); + expect(resolveExportsAt(provider, "./callbacks/manager", "import")).toBe(join(provider, "esm", "manager.js")); + expect(resolveExportsAt(provider, "./callbacks/manager", "require")).toBe(join(provider, "cjs", "manager.cjs")); + expect(resolveExportsAt(provider, "./not-exported", "import")).toBeNull(); + expect(resolveExportsAt(join(app, "nowhere"), "./callbacks/manager", "import")).toBeNull(); + }); +}); + +describe("instrument('langchain') with a nested @langchain/core", () => { + /** + * Install the adapter from a real entry point of each module system — which + * copy the application's imports reach is a property of the process — then + * call `configure` on every copy and report which ones handed it our handler. + */ + const run = (entry: "esm" | "cjs", before = "", after = ""): { seen: string[]; loaded: string[] } => { + const adapter = + entry === "esm" + ? pathToFileURL(join(root, "dist", "esm", "integrations", "langchain.js")).href + : join(root, "dist", "cjs", "integrations", "langchain.js"); + const file = join(app, entry === "esm" ? "main.mjs" : "main.cjs"); + const header = + entry === "esm" + ? `import { createRequire } from "node:module";\nimport { pathToFileURL } from "node:url";\nimport * as lc from ${JSON.stringify(adapter)};\nconst require = createRequire(import.meta.url);\nconst load = async (p) => (p.endsWith(".cjs") ? require(p) : import(pathToFileURL(p).href));\n` + : `const lc = require(${JSON.stringify(adapter)});\nconst load = async (p) => require(p);\n`; + const copies = ["@langchain/core", "lc-provider/node_modules/@langchain/core", "@acme/agents/node_modules/tools/node_modules/@langchain/core", ".pnpm/@langchain+core@1.0.4/node_modules/@langchain/core", "ancient/node_modules/@langchain/core"]; + write( + file, + `${header}(async () => {\n${before}\nawait lc.adapter.install();\n` + + `for (const dir of ${JSON.stringify(copies)}) {\n` + + ` for (const build of ["${entry === "esm" ? "esm/manager.js" : "cjs/manager.cjs"}"]) {\n` + + ` const m = await load(${JSON.stringify(modules)} + "/" + dir + "/" + build);\n` + + ` m.CallbackManager.configure(undefined);\n` + + ` }\n` + + `}\n` + + `${after}\n` + + `lc.adapter.uninstall();\n` + + `console.log(JSON.stringify({ seen: globalThis.__seen ?? [], loaded: globalThis.__loaded ?? [] }));\n` + + `})().catch((e) => { console.log(JSON.stringify({ error: String(e && e.stack) })); });\n`, + ); + const child = spawnSync(process.execPath, [file], { + cwd: app, + encoding: "utf8", + env: { ...process.env, FAILPROOFAI_HOME: join(app, ".home") }, + }); + const parsed = JSON.parse(child.stdout.trim()) as { seen?: string[]; loaded?: string[]; error?: string }; + if (parsed.error !== undefined) throw new Error(parsed.error); + return { seen: parsed.seen!, loaded: parsed.loaded! }; + }; + + it("patches every in-range copy's ES-module build for an ES-module app", () => { + const { seen, loaded } = run("esm"); + expect(seen).toEqual(["app-esm:true", "provider-esm:true", "deep-esm:true", "pnpm-esm:true", "ancient-esm:false"]); + // The CommonJS builds nothing required are never loaded. + expect(loaded.filter((name) => name.endsWith("-cjs"))).toEqual([]); + }); + + it("patches every in-range copy's CommonJS build for a CommonJS app, loading no ES module", () => { + const { seen, loaded } = run("cjs"); + expect(seen).toEqual(["app-cjs:true", "provider-cjs:true", "deep-cjs:true", "pnpm-cjs:true", "ancient-cjs:false"]); + expect(loaded.filter((name) => name.endsWith("-esm"))).toEqual([]); + }); + + it("also patches a nested copy's CommonJS build when something already required it", () => { + // An ES-module app whose CommonJS dependency pulled the provider's copy in + // before `instrument()`: both builds are live, both are patched. + const provider = JSON.stringify(join(modules, "lc-provider", "node_modules", "@langchain", "core", "cjs", "manager.cjs")); + const { seen } = run("esm", `require(${provider});`, `require(${provider}).CallbackManager.configure(undefined);`); + expect(seen).toContain("provider-esm:true"); + expect(seen).toContain("provider-cjs:true"); + }); + + it("never loads a copy outside the declared range", () => { + // Loaded by nothing but the test's own probe, which runs after install. + const { loaded } = run("esm", "", ""); + expect(loaded.slice(0, 4).sort()).toEqual(["app-esm", "deep-esm", "pnpm-esm", "provider-esm"]); + expect(loaded.slice(4)).toEqual(["ancient-esm"]); + }); +}); diff --git a/sdk/typescript/test/langchain.test.ts b/sdk/typescript/test/langchain.test.ts new file mode 100644 index 000000000..abcb35d0a --- /dev/null +++ b/sdk/typescript/test/langchain.test.ts @@ -0,0 +1,999 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import * as core from "../src/integrations/core.js"; +import { + ABANDONED_ROOT_GRACE_MS, + adapter, + captureLimitOf, + interruptIdOf, + isCancellation, + isControlFlow, + langchainHandler, + nodeOf, + promptOf, + readOptions, + resetOrphanWarning, + usageOf, +} from "../src/integrations/langchain.js"; +import { setLogger } from "../src/logger.js"; +import { agent, session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The LangChain adapter's translation table, driven with the callback + * sequences LangChain.js and LangGraph.js actually dispatch — argument order, + * metadata keys, tags and error shapes copied from recorded runs of + * `@langchain/core` 0.3.80 / 1.2.12 and `@langchain/langgraph` 0.4.10 / 1.4.17. + * + * Every expectation is the Python adapter's output for the same run + * (`sdk/python/failproofai_sdk/integrations/langchain.py`); many of these + * are ports of `sdk/python/tests/integrations/test_langchain.py`, named after + * the Python test they mirror. The real frameworks are exercised end to end in + * `integration/langchain.test.ts`; this file pins the rules one at a time, fast + * and with no framework installed. + */ + +type Handler = Record unknown>; +type Event = Record; + +let spool: Spool; +let h: Handler; + +beforeEach(() => { + spool = useSpool(); + core.setStrict(true); + core.resetFailures(); + h = langchainHandler() as Handler; +}); +afterEach(async () => { + adapter.uninstall(); + await spool.cleanup(); + core.setStrict(null); + setLogger(null); +}); + +// -- a tiny LangChain ------------------------------------------------------- + +const msg = (type: string, content: unknown, extra: Record = {}) => ({ + getType: () => type, + content, + ...extra, +}); +const human = (content: string) => msg("human", content); +const callWeather = (id = "call_1", city = "Paris") => + msg("ai", "", { tool_calls: [{ id, name: "get_weather", args: { city }, type: "tool_call" }] }); + +const GRAPH = { lc: 1, type: "not_implemented", id: ["langgraph", "pregel", "CompiledStateGraph"] }; +const MODEL = { lc: 1, type: "not_implemented", id: ["langchain", "chat_models", "scripted", "ScriptedModel"] }; +const TOOL = { lc: 1, type: "not_implemented", id: ["langchain", "tools", "DynamicStructuredTool"] }; + +/** `handleChainStart(serialized, inputs, runId, parentRunId, tags, metadata, runType, runName)`. */ +function chain( + id: string, + parent: string | undefined, + name: string, + opts: { inputs?: unknown; tags?: string[]; meta?: Record; serialized?: unknown } = {}, +): void { + h.handleChainStart!( + opts.serialized ?? { lc: 1, type: "constructor", id: ["langchain_core", "runnables", "RunnableSequence"] }, + opts.inputs ?? {}, + id, + parent, + opts.tags ?? [], + opts.meta ?? {}, + undefined, + name, + ); +} + +function root(id: string, name: string, meta: Record = {}, inputs: unknown = { messages: [human("weather?")] }) { + chain(id, undefined, name, { inputs, meta: { ls_integration: "langgraph", ...meta }, serialized: GRAPH }); +} + +/** A LangGraph node's own run: named after the node, tagged `graph:step:N`. */ +function node(id: string, parent: string, name: string, step: number, extra: Record = {}) { + chain(id, parent, name, { + inputs: extra.inputs ?? { messages: [human("weather?")] }, + tags: [`graph:step:${step}`], + meta: { + langgraph_node: name, + langgraph_step: step, + langgraph_checkpoint_ns: (extra.ns) ?? `${name}:${id}`, + ...(extra.meta as Record | undefined), + }, + }); +} + +function end(id: string, outputs: unknown = {}): void { + h.handleChainEnd!(outputs, id); +} + +function fail(id: string, error: unknown): void { + h.handleChainError!(error, id); +} + +function modelStart(id: string, parent: string | undefined, meta: Record = {}, messages = [human("weather?")]) { + h.handleChatModelStart!(MODEL, [messages], id, parent, { invocation_params: {} }, [], meta, undefined); +} + +function modelEnd(id: string, message: Record = msg("ai", "hi"), llmOutput?: Record) { + h.handleLLMEnd!({ generations: [[{ text: "", message }]], llmOutput }, id); +} + +function toolStart(id: string, parent: string | undefined, input = '{"city":"Paris"}', toolCallId?: string, meta = {}) { + h.handleToolStart!(TOOL, input, id, parent, [], meta, "get_weather", toolCallId); +} + +function toolEnd(id: string, output: unknown = "sunny in Paris"): void { + h.handleToolEnd!(output, id); +} + +const events = (): Promise => flushed(spool); +const shape = (list: Event[]): string[] => + list.map((e) => [e.agent_id, e.type, e.hook_name ?? e.tool_name ?? ""].join(" ").trim()); +const ofType = (list: Event[], type: string): Event[] => list.filter((e) => e.type === type); + +/** The weather graph, exactly as LangGraph.js 1.x dispatches it. */ +function weatherGraph(opts: { toolCallId?: boolean; meta?: Record } = {}): void { + const withId = opts.toolCallId ?? true; + root("g", "weather_graph", opts.meta); + chain("start", "g", "__start__", { tags: ["graph:step:0", "langsmith:hidden"], meta: { langgraph_node: "__start__", langgraph_step: 0 } }); + end("start"); + node("n1", "g", "agent", 1); + modelStart("m1", "n1", { langgraph_node: "agent", langgraph_step: 1 }); + modelEnd("m1", msg("ai", "", { + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + usage_metadata: { input_tokens: 12, output_tokens: 5, total_tokens: 17 }, + })); + chain("route1", "n1", "RunnableLambda", { meta: { langgraph_node: "agent", langgraph_step: 1 } }); + end("route1", { output: "tools" }); + end("n1"); + node("n2", "g", "tools", 2, { inputs: { messages: [human("weather?"), callWeather()] } }); + toolStart("t1", "n2", '{"city":"Paris"}', withId ? "call_1" : undefined, { langgraph_node: "tools" }); + toolEnd("t1", msg("tool", "sunny in Paris", { tool_call_id: "call_1", status: "success" })); + end("n2"); + node("n3", "g", "agent", 3); + modelStart("m2", "n3", { langgraph_node: "agent", langgraph_step: 3 }); + modelEnd("m2", msg("ai", "It is sunny in Paris.", { + usage_metadata: { input_tokens: 30, output_tokens: 7, total_tokens: 37 }, + response_metadata: { finish_reason: "stop" }, + })); + end("n3"); + end("g"); +} + +const GRAPH_SHAPE = [ + "weather_graph agent_start", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph hook_triggered tools", + "weather_graph tool_use get_weather", + "weather_graph tool_result get_weather", + "weather_graph hook_completed tools", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph agent_end", +]; + +class GraphInterrupt extends Error { + interrupts: unknown[]; + constructor(interrupts: unknown[]) { + super(JSON.stringify(interrupts)); + this.name = "GraphInterrupt"; + this.interrupts = interrupts; + } + get is_bubble_up(): boolean { + return true; + } +} + +const command = (resume: unknown) => ({ lg_name: "Command", resume, goto: [] }); + +// -- the mapping ------------------------------------------------------------ + +describe("the graph mapping", () => { + it("a_langgraph_node_is_a_hook_not_a_nested_agent", async () => { + weatherGraph(); + const list = await events(); + expect(shape(list)).toEqual(GRAPH_SHAPE); + for (const hook of ofType(list, "hook_triggered")) expect(hook.trigger_event).toBe("graph_node"); + // One agent, named after the graph, never a run id. + expect(new Set(list.map((e) => e.agent_id))).toEqual(new Set(["weather_graph"])); + }); + + it("the root agent_start is the session's first event and every event carries one session", async () => { + weatherGraph(); + const list = await events(); + expect(list[0]!.type).toBe("agent_start"); + expect(new Set(list.map((e) => e.session_id)).size).toBe(1); + expect(list.every((e) => e.framework === "langchain")).toBe(true); + }); + + it("intermediate runnables, edge functions and hidden nodes emit nothing", async () => { + weatherGraph(); + const names = (await events()).map((e) => e.hook_name).filter(Boolean); + expect(names).not.toContain("__start__"); + expect(names).not.toContain("RunnableLambda"); + }); + + it("carries the langgraph ids as fw_* extras, never as declared fields", async () => { + weatherGraph(); + const hook = ofType(await events(), "hook_triggered")[0]!; + expect(hook.fw_node).toBe("agent"); + expect(hook.fw_step).toBe(1); + expect(hook.fw_run_id).toBe("n1"); + expect(hook.fw_parent_run_id).toBe("g"); + expect(hook.fw_tags).toEqual(["graph:step:1"]); + }); + + it("model events pair on request_id and always carry an int duration and the tokens", async () => { + weatherGraph(); + const list = await events(); + const requests = ofType(list, "model_request"); + const responses = ofType(list, "model_response"); + expect(responses.map((r) => r.request_id)).toEqual(requests.map((r) => r.request_id)); + for (const response of responses) expect(Number.isInteger(response.duration_ms)).toBe(true); + expect(responses.map((r) => [r.input_tokens, r.output_tokens])).toEqual([ + [12, 5], + [30, 7], + ]); + expect(responses[1]!.usage).toEqual({ input_tokens: 30, output_tokens: 7, total_tokens: 37 }); + expect(responses[1]!.stop_reason).toBe("stop"); + expect(requests[0]!.model).toBe("ScriptedModel"); + }); + + it("a node that returns a Command renders the messages inside it, not their serialization envelope", async () => { + // `langchain`'s `createAgent` model node returns `{ output: [Command] }`. + // A Command is neither a message nor LangChain `Serializable` (no + // `lc_kwargs`), so the payload view used to hand it to `truncate` whole, + // which dumps it through `toJSON()` — and the messages in its `update` + // then through THEIRS, LangChain's `{lc, type: "constructor", id, kwargs}` + // envelope: class paths with the content buried one level down. + class Message { + lc_kwargs = { content: "hi" }; + content = "hi"; + getType(): string { + return "ai"; + } + toJSON(): unknown { + return { lc: 1, type: "constructor", id: ["langchain_core", "messages", "AIMessage"], kwargs: this.lc_kwargs }; + } + } + class Command { + lg_name = "Command"; + update = { messages: [new Message()] }; + goto: string[] = []; + toJSON(): unknown { + return { lg_name: this.lg_name, update: this.update, resume: undefined, goto: this.goto }; + } + } + root("g", "weather_agent"); + node("n", "g", "model_request", 1); + end("n", { output: [new Command()] }); + end("g"); + const done = ofType(await events(), "hook_completed")[0]!; + expect(done.output).toEqual({ + output: [{ lg_name: "Command", update: { messages: [{ type: "ai", content: "hi" }] }, resume: null, goto: [] }], + }); + expect(JSON.stringify(done.output)).not.toContain('"lc":1'); + }); + + it("model_request carries normalized messages with their roles", async () => { + root("g", "weather_graph"); + node("n", "g", "agent", 1); + modelStart("m", "n", {}, [ + msg("system", "be brief"), + human("weather?"), + callWeather(), + msg("tool", "sunny in Paris", { tool_call_id: "call_1" }), + ]); + modelEnd("m"); + end("n"); + end("g"); + expect(ofType(await events(), "model_request")[0]!.messages).toEqual([ + { role: "system", content: "be brief" }, + { role: "user", content: "weather?" }, + { + role: "assistant", + content: "", + tool_calls: [{ id: "call_1", name: "get_weather", args: { city: "Paris" }, type: "tool_call" }], + }, + { role: "tool", content: "sunny in Paris" }, + ]); + }); + + it("tool_use carries the MODEL's tool_call_id, and the tool's content not the message object", async () => { + weatherGraph(); + const list = await events(); + expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe("call_1"); + expect(ofType(list, "tool_use")[0]!.input).toEqual({ city: "Paris" }); + const result = ofType(list, "tool_result")[0]!; + expect(result.tool_call_id).toBe("call_1"); + expect(result.output).toBe("sunny in Paris"); + expect(result.error).toBeUndefined(); + }); + + it("recovers the tool_call_id on a core that does not pass one (0.3)", async () => { + weatherGraph({ toolCallId: false }); + expect(ofType(await events(), "tool_use")[0]!.tool_call_id).toBe("call_1"); + }); + + it("gives two identical tool calls their two different ids (0.3)", async () => { + root("g", "weather_graph"); + node("n", "g", "tools", 1, { + inputs: { + messages: [ + msg("ai", "", { + tool_calls: [ + { id: "call_a", name: "get_weather", args: { city: "Paris" } }, + { id: "call_b", name: "get_weather", args: { city: "Paris" } }, + ], + }), + ], + }, + }); + toolStart("t1", "n"); + toolStart("t2", "n"); + toolEnd("t1"); + toolEnd("t2"); + end("n"); + end("g"); + const ids = ofType(await events(), "tool_use").map((e) => e.tool_call_id); + expect(ids.sort()).toEqual(["call_a", "call_b"]); + }); + + it("falls back to the run id when no ancestor asked for the call", async () => { + root("g", "weather_graph"); + node("n", "g", "tools", 1, { inputs: { messages: [human("hi")] } }); + toolStart("t1", "n"); + toolEnd("t1"); + end("n"); + end("g"); + expect(ofType(await events(), "tool_use")[0]!.tool_call_id).toBe("t1"); + }); + + it("a_compiled_subgraph_becomes_a_nested_agent", async () => { + root("p", "parent_graph", {}, { trail: [] }); + node("pre", "p", "pre", 1, { inputs: { trail: [] } }); + end("pre"); + node("host", "p", "child", 2, { ns: "child:aaa" }); + chain("sub", "host", "child_graph", { meta: { langgraph_node: "child" }, serialized: GRAPH }); + node("inner", "sub", "inner", 1, { ns: "child:aaa|inner:bbb" }); + end("inner"); + end("sub"); + end("host"); + end("p"); + const list = await events(); + expect(shape(list)).toEqual([ + "parent_graph agent_start", + "parent_graph hook_triggered pre", + "parent_graph hook_completed pre", + "parent_graph hook_triggered child", + "parent_graph/child agent_start", + "parent_graph/child hook_triggered inner", + "parent_graph/child hook_completed inner", + "parent_graph/child agent_end", + "parent_graph hook_completed child", + "parent_graph agent_end", + ]); + expect(ofType(list, "agent_start")[1]!.parent_id).toBe("parent_graph"); + }); + + it("records a bare model call as a root agent with its model pair inside", async () => { + modelStart("m", undefined); + modelEnd("m", msg("ai", "hi", { usage_metadata: { input_tokens: 1, output_tokens: 2, total_tokens: 3 } })); + const list = await events(); + expect(shape(list)).toEqual([ + "ScriptedModel agent_start", + "ScriptedModel model_request", + "ScriptedModel model_response", + "ScriptedModel agent_end", + ]); + expect(ofType(list, "model_request")[0]!.request_id).toBe(ofType(list, "model_response")[0]!.request_id); + expect(ofType(list, "agent_start")[0]!.goal).toBe("weather?"); + }); + + it("records a bare tool call as a root agent with its tool pair inside", async () => { + toolStart("t", undefined, '{"city":"Rome"}'); + toolEnd("t", "sunny in Rome"); + expect(shape(await events())).toEqual([ + "get_weather agent_start", + "get_weather tool_use get_weather", + "get_weather tool_result get_weather", + "get_weather agent_end", + ]); + }); + + it("two_overlapping_roots_in_one_session_are_two_agents", async () => { + // `.batch()` opens one root per input, concurrently. + const handler = langchainHandler({ sessionId: "shared" }) as Handler; + expect(handler).toBe(h); + modelStart("a", undefined); + modelStart("b", undefined); + modelEnd("a"); + modelEnd("b"); + const list = await events(); + expect(ofType(list, "agent_start")).toHaveLength(2); + expect(ofType(list, "agent_end")).toHaveLength(2); + expect(ofType(list, "model_response")).toHaveLength(2); + expect(ofType(list, "human_input")).toHaveLength(0); + }); + + it("retriever output is summarized, never the document text", async () => { + root("g", "rag"); + h.handleRetrieverStart!({ id: ["x", "VectorStoreRetriever"] }, "kites", "r", "g", [], {}, undefined); + h.handleRetrieverEnd!( + [ + { pageContent: "a very long document", metadata: { source: "a.txt" } }, + { pageContent: "another", metadata: {} }, + ], + "r", + ); + end("g"); + const list = await events(); + expect(ofType(list, "tool_use")[0]!.tool_name).toBe("retriever:VectorStoreRetriever"); + expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: "kites" }); + expect(ofType(list, "tool_result")[0]!.output).toEqual({ n: 2, sources: ["a.txt", "doc[1]"] }); + }); + + it("streaming never emits per-token events; it folds into the model_response", async () => { + modelStart("m", undefined); + for (let i = 0; i < 5; i += 1) h.handleLLMNewToken!("x", {}, "m"); + modelEnd("m", msg("ai", "xxxxx")); + const list = await events(); + expect(list.map((e) => e.type)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + const response = ofType(list, "model_response")[0]!; + expect(response.fw_streamed).toBe(true); + expect(response.fw_chunks).toBe(5); + expect(Number.isInteger(response.fw_ttft_ms)).toBe(true); + }); + + it("records an intermediate chain only when includeChains names it", async () => { + adapter.uninstall(); + h = langchainHandler({ includeChains: ["summarise"] }) as Handler; + chain("p", undefined, "pipeline", { inputs: { input: "abc" } }); + chain("s1", "p", "RunnableLambda", { tags: ["seq:step:1"] }); + end("s1"); + chain("s2", "p", "summarise", { tags: ["seq:step:2"], inputs: { input: "abc" } }); + end("s2", { output: "ABC" }); + end("p"); + const list = await events(); + expect(shape(list)).toEqual([ + "pipeline agent_start", + "pipeline hook_triggered summarise", + "pipeline hook_completed summarise", + "pipeline agent_end", + ]); + expect(ofType(list, "hook_triggered")[0]!.trigger_event).toBe("pipeline"); + }); +}); + +describe("the node filter", () => { + const meta = { langgraph_node: "lookup" }; + + it("matches a node's own run", () => { + expect(nodeOf({ name: "lookup", runType: "chain", tags: ["graph:step:1"] }, meta)).toBe("lookup"); + }); + + it("a_node_named_after_its_tool_still_records_the_tool", () => { + expect(nodeOf({ name: "lookup", runType: "tool", tags: [] }, meta)).toBeNull(); + }); + + it("a_node_named_after_its_model_still_records_the_model", () => { + expect(nodeOf({ name: "lookup", runType: "chat_model", tags: [] }, meta)).toBeNull(); + }); + + it("an_inner_runnable_sharing_the_node_name_is_not_a_second_visit", () => { + expect(nodeOf({ name: "lookup", runType: "chain", tags: ["seq:step:2"] }, meta)).toBeNull(); + }); + + it("an inner runnable that merely inherits the metadata is not the node", () => { + expect(nodeOf({ name: "RunnableLambda", runType: "chain", tags: [] }, meta)).toBeNull(); + }); + + it("records the tool pair when a ToolNode is named after its tool", async () => { + root("g", "graph"); + node("n", "g", "get_weather", 1, { inputs: { messages: [callWeather()] } }); + // The tool's own run has the SAME name and inherits the node metadata. + h.handleToolStart!(TOOL, '{"city":"Paris"}', "t", "n", [], { langgraph_node: "get_weather" }, "get_weather", "call_1"); + toolEnd("t"); + end("n"); + end("g"); + expect(shape(await events())).toEqual([ + "graph agent_start", + "graph hook_triggered get_weather", + "graph tool_use get_weather", + "graph tool_result get_weather", + "graph hook_completed get_weather", + "graph agent_end", + ]); + }); +}); + +// -- failure ---------------------------------------------------------------- + +describe("failure", () => { + it("a_failed_model_call_is_reported_on_the_model_span_and_counted_once", async () => { + root("g", "weather_graph"); + node("n", "g", "agent", 1); + modelStart("m", "n"); + h.handleLLMError!(new Error("model exploded"), "m"); + fail("n", new Error("model exploded")); + fail("g", new Error("model exploded")); + const list = await events(); + expect(shape(list)).toEqual([ + "weather_graph agent_start", + "weather_graph hook_triggered agent", + "weather_graph model_request", + "weather_graph model_response", + "weather_graph hook_completed agent", + "weather_graph agent_end", + ]); + const response = ofType(list, "model_response")[0]!; + expect(response.error).toBe("Error: model exploded"); + expect(response.stop_reason).toBe("error"); + expect(ofType(list, "hook_completed")[0]!.outcome).toBe("failed"); + const endEvent = ofType(list, "agent_end")[0]!; + expect(endEvent.outcome).toBe("failed"); + expect(endEvent.summary).toBe("Error: model exploded"); + expect(ofType(list, "error")).toHaveLength(0); + }); + + it("a_failure_no_span_owns_produces_exactly_one_error_event", async () => { + // `prompt | model | parser` whose parser raised: no span owns it. + chain("p", undefined, "RunnableSequence", { inputs: { input: "x" } }); + chain("parser", "p", "StrOutputParser", { tags: ["seq:step:3"] }); + fail("parser", new TypeError("cannot parse")); + fail("p", new TypeError("cannot parse")); + const list = await events(); + expect(list.map((e) => e.type)).toEqual(["agent_start", "error", "agent_end"]); + const error = ofType(list, "error")[0]!; + expect(error.error_type).toBe("TypeError"); + // The bare message: the server prefixes the type itself. + expect(error.message).toBe("cannot parse"); + expect(ofType(list, "agent_end")[0]!.outcome).toBe("failed"); + }); + + it("a_failing_bare_tool_reports_its_error_once", async () => { + toolStart("t", undefined); + h.handleToolError!(new Error("no weather"), "t"); + const list = await events(); + expect(ofType(list, "tool_result")[0]!.error).toBe("Error: no weather"); + expect(ofType(list, "error")).toHaveLength(0); + expect(ofType(list, "agent_end")[0]!.outcome).toBe("failed"); + }); + + it("a_tool_that_fails_without_raising_is_still_an_error", async () => { + root("g", "graph"); + node("n", "g", "tools", 1, { inputs: { messages: [callWeather()] } }); + toolStart("t", "n", '{"city":"Paris"}', "call_1"); + toolEnd("t", msg("tool", "Error: no weather\n Please fix your mistakes.", { status: "error", tool_call_id: "call_1" })); + end("n"); + end("g"); + const result = ofType(await events(), "tool_result")[0]!; + expect(result.error).toBe("Error: no weather\n Please fix your mistakes."); + expect(result.output).toBe("Error: no weather\n Please fix your mistakes."); + }); + + it("a cancelled run ends cancelled, with no error event", async () => { + root("g", "graph"); + node("n", "g", "agent", 1); + const abort = new Error("This operation was aborted"); + abort.name = "AbortError"; + fail("n", abort); + fail("g", abort); + const list = await events(); + expect(ofType(list, "hook_completed")[0]!.outcome).toBe("cancelled"); + expect(ofType(list, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(ofType(list, "error")).toHaveLength(0); + }); + + it("closes a root LangGraph abandoned after an abort, once it has been silent", async () => { + vi.useFakeTimers(); + try { + root("g", "abort_graph"); + node("n", "g", "slow", 1); + const abort = new Error("This operation was aborted"); + abort.name = "AbortError"; + fail("n", abort); + // LangGraph.js 1.x: no end callback for the root, ever. + vi.advanceTimersByTime(ABANDONED_ROOT_GRACE_MS - 1); + expect(ofType(await events(), "agent_end")).toHaveLength(0); + vi.advanceTimersByTime(1); + const list = await events(); + expect(list.map((e) => e.type)).toEqual(["agent_start", "hook_triggered", "hook_completed", "agent_end"]); + expect(ofType(list, "agent_end")[0]!.outcome).toBe("cancelled"); + expect(ofType(list, "error")).toHaveLength(0); + } finally { + vi.useRealTimers(); + } + }); + + it("does not reap a root that carried on after a node's own abort (a retry)", async () => { + vi.useFakeTimers(); + try { + root("g", "graph"); + node("n1", "g", "fetch", 1); + const abort = new Error("timed out"); + abort.name = "AbortError"; + fail("n1", abort); + vi.advanceTimersByTime(500); + node("n2", "g", "fetch", 1); // the retry + vi.advanceTimersByTime(ABANDONED_ROOT_GRACE_MS * 2); + expect(ofType(await events(), "agent_end")).toHaveLength(0); + end("n2"); + end("g"); + const list = await events(); + expect(ofType(list, "agent_end").map((e) => e.outcome)).toEqual(["success"]); + } finally { + vi.useRealTimers(); + } + }); + + it("classifies control flow and cancellation", () => { + expect(isControlFlow(new GraphInterrupt([]))).toBe(true); + const named = new Error("x"); + named.name = "ParentCommand"; + expect(isControlFlow(named)).toBe(true); + expect(isControlFlow(new Error("x"))).toBe(false); + expect(isCancellation(new Error("Aborted"))).toBe(true); + expect(isCancellation(new Error("Abort"))).toBe(true); + expect(isCancellation(Object.assign(new Error("x"), { code: "ABORT_ERR" }))).toBe(true); + expect(isCancellation(new Error("x"))).toBe(false); + }); + + it("open leaves are closed when the root run ends without them", async () => { + root("g", "graph"); + node("n", "g", "tools", 1, { inputs: { messages: [callWeather()] } }); + toolStart("t", "n", '{"city":"Paris"}', "call_1"); + // The framework skipped both end callbacks. + end("g"); + const list = await events(); + const result = ofType(list, "tool_result")[0]!; + expect(result.fw_incomplete).toBe(true); + expect(ofType(list, "hook_completed")[0]!.outcome).toBe("cancelled"); + expect(list.at(-1)!.type).toBe("agent_end"); + }); +}); + +// -- human in the loop ------------------------------------------------------ + +function interruptRun(rootId = "r1", threadId = "t-1"): void { + root(rootId, "hitl_graph", { thread_id: threadId }, { trail: [] }); + node(`${rootId}-approve`, rootId, "approve", 2, { inputs: { trail: ["plan"] } }); + fail(`${rootId}-approve`, new GraphInterrupt([{ id: "int-1", value: { prompt: "ship it?", options: ["yes", "no"] } }])); + end(rootId, { trail: ["plan"] }); +} + +describe("human in the loop", () => { + it("interrupt_and_resume_emit_both_pairs_in_order", async () => { + interruptRun(); + root("r2", "hitl_graph", { thread_id: "t-1" }, command("yes")); + node("r2-approve", "r2", "approve", 2); + end("r2-approve", { trail: ["approve:yes"] }); + end("r2"); + const list = await events(); + expect(shape(list)).toEqual([ + "hitl_graph agent_start", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph human_wait", + "hitl_graph agent_pause", + "hitl_graph agent_resume", + "hitl_graph human_input", + "hitl_graph hook_triggered approve", + "hitl_graph hook_completed approve", + "hitl_graph agent_end", + ]); + expect(ofType(list, "hook_completed")[0]!.outcome).toBe("paused"); + const wait = ofType(list, "human_wait")[0]!; + expect([wait.input_id, wait.prompt, wait.options]).toEqual(["int-1", "ship it?", ["yes", "no"]]); + const answer = ofType(list, "human_input")[0]!; + expect([answer.input_id, answer.response, answer.fw_prompt]).toEqual(["int-1", "yes", "ship it?"]); + expect(ofType(list, "agent_end")[0]!.outcome).toBe("success"); + expect(new Set(list.map((e) => e.session_id))).toEqual(new Set(["t-1"])); + }); + + it("an_interrupt_is_control_flow_not_an_error", async () => { + interruptRun(); + const list = await events(); + expect(ofType(list, "error")).toHaveLength(0); + // Deliberately still open: the resume closes it. + expect(ofType(list, "agent_end")).toHaveLength(0); + }); + + it("the_two_interrupt_paths_do_not_double_emit", async () => { + root("r1", "hitl_graph", { thread_id: "t-1" }, { trail: [] }); + node("a", "r1", "approve", 2); + const interrupts = [{ id: "int-1", value: "ok?" }]; + fail("a", new GraphInterrupt(interrupts)); + // LangGraph.js >= 1 also delivers it through the lifecycle callback. + h.handleInterrupt!({ runId: "r1", status: "pending", checkpointNs: [], interrupts }); + end("r1"); + const list = await events(); + expect(ofType(list, "human_wait")).toHaveLength(1); + expect(ofType(list, "agent_pause")).toHaveLength(1); + }); + + it("interrupt events survive without the graph lifecycle callbacks", async () => { + adapter.uninstall(); + h = langchainHandler({ graphCallbacks: false }) as Handler; + expect(h[Symbol.for("langgraph.graph_callback_handler")]).toBe(false); + interruptRun(); + expect(ofType(await events(), "human_wait")).toHaveLength(1); + }); + + it("carries the LangGraph lifecycle marker by default", () => { + expect(h[Symbol.for("langgraph.graph_callback_handler")]).toBe(true); + }); + + it("an_unrelated_run_during_a_pause_is_not_read_as_the_approval", async () => { + interruptRun(); + // A different graph on the same thread, with FRESH state. + root("other", "other", { thread_id: "t-1" }, { vals: [] }); + end("other"); + const list = await events(); + expect(ofType(list, "agent_resume")).toHaveLength(0); + expect(ofType(list, "human_input")).toHaveLength(0); + expect(ofType(list, "agent_start").map((e) => e.agent_id)).toContain("other"); + }); + + it("a_none_input_is_still_a_continuation_of_the_pause", async () => { + interruptRun(); + root("r2", "hitl_graph", { thread_id: "t-1" }, { input: null }); + end("r2"); + const list = await events(); + expect(ofType(list, "agent_start")).toHaveLength(1); + expect(ofType(list, "agent_resume")).toHaveLength(1); + }); + + it("an_unrelated_runnable_invoked_with_none_is_not_the_humans_approval", async () => { + interruptRun(); + // `someRunnable.invoke(null)` inside the same session scope: the same + // `{input: null}` shape, but no graph metadata. + langchainHandler({ sessionId: "t-1" }); + chain("hb", undefined, "heartbeat", { inputs: { input: null } }); + end("hb"); + const list = await events(); + expect(ofType(list, "agent_resume")).toHaveLength(0); + }); + + it("closes a paused agent as cancelled at uninstall", async () => { + interruptRun(); + adapter.uninstall(); + const list = await events(); + expect(list.at(-1)!.type).toBe("agent_end"); + expect(list.at(-1)!.outcome).toBe("cancelled"); + }); + + it("drops the prompt and the answer under captureContent: false", async () => { + adapter.uninstall(); + h = langchainHandler({ captureContent: false }) as Handler; + interruptRun(); + root("r2", "hitl_graph", { thread_id: "t-1" }, command("yes")); + end("r2"); + const list = await events(); + expect(ofType(list, "human_wait")[0]!.prompt).toBeUndefined(); + expect(ofType(list, "human_wait")[0]!.options).toBeUndefined(); + expect(ofType(list, "human_input")[0]!.response).toBeUndefined(); + expect(ofType(list, "human_input")[0]!.fw_prompt).toBeUndefined(); + }); + + it("reads a prompt out of the interrupt payload", () => { + expect(promptOf({ question: "ok?", options: ["y", 2] })).toEqual({ prompt: "ok?", options: ["y", "2"] }); + expect(promptOf("plain")).toEqual({ prompt: "plain" }); + expect(promptOf({ record: 1 })).toEqual({ prompt: '{"record":1}', options: undefined }); + }); + + it("derives no cross-process pause id when LangGraph is not installed", () => { + // The remote path degrades to "no resume pair", never to a made-up id. + expect(interruptIdOf("approve:1234")).toBeNull(); + }); +}); + +// -- sessions --------------------------------------------------------------- + +describe("session resolution", () => { + it("an explicit sessionId option wins over everything", async () => { + adapter.uninstall(); + h = langchainHandler({ sessionId: "opt" }) as Handler; + weatherGraph({ meta: { failproofai_sdk_session_id: "meta", thread_id: "th" } }); + expect(new Set((await events()).map((e) => e.session_id))).toEqual(new Set(["opt"])); + }); + + it("session_id_prefers_the_documented_metadata_key_over_thread_id", async () => { + weatherGraph({ meta: { failproofai_sdk_session_id: "meta", thread_id: "th" } }); + expect(new Set((await events()).map((e) => e.session_id))).toEqual(new Set(["meta"])); + }); + + it("session_id_falls_back_to_thread_id", async () => { + weatherGraph({ meta: { thread_id: "th" } }); + expect(new Set((await events()).map((e) => e.session_id))).toEqual(new Set(["th"])); + }); + + it("session_id_falls_back_to_the_root_run_id", async () => { + weatherGraph(); + expect(new Set((await events()).map((e) => e.session_id))).toEqual(new Set(["g"])); + }); + + it("an_ambient_agent_scope_and_the_adapter_produce_one_tree", async () => { + await session({ sessionId: "req-1" }, () => + agent("planner", () => { + weatherGraph({ meta: { thread_id: "th" } }); + }), + ); + const list = await events(); + expect(new Set(list.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + const graphStart = ofType(list, "agent_start").find((e) => e.agent_id === "weather_graph")!; + expect(graphStart.parent_id).toBe("planner"); + }); + + it("a graph wrapped in an agent() of its own name joins it instead of nesting a copy", async () => { + // The wrap every tester reached for — to own the session id and print it. + // It used to give each run two agent_start/agent_end pairs, the graph's + // listing the wrapper (itself) as its parent. + await session({ sessionId: "req-2" }, () => + agent("weather_graph", () => { + weatherGraph({ meta: { thread_id: "th" } }); + }), + ); + const list = await events(); + expect(ofType(list, "agent_start").map((e) => [e.agent_id, e.parent_id ?? null])).toEqual([ + ["weather_graph", null], + ]); + expect(ofType(list, "agent_end")).toHaveLength(1); + expect(list[0]!.type).toBe("agent_start"); + expect(list.at(-1)!.type).toBe("agent_end"); + // Everything the graph recorded is still there, on the one agent. + expect(ofType(list, "tool_use").length).toBeGreaterThan(0); + expect(new Set(list.map((e) => e.agent_id))).toEqual(new Set(["weather_graph"])); + }); +}); + +// -- options and teardown --------------------------------------------------- + +describe("options", () => { + it("captureContent: false keeps structure, durations and tokens, and drops the rest", async () => { + adapter.uninstall(); + h = langchainHandler({ captureContent: false }) as Handler; + weatherGraph(); + const list = await events(); + expect(shape(list)).toEqual(GRAPH_SHAPE); + for (const event of list) { + for (const key of ["goal", "input", "output", "messages", "content"]) { + expect(event[key], `${String(event.type)}.${key}`).toBeUndefined(); + } + } + expect(ofType(list, "model_response").map((e) => e.output_tokens)).toEqual([5, 7]); + }); + + it("captureLimit bounds captured values", async () => { + adapter.uninstall(); + h = langchainHandler({ captureLimit: 20 }) as Handler; + toolStart("t", undefined); + toolEnd("t", "x".repeat(200)); + const output = ofType(await events(), "tool_result")[0]!.output as string; + expect(output.length).toBeLessThanOrEqual(20); + expect(output.endsWith(core.TRUNCATION_MARKER)).toBe(true); + }); + + it("says so, once, when a run's parent was never seen — the unawaited-instrument() trace", () => { + // `void instrument()` then `graph.invoke()`: the root starts before the + // callback exists, so its nodes arrive under a parent nobody saw and one + // of them became the session's agent with nothing said. + resetOrphanWarning(); + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + node("n1", "never-seen-root", "bump", 1); + node("n2", "never-seen-root", "bump", 2); + const orphan = warn.mock.calls.map((c) => String(c[0])).filter((m) => m.includes("never saw")); + expect(orphan).toHaveLength(1); + expect(orphan[0]).toContain("await failproofai.instrument()"); + }); + + it("says so when a graph node arrives as a root — the other shape of an unawaited instrument()", () => { + // Live: the graph's root started before the callback existed, so the + // handler it built never reached the nodes' parent chain, and each node + // arrived with no parent at all and became the session's agent. + resetOrphanWarning(); + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + chain("n1", undefined, "agent", { tags: ["graph:step:1"], meta: { langgraph_node: "agent", langgraph_step: 1 } }); + expect(warn.mock.calls.filter((c) => String(c[0]).includes("never saw"))).toHaveLength(1); + }); + + it("does not warn about orphans on an ordinary graph run", () => { + resetOrphanWarning(); + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + root("r", "support"); + node("n1", "r", "agent", 1); + end("n1"); + end("r"); + expect(warn.mock.calls.filter((c) => String(c[0]).includes("never saw"))).toEqual([]); + }); + + it("an unusable captureLimit falls back rather than throwing", () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + expect(captureLimitOf(undefined)).toBe(core.FIELD_LIMIT); + expect(captureLimitOf("4096")).toBe(4096); + for (const bad of [Infinity, Number.NaN, -1, 0, 1.5, "lots", {}]) { + expect(captureLimitOf(bad)).toBe(core.FIELD_LIMIT); + } + }); + + it("reads Python's option names in camelCase, and ignores other adapters' options", () => { + const options = readOptions({ + sessionId: "s", + includeChains: "one", + captureContent: false, + graphCallbacks: false, + steps: false, + }); + expect(options.sessionId).toBe("s"); + expect([...options.includeChains]).toEqual(["one"]); + expect(options.captureContent).toBe(false); + expect(options.graphCallbacks).toBe(false); + expect(readOptions({ includeChains: ["a", "b"] }).includeChains).toEqual(new Set(["a", "b"])); + }); + + it("uninstall stops recording, even through a handler already handed out", async () => { + adapter.uninstall(); + modelStart("m", undefined); + modelEnd("m"); + expect(await events()).toEqual([]); + }); + + it("asking for the handler again after uninstall records again", async () => { + adapter.uninstall(); + h = langchainHandler() as Handler; + modelStart("m", undefined); + modelEnd("m"); + expect((await events()).map((e) => e.type)).toEqual([ + "agent_start", + "model_request", + "model_response", + "agent_end", + ]); + }); + + it("follows strict mode for LangChain's own handler firewall", () => { + core.setStrict(false); + expect(h.raiseError).toBe(false); + core.setStrict(true); + expect(h.raiseError).toBe(true); + }); + + it("is awaited, so its events stay in the caller's async context and order", () => { + expect(h.awaitHandlers).toBe(true); + }); +}); + +describe("token counts", () => { + it("prefers usage_metadata, with its detail", () => { + expect( + usageOf({ + generations: [[{ message: { usage_metadata: { input_tokens: 3, output_tokens: 4, total_tokens: 7, input_token_details: { cache_read: 1 } } } }]], + }), + ).toEqual({ input_tokens: 3, output_tokens: 4, total_tokens: 7, input_token_details: { cache_read: 1 } }); + }); + + it("falls back to llmOutput in both spellings", () => { + expect(usageOf({ generations: [[{ text: "" }]], llmOutput: { tokenUsage: { promptTokens: 9, completionTokens: 8 } } })).toEqual({ + input_tokens: 9, + output_tokens: 8, + total_tokens: 17, + }); + expect(usageOf({ generations: [], llmOutput: { token_usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 } } })).toEqual({ + input_tokens: 1, + output_tokens: 2, + total_tokens: 3, + }); + expect(usageOf({ generations: [] })).toBeUndefined(); + }); +}); diff --git a/sdk/typescript/test/llamaindex.test.ts b/sdk/typescript/test/llamaindex.test.ts new file mode 100644 index 000000000..55d401527 --- /dev/null +++ b/sdk/typescript/test/llamaindex.test.ts @@ -0,0 +1,1760 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { performance } from "node:perf_hooks"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import * as compat from "../src/integrations/compat.js"; +import * as core from "../src/integrations/core.js"; +import { + adapter, + attach, + eventCallerStorage, + parseOptions, + summarizeNodes, + usageOf, + type AsyncContextModule, + type GlobalModule, + type RetrieverModule, + type WorkflowModule, +} from "../src/integrations/llamaindex.js"; +import { runtime } from "../src/runtime.js"; +import { agent as agentScope, session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The LlamaIndex.TS adapter's logic, against stand-ins for the two framework + * surfaces it attaches to: the callback bus and the workflow runtime's context + * middleware. `integration/llamaindex.test.ts` proves the same against real + * releases; this proves the cases a real scripted run cannot reach cheaply — + * a model call that throws, a stale leaf, a subscriber that is not there. + */ + +type Event = Record; + +/** `Settings.callbackManager`, dispatching synchronously with an EventCaller chain. */ +class FakeBus { + private readonly handlers = new Map void>>(); + on(event: string, handler: (event: unknown) => void): this { + this.handlers.set(event, [...(this.handlers.get(event) ?? []), handler]); + return this; + } + off(event: string, handler: (event: unknown) => void): this { + this.handlers.set( + event, + (this.handlers.get(event) ?? []).filter((h) => h !== handler), + ); + return this; + } + emit(event: string, detail: Event, callers: unknown[] = []): void { + for (const handler of this.handlers.get(event) ?? []) { + handler({ detail, reason: { computedCallers: callers } }); + } + } + /** Dispatch with a real-shaped `EventCaller` as `reason` (see `EventCaller` below). */ + emitReason(event: string, detail: Event, reason: unknown): void { + for (const handler of this.handlers.get(event) ?? []) { + handler({ detail, reason }); + } + } + count(): number { + return [...this.handlers.values()].reduce((sum, list) => sum + list.length, 0); + } +} + +function subscribable() { + const subs = new Set<(...args: never[]) => unknown>(); + return { + subs, + subscribe(callback: (...args: never[]) => unknown) { + subs.add(callback); + return () => subs.delete(callback); + }, + }; +} + +/** A workflow-core context, reduced to the two middleware hooks and a step runner. */ +function fakeContext() { + const callContext = subscribable(); + const sendEvent = subscribable(); + return { + __internal__call_context: callContext, + __internal__call_send_event: sendEvent, + step(handler: (...args: unknown[]) => unknown, input: unknown): unknown { + const cbs = [...callContext.subs] as Array<(c: unknown, next: (c: unknown) => void) => void>; + let i = 0; + let result: unknown; + const next = (context: unknown): void => { + if (i === cbs.length) { + const c = context as { handler: (...a: unknown[]) => unknown; inputs: unknown[] }; + result = c.handler(this, ...c.inputs); + return; + } + cbs[i++]!(context, next); + }; + next({ handler, inputs: [input] }); + return result; + }, + send(event: unknown): void { + for (const sub of sendEvent.subs) (sub as (e: unknown, h: unknown) => void)(event, {}); + }, + }; +} + +type Ctx = ReturnType; +const ev = (kind: string, data: Event = {}) => ({ kind, data }); +const is = (kind: string) => ({ include: (event: unknown) => (event as { kind?: unknown })?.kind === kind }); + +/** An `AgentWorkflow`: step handlers are instance arrow fields, as in the real one. */ +class AgentWorkflow { + workflow: { createContext: () => Ctx }; + agents: Map; + rootAgentName: string; + done: Promise = Promise.resolve(); + input: unknown; + program: (ctx: Ctx, self: AgentWorkflow) => Promise; + handleInputStep: (ctx: unknown, event: unknown) => Promise = async () => {}; + runAgentStep: (ctx: unknown, event: unknown) => Promise = async () => {}; + executeToolCalls: (ctx: unknown, event: unknown) => Promise = async () => {}; + + constructor(names: string[], program: AgentWorkflow["program"]) { + this.workflow = { createContext: () => fakeContext() }; + this.agents = new Map(names.map((name) => [name, { llm: { metadata: { model: `${name}-model` } } }])); + this.rootAgentName = names[0]!; + this.program = program; + } + + runStream(input: unknown): string { + this.input = input; + const ctx = this.workflow.createContext(); + this.done = this.program(ctx, this); + return "stream"; + } +} + +const workflowModule = (): WorkflowModule => ({ + AgentWorkflow, + stopAgentEvent: is("stop"), + agentToolCallEvent: is("toolCall"), + agentToolCallResultEvent: is("toolResult"), +}); + +let spool: Spool; +let bus: FakeBus; + +function install(options: Record = {}, workflows: WorkflowModule[] = [workflowModule()]) { + bus = new FakeBus(); + return attach({ reaperInterval: 0, ...options }, { + globals: [{ Settings: { callbackManager: bus } }], + workflows, + }); +} + +const shape = (events: Event[]): string[] => + events.map((e) => [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim()); + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); + core.setStrict(true); +}); +afterEach(async () => { + adapter.uninstall(); + core.setStrict(null); + compat.setStrictIntegrations(null); + compat.resetWarnings(); + await spool.cleanup(); + setLogger(null); +}); + +describe("usage extraction", () => { + it("finds a streamed call's usage on the chunk that carries it", () => { + // `wrapLLMEvent` hands `llm-end` the ARRAY of chunks as `raw`; OpenAI puts + // the usage on the last, content-less one. + const usage = usageOf({ + raw: [{ delta: "hi", raw: { choices: [] } }, { delta: "", raw: { usage: { prompt_tokens: 7, completion_tokens: 3 } } }], + }); + expect(usage).toEqual({ usage: { prompt_tokens: 7, completion_tokens: 3 }, inputTokens: 7, outputTokens: 3 }); + }); + + it("reads a non-streamed response and the camelCase spellings", () => { + expect(usageOf({ raw: { usage: { inputTokens: 4, outputTokens: 2 } } })).toMatchObject({ inputTokens: 4, outputTokens: 2 }); + expect(usageOf({ raw: { usageMetadata: { promptTokenCount: 9, candidatesTokenCount: 1 } } })).toMatchObject({ + inputTokens: 9, + outputTokens: 1, + }); + }); + + it("ships an unrecognised usage object without inventing token counts", () => { + expect(usageOf({ raw: { usage: { units: 12 } } })).toEqual({ usage: { units: 12 }, inputTokens: undefined, outputTokens: undefined }); + expect(usageOf({ raw: null })).toEqual({}); + expect(usageOf(undefined)).toEqual({}); + }); +}); + +describe("summarizeNodes", () => { + it("keeps the count, the scores and a prefix of the top few", () => { + const nodes = Array.from({ length: 7 }, (_, i) => ({ + node: { id_: `n${i}`, getContent: () => "x".repeat(500) }, + score: i / 10, + })); + const summary = summarizeNodes(nodes); + expect(summary.num_nodes).toBe(7); + expect(summary.top).toHaveLength(5); + expect(summary.top[0]).toMatchObject({ id: "n0", score: 0 }); + expect(String(summary.top[0]!.text).length).toBeLessThanOrEqual(200); + expect(summarizeNodes(undefined)).toEqual({ num_nodes: 0, top: [] }); + }); +}); + +describe("options", () => { + it("mirrors the Python adapter's options, in camelCase", () => { + expect(parseOptions({})).toEqual({ + captureMessages: true, + steps: true, + embeddings: false, + staleAfter: 600, + reaperInterval: 30, + captureLimit: undefined, + }); + expect(parseOptions({ captureMessages: false, steps: false, embeddings: true, staleAfter: 5, reaperInterval: 0 })).toMatchObject({ + captureMessages: false, + steps: false, + embeddings: true, + staleAfter: 5, + reaperInterval: 0, + }); + }); +}); + +describe("the callback bus", () => { + class LLMAgent { + llm = { metadata: { model: "legacy-model" } }; + } + + it("records a legacy task as ONE agent across every step, not one per step", async () => { + install(); + const runner = new LLMAgent(); + const step1 = { id: "s1", prevStep: null, context: { store: { messages: [{ content: "weather?" }] } } }; + const step2 = { id: "s2", prevStep: step1, context: {} }; + // `agent-start` fires for EVERY step, `agent-end` only after the last. + bus.emit("agent-start", { startStep: step1 }, [runner]); + bus.emit("llm-start", { id: "m1", messages: [] }, [runner]); + bus.emit("llm-end", { id: "m1", response: { message: { content: "" }, raw: { usage: { prompt_tokens: 1, completion_tokens: 1 } } } }, [runner]); + bus.emit("llm-tool-call", { toolCall: { id: "t1", name: "get_weather", input: { city: "Rome" } } }, [runner]); + bus.emit("agent-start", { startStep: step2 }, [runner]); + // `callTool` dispatched no `llm-tool-result` — the tool threw. The next model + // call carries its result, with `isError`. + bus.emit( + "llm-start", + { id: "m2", messages: [{ role: "user", content: "x", options: { toolResult: { id: "t1", result: "Error: boom", isError: true } } }] }, + [runner], + ); + bus.emit("llm-end", { id: "m2", response: { message: { content: "It is sunny." } } }, [runner]); + bus.emit("agent-end", { endStep: step2 }, [runner]); + + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "LLMAgent agent_start", + "LLMAgent model_request", + "LLMAgent model_response", + "LLMAgent tool_use get_weather", + "LLMAgent tool_result get_weather", + "LLMAgent model_request", + "LLMAgent model_response", + "LLMAgent agent_end", + ]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + expect(events.find((e) => e.type === "tool_result")!.error).toBe("Error: boom"); + expect(events.find((e) => e.type === "model_request")!.model).toBe("legacy-model"); + const end = events.find((e) => e.type === "agent_end")!; + expect(end.outcome).toBe("success"); + expect(end.summary).toBe("It is sunny."); + expect(events[0]!.goal).toBe("weather?"); + }); + + it("defers a run's end until its streamed model call has been consumed", async () => { + install(); + const runner = new LLMAgent(); + const step = { id: "s1", prevStep: null }; + bus.emit("agent-start", { startStep: step }, [runner]); + bus.emit("llm-start", { id: "m1", messages: [] }, [runner]); + bus.emit("llm-stream", { id: "m1", chunk: { delta: "a" } }, [runner]); + // The last step returns a stream: `agent-end` fires before anyone reads it. + bus.emit("agent-end", { endStep: step }, [runner]); + bus.emit( + "llm-end", + { id: "m1", response: { message: { content: "a" }, raw: [{ raw: {} }, { raw: { usage: { input_tokens: 5, output_tokens: 2 } } }] } }, + [runner], + ); + const events = await flushed(spool); + expect(shape(events)).toEqual(["LLMAgent agent_start", "LLMAgent model_request", "LLMAgent model_response", "LLMAgent agent_end"]); + const response = events.find((e) => e.type === "model_response")!; + expect([response.input_tokens, response.output_tokens]).toEqual([5, 2]); + expect(response.fw_chunks).toBe(2); + expect(typeof response.fw_ttft_ms).toBe("number"); + expect(typeof response.duration_ms).toBe("number"); + }); + + it("makes a bare model call its own run, named after the model's class", async () => { + install(); + class OpenAI { + metadata = { model: "gpt-x" }; + } + const llm = new OpenAI(); + bus.emit("llm-start", { id: "m1", messages: [{ role: "user", content: "hi" }] }, [llm]); + bus.emit("llm-end", { id: "m1", response: { message: { content: "hello" } } }, [llm]); + const events = await flushed(spool); + expect(shape(events)).toEqual(["OpenAI agent_start", "OpenAI model_request", "OpenAI model_response", "OpenAI agent_end"]); + expect(events.find((e) => e.type === "model_request")!.model).toBe("gpt-x"); + expect(events[1]!.request_id).toBe("m1"); + expect(events[2]!.request_id).toBe("m1"); + }); + + it("joins an enclosing session() scope instead of inventing one", async () => { + install(); + await session({ sessionId: "req-9" }, () => { + bus.emit("llm-start", { id: "m1", messages: [] }); + bus.emit("llm-end", { id: "m1", response: {} }); + }); + const events = await flushed(spool); + expect(shape(events)).toEqual(["llm agent_start", "llm model_request", "llm model_response", "llm agent_end"]); + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["req-9"])); + }); + + it("records a top-level query engine call as a run, with its retrieval inside it", async () => { + install(); + class RetrieverQueryEngine {} + const engine = new RetrieverQueryEngine(); + bus.emit("query-start", { id: "q1", query: "what?" }, [engine]); + bus.emit("retrieve-start", { id: "r1", query: { query: "what?" } }, [engine]); + bus.emit("retrieve-end", { id: "r1", nodes: [{ node: { id_: "a", text: "alpha" }, score: 0.9 }] }, [engine]); + bus.emit("synthesize-start", { id: "y1" }, [engine]); + bus.emit("query-end", { id: "q1", response: { message: { content: "answer" } } }, [engine]); + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "RetrieverQueryEngine agent_start", + "RetrieverQueryEngine tool_use retriever", + "RetrieverQueryEngine tool_result retriever", + "RetrieverQueryEngine agent_end", + ]); + expect(events[2]!.output).toEqual({ num_nodes: 1, top: [{ id: "a", score: 0.9, text: "alpha" }] }); + expect(events[3]!.summary).toBe("answer"); + }); + + it("suffixes a repeated tool call id within one run rather than pairing it wrongly", async () => { + install(); + const runner = new LLMAgent(); + bus.emit("agent-start", { startStep: { id: "s1" } }, [runner]); + for (let i = 0; i < 2; i += 1) { + bus.emit("llm-tool-call", { toolCall: { id: "c1", name: "t", input: {} } }, [runner]); + bus.emit("llm-tool-result", { toolCall: { id: "c1" }, toolResult: { output: i, isError: false } }, [runner]); + } + bus.emit("agent-end", { endStep: { id: "s1" } }, [runner]); + const events = await flushed(spool); + expect(events.filter((e) => e.type === "tool_use").map((e) => e.tool_call_id)).toEqual(["c1", "c1#1"]); + expect(events.filter((e) => e.type === "tool_result").map((e) => e.tool_call_id)).toEqual(["c1", "c1#1"]); + }); + + it("drops every payload under captureMessages: false, keeping structure and tokens", async () => { + install({ captureMessages: false }); + bus.emit("llm-start", { id: "m1", messages: [{ role: "user", content: "secret" }] }); + bus.emit("llm-end", { id: "m1", response: { message: { content: "secret" }, raw: { usage: { prompt_tokens: 2, completion_tokens: 1 } } } }); + const events = await flushed(spool); + expect(JSON.stringify(events)).not.toContain("secret"); + expect(events.find((e) => e.type === "model_response")!.input_tokens).toBe(2); + }); + + it("closes a leaf nobody will close once it is stale, and ends the bare run it opened", async () => { + // A model call that throws has no `llm-end`: wrapLLMEvent has no error path. + const handle = install({ staleAfter: 60 }); + bus.emit("llm-start", { id: "m1", messages: [] }); + expect(handle.sweep(performance.now())).toBe(0); + expect(handle.sweep(performance.now() + 61_000)).toBe(1); + const events = await flushed(spool); + expect(shape(events)).toEqual(["llm agent_start", "llm model_request", "llm model_response", "llm agent_end"]); + expect(events[2]!.fw_closed_by).toBe("stale"); + // We never learned how that call ended; it is not a success. + expect(events[3]!.outcome).toBe("cancelled"); + }); + + it("ends a legacy task that will never send agent-end once it has been silent too long", async () => { + // A legacy step that throws dispatches nothing: no `llm-end`, no `agent-end`. + const handle = install({ staleAfter: 60 }); + const runner = new LLMAgent(); + bus.emit("agent-start", { startStep: { id: "s1" } }, [runner]); + expect(handle.sweep(performance.now() + 30_000)).toBe(0); + expect(handle.sweep(performance.now() + 61_000)).toBe(1); + const events = await flushed(spool); + expect(shape(events)).toEqual(["LLMAgent agent_start", "LLMAgent agent_end"]); + expect(events[1]).toMatchObject({ outcome: "cancelled", fw_run_id: expect.any(String) as unknown }); + // And a later event for that task starts nothing stale: it is forgotten. + bus.emit("agent-end", { endStep: { id: "s1" } }, [runner]); + expect(await flushed(spool)).toHaveLength(2); + }); + + it("makes a bare tool call its own run, named after the tool", async () => { + install(); + bus.emit("llm-tool-call", { toolCall: { id: "c1", name: "get_weather", input: { city: "Rome" } } }); + bus.emit("llm-tool-result", { toolCall: { id: "c1" }, toolResult: { output: "sunny", isError: false } }); + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "get_weather agent_start", + "get_weather tool_use get_weather", + "get_weather tool_result get_weather", + "get_weather agent_end", + ]); + expect(events[2]!.output).toBe("sunny"); + expect(events[3]!.outcome).toBe("success"); + }); + + it("detaches from the bus on uninstall and closes what is open as cancelled", async () => { + install(); + bus.emit("llm-start", { id: "m1", messages: [] }); + expect(bus.count()).toBeGreaterThan(0); + adapter.uninstall(); + expect(bus.count()).toBe(0); + const events = await flushed(spool); + expect(shape(events)).toEqual(["llm agent_start", "llm model_request", "llm model_response", "llm agent_end"]); + expect(events[3]!.outcome).toBe("cancelled"); + }); +}); + +describe("the workflow runtime", () => { + const model = (id: string, content = "", callers: unknown[] = []) => { + bus.emit("llm-start", { id, messages: [] }, callers); + bus.emit("llm-end", { id, response: { message: { content }, raw: { usage: { prompt_tokens: 3, completion_tokens: 1 } } } }, callers); + }; + + it("records a run as the agent, its steps as hooks, and closes it on the stop event", async () => { + install(); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + await ctx.step(self.handleInputStep, ev("start", { userInput: "q" })); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent" })); + await ctx.step(self.executeToolCalls, ev("toolCalls", { agentName: "Agent" })); + ctx.send(ev("stop", { result: "done", state: { memory: "huge" } })); + }); + wf.runAgentStep = async () => { + model("m1"); + }; + wf.executeToolCalls = async () => { + // The runtime announces the call, THEN `callTool` dispatches on the bus + // too (workflow ≥1.1.2x): one call, recorded once. + current!.send(ev("toolCall", { toolName: "get_weather", toolId: "call_1", toolKwargs: { city: "Paris" } })); + bus.emit("llm-tool-call", { toolCall: { id: "call_1", name: "get_weather", input: { city: "Paris" } } }); + bus.emit("llm-tool-result", { toolCall: { id: "call_1" }, toolResult: { output: "sunny", isError: false } }); + current!.send(ev("toolResult", { toolId: "call_1", toolOutput: { result: "sunny", isError: false }, raw: "sunny" })); + }; + let current: Ctx | null = null; + const create = wf.workflow.createContext; + wf.workflow.createContext = () => (current = create()); + expect(wf.runStream("q")).toBe("stream"); + await wf.done; + + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "Agent agent_start", + "Agent hook_triggered handleInputStep", + "Agent hook_completed handleInputStep", + "Agent hook_triggered runAgentStep", + "Agent model_request", + "Agent model_response", + "Agent hook_completed runAgentStep", + "Agent hook_triggered executeToolCalls", + "Agent tool_use get_weather", + "Agent tool_result get_weather", + "Agent hook_completed executeToolCalls", + "Agent agent_end", + ]); + expect(events.find((e) => e.type === "hook_triggered")!.trigger_event).toBe("workflow_step"); + // No `llm-start` caller chain here: the model comes from the agent the step names. + expect(events.find((e) => e.type === "model_request")!.model).toBe("Agent-model"); + expect(events.find((e) => e.type === "tool_result")!.output).toBe("sunny"); + expect(events.at(-1)!.summary).toBe("done"); + expect(events[0]!.goal).toBe("q"); + }); + + it("opens a nested agent per agent holding the turn, closing it on handoff", async () => { + install(); + const wf = new AgentWorkflow(["triage", "forecaster"], async (ctx, self) => { + await ctx.step(self.handleInputStep, ev("start")); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "triage" })); + // A tool step names `agentName`, not `currentAgentName`; a step naming + // neither keeps whoever holds the turn. + await ctx.step(self.executeToolCalls, ev("toolCalls", { agentName: "triage" })); + await ctx.step(self.handleInputStep, ev("other")); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "forecaster" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.runAgentStep = async () => { + model(`m${Math.random()}`); + }; + wf.runStream("q"); + await wf.done; + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "AgentWorkflow agent_start", + "AgentWorkflow hook_triggered handleInputStep", + "AgentWorkflow hook_completed handleInputStep", + "triage agent_start", + "triage hook_triggered runAgentStep", + "triage model_request", + "triage model_response", + "triage hook_completed runAgentStep", + "triage hook_triggered executeToolCalls", + "triage hook_completed executeToolCalls", + "triage hook_triggered handleInputStep", + "triage hook_completed handleInputStep", + "triage agent_end", + "forecaster agent_start", + "forecaster hook_triggered runAgentStep", + "forecaster model_request", + "forecaster model_response", + "forecaster hook_completed runAgentStep", + "forecaster agent_end", + "AgentWorkflow agent_end", + ]); + for (const start of events.filter((e) => e.type === "agent_start").slice(1)) { + expect(start.parent_id).toBe("AgentWorkflow"); + } + }); + + it("fails the step, its open model call and the run when a step throws", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + install(); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + await Promise.resolve(ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent" }))).catch(() => undefined); + }); + wf.runAgentStep = async () => { + // A provider that throws: `llm-start` went out, `llm-end` never will. + bus.emit("llm-start", { id: "m1", messages: [] }); + throw new Error("model exploded"); + }; + wf.runStream("q"); + await wf.done; + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "Agent agent_start", + "Agent hook_triggered runAgentStep", + "Agent model_request", + "Agent model_response", + "Agent hook_completed runAgentStep", + "Agent agent_end", + ]); + expect(events[3]!.error).toMatch(/model exploded/); + expect(events[4]!.outcome).toBe("failed"); + expect(events[5]!.outcome).toBe("failed"); + expect(events.filter((e) => e.type === "error")).toEqual([]); + }); + + it("reports a step failure no hook or leaf carries as ONE error event (steps: false)", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + install({ steps: false }); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + await Promise.resolve(ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent" }))).catch(() => undefined); + }); + wf.runAgentStep = async () => { + throw new TypeError("bad state"); + }; + wf.runStream("q"); + await wf.done; + const events = await flushed(spool); + expect(shape(events)).toEqual(["Agent agent_start", "Agent error", "Agent agent_end"]); + expect(events[1]).toMatchObject({ error_type: "TypeError", message: "bad state" }); + expect(events[2]).toMatchObject({ outcome: "failed", summary: "TypeError: bad state" }); + }); + + it("reports a runStream() that throws before any step ran", async () => { + install(); + const wf = new AgentWorkflow(["Agent"], async () => {}); + wf.workflow.createContext = () => { + throw new Error("No agents added to workflow"); + }; + expect(() => wf.runStream("q")).toThrow("No agents added to workflow"); + const events = await flushed(spool); + expect(shape(events)).toEqual(["Agent agent_start", "Agent error", "Agent agent_end"]); + expect(events[2]!.outcome).toBe("failed"); + }); + + it("records a tool that threw from the runtime's own tool-result event", async () => { + install(); + let current: Ctx | null = null; + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + current = ctx; + await ctx.step(self.executeToolCalls, ev("toolCalls", { agentName: "Agent" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.executeToolCalls = async () => { + current!.send(ev("toolCall", { toolName: "broken", toolId: "c1", toolKwargs: {} })); + // `callTool` catches the throw and dispatches no `llm-tool-result`. + current!.send(ev("toolResult", { toolId: "c1", toolOutput: { result: "Error: Error: Error: tool exploded", isError: true } })); + }; + wf.runStream("q"); + await wf.done; + const events = await flushed(spool); + const result = events.find((e) => e.type === "tool_result")!; + expect(result.error).toBe("Error: tool exploded"); + expect(result.tool_call_id).toBe("c1"); + expect(events.at(-1)!.outcome).toBe("success"); + }); + + it.each([ + // llamaindex 0.12's prettifyError writes `Error(): `, and the + // workflow prefixes `Error: ` again — recorded verbatim, it read + // "Error: Error(Error): unknown region: latam". + ["Error: Error(Error): unknown region: latam", "Error: unknown region: latam"], + ["Error: Error(TypeError): bad input", "TypeError: bad input"], + ["Error: Error: Error(RangeError): out of range", "RangeError: out of range"], + ])("records %j from a failed tool as %j", async (raw, expected) => { + install(); + let current: Ctx | null = null; + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + current = ctx; + await ctx.step(self.executeToolCalls, ev("toolCalls", { agentName: "Agent" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.executeToolCalls = async () => { + current!.send(ev("toolCall", { toolName: "broken", toolId: "c1", toolKwargs: {} })); + current!.send(ev("toolResult", { toolId: "c1", toolOutput: { result: raw, isError: true } })); + }; + wf.runStream("q"); + await wf.done; + const events = await flushed(spool); + expect(events.find((e) => e.type === "tool_result")!.error).toBe(expected); + }); + + it("keeps correlating under steps: false, emitting no hooks", async () => { + install({ steps: false }); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.runAgentStep = async () => { + model("m1"); + }; + wf.runStream("q"); + await wf.done; + expect(shape(await flushed(spool))).toEqual([ + "Agent agent_start", + "Agent model_request", + "Agent model_response", + "Agent agent_end", + ]); + }); + + it("records the run but not its steps when the context has no call-context hook", async () => { + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + compat.setStrictIntegrations(false); + install(); + const wf = new AgentWorkflow(["Agent"], async (ctx) => { + ctx.send(ev("stop", { result: "ok" })); + }); + wf.workflow.createContext = () => { + const ctx = fakeContext(); + delete (ctx as Partial).__internal__call_context; + return ctx; + }; + wf.runStream("q"); + await wf.done; + expect(shape(await flushed(spool))).toEqual(["Agent agent_start", "Agent agent_end"]); + }); + + it("restores the prototype and cancels an open run on uninstall", async () => { + const proto = AgentWorkflow.prototype as unknown as Record; + const original = proto.runStream; + install(); + expect(proto.runStream).not.toBe(original); + const wf = new AgentWorkflow(["Agent"], async () => {}); + wf.runStream("q"); + adapter.uninstall(); + expect(proto.runStream).toBe(original); + const events = await flushed(spool); + expect(shape(events)).toEqual(["Agent agent_start", "Agent agent_end"]); + expect(events[1]!.outcome).toBe("cancelled"); + }); + + it("records nothing from a context that outlives uninstall()", async () => { + install(); + let release!: () => void; + const gate = new Promise((resolve) => (release = resolve)); + let current: Ctx | null = null; + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + current = ctx; + await gate; + ctx.send(ev("toolCall", { toolName: "late", toolId: "c9", toolKwargs: {} })); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.runAgentStep = async () => { + model("m-late"); + }; + wf.runStream("q"); + adapter.uninstall(); + release(); + await wf.done; + expect(current).not.toBeNull(); + expect(shape(await flushed(spool))).toEqual(["Agent agent_start", "Agent agent_end"]); + }); + + it("leaves a workflow whose context it cannot reach unrecorded as a run, and says so", async () => { + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + compat.setStrictIntegrations(false); + install(); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + await ctx.step(self.runAgentStep, ev("setup")); + }); + // A `createContext` on a prototype, not an own writable property. + const create = wf.workflow.createContext; + wf.workflow = Object.create({ createContext: create }) as AgentWorkflow["workflow"]; + wf.runAgentStep = async () => { + model("m1"); + }; + wf.runStream("q"); + await wf.done; + // No run that would never end — the model call is recorded as its own run. + expect(shape(await flushed(spool))).toEqual(["llm agent_start", "llm model_request", "llm model_response", "llm agent_end"]); + expect(warn).toHaveBeenCalled(); + }); + + it("never changes what runStream returns or throws", () => { + install(); + const wf = new AgentWorkflow(["Agent"], async () => {}); + const failure = new Error("No agents added to workflow"); + wf.workflow.createContext = () => { + throw failure; + }; + expect(() => wf.runStream("q")).toThrow(failure); + // The createContext override is removed again, whatever happened. + expect(Object.getOwnPropertyDescriptor(wf.workflow, "createContext")!.value).toBeTypeOf("function"); + }); +}); + +/** + * LlamaIndex's own `EventCaller`, as `@llamaindex/core/global` builds it: a + * FRESH object per `@wrapEventCaller` invocation, chained to the invocation it + * ran inside. Two concurrent `engine.query()` calls share `caller` (the engine) + * and nothing else. + */ +class EventCaller { + constructor( + readonly caller: unknown, + readonly parent: EventCaller | null = null, + ) {} + get computedCallers(): unknown[] { + const callers: unknown[] = []; + // eslint-disable-next-line @typescript-eslint/no-this-alias -- walking the chain from here + for (let node: EventCaller | null = this; node !== null; node = node.parent) callers.push(node.caller); + return callers; + } +} + +/** Dispatch the way the real bus does: `reason` is the EventCaller bound at dispatch. */ +function emitIn(event: string, detail: Event, reason: EventCaller | null): void { + bus.emitReason(event, detail, reason); +} + +const bySession = (events: Event[]): Map => { + const out = new Map(); + for (const e of events) out.set(e.session_id, [...(out.get(e.session_id) ?? []), shape([e])[0]!]); + return out; +}; + +class LLMAgentStandIn { + llm = { metadata: { model: "legacy-model" } }; +} +class OpenAIStandIn { + metadata = { model: "gpt-x" }; +} +class RetrieverQueryEngineStandIn {} + +describe("concurrent runs on ONE shared object", () => { + const answer = (text: string) => ({ message: { content: text }, raw: { usage: { prompt_tokens: 1, completion_tokens: 1 } } }); + + it("keeps two interleaved queries on one engine in two sessions", async () => { + install(); + const engine = new RetrieverQueryEngineStandIn(); + const llm = new OpenAIStandIn(); + // One engine, built once, serving two requests at the same time. + const a = new EventCaller(engine); + const b = new EventCaller(engine); + emitIn("query-start", { id: "qA", query: "question A" }, a); + emitIn("query-start", { id: "qB", query: "question B" }, b); + emitIn("retrieve-start", { id: "rB", query: "retrieval B" }, b); + emitIn("retrieve-start", { id: "rA", query: "retrieval A" }, a); + emitIn("retrieve-end", { id: "rA", nodes: [] }, a); + emitIn("retrieve-end", { id: "rB", nodes: [{ node: { id_: "b" } }] }, b); + emitIn("llm-start", { id: "mB", messages: [{ role: "user", content: "B" }] }, new EventCaller(llm, b)); + emitIn("llm-start", { id: "mA", messages: [{ role: "user", content: "A" }] }, new EventCaller(llm, a)); + emitIn("llm-end", { id: "mA", response: answer("answer A") }, new EventCaller(llm, a)); + emitIn("llm-end", { id: "mB", response: answer("answer B") }, new EventCaller(llm, b)); + emitIn("query-end", { id: "qB", response: { response: "answer B" } }, b); + emitIn("query-end", { id: "qA", response: { response: "answer A" } }, a); + + const events = await flushed(spool); + const sessions = bySession(events); + expect(sessions.size).toBe(2); + const run = [ + "RetrieverQueryEngineStandIn agent_start", + "RetrieverQueryEngineStandIn tool_use retriever", + "RetrieverQueryEngineStandIn tool_result retriever", + "RetrieverQueryEngineStandIn model_request", + "RetrieverQueryEngineStandIn model_response", + "RetrieverQueryEngineStandIn agent_end", + ]; + for (const shapeOf of sessions.values()) expect(shapeOf).toEqual(run); + const sessionOf = (pred: (e: Event) => boolean) => events.find(pred)!.session_id; + const sa = sessionOf((e) => e.type === "agent_start" && e.goal === "question A"); + const sb = sessionOf((e) => e.type === "agent_start" && e.goal === "question B"); + expect(sa).not.toBe(sb); + expect(sessionOf((e) => e.type === "tool_use" && JSON.stringify(e.input).includes("retrieval A"))).toBe(sa); + expect(sessionOf((e) => e.type === "tool_use" && JSON.stringify(e.input).includes("retrieval B"))).toBe(sb); + expect(sessionOf((e) => e.type === "model_response" && e.content === "answer A")).toBe(sa); + expect(sessionOf((e) => e.type === "model_response" && e.content === "answer B")).toBe(sb); + expect(sessionOf((e) => e.type === "agent_end" && e.summary === "answer A")).toBe(sa); + expect(sessionOf((e) => e.type === "agent_end" && e.summary === "answer B")).toBe(sb); + for (const e of events.filter((x) => x.type === "agent_start")) expect(e.parent_id ?? null).toBeNull(); + }); + + it("does not nest a second query under the first when the bus gives only the caller list", async () => { + // The reviewer's repro: no EventCaller chain, only `computedCallers`. A run + // that the SAME object owns cannot be told apart from a concurrent sibling + // there, and a sibling is the normal case — so it is never taken as the parent. + install(); + const engine = new RetrieverQueryEngineStandIn(); + bus.emit("query-start", { id: "qA", query: "question from user A" }, [engine]); + bus.emit("query-start", { id: "qB", query: "question from user B" }, [engine]); + bus.emit("query-end", { id: "qB", response: "answer B" }, [engine]); + bus.emit("query-end", { id: "qA", response: "answer A" }, [engine]); + const events = await flushed(spool); + const starts = events.filter((e) => e.type === "agent_start"); + expect(starts.map((e) => e.goal)).toEqual(["question from user A", "question from user B"]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(2); + const ends = events.filter((e) => e.type === "agent_end"); + expect(ends.map((e) => e.summary)).toEqual(["answer B", "answer A"]); + expect(ends[0]!.session_id).toBe(starts[1]!.session_id); + expect(ends[1]!.session_id).toBe(starts[0]!.session_id); + }); + + it("keeps two interleaved legacy chats on one LLMAgent in two sessions", async () => { + install(); + const runner = new LLMAgentStandIn(); + const llm = new OpenAIStandIn(); + const a = new EventCaller(runner); + const b = new EventCaller(runner); + const stepA = { id: "a1", prevStep: null, context: { store: { messages: [{ content: "goal A" }] } } }; + const stepB = { id: "b1", prevStep: null, context: { store: { messages: [{ content: "goal B" }] } } }; + emitIn("agent-start", { startStep: stepA }, a); + emitIn("agent-start", { startStep: stepB }, b); + // B is the newest run on the runner; A's calls must still go to A. + emitIn("llm-start", { id: "mA", messages: [] }, new EventCaller(llm, a)); + emitIn("llm-end", { id: "mA", response: answer("") }, new EventCaller(llm, a)); + emitIn("llm-tool-call", { toolCall: { id: "tA", name: "tool_a", input: {} } }, a); + emitIn("llm-start", { id: "mB", messages: [] }, new EventCaller(llm, b)); + emitIn("llm-tool-result", { toolCall: { id: "tA" }, toolResult: { output: "ra", isError: false } }, a); + emitIn("llm-end", { id: "mB", response: answer("done B") }, new EventCaller(llm, b)); + emitIn("agent-end", { endStep: stepB }, b); + emitIn("llm-start", { id: "mA2", messages: [] }, new EventCaller(llm, a)); + emitIn("llm-end", { id: "mA2", response: answer("done A") }, new EventCaller(llm, a)); + emitIn("agent-end", { endStep: stepA }, a); + + const events = await flushed(spool); + const starts = events.filter((e) => e.type === "agent_start"); + expect(starts.map((e) => e.goal)).toEqual(["goal A", "goal B"]); + for (const start of starts) expect(start.parent_id ?? null).toBeNull(); + const [sa, sb] = starts.map((e) => e.session_id); + expect(sa).not.toBe(sb); + const sessions = bySession(events); + expect(sessions.get(sa)).toEqual([ + "LLMAgentStandIn agent_start", + "LLMAgentStandIn model_request", + "LLMAgentStandIn model_response", + "LLMAgentStandIn tool_use tool_a", + "LLMAgentStandIn tool_result tool_a", + "LLMAgentStandIn model_request", + "LLMAgentStandIn model_response", + "LLMAgentStandIn agent_end", + ]); + expect(sessions.get(sb)).toEqual([ + "LLMAgentStandIn agent_start", + "LLMAgentStandIn model_request", + "LLMAgentStandIn model_response", + "LLMAgentStandIn agent_end", + ]); + expect(events.find((e) => e.type === "agent_end" && e.session_id === sa)!.summary).toBe("done A"); + expect(events.find((e) => e.type === "agent_end" && e.session_id === sb)!.summary).toBe("done B"); + }); + + it("still nests a query INSIDE another invocation's chain", async () => { + // A sub-engine queried from inside an outer engine's query (SubQuestion- + // QueryEngine), and the SAME engine re-entered from inside its own query: + // both run inside the outer invocation, so neither opens a run. + install(); + const outer = new RetrieverQueryEngineStandIn(); + const inner = new RetrieverQueryEngineStandIn(); + const top = new EventCaller(outer); + emitIn("query-start", { id: "q1", query: "outer" }, top); + const sub = new EventCaller(inner, top); + emitIn("query-start", { id: "q2", query: "inner" }, sub); + emitIn("retrieve-start", { id: "r2", query: "inner" }, sub); + emitIn("retrieve-end", { id: "r2", nodes: [] }, sub); + emitIn("query-end", { id: "q2", response: "inner answer" }, sub); + const again = new EventCaller(outer, top); + emitIn("query-start", { id: "q3", query: "again" }, again); + emitIn("query-end", { id: "q3", response: "again answer" }, again); + emitIn("query-end", { id: "q1", response: "outer answer" }, top); + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "RetrieverQueryEngineStandIn agent_start", + "RetrieverQueryEngineStandIn tool_use retriever", + "RetrieverQueryEngineStandIn tool_result retriever", + "RetrieverQueryEngineStandIn agent_end", + ]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + }); + + it("keeps two concurrent workflow runs of one shared agent apart", async () => { + install(); + const gates: Array<() => void> = []; + const wait = () => new Promise((resolve) => gates.push(resolve)); + const wf = new AgentWorkflow(["Agent"], async (ctx, self) => { + const input = String(self.input); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "Agent", q: input })); + ctx.send(ev("stop", { result: `answer ${input}` })); + }); + wf.runAgentStep = async (_ctx, event) => { + const q = String((event as { data: { q: string } }).data.q); + await wait(); + bus.emit("llm-start", { id: `m-${q}`, messages: [{ role: "user", content: q }] }); + await wait(); + bus.emit("llm-end", { id: `m-${q}`, response: answer(`reply ${q}`) }); + }; + wf.runStream("A"); + const doneA = wf.done; + wf.runStream("B"); + const doneB = wf.done; + // Interleave: each round releases the waiting calls newest first, so B's + // model call starts before A's. + for (let i = 0; i < 10; i += 1) { + await new Promise((resolve) => setTimeout(resolve, 0)); + for (const release of gates.splice(0).reverse()) release(); + } + await Promise.all([doneA, doneB]); + const events = await flushed(spool); + const starts = events.filter((e) => e.type === "agent_start"); + expect(starts.map((e) => e.goal)).toEqual(["A", "B"]); + const [sa, sb] = starts.map((e) => e.session_id); + expect(sa).not.toBe(sb); + const requests = events.filter((e) => e.type === "model_request"); + expect(requests.map((e) => e.session_id)).toEqual([sb, sa]); + const sessions = bySession(events); + const run = [ + "Agent agent_start", + "Agent hook_triggered runAgentStep", + "Agent model_request", + "Agent model_response", + "Agent hook_completed runAgentStep", + "Agent agent_end", + ]; + expect(sessions.get(sa)).toEqual(run); + expect(sessions.get(sb)).toEqual(run); + expect(events.find((e) => e.type === "model_response" && e.content === "reply A")!.session_id).toBe(sa); + expect(events.find((e) => e.type === "model_response" && e.content === "reply B")!.session_id).toBe(sb); + expect(events.find((e) => e.type === "agent_end" && e.session_id === sa)!.summary).toBe("answer A"); + expect(events.find((e) => e.type === "agent_end" && e.session_id === sb)!.summary).toBe("answer B"); + }); +}); + +describe("attach()", () => { + it("refuses to replace an install that is still live, leaving it removable", () => { + install(); + const live = bus; + const subscribed = live.count(); + const proto = AgentWorkflow.prototype as unknown as Record; + const patched = proto.runStream; + const second = new FakeBus(); + expect(() => + attach({ reaperInterval: 0 }, { globals: [{ Settings: { callbackManager: second } }], workflows: [workflowModule()] }), + ).toThrow(/already installed/); + // Nothing of the refused attach happened, and the live one is untouched. + expect(second.count()).toBe(0); + expect(live.count()).toBe(subscribed); + expect(proto.runStream).toBe(patched); + // So uninstall() still reaches every subscription and the patch. + adapter.uninstall(); + expect(live.count()).toBe(0); + expect(proto.runStream).not.toBe(patched); + // And once removed, attaching again works. + install(); + expect(bus.count()).toBe(subscribed); + }); +}); + +describe("state after a run ends", () => { + it("leaves no residue after 20k completed runs of every kind", async () => { + // Every per-run entry — the adapter's own maps AND the tracker's links — + // must go when its run ends. A leaked link is not just memory: at the + // tracker's FIFO cap the next eviction takes a LIVE run's link, and its + // events drop. + const original = runtime.event; + runtime.event = new Proxy({}, { get: () => () => undefined }) as typeof runtime.event; + try { + const handle = install({ staleAfter: 60 }); + const engine = new RetrieverQueryEngineStandIn(); + const runner = new LLMAgentStandIn(); + const llm = new OpenAIStandIn(); + const N = 20_000; + for (let i = 0; i < N; i += 1) { + switch (i % 6) { + case 0: { + // A query, with its retrieval and model call. + const q = new EventCaller(engine); + emitIn("query-start", { id: `q${i}`, query: "q" }, q); + emitIn("retrieve-start", { id: `r${i}`, query: "q" }, q); + emitIn("retrieve-end", { id: `r${i}`, nodes: [] }, q); + emitIn("llm-start", { id: `m${i}`, messages: [] }, new EventCaller(llm, q)); + emitIn("llm-end", { id: `m${i}`, response: {} }, new EventCaller(llm, q)); + emitIn("query-end", { id: `q${i}`, response: "a" }, q); + break; + } + case 1: { + // A two-step legacy task with a tool call. + const c = new EventCaller(runner); + const s1 = { id: `s${i}a`, prevStep: null }; + const s2 = { id: `s${i}b`, prevStep: s1 }; + emitIn("agent-start", { startStep: s1 }, c); + emitIn("llm-tool-call", { toolCall: { id: `t${i}`, name: "t", input: {} } }, c); + emitIn("llm-tool-result", { toolCall: { id: `t${i}` }, toolResult: { output: 1, isError: false } }, c); + emitIn("agent-start", { startStep: s2 }, c); + emitIn("agent-end", { endStep: s2 }, c); + break; + } + case 2: { + // A bare model call. + bus.emit("llm-start", { id: `b${i}`, messages: [] }, [llm]); + bus.emit("llm-end", { id: `b${i}`, response: {} }, [llm]); + break; + } + case 3: + case 4: { + // A workflow run: steps, a model call, a handoff to a sub-agent — + // once with hooks and once under steps: false's twin path (a step + // that throws, so no hook_completed success). + const fail = i % 6 === 4; + const wf = new AgentWorkflow(["triage", "forecaster"], async (ctx, self) => { + await ctx.step(self.handleInputStep, ev("start")); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "triage" })); + await ctx.step(self.runAgentStep, ev("setup", { currentAgentName: "forecaster" })); + ctx.send(ev("stop", { result: "ok" })); + }); + wf.runAgentStep = async () => { + bus.emit("llm-start", { id: `w${i}`, messages: [] }); + bus.emit("llm-end", { id: `w${i}`, response: {} }); + if (fail) throw new Error("step failed"); + }; + wf.runStream("q"); + await wf.done.catch(() => undefined); + break; + } + default: { + // A task that never ends on its own, with a model call left open: + // the reaper closes both. + const c = new EventCaller(runner); + emitIn("agent-start", { startStep: { id: `o${i}`, prevStep: null } }, c); + emitIn("llm-start", { id: `o${i}`, messages: [] }, new EventCaller(llm, c)); + handle.sweep(performance.now() + 61_000); + } + } + } + const residue = handle.residue(); + expect({ runs: residue.runs, leaves: residue.leaves, tasks: residue.tasks, queries: residue.queries }).toEqual({ + runs: 0, + leaves: 0, + tasks: 0, + queries: 0, + }); + expect(residue.tracker.openAgents()).toEqual([]); + expect((residue.tracker as unknown as { links: Map }).links.size).toBe(0); + } finally { + runtime.event = original; + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// Invocation boundaries, retriever names and plain workflows +// --------------------------------------------------------------------------- + +/** + * `flushed()`, made deterministic: `flushNow()` hands back a flush the + * interval timer already started — one that drained the queue BEFORE the + * newest events — so a single call can return without them. The second call + * waits for a flush that started after it. + */ +async function drained(): Promise>> { + await runtime.writer.flushNow(); + return flushed(spool); +} + +/** + * `@llamaindex/core/global` as the adapter meets it: a REAL `AsyncLocalStorage` + * of `EventCaller`s (module-private there too), `getEventCaller`, and the + * `withEventCaller` every `@wrapEventCaller` method runs through. + */ +function coreGlobal() { + const storage = new AsyncLocalStorage(); + const getEventCaller = (): EventCaller | null => storage.getStore() ?? null; + const withEventCaller = (caller: unknown, fn: () => T): T => + storage.run(new EventCaller(caller, getEventCaller()), fn); + return { storage, getEventCaller, withEventCaller }; +} + +let li: ReturnType; + +function installCore( + options: Record = {}, + extra: { retrievers?: RetrieverModule[]; asyncContexts?: AsyncContextModule[]; workflows?: WorkflowModule[] } = {}, +) { + bus = new FakeBus(); + li = coreGlobal(); + const globals: GlobalModule[] = [{ Settings: { callbackManager: bus }, getEventCaller: li.getEventCaller }]; + return attach({ reaperInterval: 0, ...options }, { workflows: [], ...extra, globals }); +} + +/** Dispatch the way `CallbackManager.dispatchEvent` does: with the EventCaller bound NOW. */ +const dispatch = (event: string, detail: Event): void => bus.emitReason(event, detail, li.getEventCaller()); +const tick = () => new Promise((resolve) => setTimeout(resolve, 1)); +/** A property read for an identity check, so a method is compared, never called. */ +const member = (target: object, key: string): unknown => (target as Record)[key]; + +/** A first-party provider: `chat` decorated `@wrapEventCaller @wrapLLMEvent`. */ +class OpenAI { + metadata = { model: "gpt-x" }; + constructor(private readonly reply = "It is sunny.") {} + chat(options: { fail?: string; stream?: boolean } = {}): Promise { + return li.withEventCaller(this, async () => { + const id = `m-${Math.random()}`; + dispatch("llm-start", { id, messages: [{ role: "user", content: "weather?" }] }); + await tick(); + if (options.fail) throw new Error(options.fail); + const usage = { prompt_tokens: 9, completion_tokens: 4 }; + if (options.stream) { + const reply = this.reply; + return (async function* () { + yield { delta: reply }; + await tick(); + dispatch("llm-end", { id, response: { message: { content: reply }, raw: [{ raw: { usage } }] } }); + })(); + } + dispatch("llm-end", { id, response: { message: { content: this.reply }, raw: { usage } } }); + return { message: { content: this.reply } }; + }); + } +} + +/** A chat engine: `chat` is `@wrapEventCaller` and dispatches nothing of its own. */ +class ContextChatEngine { + constructor(private readonly llm: OpenAI) {} + chat(options: { fail?: string; stream?: boolean; retrieveFails?: boolean } = {}): Promise { + return li.withEventCaller(this, async () => { + const rid = `r-${Math.random()}`; + dispatch("retrieve-start", { id: rid, query: { query: "weather?" } }); + await tick(); + dispatch("retrieve-end", { id: rid, nodes: [{ node: { id_: "n1", text: "Paris is sunny." }, score: 1 }] }); + const response = await this.llm.chat(options); + return options.stream ? response : { message: { content: (response as { message: { content: string } }).message.content } }; + }); + } +} + +describe("invocation boundaries", () => { + it("records a chat engine — no bus event of its own — as ONE run named after it", async () => { + installCore(); + class SimpleChat { + constructor(private readonly llm: OpenAI) {} + chat(): Promise { + return li.withEventCaller(this, async () => { + const r = rid(); + dispatch("retrieve-start", { id: r, query: { query: "weather?" } }); + dispatch("retrieve-end", { id: r, nodes: [] }); + await tick(); + return this.llm.chat(); + }); + } + } + let n = 0; + const rid = () => `r${(n += 1)}`; + const out = await new SimpleChat(new OpenAI()).chat(); + expect(out).toEqual({ message: { content: "It is sunny." } }); + const events = await drained(); + expect(shape(events)).toEqual([ + "SimpleChat agent_start", + "SimpleChat tool_use retriever", + "SimpleChat tool_result retriever", + "SimpleChat model_request", + "SimpleChat model_response", + "SimpleChat agent_end", + ]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + const end = events.at(-1)!; + expect(end).toMatchObject({ outcome: "success", summary: "It is sunny." }); + expect(events.find((e) => e.type === "model_request")!.model).toBe("gpt-x"); + }); + + it("ends a streamed chat when its stream has been read, not when chat() returns", async () => { + installCore(); + class StreamChat { + constructor(private readonly llm: OpenAI) {} + chat(): Promise { + return li.withEventCaller(this, () => this.llm.chat({ stream: true })); + } + } + const stream = (await new StreamChat(new OpenAI()).chat()) as AsyncIterable; + let events = await drained(); + // chat() has returned; the model call is still being read. + expect(events.filter((e) => e.type === "agent_end")).toEqual([]); + for await (const chunk of stream) { + void chunk; + // drain + } + events = await drained(); + expect(shape(events)).toEqual([ + "StreamChat agent_start", + "StreamChat model_request", + "StreamChat model_response", + "StreamChat agent_end", + ]); + expect([events[2]!.input_tokens, events[2]!.output_tokens]).toEqual([9, 4]); + expect(events[3]).toMatchObject({ outcome: "success", summary: "It is sunny." }); + }); + + it("closes a model call that threw on its response, and fails the run it ended — reported once", async () => { + // `wrapLLMEvent` has no error path: no `llm-end`. The invocation throwing + // is the only signal, and it used to leave both open until the reaper. + installCore(); + const engine = new ContextChatEngine(new OpenAI()); + await expect(engine.chat({ fail: "rate limited" })).rejects.toThrow("rate limited"); + const events = await drained(); + expect(shape(events)).toEqual([ + "ContextChatEngine agent_start", + "ContextChatEngine tool_use retriever", + "ContextChatEngine tool_result retriever", + "ContextChatEngine model_request", + "ContextChatEngine model_response", + "ContextChatEngine agent_end", + ]); + expect(events.find((e) => e.type === "model_response")!.error).toBe("Error: rate limited"); + const end = events.at(-1)!; + expect(end).toMatchObject({ outcome: "failed", summary: "Error: rate limited" }); + // The leaves carry the failure; an `error` event on top would count it twice. + expect(events.filter((e) => e.type === "error")).toEqual([]); + }); + + it("fails a bare model call that threw, instead of leaving it for the reaper", async () => { + installCore(); + await expect(new OpenAI().chat({ fail: "boom" })).rejects.toThrow("boom"); + const events = await drained(); + expect(shape(events)).toEqual(["OpenAI agent_start", "OpenAI model_request", "OpenAI model_response", "OpenAI agent_end"]); + expect(events[2]!.error).toBe("Error: boom"); + expect(events[3]!.outcome).toBe("failed"); + }); + + it("fails a legacy task whose step threw — it never sends agent-end — straight away", async () => { + installCore(); + class LLMAgent { + llm = { metadata: { model: "legacy-model" } }; + chat(): Promise { + return li.withEventCaller(this, async () => { + dispatch("agent-start", { startStep: { id: "s1", prevStep: null } }); + await tick(); + throw new Error("step exploded"); + }); + } + } + await expect(new LLMAgent().chat()).rejects.toThrow("step exploded"); + const events = await drained(); + expect(shape(events)).toEqual(["LLMAgent agent_start", "LLMAgent error", "LLMAgent agent_end"]); + expect(events[1]).toMatchObject({ error_type: "Error", message: "step exploded" }); + expect(events[2]).toMatchObject({ outcome: "failed", summary: "Error: step exploded" }); + }); + + it("roots a query made first inside another invocation at that invocation", async () => { + // CondenseQuestionChatEngine's shape, when its first act is the query. + installCore(); + class QueryEngine { + query(): Promise { + return li.withEventCaller(this, async () => { + dispatch("query-start", { id: "q1", query: "weather?" }); + await tick(); + dispatch("query-end", { id: "q1", response: { message: { content: "notes" } } }); + return "notes"; + }); + } + } + class CondenseChat { + chat(): Promise { + return li.withEventCaller(this, async () => { + await new QueryEngine().query(); + return new OpenAI("final").chat(); + }); + } + } + await new CondenseChat().chat(); + const events = await drained(); + expect(shape(events)).toEqual([ + "CondenseChat agent_start", + "CondenseChat model_request", + "CondenseChat model_response", + "CondenseChat agent_end", + ]); + expect(events.at(-1)!.summary).toBe("final"); + }); + + it("keeps concurrent chats on ONE shared engine in separate sessions", async () => { + installCore(); + const engine = new ContextChatEngine(new OpenAI()); + await Promise.all(Array.from({ length: 10 }, () => engine.chat())); + const events = await drained(); + const sessions = bySession(events); + expect(sessions.size).toBe(10); + for (const list of sessions.values()) { + expect(list).toEqual([ + "ContextChatEngine agent_start", + "ContextChatEngine tool_use retriever", + "ContextChatEngine tool_result retriever", + "ContextChatEngine model_request", + "ContextChatEngine model_response", + "ContextChatEngine agent_end", + ]); + } + }); + + it("never changes what an invocation returns or throws", async () => { + installCore(); + const value = { answer: 42 }; + expect(li.withEventCaller({}, () => value)).toBe(value); + const iterable = (async function* () { + yield 1; + })(); + expect(li.withEventCaller({}, () => iterable)).toBe(iterable); + await expect(li.withEventCaller({}, async () => value)).resolves.toBe(value); + const error = new Error("nope"); + await expect(li.withEventCaller({}, () => Promise.reject(error))).rejects.toBe(error); + expect(() => + li.withEventCaller({}, () => { + throw error; + }), + ).toThrow(error); + // A thenable that is not a native promise is handed back untouched. + const thenable = { then: (resolve: (v: number) => void) => resolve(7) }; + expect(li.withEventCaller({}, () => thenable)).toBe(thenable); + // And the EventCaller is still the one the callback sees. + expect(li.withEventCaller("me", () => li.getEventCaller()!.caller)).toBe("me"); + }); + + it("still reports a rejection nobody handles, as the application would have seen it", async () => { + installCore(); + const seen: unknown[] = []; + const onUnhandled = (reason: unknown) => seen.push(reason); + process.on("unhandledRejection", onUnhandled); + try { + const error = new Error("nobody catches me"); + void li.withEventCaller({}, () => Promise.reject(error)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(seen).toContain(error); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("finds LlamaIndex's storage, touches nothing else, and restores it on uninstall", () => { + const g = coreGlobal(); + const getStore = member(AsyncLocalStorage.prototype, "getStore"); + expect(eventCallerStorage({ getEventCaller: g.getEventCaller })).toBe(g.storage); + expect(member(AsyncLocalStorage.prototype, "getStore")).toBe(getStore); + expect(eventCallerStorage({})).toBeNull(); + expect(eventCallerStorage({ getEventCaller: () => null })).toBeNull(); + + installCore(); + const protoRun = member(AsyncLocalStorage.prototype, "run"); + expect(Object.prototype.hasOwnProperty.call(li.storage, "run")).toBe(true); + expect(member(li.storage, "run")).not.toBe(protoRun); + expect(member(new AsyncLocalStorage(), "run")).toBe(protoRun); + adapter.uninstall(); + expect(member(li.storage, "run")).toBe(protoRun); + // Uninstalled: an invocation opens nothing. + return new OpenAI().chat().then(async () => { + expect(await drained()).toEqual([]); + }); + }); + + it("falls back to a bare run for an invocation it never saw start", async () => { + // A chain built outside the hooked storage (an invocation already running + // at instrument() time): no end signal, so the one-leaf rule. + installCore(); + const engine = new RetrieverQueryEngineStandIn(); + const outer = new EventCaller(engine); + emitIn("llm-start", { id: "m1", messages: [] }, new EventCaller(new OpenAIStandIn(), outer)); + emitIn("llm-end", { id: "m1", response: { message: { content: "x" } } }, new EventCaller(new OpenAIStandIn(), outer)); + const events = await drained(); + expect(shape(events)).toEqual([ + "OpenAIStandIn agent_start", + "OpenAIStandIn model_request", + "OpenAIStandIn model_response", + "OpenAIStandIn agent_end", + ]); + }); +}); + +describe("retrieval names", () => { + /** `@llamaindex/core/retriever`: `retrieve()` dispatches, `_retrieve()` is the subclass's. */ + class BaseRetriever { + async retrieve(query: string): Promise { + const id = `r-${query}`; + dispatch("retrieve-start", { id, query: { query } }); + const nodes = await this._retrieve(query); + dispatch("retrieve-end", { id, nodes }); + return nodes; + } + async _retrieve(query: string): Promise { + void query; + return []; + } + } + class VectorIndexRetriever extends BaseRetriever { + override async _retrieve(query: string): Promise { + await tick(); + return [{ node: { id_: "n1", text: `notes on ${query}` }, score: 0.5 }]; + } + } + + it("names a retrieval after the retriever's class, as Python does — even one built earlier", async () => { + const retriever = new VectorIndexRetriever(); + const original = member(BaseRetriever.prototype, "retrieve"); + installCore({}, { retrievers: [{ BaseRetriever }] }); + expect(member(BaseRetriever.prototype, "retrieve")).not.toBe(original); + await retriever.retrieve("Paris"); + const events = await drained(); + expect(shape(events)).toEqual([ + "VectorIndexRetriever agent_start", + "VectorIndexRetriever tool_use VectorIndexRetriever", + "VectorIndexRetriever tool_result VectorIndexRetriever", + "VectorIndexRetriever agent_end", + ]); + expect(events[1]!.input).toEqual({ query: "Paris" }); + expect(events[2]!.output).toEqual({ num_nodes: 1, top: [{ id: "n1", score: 0.5, text: "notes on Paris" }] }); + adapter.uninstall(); + expect(member(BaseRetriever.prototype, "retrieve")).toBe(original); + }); + + it("keeps each concurrent retrieval's own name", async () => { + class KeywordRetriever extends BaseRetriever {} + installCore({}, { retrievers: [{ BaseRetriever }] }); + await Promise.all([new VectorIndexRetriever().retrieve("a"), new KeywordRetriever().retrieve("b")]); + const events = await drained(); + const uses = events.filter((e) => e.type === "tool_use").map((e) => [e.tool_name, (e.input as { query: string }).query]); + expect(uses.sort()).toEqual([ + ["KeywordRetriever", "b"], + ["VectorIndexRetriever", "a"], + ]); + }); +}); + +/** + * workflow-core ≥1.1, reduced to what the adapter meets: an exported + * `AsyncContext.Variable` class, and a runtime that runs each step handler as + * `handlerVariable.run(handlerContext, …)` and sends the handler's output on + * once its promise settles — through the runtime's OWN `.then`, as the real + * one does, which is why a run cannot end the moment a step does. + */ +function workflowCore() { + class Variable { + private readonly als = new AsyncLocalStorage(); + run(value: unknown, fn: () => T): T { + return this.als.run(value, fn); + } + } + const module: AsyncContextModule = { AsyncContext: { Variable } }; + type Ev = { kind: string; data?: unknown }; + const createWorkflow = () => { + const listeners = new Map unknown>(); + return { + handle(kind: string, handler: (ctx: unknown, event: Ev) => unknown) { + listeners.set(kind, handler); + }, + createContext() { + const handlerVariable = new Variable(); + const sent: Ev[] = []; + const root: Record = { handler: null, inputs: [], outputs: [], prev: null, next: new Set() }; + const send = (event: Ev, parent: Record): void => { + sent.push(event); + const handler = listeners.get(event.kind); + if (!handler) return; + const hc: Record = { + handler, + inputs: [event], + outputs: [], + prev: parent, + next: new Set(), + pending: null, + get root() { + return root; + }, + }; + (parent.next as Set).add(hc); + handlerVariable.run(hc, () => { + const result = (hc.handler as (c: unknown, e: Ev) => unknown)({}, event); + if (result instanceof Promise) { + hc.pending = result.then((out: Ev | undefined) => { + if (out) send(out, hc); + return out; + }); + } else if (result) { + send(result as Ev, hc); + } + }); + }; + return { + sendEvent: (event: Ev) => send(event, root), + sent, + async until(kind: string): Promise { + while (!sent.some((e) => e.kind === kind)) await new Promise((resolve) => setImmediate(resolve)); + }, + }; + }, + }; + }; + return { module, Variable, createWorkflow }; +} + +describe("plain workflows", () => { + const twoSteps = (wc: ReturnType, fail = false) => { + const wf = wc.createWorkflow(); + wf.handle("start", async function research(_ctx, event) { + await tick(); + return { kind: "researched", data: `notes on ${String(event.data)}` }; + }); + wf.handle("researched", async function answer() { + await new OpenAI("It is sunny in Paris.").chat(); + if (fail) throw new Error("answer failed"); + return { kind: "stop", data: "It is sunny in Paris." }; + }); + return wf; + }; + + it("records a createWorkflow() run as an agent with its steps as hooks", async () => { + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module] }); + const ctx = twoSteps(wc).createContext(); + ctx.sendEvent({ kind: "start", data: "weather in Paris?" }); + await ctx.until("stop"); + const events = await drained(); + expect(shape(events)).toEqual([ + "Workflow agent_start", + "Workflow hook_triggered research", + "Workflow hook_completed research", + "Workflow hook_triggered answer", + "Workflow model_request", + "Workflow model_response", + "Workflow hook_completed answer", + "Workflow agent_end", + ]); + expect(events[0]!.goal).toBe("weather in Paris?"); + expect(events.find((e) => e.type === "hook_triggered")!.trigger_event).toBe("workflow_step"); + expect(events.at(-1)).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." }); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + }); + + it("ends the run before the code awaiting it goes on, inside an agent() scope", async () => { + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module] }); + await agentScope("forecast_flow", async () => { + const ctx = twoSteps(wc).createContext(); + ctx.sendEvent({ kind: "start", data: "q" }); + await ctx.until("stop"); + }); + const events = await drained(); + expect(shape(events).filter((s) => s.includes("agent_"))).toEqual([ + "forecast_flow agent_start", + "Workflow agent_start", + "Workflow agent_end", + "forecast_flow agent_end", + ]); + expect(events.find((e) => e.agent_id === "Workflow" && e.type === "agent_start")!.parent_id).toBe("forecast_flow"); + }); + + it("fails the step and the run when a step throws", async () => { + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module] }); + const ctx = twoSteps(wc, true).createContext(); + const onUnhandled = () => undefined; + process.on("unhandledRejection", onUnhandled); + try { + ctx.sendEvent({ kind: "start", data: "q" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + } finally { + process.off("unhandledRejection", onUnhandled); + } + const events = await drained(); + const failed = events.find((e) => e.type === "hook_completed" && e.hook_name === "answer")!; + expect(failed).toMatchObject({ outcome: "failed", error: "Error: answer failed" }); + expect(events.at(-1)).toMatchObject({ type: "agent_end", outcome: "failed" }); + expect(events.filter((e) => e.type === "error")).toEqual([]); + }); + + it("keeps two concurrent contexts of one workflow in two sessions", async () => { + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module] }); + const wf = twoSteps(wc); + const a = wf.createContext(); + const b = wf.createContext(); + a.sendEvent({ kind: "start", data: "A" }); + b.sendEvent({ kind: "start", data: "B" }); + await Promise.all([a.until("stop"), b.until("stop")]); + const events = await drained(); + const sessions = bySession(events); + expect(sessions.size).toBe(2); + for (const list of sessions.values()) expect(list[0]).toBe("Workflow agent_start"); + expect(events.filter((e) => e.type === "agent_start").map((e) => e.goal).sort()).toEqual(["A", "B"]); + }); + + it("records a later burst (an event sent from outside) as a new run", async () => { + // Documented: a plain workflow has no end of its own, so idle is the end. + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module] }); + const wf = wc.createWorkflow(); + wf.handle("ask", async function ask(_ctx, event) { + await tick(); + return { kind: "asked", data: event.data }; + }); + const ctx = wf.createContext(); + ctx.sendEvent({ kind: "ask", data: "first" }); + await ctx.until("asked"); + await tick(); + ctx.sendEvent({ kind: "ask", data: "second" }); + await new Promise((resolve) => setTimeout(resolve, 10)); + const events = await drained(); + expect(events.filter((e) => e.type === "agent_start").map((e) => e.goal)).toEqual(["first", "second"]); + expect(events.filter((e) => e.type === "agent_end")).toHaveLength(2); + }); + + it("leaves an AgentWorkflow's steps to the agent path, and any other value alone", async () => { + const wc = workflowCore(); + installCore({}, { asyncContexts: [wc.module], workflows: [workflowModule()] }); + const agentWf = new AgentWorkflow(["Agent"], async () => {}); + agentWf.runStream("q"); + const wf = wc.createWorkflow(); + wf.handle("start", agentWf.runAgentStep); + const ctx = wf.createContext(); + ctx.sendEvent({ kind: "start" }); + await tick(); + const variable = new wc.Variable(); + expect(variable.run({ not: "a handler context" }, () => 5)).toBe(5); + const events = await drained(); + expect(events.filter((e) => e.agent_id === "Workflow")).toEqual([]); + }); + + it("restores the Variable prototype on uninstall", () => { + const wc = workflowCore(); + const original = (wc.Variable.prototype as unknown as Record).run; + installCore({}, { asyncContexts: [wc.module] }); + expect((wc.Variable.prototype as unknown as Record).run).not.toBe(original); + adapter.uninstall(); + expect((wc.Variable.prototype as unknown as Record).run).toBe(original); + }); +}); + +describe("@llamaindex/openai streaming usage", () => { + it("records usage from the final chunk that stream_options.include_usage adds", async () => { + // @llamaindex/openai (0.1.61 – 0.4.23) streams chat completions and, when + // the request carries `stream_options: {include_usage: true}` (set it with + // `new OpenAI({additionalChatOptions: {stream_options: {include_usage: true}}})`), + // yields OpenAI's content-less usage chunk as `{raw: part, delta: ""}` — + // no `options`. Without that option OpenAI never sends it, and there is no + // number to record. These are the exact chunks it yields. + installCore(); + const part = (choices: unknown[], usage: unknown = null) => ({ + id: "chatcmpl-1", + object: "chat.completion.chunk", + model: "gpt-4o-mini", + choices, + usage, + }); + const chunks = [ + { raw: part([{ index: 0, delta: { role: "assistant", content: "It is" }, finish_reason: null }]), options: {}, delta: "It is" }, + { raw: part([{ index: 0, delta: { content: " sunny." }, finish_reason: null }]), options: {}, delta: " sunny." }, + { raw: part([{ index: 0, delta: {}, finish_reason: "stop" }]), options: {}, delta: "" }, + { + raw: part([], { + prompt_tokens: 21, + completion_tokens: 4, + total_tokens: 25, + prompt_tokens_details: { cached_tokens: 0 }, + }), + delta: "", + }, + ]; + const llm = new OpenAIStandIn(); + // What `wrapLLMEvent` hands `llm-end` for a stream: every chunk as `raw`. + await li.withEventCaller(llm, async () => { + dispatch("llm-start", { id: "s1", messages: [{ role: "user", content: "weather?" }] }); + for (const chunk of chunks) dispatch("llm-stream", { id: "s1", chunk }); + dispatch("llm-end", { id: "s1", response: { message: { content: "It is sunny.", role: "assistant", options: {} }, raw: chunks } }); + }); + const events = await drained(); + const response = events.find((e) => e.type === "model_response")!; + expect([response.input_tokens, response.output_tokens]).toEqual([21, 4]); + expect(response.usage).toMatchObject({ prompt_tokens: 21, completion_tokens: 4, total_tokens: 25 }); + expect(response.stop_reason).toBe("stop"); + expect(response.fw_chunks).toBe(4); + }); + + it("records no token counts — and invents none — for the same stream without include_usage", async () => { + installCore(); + const chunks = [ + { raw: { choices: [{ delta: { content: "hi" }, finish_reason: null }] }, options: {}, delta: "hi" }, + { raw: { choices: [{ delta: {}, finish_reason: "stop" }] }, options: {}, delta: "" }, + ]; + bus.emit("llm-start", { id: "s2", messages: [] }, [new OpenAIStandIn()]); + bus.emit("llm-end", { id: "s2", response: { message: { content: "hi" }, raw: chunks } }, [new OpenAIStandIn()]); + const response = (await drained()).find((e) => e.type === "model_response")!; + expect(response.input_tokens).toBeUndefined(); + expect(response.output_tokens).toBeUndefined(); + expect(response.stop_reason).toBe("stop"); + }); +}); + +describe("state after invocation runs and plain workflows end", () => { + it("leaves no residue after thousands of chat and workflow runs", async () => { + const original = runtime.event; + runtime.event = new Proxy({}, { get: () => () => undefined }) as typeof runtime.event; + try { + const wc = workflowCore(); + const handle = installCore({}, { asyncContexts: [wc.module] }); + const engine = new ContextChatEngine(new OpenAI()); + const wf = wc.createWorkflow(); + wf.handle("start", async function step() { + await new OpenAI().chat(); + return { kind: "stop" }; + }); + for (let i = 0; i < 300; i += 1) { + const ctx = wf.createContext(); + ctx.sendEvent({ kind: "start" }); + await Promise.all([engine.chat(), engine.chat({ stream: true }).then(async (s) => { + for await (const chunk of s as AsyncIterable) { + void chunk; + // drain + } + }), engine.chat({ fail: "x" }).catch(() => undefined), ctx.until("stop")]); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + const residue = handle.residue(); + expect({ runs: residue.runs, leaves: residue.leaves }).toEqual({ runs: 0, leaves: 0 }); + expect(residue.tracker.openAgents()).toEqual([]); + expect((residue.tracker as unknown as { links: Map }).links.size).toBe(0); + } finally { + runtime.event = original; + } + }, 60_000); +}); diff --git a/sdk/typescript/test/mastra-coverage.test.ts b/sdk/typescript/test/mastra-coverage.test.ts new file mode 100644 index 000000000..0ee202bf5 --- /dev/null +++ b/sdk/typescript/test/mastra-coverage.test.ts @@ -0,0 +1,501 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import * as core from "../src/integrations/core.js"; +import { _internals, adapter } from "../src/integrations/mastra.js"; +import { agent as agentScope, session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The Mastra surfaces beyond a bare `generate()` / `stream()` — memory + * threads, agent networks, processor tripwires, workflow suspend/resume — + * against hand-written stand-ins for `@mastra/core` shaped like the real + * call sequences (the module loader is the only thing mocked). + * `integration/mastra.test.ts` proves each against real 0.x and 1.x releases. + */ + +const modules = vi.hoisted(() => ({ agent: [] as unknown[], workflows: [] as unknown[] })); + +vi.mock("../src/integrations/compat.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requireModuleCopies: async (specifier: string): Promise => + specifier.endsWith("/agent") ? modules.agent : modules.workflows, + }; +}); + +const { + conversationOf, + isRoutingAgent, + suspendedStepsOf, + resumeTargetOf, + humanText, +} = _internals; + +// --------------------------------------------------------------------------- +// Stand-ins +// --------------------------------------------------------------------------- + +class FakeModel { + modelId = "fake-model"; + constructor(private usage: [number, number] = [3, 4]) {} + async doGenerate(): Promise { + return { + content: [{ type: "text", text: "sunny" }], + finishReason: "stop", + usage: { inputTokens: this.usage[0], outputTokens: this.usage[1] }, + }; + } +} + +interface StreamOptions { + onFinish?: (result: unknown) => unknown; + onError?: (error: unknown) => unknown; + onAbort?: (event: unknown) => unknown; +} + +/** + * A stream's output the way Mastra's is: consuming it calls `onFinish` unless + * a processor blocked it, and then finishes — which only `_waitUntilFinished()` + * reports. + */ +class FakeOutput { + tripwire: unknown = undefined; + private finished: () => void = () => undefined; + private readonly done = new Promise((resolve) => (this.finished = resolve)); + constructor(private readonly options: StreamOptions = {}) {} + _waitUntilFinished(): Promise { + return this.done; + } + consume(): void { + if (!this.tripwire) this.options.onFinish?.({ finishReason: "stop" }); + this.finished(); + } +} + +class FakeAgent { + model: FakeModel; + /** What `__runInputProcessors` reports, in the 0.x or 1.x shape. */ + guard: Record = {}; + /** Runs inside the agent's own call, before its model step. */ + inside: (() => Promise) | null = null; + /** What each generate() / network() call was handed. */ + calls: unknown[][] = []; + constructor( + public name: string, + public id: string = name, + usage?: [number, number], + ) { + this.model = new FakeModel(usage); + } + + resolveModelConfig(): FakeModel { + return this.model; + } + + convertTools(): Record { + return {}; + } + + __runInputProcessors(): Promise> { + return Promise.resolve(this.guard); + } + + async generate(prompt: string, options?: unknown): Promise { + this.calls.push([prompt, options]); + await this.inside?.(); + const model = await this.resolveModelConfig(); + await model.doGenerate(); + return { text: "sunny", finishReason: "stop" }; + } + + /** A stream whose input processors ran first; a blocked one never calls the model. */ + async stream(_prompt: string, options: StreamOptions = {}): Promise { + const guard = await this.__runInputProcessors(); + const output = new FakeOutput(options); + if (guard.tripwire || guard.tripwireTriggered) { + output.tripwire = guard.tripwire ?? true; + return output; + } + const model = await this.resolveModelConfig(); + await model.doGenerate(); + return output; + } + + /** + * `agent.network()` as Mastra runs it: a fresh `routing-agent` on this + * agent's model decides, a sub-agent runs, the router judges it complete, + * and the stream settles its `status` by itself. + */ + async network(prompt: string, options?: unknown): Promise<{ status: Promise }> { + this.calls.push([prompt, options]); + const route = (): FakeAgent => { + const router = new FakeAgent("Routing Agent", "routing-agent"); + router.model = this.model; + return router; + }; + let settle!: (status: string) => void; + let fail!: (error: unknown) => void; + const status = new Promise((resolve, reject) => { + settle = resolve; + fail = reject; + }); + void (async () => { + await route().generate("route"); + if (this.name === "broken") { + fail(new Error("network exploded")); + return; + } + await new FakeAgent("helper", "helper", [11, 7]).generate("weather"); + await route().generate("complete?"); + settle("success"); + })(); + return { status }; + } +} + +class FakeEngine { + static inner: ((runId: string) => Promise) | null = null; + async executeStep(params: { step: { id: string }; executionContext: { runId: string } }): Promise { + if (params.step.id === "agent" && FakeEngine.inner) await FakeEngine.inner(params.executionContext.runId); + return { result: { status: params.step.id === "approve-wait" ? "suspended" : "success", output: { ok: true } } }; + } +} + +/** A workflow run whose `_start` / `_resume` resolve with the given results. */ +class FakeRun { + workflowId = "approval-flow"; + resumedWith: unknown[] = []; + constructor( + public runId: string, + private results: unknown[], + private startSteps: string[] = ["city", "approve-wait"], + ) {} + + private async next(steps: string[]): Promise { + const engine = new FakeEngine(); + for (const id of steps) await engine.executeStep({ step: { id }, executionContext: { runId: this.runId } }); + return this.results.shift(); + } + + _start(): Promise { + return this.next(this.startSteps); + } + + _resume(params: unknown): Promise { + this.resumedWith.push(params); + return this.next(["approve", "done"]); + } +} + +const suspended = (...ids: string[]): Record => ({ + status: "suspended", + suspended: ids.map((id) => [id]), + steps: Object.fromEntries(ids.map((id) => [id, { status: "suspended", suspendPayload: { prompt: `Approve ${id}?` } }])), +}); + +// --------------------------------------------------------------------------- + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); + FakeEngine.inner = null; + modules.agent = [{ Agent: FakeAgent }]; + modules.workflows = [{ Run: FakeRun, DefaultExecutionEngine: FakeEngine }]; +}); +afterEach(async () => { + adapter.uninstall(); + await spool.cleanup(); + setLogger(null); +}); + +const shape = (events: Array>): string[] => + events.map((e) => `${String(e.agent_id)} ${String(e.type)}`); + +describe("readers", () => { + it("reads a memory thread and resource in both majors' shapes", () => { + expect(conversationOf({ memory: { thread: "t-1", resource: "u-1" } })).toEqual({ thread: "t-1", resource: "u-1" }); + expect(conversationOf({ memory: { thread: { id: "t-2", title: "x" }, resource: "u-1" } })).toEqual({ + thread: "t-2", + resource: "u-1", + }); + // 0.x's deprecated top-level options. + expect(conversationOf({ threadId: "t-3", resourceId: "u-3" })).toEqual({ thread: "t-3", resource: "u-3" }); + expect(conversationOf({ memory: { thread: "" } })).toEqual({ thread: undefined, resource: undefined }); + expect(conversationOf(undefined)).toEqual({ thread: undefined, resource: undefined }); + }); + + it("recognises a network's router by its id or name, and nothing else", () => { + expect(isRoutingAgent({ id: "routing-agent", name: "Routing Agent" })).toBe(true); + expect(isRoutingAgent({ name: "routing-agent" })).toBe(true); + expect(isRoutingAgent({ id: "router", name: "Router" })).toBe(false); + }); + + it("reads which steps a suspended result waits on, and what each asked", () => { + expect(suspendedStepsOf(suspended("approve"))).toEqual([{ path: "approve", payload: { prompt: "Approve approve?" } }]); + // A step inside a nested workflow: the path, with the outer step's payload. + expect( + suspendedStepsOf({ suspended: [["inner", "approve"]], suspendPayload: { inner: { prompt: "nested?" } } }), + ).toEqual([{ path: "inner.approve", payload: { prompt: "nested?" } }]); + // No `suspended` list: the steps' own status. + expect(suspendedStepsOf({ steps: { a: { status: "success" }, b: { status: "suspended", suspendPayload: 1 } } })).toEqual([ + { path: "b", payload: 1 }, + ]); + expect(suspendedStepsOf(undefined)).toEqual([]); + }); + + it("reads the step a resume names, in every form Mastra accepts", () => { + expect(resumeTargetOf({ step: "approve", resumeData: { ok: true } })).toEqual({ path: "approve", answer: { ok: true } }); + expect(resumeTargetOf({ step: { id: "approve" } }).path).toBe("approve"); + expect(resumeTargetOf({ step: ["inner", { id: "approve" }] }).path).toBe("inner.approve"); + expect(resumeTargetOf({ resumeData: 1 })).toEqual({ path: undefined, answer: 1 }); + }); + + it("renders what a human read or wrote as text", () => { + expect(humanText("yes")).toBe("yes"); + expect(humanText({ prompt: "Approve?", extra: 1 })).toBe("Approve?"); + expect(humanText({ approved: true })).toBe('{"approved":true}'); + expect(humanText(undefined)).toBeUndefined(); + }); +}); + +describe("memory threads", () => { + it("makes a root run's memory thread its session", async () => { + await adapter.install({}); + const agent = new FakeAgent("weather-agent"); + await agent.generate("Paris", { memory: { thread: "thread-42", resource: "user-7" } }); + await agent.generate("Rome", { threadId: "thread-42", resourceId: "user-7" }); + const events = await flushed(spool); + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["thread-42"])); + const starts = events.filter((e) => e.type === "agent_start"); + expect(starts).toHaveLength(2); + for (const start of starts) expect(start).toMatchObject({ fw_thread_id: "thread-42", fw_resource_id: "user-7" }); + }); + + it("lets an enclosing scope win over the thread, and nests a threaded sub-run", async () => { + await adapter.install({}); + await session({ sessionId: "req-1" }, () => + new FakeAgent("weather-agent").generate("Paris", { memory: { thread: "thread-42", resource: "u" } }), + ); + await agentScope("planner", {}, () => + new FakeAgent("weather-agent").generate("Paris", { memory: { thread: "thread-43", resource: "u" } }), + ); + const events = await flushed(spool); + expect(events.filter((e) => e.session_id === "thread-42" || e.session_id === "thread-43")).toEqual([]); + const starts = events.filter((e) => e.type === "agent_start" && e.agent_id === "weather-agent"); + expect(starts.map((e) => [e.session_id === "req-1", e.parent_id, e.fw_thread_id])).toEqual([ + [true, undefined, "thread-42"], + [false, "planner", "thread-43"], + ]); + }); +}); + +describe("processor tripwires", () => { + for (const [label, guard] of [ + ["1.x", { tripwire: { reason: "blocked", processorId: "guard" } }], + ["0.x", { tripwireTriggered: true, tripwireReason: "blocked" }], + ] as const) { + it(`closes a stream an input processor blocks, rejected (${label})`, async () => { + await adapter.install({}); + const agent = new FakeAgent("weather-agent"); + agent.guard = guard; + const run = await agent.stream("Paris", {}); + run.consume(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const events = await flushed(spool); + expect(shape(events)).toEqual(["weather-agent agent_start", "weather-agent agent_end"]); + expect(events[1]!.outcome).toBe("rejected"); + expect(_internals.openSpans().agents).toBe(0); + }); + } + + it("closes a blocked stream from its finish when nothing reports the block earlier", async () => { + await adapter.install({}); + const agent = new FakeAgent("weather-agent"); + const run = await agent.stream("Paris", {}); + // Blocked later, on the output stream: no callback, only the finish. + run.tripwire = { reason: "answer blocked" }; + run.consume(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const events = await flushed(spool); + expect(events.filter((e) => e.type === "agent_end").map((e) => e.outcome)).toEqual(["rejected"]); + }); + + it("closes an unblocked stream once, from its callback", async () => { + await adapter.install({}); + const run = await new FakeAgent("weather-agent").stream("Paris", {}); + run.consume(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const events = await flushed(spool); + expect(events.filter((e) => e.type === "agent_end").map((e) => e.outcome)).toEqual(["success"]); + }); +}); + +describe("agent networks", () => { + it("records a network as one agent whose router steps are its own", async () => { + await adapter.install({}); + const planner = new FakeAgent("planner", "planner", [50, 10]); + const stream = await planner.network("Weather?", { memory: { thread: "thread-net", resource: "u" } }); + await stream.status; + await new Promise((resolve) => setTimeout(resolve, 0)); + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "planner agent_start", + "planner model_request", + "planner model_response", + "helper agent_start", + "helper model_request", + "helper model_response", + "helper agent_end", + "planner model_request", + "planner model_response", + "planner agent_end", + ]); + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["thread-net"])); + expect(events[0]).toMatchObject({ fw_method: "network" }); + expect(events[3]).toMatchObject({ parent_id: "planner" }); + expect(events.at(-1)).toMatchObject({ outcome: "success" }); + expect(_internals.openSpans().agents).toBe(0); + }); + + it("closes a network whose stream fails, failed", async () => { + await adapter.install({}); + const stream = await new FakeAgent("broken").network("Weather?"); + await expect(stream.status).rejects.toThrow("network exploded"); + await new Promise((resolve) => setTimeout(resolve, 0)); + const events = await flushed(spool); + expect(events.at(-1)).toMatchObject({ type: "agent_end", agent_id: "broken", outcome: "failed" }); + }); + + it("records a routing-agent called outside any network as an agent like any other", async () => { + await adapter.install({}); + await new FakeAgent("Routing Agent", "routing-agent").generate("hi"); + const events = await flushed(spool); + expect(shape(events)).toEqual([ + "Routing Agent agent_start", + "Routing Agent model_request", + "Routing Agent model_response", + "Routing Agent agent_end", + ]); + }); +}); + +describe("workflow suspend / resume", () => { + it("pauses on suspend and continues the same span on resume", async () => { + await adapter.install({}); + const run = new FakeRun("run-1", [suspended("approve"), { status: "success" }]); + await run._start(); + expect(_internals.pausedRuns()).toBe(1); + await run._resume({ step: "approve", resumeData: { approved: true } }); + expect(_internals.pausedRuns()).toBe(0); + + const events = await flushed(spool); + expect(events.map((e) => e.type)).toEqual([ + "agent_start", + "hook_triggered", + "hook_completed", + "hook_triggered", + "hook_completed", + "human_wait", + "agent_pause", + "agent_resume", + "human_input", + "hook_triggered", + "hook_completed", + "hook_triggered", + "hook_completed", + "agent_end", + ]); + // A root run's session is its run id: what its resume is sure to share. + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["run-1"])); + const wait = events.find((e) => e.type === "human_wait")!; + expect(wait).toMatchObject({ input_id: "run-1:approve", prompt: "Approve approve?", reason: "mastra_suspend" }); + expect(events.find((e) => e.type === "agent_pause")).toMatchObject({ pause_id: "run-1:approve" }); + expect(events.find((e) => e.type === "agent_resume")).toMatchObject({ pause_id: "run-1:approve" }); + expect(events.find((e) => e.type === "human_input")).toMatchObject({ + input_id: "run-1:approve", + response: '{"approved":true}', + }); + expect(events.at(-1)).toMatchObject({ type: "agent_end", outcome: "success" }); + expect(_internals.openSpans().agents).toBe(0); + }); + + it("pauses again only for steps newly suspended, and resumes only the step named", async () => { + await adapter.install({}); + const run = new FakeRun("run-2", [suspended("a", "b"), suspended("b"), { status: "success" }]); + await run._start(); + await run._resume({ step: "a", resumeData: "yes" }); + await run._resume({ step: "b", resumeData: "no" }); + const events = await flushed(spool); + const pick = (type: string): unknown[] => events.filter((e) => e.type === type).map((e) => e.input_id ?? e.pause_id); + expect(pick("human_wait")).toEqual(["run-2:a", "run-2:b"]); + expect(pick("agent_resume")).toEqual(["run-2:a", "run-2:b"]); + expect(events.filter((e) => e.type === "human_input").map((e) => e.response)).toEqual(["yes", "no"]); + expect(events.filter((e) => e.type === "agent_start")).toHaveLength(1); + expect(events.filter((e) => e.type === "agent_end")).toHaveLength(1); + }); + + it("closes a pause opened by another process under the same ids", async () => { + await adapter.install({}); + await new FakeRun("run-3", [suspended("approve")])._start(); + // This process forgets it — as a different worker would never have seen it. + adapter.uninstall(); + await adapter.install({}); + await new FakeRun("run-3", [{ status: "success" }])._resume({ step: "approve", resumeData: { approved: true } }); + + const events = await flushed(spool); + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["run-3"])); + const resumed = events.slice(events.findIndex((e) => e.fw_method === "resume")); + expect(resumed.map((e) => e.type)).toEqual([ + "agent_start", + "agent_resume", + "human_input", + "hook_triggered", + "hook_completed", + "hook_triggered", + "hook_completed", + "agent_end", + ]); + expect(resumed[1]).toMatchObject({ pause_id: "run-3:approve", fw_resumed_elsewhere: true }); + expect(resumed[2]).toMatchObject({ input_id: "run-3:approve", fw_resumed_elsewhere: true }); + }); + + it("keeps a nested run's session: the run id is only a ROOT run's session", async () => { + await adapter.install({}); + await session({ sessionId: "req-1" }, () => new FakeRun("run-4", [{ status: "success" }])._start()); + const events = await flushed(spool); + expect(new Set(events.map((e) => e.session_id))).toEqual(new Set(["req-1"])); + }); +}); + +describe("workflow steps", () => { + it("records only the run's own steps, not those of an agent run handed its id", async () => { + await adapter.install({}); + // An agent inside a step executes its loop as internal workflows whose + // steps carry the SAME run id when the agent is handed it — as 0.x's + // network does with its sub-agents. Those run in the agent's frame. + FakeEngine.inner = async (runId: string) => { + const helper = new FakeAgent("helper"); + helper.inside = () => new FakeEngine().executeStep({ step: { id: "llm-execution" }, executionContext: { runId } }); + await helper.generate("weather"); + await new FakeEngine().executeStep({ step: { id: "after-agent" }, executionContext: { runId } }); + }; + await new FakeRun("run-5", [{ status: "success" }], ["city", "agent"])._start(); + const events = await flushed(spool); + expect(events.filter((e) => e.type === "hook_triggered").map((e) => e.hook_name)).toEqual([ + "city", + "agent", + "after-agent", + ]); + expect(events.filter((e) => e.agent_id === "helper").map((e) => e.type)).toEqual([ + "agent_start", + "model_request", + "model_response", + "agent_end", + ]); + }); +}); diff --git a/sdk/typescript/test/mastra-lifecycle.test.ts b/sdk/typescript/test/mastra-lifecycle.test.ts new file mode 100644 index 000000000..20d63e943 --- /dev/null +++ b/sdk/typescript/test/mastra-lifecycle.test.ts @@ -0,0 +1,479 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import * as core from "../src/integrations/core.js"; +import { _internals, adapter, wrapTool } from "../src/integrations/mastra.js"; +import { session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The Mastra adapter's lifecycle: what `uninstrument()` stops, and what it + * leaves behind. + * + * `instrument()` hands Mastra objects that outlive it — a model behind a + * proxy, tools behind wrappers, a stream the caller is still reading — so + * restoring the prototypes is not enough on its own. These run the adapter + * against a hand-written stand-in for `@mastra/core` (the module loader is the + * only thing mocked), so each case controls exactly when a model step or a + * stream ends relative to the teardown. `integration/mastra.test.ts` proves + * the same against real releases. + */ + +const modules = vi.hoisted(() => ({ agent: [] as unknown[], workflows: [] as unknown[] })); + +vi.mock("../src/integrations/compat.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requireModuleCopies: async (specifier: string): Promise => + specifier.endsWith("/agent") ? modules.agent : modules.workflows, + }; +}); + +// --------------------------------------------------------------------------- +// A stand-in for @mastra/core: the four prototype methods the adapter patches, +// driven the way Mastra drives them. +// --------------------------------------------------------------------------- + +type Part = Record; + +/** A released-on-demand barrier, so a test decides when a stream moves on. */ +function barrier(): { wait: Promise; open: () => void } { + let open!: () => void; + const wait = new Promise((resolve) => (open = resolve)); + return { wait, open }; +} + +class FakeModel { + modelId = "fake-model"; + provider = "fake"; + fail = false; + /** When set, a stream stops after its first part until this opens. */ + gate: Promise | null = null; + + async doGenerate(): Promise { + if (this.fail) throw new Error("model exploded"); + return { + content: [{ type: "text", text: "sunny" }], + finishReason: "stop", + usage: { inputTokens: 3, outputTokens: 4 }, + }; + } + + async doStream(): Promise { + const parts: Part[] = [ + { type: "response-metadata", modelId: "fake-model" }, + { type: "text-delta", delta: "sunny" }, + { type: "finish", finishReason: "stop", usage: { inputTokens: 5, outputTokens: 6 } }, + ]; + const gate = this.gate; + let index = 0; + return { + stream: new ReadableStream({ + async pull(controller) { + if (index === 1 && gate) await gate; + if (index >= parts.length) { + controller.close(); + return; + } + controller.enqueue(parts[index++]!); + }, + }), + }; + } +} + +interface Resolved { + model: FakeModel; + tools: Record Promise }>; +} + +interface StreamOptions { + onFinish?: (result: unknown) => unknown; + onError?: (error: unknown) => unknown; + onAbort?: (event: unknown) => unknown; +} + +class FakeAgent { + name = "fake-agent"; + model = new FakeModel(); + toolFails = false; + /** What the last run resolved — held on to, the way a reused agent does. */ + resolved: Resolved | null = null; + /** + * What the next run resolves to, when a test decides. Fields rather than + * instance methods: an own `convertTools` would shadow the patched + * prototype method, and the adapter would never see the run. + */ + nextModel: FakeModel | null = null; + nextTools: (() => Resolved["tools"]) | null = null; + + resolveModelConfig(): FakeModel { + return this.nextModel ?? this.model; + } + + convertTools(): Resolved["tools"] { + if (this.nextTools) return this.nextTools(); + return { + weather: { + execute: async (input: unknown) => { + if (this.toolFails) throw new Error("tool exploded"); + return { ...(input as object), forecast: "sunny" }; + }, + }, + }; + } + + private async resolve(): Promise { + const model = await this.resolveModelConfig(); + const tools = await this.convertTools(); + this.resolved = { model, tools }; + return this.resolved; + } + + async generate(prompt: string): Promise { + const { model, tools } = await this.resolve(); + await model.doGenerate(); + try { + await tools.weather!.execute({ city: prompt }, { toolCallId: "call_1" }); + } catch { + // Mastra hands a tool's failure to the model and carries on. + } + await model.doGenerate(); + return { text: "sunny", finishReason: "stop" }; + } + + /** + * One streamed step. The returned reader is the step's own stream; the + * caller ends the run with `finish()` / `abort()`, the way Mastra calls + * `onFinish` / `onAbort` once the caller has consumed or cancelled it. + */ + async stream( + _prompt: string, + options: StreamOptions = {}, + ): Promise<{ reader: ReadableStreamDefaultReader; finish: () => void; abort: () => Promise }> { + const { model } = await this.resolve(); + const { stream } = (await model.doStream()) as { stream: ReadableStream }; + const reader = stream.getReader(); + return { + reader, + finish: () => void options.onFinish?.({ finishReason: "stop" }), + abort: async () => { + await reader.cancel("caller went away"); + options.onAbort?.({}); + }, + }; + } +} + +class FakeEngine { + /** When set, a step called `wait` parks here until it opens. */ + static gate: { entered: () => void; wait: Promise } | null = null; + + async executeStep(params: { step: { id: string }; executionContext: { runId: string } }): Promise { + if (params.step.id === "wait" && FakeEngine.gate) { + FakeEngine.gate.entered(); + await FakeEngine.gate.wait; + } + if (params.step.id === "explode") throw new Error("step threw"); + return { result: { status: params.step.id === "fail" ? "failed" : "success", output: { ok: true }, error: "step failed" } }; + } +} + +class FakeRun { + workflowId = "fake-flow"; + constructor( + public runId: string, + private steps: string[], + ) {} + + async _start(): Promise { + const engine = new FakeEngine(); + let status = "success"; + for (const id of this.steps) { + const out = (await engine.executeStep({ step: { id }, executionContext: { runId: this.runId } })) as { + result: { status: string }; + }; + if (out.result.status === "failed") status = "failed"; + } + return { status }; + } +} + +async function readAll(reader: ReadableStreamDefaultReader): Promise { + const parts: Part[] = []; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) return parts; + parts.push(chunk.value); + } +} + +/** Resolve once the adapter's observed stream is parked on the gate. */ +const settle = (): Promise => new Promise((resolve) => setTimeout(resolve, 10)); + +// --------------------------------------------------------------------------- + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); + modules.agent = [{ Agent: FakeAgent }]; + modules.workflows = [{ Run: FakeRun, DefaultExecutionEngine: FakeEngine }]; +}); +afterEach(async () => { + adapter.uninstall(); + await spool.cleanup(); + setLogger(null); +}); + +const types = (events: Array>): string[] => events.map((e) => String(e.type)); + +describe("uninstrument()", () => { + it("turns the model proxy and tool wrappers a run already built into pass-throughs", async () => { + await adapter.install({}); + const agent = new FakeAgent(); + await session({ sessionId: "s1" }, () => agent.generate("Paris")); + const recorded = (await flushed(spool)).length; + expect(recorded).toBe(8); + + adapter.uninstall(); + expect(_internals.isEnabled()).toBe(false); + + // What the agent held on to from its instrumented run, used afterwards — + // inside a scope, so a stray event would have a session to land in. + const { model, tools } = agent.resolved!; + await session({ sessionId: "s1" }, async () => { + await expect(model.doGenerate()).resolves.toMatchObject({ finishReason: "stop" }); + await expect(tools.weather!.execute({ city: "Rome" }, { toolCallId: "call_9" })).resolves.toEqual({ + city: "Rome", + forecast: "sunny", + }); + const { stream } = (await model.doStream()) as { stream: ReadableStream }; + expect(await readAll(stream.getReader())).toHaveLength(3); + await agent.generate("Oslo"); + }); + expect((await flushed(spool)).length).toBe(recorded); + }); + + it("closes a stream in flight as cancelled and records nothing after it", async () => { + await adapter.install({}); + const agent = new FakeAgent(); + const gate = barrier(); + agent.model.gate = gate.wait; + + await session({ sessionId: "s1" }, async () => { + const run = await agent.stream("Paris", {}); + const first = await run.reader.read(); + expect(first.value).toMatchObject({ type: "response-metadata" }); + await settle(); + + adapter.uninstall(); + const atTeardown = await flushed(spool); + expect(types(atTeardown)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + const response = atTeardown[2]!; + expect(response.stop_reason).toBe("incomplete"); + expect(response.fw_incomplete).toBe(true); + expect(atTeardown[3]!.outcome).toBe("cancelled"); + + // The stream is the caller's: it still delivers everything. + gate.open(); + expect(await readAll(run.reader)).toHaveLength(2); + run.finish(); + }); + expect(types(await flushed(spool))).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(_internals.openSpans()).toEqual({ models: 0, tools: 0, steps: 0, workflowRuns: 0, agents: 0 }); + }); + + it("closes an open tool call as incomplete", async () => { + await adapter.install({}); + const gate = barrier(); + const entered = barrier(); + const agent = new FakeAgent(); + agent.nextTools = () => ({ + weather: { + execute: async () => { + entered.open(); + await gate.wait; + return { forecast: "sunny" }; + }, + }, + }); + const pending = session({ sessionId: "s1" }, () => agent.generate("Paris")); + await entered.wait; + adapter.uninstall(); + gate.open(); + await pending; + + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "tool_use", "tool_result", "agent_end"]); + expect(events[4]!.fw_incomplete).toBe(true); + expect(events[4]!.tool_call_id).toBe("call_1"); + expect(events[5]!.outcome).toBe("cancelled"); + }); + + it("closes an open workflow step as cancelled, and the run with it", async () => { + await adapter.install({}); + const gate = barrier(); + const entered = barrier(); + FakeEngine.gate = { entered: entered.open, wait: gate.wait }; + try { + const pending = new FakeRun("r1", ["a", "wait", "b"])._start(); + await entered.wait; + adapter.uninstall(); + gate.open(); + await pending; + } finally { + FakeEngine.gate = null; + } + const events = await flushed(spool); + expect(types(events)).toEqual([ + "agent_start", + "hook_triggered", + "hook_completed", + "hook_triggered", + "hook_completed", + "agent_end", + ]); + expect(events[4]).toMatchObject({ hook_name: "wait", outcome: "cancelled", fw_incomplete: true }); + expect(events[5]!.outcome).toBe("cancelled"); + expect(_internals.openSpans()).toEqual({ models: 0, tools: 0, steps: 0, workflowRuns: 0, agents: 0 }); + }); + + it("re-instruments cleanly: one recording, and the old run's objects stay silent", async () => { + await adapter.install({}); + const old = new FakeAgent(); + await session({ sessionId: "s0" }, () => old.generate("Paris")); + adapter.uninstall(); + await adapter.install({}); + expect(_internals.isEnabled()).toBe(true); + + const before = (await flushed(spool)).length; + await session({ sessionId: "s1" }, async () => { + await new FakeAgent().generate("Rome"); + // The first installation's proxy and wrapper, reused under the second. + await old.resolved!.model.doGenerate(); + await old.resolved!.tools.weather!.execute({ city: "Oslo" }, { toolCallId: "call_2" }); + }); + const after = (await flushed(spool)).slice(before); + expect(types(after)).toEqual([ + "agent_start", + "model_request", + "model_response", + "tool_use", + "tool_result", + "model_request", + "model_response", + "agent_end", + ]); + expect(new Set(after.map((e) => e.session_id))).toEqual(new Set(["s1"])); + }); + + it("re-wraps a model an agent resolved under an earlier installation", async () => { + // A reused agent can hand the patched resolveModelConfig the proxy it + // cached last time; that proxy belongs to a dead installation and would + // pass everything through. + await adapter.install({}); + const agent = new FakeAgent(); + await session({ sessionId: "s0" }, () => agent.generate("Paris")); + adapter.uninstall(); + await adapter.install({}); + const cached = agent.resolved!; + agent.nextModel = cached.model; + agent.nextTools = () => cached.tools; + + const before = (await flushed(spool)).length; + await session({ sessionId: "s1" }, () => agent.generate("Rome")); + const after = (await flushed(spool)).slice(before); + // Attributed to THIS run's agent, not to whatever scope happens to be open. + expect(new Set(after.map((e) => e.agent_id))).toEqual(new Set(["fake-agent"])); + expect(types(after)).toEqual([ + "agent_start", + "model_request", + "model_response", + "tool_use", + "tool_result", + "model_request", + "model_response", + "agent_end", + ]); + }); + + it("honours instrument()'s options even when wrapTool() recorded first", async () => { + const tool = wrapTool({ id: "early", execute: async () => "ok" }); + await tool.execute(); + await adapter.install({ captureLimit: 16 }); + await new FakeAgent().generate("x".repeat(500)); + const start = (await flushed(spool)).find((e) => e.type === "agent_start" && e.agent_id === "fake-agent")!; + expect(String(start.goal).length).toBeLessThan(100); + }); + + it("leaves a hand-wrapped tool working on its own", async () => { + await adapter.install({}); + adapter.uninstall(); + const tool = wrapTool({ id: "solo", execute: async () => "ok" }); + await expect(tool.execute()).resolves.toBe("ok"); + expect(types(await flushed(spool))).toEqual(["agent_start", "tool_use", "tool_result", "agent_end"]); + }); +}); + +describe("bookkeeping", () => { + const empty = { models: 0, tools: 0, steps: 0, workflowRuns: 0, agents: 0 }; + + it("forgets every model and tool call once the run ends — success, model failure, tool failure", async () => { + await adapter.install({}); + await new FakeAgent().generate("Paris"); + expect(_internals.openSpans()).toEqual(empty); + + const failing = new FakeAgent(); + failing.model.fail = true; + await expect(failing.generate("Paris")).rejects.toThrow("model exploded"); + expect(_internals.openSpans()).toEqual(empty); + + const broken = new FakeAgent(); + broken.toolFails = true; + await broken.generate("Paris"); + expect(_internals.openSpans()).toEqual(empty); + }); + + it("forgets a streamed step whether it is consumed or cancelled", async () => { + await adapter.install({}); + const consumed = await new FakeAgent().stream("Paris", {}); + await readAll(consumed.reader); + consumed.finish(); + expect(_internals.openSpans()).toEqual(empty); + + const agent = new FakeAgent(); + const cancelled = await agent.stream("Paris", {}); + await cancelled.abort(); + expect(_internals.openSpans()).toEqual(empty); + const events = await flushed(spool); + expect(events.at(-1)).toMatchObject({ type: "agent_end", outcome: "cancelled" }); + }); + + it("holds a never-consumed stream open only until uninstrument(), which closes it", async () => { + await adapter.install({}); + await new FakeAgent().stream("Paris", {}); + expect(_internals.openSpans()).toEqual({ ...empty, models: 1, agents: 1 }); + adapter.uninstall(); + expect(_internals.openSpans()).toEqual(empty); + const events = await flushed(spool); + expect(types(events)).toEqual(["agent_start", "model_request", "model_response", "agent_end"]); + expect(events[3]!.outcome).toBe("cancelled"); + }); + + it("forgets a workflow run and its steps however the run ends", async () => { + await adapter.install({}); + await new FakeRun("r1", ["a", "b"])._start(); + expect(_internals.openSpans()).toEqual(empty); + await new FakeRun("r2", ["a", "fail"])._start(); + expect(_internals.openSpans()).toEqual(empty); + await expect(new FakeRun("r3", ["explode"])._start()).rejects.toThrow("step threw"); + expect(_internals.openSpans()).toEqual(empty); + + const events = await flushed(spool); + expect(events.filter((e) => e.type === "agent_end").map((e) => e.outcome)).toEqual(["success", "failed", "failed"]); + }); +}); diff --git a/sdk/typescript/test/mastra.test.ts b/sdk/typescript/test/mastra.test.ts new file mode 100644 index 000000000..00d2d7094 --- /dev/null +++ b/sdk/typescript/test/mastra.test.ts @@ -0,0 +1,285 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import * as core from "../src/integrations/core.js"; +import { _internals, wrapTool } from "../src/integrations/mastra.js"; +import { session } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The Mastra adapter's translation tables, against the shapes Mastra and the + * AI SDK model contract actually produce. `integration/mastra.test.ts` proves + * the patches reach real releases; this proves what they read once there, + * including the shapes no fixture happens to hit (V1 and V3 models, 0.20's + * unwrapped step results). + */ + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); +}); +afterEach(async () => { + await spool.cleanup(); + setLogger(null); +}); + +const { + usageOf, + finishReasonOf, + promptOf, + generateOutcome, + foldStreamPart, + toolCallArgs, + isInternalRun, + workflowOutcome, + stepOutcome, +} = _internals; + +describe("reading a model step", () => { + it("reads token counts from every LanguageModel version", () => { + // V2: plain numbers. + expect(usageOf({ inputTokens: 11, outputTokens: 7, totalTokens: 18 })).toEqual({ inputTokens: 11, outputTokens: 7 }); + // V1: the AI SDK 4 names. + expect(usageOf({ promptTokens: 3, completionTokens: 4 })).toEqual({ inputTokens: 3, outputTokens: 4 }); + // V3: a breakdown whose `total` is the count. + expect(usageOf({ inputTokens: { total: 20, noCache: 5 }, outputTokens: { total: 9, text: 9 } })).toEqual({ + inputTokens: 20, + outputTokens: 9, + }); + // Never a float or a negative: the column is a u32 and drops either whole. + expect(usageOf({ inputTokens: 1.6, outputTokens: -1 })).toEqual({ inputTokens: 2, outputTokens: undefined }); + expect(usageOf(undefined)).toEqual({}); + }); + + it("reads the finish reason as a string in every version", () => { + expect(finishReasonOf("tool-calls")).toBe("tool-calls"); + expect(finishReasonOf({ unified: "stop", raw: "end_turn" })).toBe("stop"); + expect(finishReasonOf({ raw: "end_turn" })).toBe("end_turn"); + expect(finishReasonOf({})).toBeUndefined(); + expect(finishReasonOf("")).toBeUndefined(); + }); + + it("splits the provider prompt into system, messages and tools", () => { + const prompt = promptOf({ + prompt: [ + { role: "system", content: "Answer weather questions." }, + { role: "user", content: [{ type: "text", text: "Weather " }, { type: "text", text: "in Paris?" }] }, + { + role: "assistant", + content: [{ type: "tool-call", toolCallId: "call_1", toolName: "weather", input: { city: "Paris" } }], + }, + { + role: "tool", + content: [{ type: "tool-result", toolCallId: "call_1", toolName: "weather", output: { value: "sunny" } }], + }, + ], + tools: [{ type: "function", name: "weather", description: "Current weather", inputSchema: {} }], + }); + expect(prompt.system).toBe("Answer weather questions."); + expect(prompt.messages).toEqual([ + { role: "user", content: "Weather in Paris?" }, + { role: "assistant", content: [{ type: "tool_call", id: "call_1", name: "weather", input: { city: "Paris" } }] }, + { role: "tool", content: [{ type: "tool_result", id: "call_1", name: "weather", output: { value: "sunny" } }] }, + ]); + expect(prompt.tools).toEqual([{ name: "weather", description: "Current weather" }]); + // V1 puts the tools under `mode`. + expect(promptOf({ prompt: [], mode: { type: "regular", tools: [{ name: "t" }] } }).tools).toEqual([ + { name: "t", description: undefined }, + ]); + }); + + it("reads a finished generation in the V2 and V1 shapes", () => { + expect( + generateOutcome({ + content: [ + { type: "text", text: "Checking." }, + { type: "tool-call", toolCallId: "call_1", toolName: "weather", input: '{"city":"Paris"}' }, + ], + finishReason: "tool-calls", + usage: { inputTokens: 11, outputTokens: 7 }, + response: { modelId: "gpt-x-2026" }, + }), + ).toEqual({ + model: "gpt-x-2026", + finishReason: "tool-calls", + usage: { inputTokens: 11, outputTokens: 7 }, + text: "Checking.", + toolCalls: [{ id: "call_1", name: "weather", input: '{"city":"Paris"}' }], + }); + const v1 = generateOutcome({ + text: "Hi", + toolCalls: [{ toolCallId: "c", toolName: "t", args: "{}" }], + finishReason: "stop", + usage: { promptTokens: 1, completionTokens: 2 }, + }); + expect(v1.text).toBe("Hi"); + expect(v1.toolCalls).toEqual([{ id: "c", name: "t", input: "{}" }]); + expect(usageOf(v1.usage)).toEqual({ inputTokens: 1, outputTokens: 2 }); + }); + + it("assembles a streamed step from its parts", () => { + const outcome = {}; + for (const part of [ + { type: "stream-start", warnings: [] }, + { type: "response-metadata", id: "r1", modelId: "served-model" }, + { type: "text-delta", id: "t", delta: "It is " }, + { type: "text-delta", textDelta: "sunny." }, // V1 spelling + { type: "tool-call", toolCallId: "call_2", toolName: "weather", input: "{}" }, + { type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: { inputTokens: 5, outputTokens: 6 } }, + ]) { + foldStreamPart(outcome, part); + } + expect(outcome).toEqual({ + model: "served-model", + text: "It is sunny.", + toolCalls: [{ id: "call_2", name: "weather", input: "{}" }], + finishReason: "stop", + usage: { inputTokens: 5, outputTokens: 6 }, + }); + const failed: { error?: unknown } = {}; + foldStreamPart(failed, { type: "error", error: new Error("rate limited") }); + expect((failed.error as Error).message).toBe("rate limited"); + }); +}); + +describe("reading a tool call", () => { + it("reads 1.x's execute(input, context) with the model's id under context.agent", () => { + expect(toolCallArgs([{ city: "Paris" }, { agent: { toolCallId: "call_9" }, requestContext: {} }])).toEqual({ + input: { city: "Paris" }, + toolCallId: "call_9", + }); + // A direct call from user code passes the input alone. + expect(toolCallArgs([{ city: "Rome" }])).toEqual({ input: { city: "Rome" }, toolCallId: undefined }); + }); + + it("reads 0.x's execute({ context, runtimeContext }, { toolCallId })", () => { + expect(toolCallArgs([{ context: { city: "Paris" }, runtimeContext: {} }, { toolCallId: "call_3" }])).toEqual({ + input: { city: "Paris" }, + toolCallId: "call_3", + }); + expect(toolCallArgs([{ context: { city: "Rome" } }])).toEqual({ input: { city: "Rome" }, toolCallId: undefined }); + }); + + it("does not mistake a 1.x input that has a `context` field for the 0.x envelope", () => { + expect(toolCallArgs([{ context: "long", query: "q" }, { agent: { toolCallId: "c1" } }])).toEqual({ + input: { context: "long", query: "q" }, + toolCallId: "c1", + }); + }); +}); + +describe("reading a workflow", () => { + it("recognises the runs Mastra marks as its own plumbing", () => { + expect(isInternalRun({ isInternalWorkflow: true })).toBe(true); + // The agentic loop's workflows carry InternalSpans.WORKFLOW (1) or ALL (15). + expect(isInternalRun({ tracingPolicy: { internal: 1 } })).toBe(true); + expect(isInternalRun({ tracingPolicy: { internal: 15 } })).toBe(true); + // AGENT | TOOL without the WORKFLOW bit is a user workflow that hides other spans. + expect(isInternalRun({ tracingPolicy: { internal: 6 } })).toBe(false); + expect(isInternalRun({ workflowId: "weather-flow" })).toBe(false); + }); + + it("maps a run status onto an agent_end outcome the server counts correctly", () => { + expect(workflowOutcome("success")).toBe("success"); + expect(workflowOutcome("failed")).toBe("failed"); + expect(workflowOutcome("canceled")).toBe("cancelled"); + expect(workflowOutcome("tripwire")).toBe("rejected"); + expect(workflowOutcome("suspended")).toBe("suspended"); + expect(workflowOutcome(undefined)).toBe("success"); + }); + + it("reads a step result in the 0.24+ wrapped shape and the older bare one", () => { + expect(stepOutcome({ result: { status: "success", output: { city: "Paris" } }, stepResults: {} })).toEqual({ + outcome: "success", + output: { city: "Paris" }, + }); + expect(stepOutcome({ status: "success", output: 1 })).toEqual({ outcome: "success", output: 1 }); + // 1.x hands the error over as an object, 0.x as a string with its stack. + expect(stepOutcome({ result: { status: "failed", error: { name: "Error", message: "step exploded" } } })).toEqual({ + outcome: "failed", + error: "Error: step exploded", + }); + expect(stepOutcome({ status: "failed", error: "Error: step exploded\n at execute (file.js:1:1)" })).toEqual({ + outcome: "failed", + error: "Error: step exploded", + }); + expect(stepOutcome(undefined, new TypeError("boom"))).toEqual({ outcome: "failed", error: "TypeError: boom" }); + expect(stepOutcome({ result: { status: "suspended" } })).toEqual({ outcome: "suspended", output: undefined }); + }); +}); + +describe("wrapTool", () => { + class Tool { + id = "weather"; + // 1.x's signature: the input, then the execution context. + execute = async (input: { city: string }, context?: unknown) => { + void context; + return { city: input.city, forecast: "sunny" }; + }; + describe(): string { + return `tool ${this.id}`; + } + } + + it("keeps the tool a Tool, so Mastra's own checks still accept it", () => { + const tool = new Tool(); + const wrapped = wrapTool(tool); + expect(wrapped).not.toBe(tool); + expect(wrapped).toBeInstanceOf(Tool); + expect(wrapped.describe()).toBe("tool weather"); + expect(wrapTool(wrapped)).toBe(wrapped); + }); + + it("records a call made with nothing open as its own run, named after the tool", async () => { + const wrapped = wrapTool(new Tool()); + await expect(wrapped.execute({ city: "Rome" })).resolves.toEqual({ city: "Rome", forecast: "sunny" }); + const events = await flushed(spool); + expect(events.map((e) => `${String(e.agent_id)} ${String(e.type)}`)).toEqual([ + "weather agent_start", + "weather tool_use", + "weather tool_result", + "weather agent_end", + ]); + expect(new Set(events.map((e) => e.session_id)).size).toBe(1); + expect(events[1]!.input).toEqual({ city: "Rome" }); + expect(events[1]!.tool_call_id).toBe(events[2]!.tool_call_id); + expect(events[3]!.outcome).toBe("success"); + }); + + it("closes its own run failed when the tool throws, and re-throws", async () => { + const wrapped = wrapTool({ + id: "flaky", + execute: async (): Promise => { + throw new RangeError("out of range"); + }, + }); + await expect(wrapped.execute()).rejects.toThrow("out of range"); + const events = await flushed(spool); + expect(events.map((e) => e.type)).toEqual(["agent_start", "tool_use", "tool_result", "agent_end"]); + expect(events[2]!.error).toBe("RangeError: out of range"); + expect(events[3]!.outcome).toBe("failed"); + }); + + it("records a call inside a scope under that scope, with no run of its own", async () => { + const wrapped = wrapTool(new Tool()); + await session({ sessionId: "s1" }, () => + wrapped.execute({ city: "Faro" }, { agent: { toolCallId: "call_7" } }), + ); + const events = await flushed(spool); + expect(events.map((e) => e.type)).toEqual(["tool_use", "tool_result"]); + expect(events[0]!.session_id).toBe("s1"); + expect(events[0]!.tool_call_id).toBe("call_7"); + }); + + it("records a returned failure as a failure", async () => { + // What Mastra 0.x's tool builder hands back instead of throwing. + const wrapped = wrapTool({ id: "validated", execute: async () => ({ error: true, message: "bad input" }) }); + await session({ sessionId: "s1" }, () => wrapped.execute()); + const result = (await flushed(spool)).find((e) => e.type === "tool_result")!; + expect(result.error).toBe("Error: bad input"); + expect(result.output).toBeUndefined(); + }); +}); diff --git a/sdk/typescript/test/next.test.ts b/sdk/typescript/test/next.test.ts new file mode 100644 index 000000000..d9f2b9e67 --- /dev/null +++ b/sdk/typescript/test/next.test.ts @@ -0,0 +1,109 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { nextExternalsGap, resetNextWarnings } from "../src/integrations/index.js"; +import { NEXT_EXTERNALS_ENV, NEXT_EXTERNAL_PACKAGES, withFailproofai } from "../src/next.js"; + +/** + * `withFailproofai(nextConfig)` and the warning `instrument()` gives a Next.js + * server that bundles what an adapter attaches to. + * + * The failure these guard is silent: `next build` bundles LangChain, Mastra + * and LlamaIndex by default, `instrument()` patches the node_modules copy the + * app never runs, reports success, and records nothing. + */ + +const ENV_KEYS = [NEXT_EXTERNALS_ENV, "__NEXT_PRIVATE_STANDALONE_CONFIG", "NEXT_RUNTIME"] as const; +let saved: Record; + +beforeEach(() => { + saved = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])); + for (const key of ENV_KEYS) delete process.env[key]; + resetNextWarnings(); +}); +afterEach(() => { + for (const key of ENV_KEYS) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } +}); + +describe("withFailproofai", () => { + it("adds the packages instrument() needs, keeping the app's own list and the rest of its config", () => { + const config = withFailproofai({ reactStrictMode: true, serverExternalPackages: ["sharp", "@mastra/core"] }); + expect(config.reactStrictMode).toBe(true); + expect(config.serverExternalPackages[0]).toBe("sharp"); + for (const pkg of NEXT_EXTERNAL_PACKAGES) expect(config.serverExternalPackages).toContain(pkg); + // No duplicates, even for one the app already listed. + expect(new Set(config.serverExternalPackages).size).toBe(config.serverExternalPackages.length); + }); + + it("works with no config at all", () => { + expect(withFailproofai()).toEqual({ + serverExternalPackages: [...NEXT_EXTERNAL_PACKAGES], + }); + }); + + it("wraps a config function, sync and async, without calling it early", async () => { + let calls = 0; + const sync = withFailproofai((phase: string) => { + calls += 1; + return { env: { PHASE: phase } }; + }); + expect(calls).toBe(0); + const out = sync("phase-production-build"); + expect(out.env).toEqual({ PHASE: "phase-production-build" }); + expect(out.serverExternalPackages).toContain("@langchain/core"); + + const asyncConfig = withFailproofai(async () => ({ poweredByHeader: false })); + const resolved = await asyncConfig(); + expect(resolved.poweredByHeader).toBe(false); + expect(resolved.serverExternalPackages).toContain("@mastra/core"); + }); + + it("leaves out a package the app transpiles, which Next rejects in both lists", () => { + const config = withFailproofai({ transpilePackages: ["@mastra/core"] }); + expect(config.serverExternalPackages).not.toContain("@mastra/core"); + expect(config.serverExternalPackages).toContain("@langchain/core"); + }); + + it("records what it externalized, for instrument() in the same server process", () => { + withFailproofai({}); + expect(process.env[NEXT_EXTERNALS_ENV]!.split(",")).toEqual([...NEXT_EXTERNAL_PACKAGES]); + }); +}); + +describe("nextExternalsGap", () => { + it("is unknown (null) with no wrapper marker and no standalone config", () => { + expect(nextExternalsGap("langchain")).toBeNull(); + }); + + it("is empty once withFailproofai has run", () => { + withFailproofai({}); + expect(nextExternalsGap("langchain")).toEqual([]); + expect(nextExternalsGap("mastra")).toEqual([]); + expect(nextExternalsGap("llamaindex")).toEqual([]); + }); + + it("names what is missing when the app transpiles a framework the adapter needs external", () => { + withFailproofai({ transpilePackages: ["@mastra/core"] }); + expect(nextExternalsGap("mastra")).toEqual(["@mastra/core"]); + expect(nextExternalsGap("langchain")).toEqual([]); + }); + + it("reads a standalone server's resolved config, where next.config is not evaluated", () => { + process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify({ + serverExternalPackages: ["@failproofai/sdk", "@langchain/core"], + }); + expect(nextExternalsGap("langchain")).toEqual([]); + expect(nextExternalsGap("mastra")).toEqual(["@mastra/core"]); + }); + + it("trusts a hand-set FAILPROOFAI_NEXT_EXTERNALS=1", () => { + process.env[NEXT_EXTERNALS_ENV] = "1"; + expect(nextExternalsGap("llamaindex")).toEqual([]); + }); + + it("never applies to the Vercel AI SDK, which reaches a bundled copy", () => { + expect(nextExternalsGap("ai")).toEqual([]); + }); +}); diff --git a/sdk/typescript/test/packaging.test.ts b/sdk/typescript/test/packaging.test.ts new file mode 100644 index 000000000..6d1395ca7 --- /dev/null +++ b/sdk/typescript/test/packaging.test.ts @@ -0,0 +1,312 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { VERSION } from "../src/version.js"; +import { runNode, indexUrl } from "./helpers.js"; + +/** + * What a consumer actually gets. + * + * Everything else in this suite runs against `src/`. These cases run against + * `dist/` and `package.json`, because the failures they guard — a missing + * export condition, a CommonJS build Node reads as ESM, a dependency somebody + * added with `--save` — are invisible from inside the source tree and total + * from outside it. + */ + +/** An `exports` value: a target, or conditions that nest (`require.types`). */ +type Conditions = string | { [condition: string]: Conditions }; + +/** Every `[condition path, target]` under one `exports` entry. */ +const targetsOf = (value: Conditions, path: string[] = []): Array<[string[], string]> => + typeof value === "string" + ? [[path, value]] + : Object.entries(value).flatMap(([condition, next]) => targetsOf(next, [...path, condition])); + +/** The single target the given condition path selects, e.g. `["require", "default"]`. */ +const targetAt = (value: Conditions, ...path: string[]): string => { + const found = targetsOf(value).filter(([p]) => p.join(".") === path.join(".")); + if (found.length !== 1) throw new Error(`no single target at ${path.join(".")}`); + return found[0]![1]; +}; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { + name: string; + version: string; + type: string; + bin: Record; + exports: Record; + main: string; + types: string; + typesVersions?: Record>; + dependencies?: Record; + optionalDependencies?: Record; + peerDependencies?: Record; + peerDependenciesMeta?: Record; + files: string[]; +}; + +describe("zero runtime dependencies", () => { + it("declares none, because every one would be a constraint the host inherits", () => { + // This package installs into other people's agent processes. A dependency + // we declare is a version they have to resolve against, in the process + // whose reliability we are supposed to be improving. + expect(manifest.dependencies ?? {}).toEqual({}); + expect(manifest.optionalDependencies ?? {}).toEqual({}); + }); + + it("marks every peer dependency optional, so npm installs none of them", () => { + for (const name of Object.keys(manifest.peerDependencies ?? {})) { + expect(manifest.peerDependenciesMeta?.[name]?.optional).toBe(true); + } + }); + + it("imports nothing outside node: builtins", () => { + const offenders: string[] = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + walk(path); + continue; + } + if (!entry.name.endsWith(".ts")) continue; + const source = readFileSync(path, "utf8"); + for (const match of source.matchAll(/^\s*import\s[^;]*?from\s+"([^"]+)"/gm)) { + const specifier = match[1]!; + const bare = !specifier.startsWith(".") && !specifier.startsWith("node:"); + if (bare) offenders.push(`${path}: ${specifier}`); + } + } + }; + walk(join(root, "src")); + // Framework imports are DYNAMIC and guarded — a static one would make + // `import "@failproofai/sdk"` pull LangChain into every process. + expect(offenders).toEqual([]); + }); +}); + +describe("the published surface", () => { + it("builds every path its exports map promises", () => { + for (const [entry, conditions] of Object.entries(manifest.exports)) { + for (const [path, target] of targetsOf(conditions)) { + expect(existsSync(join(root, target)), `${entry}.${path.join(".")} -> ${target}`).toBe(true); + } + } + }); + + it("hands each module system the declarations of its own half", () => { + // A `.d.ts` takes its module format from the nearest package.json, exactly + // like a `.js`. ESM declarations under `require` told every CommonJS + // project on `module: node16` that a CommonJS file was an ES module + // (TS1479 on every import). + for (const [entry, conditions] of Object.entries(manifest.exports)) { + if (entry === "./package.json") continue; + if (entry === "./sandbox-worker") { + // CommonJS only, for both module systems: types and code agree. + for (const [path, target] of targetsOf(conditions)) { + expect(target.startsWith("./dist/cjs/"), `${entry}.${path.join(".")}`).toBe(true); + } + continue; + } + for (const [half, dir] of [ + ["import", "./dist/esm/"], + ["require", "./dist/cjs/"], + ] as const) { + const types = targetAt(conditions, half, "types"); + const code = targetAt(conditions, half, "default"); + expect(types.startsWith(dir), `${entry}.${half}.types -> ${types}`).toBe(true); + expect(code.startsWith(dir), `${entry}.${half}.default -> ${code}`).toBe(true); + expect(types, entry).toBe(code.replace(/\.js$/, ".d.ts")); + } + } + }); + + it("gives moduleResolution: node (node10) every subpath, as CommonJS declarations", () => { + // node10 ignores `exports`; `typesVersions` is its only route to a subpath, + // and `types` to the root. A node10 project compiles to CommonJS. + expect(manifest.types).toBe("./dist/cjs/index.d.ts"); + expect(manifest.main).toBe("./dist/cjs/index.js"); + const mapping = manifest.typesVersions?.["*"] ?? {}; + const subpaths = Object.keys(manifest.exports) + .filter((entry) => entry !== "." && entry !== "./package.json") + .map((entry) => entry.slice(2)); + expect(Object.keys(mapping).sort()).toEqual(subpaths.sort()); + for (const subpath of subpaths) { + const conditions = manifest.exports[`./${subpath}`]!; + const expected = + subpath === "sandbox-worker" + ? targetAt(conditions, "types") + : targetAt(conditions, "require", "types"); + expect(mapping[subpath], subpath).toEqual([expected]); + } + }); + + it("builds the CommonJS declarations the require conditions point at", () => { + const cjs = readFileSync(join(root, "dist/cjs/index.d.ts"), "utf8"); + expect(cjs).toContain("export declare function configure"); + expect(existsSync(join(root, "dist/cjs/evaluator/sandbox-worker.d.ts"))).toBe(true); + }); + + it("tells Node which half of the dual build is CommonJS", () => { + // The package is `"type": "module"`, so without this every `.js` under + // `dist/cjs` is read as ESM and its `require` calls are a syntax error. + expect(manifest.type).toBe("module"); + expect(JSON.parse(readFileSync(join(root, "dist/cjs/package.json"), "utf8"))).toEqual({ + type: "commonjs", + }); + expect(JSON.parse(readFileSync(join(root, "dist/esm/package.json"), "utf8"))).toEqual({ + type: "module", + }); + }); + + it("ships an executable CLI", () => { + const cli = join(root, manifest.bin["failproofai-evaluator"]!); + expect(existsSync(cli)).toBe(true); + expect(statSync(cli).mode & 0o111).toBeGreaterThan(0); + expect(readFileSync(cli, "utf8").startsWith("#!/usr/bin/env node")).toBe(true); + }); + + it("prints usage rather than a stack trace when run with no arguments", () => { + const cli = join(root, manifest.bin["failproofai-evaluator"]!); + const output = execFileSync(process.execPath, [cli, "--help"], { encoding: "utf8" }); + expect(output).toContain("Usage: failproofai-evaluator"); + expect(output).toContain("FAILPROOFAI_EVALUATOR_URL"); + }); + + it("keeps the version in one place", () => { + expect(manifest.version).toBe(VERSION); + }); + + it("includes only what a consumer needs", () => { + expect(manifest.files).toContain("dist/"); + expect(manifest.files).not.toContain("src/"); + expect(manifest.files).not.toContain("test/"); + }); +}); + +describe("both module systems load it", () => { + it("works as ESM", async () => { + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + if (typeof fp.configure !== "function") throw new Error("configure is missing"); + if (typeof fp.agent !== "function") throw new Error("agent is missing"); + if (typeof fp.event.toolUse !== "function") throw new Error("event.toolUse is missing"); + console.log(fp.version); + `); + expect(child.stderr).toBe(""); + expect(child.stdout.trim()).toBe(VERSION); + }); + + it("works as CommonJS", async () => { + const cjs = JSON.stringify(join(root, "dist/cjs/index.js")); + const child = await runNode(` + const { createRequire } = await import("node:module"); + const require_ = createRequire(${JSON.stringify(join(root, "anchor.js"))}); + const fp = require_(${cjs}); + if (typeof fp.configure !== "function") throw new Error("configure is missing"); + if (typeof fp.session !== "function") throw new Error("session is missing"); + console.log(fp.version); + `); + expect(child.stderr).toBe(""); + expect(child.stdout.trim()).toBe(VERSION); + }); + + it("loads the evaluator and the adapters from their subpaths", async () => { + const require_ = createRequire(join(root, "anchor.js")); + for (const subpath of ["./evaluator", "./ai", "./mastra", "./langchain", "./llamaindex"]) { + const target = targetAt(manifest.exports[subpath]!, "require", "default"); + expect(existsSync(join(root, target))).toBe(true); + expect(() => require_(join(root, target))).not.toThrow(); + } + }); + + it("applies configure() from one copy to every copy in the process", async () => { + // Next.js without withFailproofai bundles a copy per route beside the one + // instrumentation.ts configures; the dual build loads ESM and CommonJS + // side by side. Per-copy settings sent the route's events out as `dev`, + // into whatever spool that copy defaulted to. + const dir = mkdtempSync(join(tmpdir(), "fpai-copies-")); + try { + const child = await runNode(` + const esm = await import(${JSON.stringify(indexUrl())}); + const { createRequire } = await import("node:module"); + const cjs = createRequire(${JSON.stringify(join(root, "anchor.js"))})(${JSON.stringify(join(root, "dist/cjs/index.js"))}); + if (esm.configure === cjs.configure) throw new Error("expected two copies"); + esm.configure({ environment: "prod-eu", baseDir: ${JSON.stringify(dir)} }); + cjs.event.agentStart({ sessionId: "copies" }); + await cjs.flush(); + `); + expect(child.stderr).toBe(""); + const events = readdirSync(join(dir, "events")) + .flatMap((f) => readFileSync(join(dir, "events", f), "utf8").split("\n").filter(Boolean)) + .map((line) => JSON.parse(line) as Record); + expect(events.map((e) => [e.session_id, e.environment])).toEqual([["copies", "prod-eu"]]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("runs an evaluator module written as CommonJS or as ESM through the bin", () => { + // The bin is the ESM build. A CommonJS evals file gets `Evaluator` from + // `dist/cjs` — a second copy of the class — so an `instanceof` check in the + // loader refused it with "resolved to Evaluator, not an Evaluator", and the + // commonest setup (plain `tsc` output, no "type": "module") could not start. + const dir = mkdtempSync(join(tmpdir(), "fpai-evals-")); + const cjsEntry = JSON.stringify(join(root, targetAt(manifest.exports["./evaluator"]!, "require", "default"))); + const esmEntry = JSON.stringify( + pathToFileURL(join(root, targetAt(manifest.exports["./evaluator"]!, "import", "default"))).href, + ); + writeFileSync( + join(dir, "evals.cjs"), + `const { Evaluator } = require(${cjsEntry});\nexports.app = new Evaluator({ name: "cjs", version: "1" });\n`, + ); + writeFileSync( + join(dir, "evals.mjs"), + `import { Evaluator } from ${esmEntry};\nexport const app = new Evaluator({ name: "esm", version: "1" });\n`, + ); + writeFileSync(join(dir, "not-evals.cjs"), "exports.app = { runFromEnv() {} };\n"); + + const run = (file: string) => { + const env = { ...process.env }; + delete env.FAILPROOFAI_EVALUATOR_URL; + delete env.FAILPROOFAI_EVALUATOR_TOKEN; + try { + execFileSync(process.execPath, [join(root, manifest.bin["failproofai-evaluator"]!), join(dir, file)], { + env, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + return ""; + } catch (error) { + return String((error as { stderr?: string }).stderr ?? error); + } + }; + try { + // Loading succeeded when the worker gets as far as reading its config. + expect(run("evals.cjs")).toContain("FAILPROOFAI_EVALUATOR_URL is required"); + expect(run("evals.mjs")).toContain("FAILPROOFAI_EVALUATOR_URL is required"); + expect(run("not-evals.cjs")).toContain("not an Evaluator"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not pull a framework into the process just by being imported", async () => { + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + const loaded = Object.keys(await import("node:module").then((m) => m.createRequire(process.cwd() + "/x.js").cache)); + const frameworks = loaded.filter((p) => /node_modules[\\\\/](@langchain|ai|@mastra|llamaindex)[\\\\/]/.test(p)); + console.log(JSON.stringify(frameworks)); + `); + expect(child.stderr).toBe(""); + expect(JSON.parse(child.stdout.trim())).toEqual([]); + }); +}); diff --git a/sdk/typescript/test/redaction.test.ts b/sdk/typescript/test/redaction.test.ts new file mode 100644 index 000000000..467a742d5 --- /dev/null +++ b/sdk/typescript/test/redaction.test.ts @@ -0,0 +1,171 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { redactJsonLine, redactionEnabled, scrubString } from "../src/redact.js"; +import { runtime } from "../src/runtime.js"; +import { useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * The SDK redacts before bytes reach disk; the daemon redacts again before + * upload. Both sides implement the same rules, so these cases are the contract + * between them — a rule that matches here and not there means a credential + * sitting in a spool file waiting for a collector that will not clean it. + * + * ## Why the fixtures are assembled rather than written out + * + * A credential-shaped literal in a source file is a credential-shaped literal, + * whatever the comment beside it says. This repository scans itself for exactly + * these prefixes (and so does every CI secret scanner worth having), so a test + * that spelled one out would trip its own tooling and train everybody to wave + * the alert through. `fake()` builds the same bytes at runtime from parts that + * are individually meaningless. + */ +const fake = (...parts: string[]): string => parts.join(""); + +const ANTHROPIC = fake("sk-", "ant-", "api03-", "A".repeat(20)); +const GITHUB = fake("ghp", "_", "A".repeat(24)); +const GITHUB_OTHER = fake("ghp", "_", "B".repeat(24)); +const AWS = fake("AKIA", "IOSFODNN7EXAMPLE1234"); +const SLACK = fake("xoxb", "-1234567890-ABCDEFGHIJKLMNOP"); + +describe("scrubString", () => { + const redacted = (value: string): string => scrubString(value)[0]; + + it("redacts provider key prefixes at a token boundary", () => { + expect(redacted(`use ${ANTHROPIC} now`)).toBe("use [redacted:anthropic-key] now"); + expect(redacted(GITHUB)).toBe("[redacted:github-token]"); + expect(redacted(AWS)).toBe("[redacted:aws-access-key-id]"); + expect(redacted(SLACK)).toBe("[redacted:slack-token]"); + }); + + it("leaves a prefix that is too short to be a key", () => { + expect(redacted(fake("sk-", "short"))).toBe("sk-short"); + }); + + it("does not fire mid-token, where a match would be a coincidence", () => { + expect(redacted(`x${ANTHROPIC}`)).toBe(`x${ANTHROPIC}`); + }); + + it("redacts a three-segment JWT", () => { + const jwt = fake("eyJ", "hbGciOiJIUzI1NiJ9", ".", "a".repeat(30), ".", "b".repeat(20)); + expect(redacted(jwt)).toBe("[redacted:jwt]"); + }); + + it("redacts a bearer token, case-insensitively on the scheme", () => { + expect(redacted('Authorization: Bearer abcdefghijkl"')).toBe( + 'Authorization: [redacted:bearer-token]"', + ); + expect(redacted("authorization: bearer abcdefghijkl")).toBe( + "authorization: [redacted:bearer-token]", + ); + }); + + it("redacts a secret-shaped assignment but not an ordinary one", () => { + expect(redacted("API_KEY=abcdefghijklmnop")).toBe("API_KEY=[redacted:secret-assignment]"); + expect(redacted('DB_PASSWORD="hunter2hunter2"')).toBe( + 'DB_PASSWORD="[redacted:secret-assignment]"', + ); + // `key` on its own is too weak a name to be a secret; only a compound one + // counts, or the redactor eats every ordinary `key=` in a tool output. + expect(redacted("key=abcdefghijklmnop")).toBe("key=abcdefghijklmnop"); + expect(redacted("NAME=abcdefghijklmnop")).toBe("NAME=abcdefghijklmnop"); + }); + + it("leaves an interpolated or placeholder value alone", () => { + expect(redacted("API_KEY=${SECRET_FROM_VAULT}")).toBe("API_KEY=${SECRET_FROM_VAULT}"); + expect(redacted("API_KEY=")).toBe("API_KEY="); + }); + + it("returns the original string when nothing matched", () => { + const value = "a perfectly ordinary tool output"; + const [scrubbed, hits] = scrubString(value); + expect(hits).toBe(0); + expect(scrubbed).toBe(value); + }); + + it("counts UTF-8 bytes, not code units, for the length thresholds", () => { + // Two 3-byte characters is 6 bytes — under the bearer minimum of 8 — but a + // naive code-unit count would see 2 and a naive character count would see + // 2 as well, so only a byte count refuses this one and accepts the next. + expect(redacted("Bearer 日本")).toBe("Bearer 日本"); + expect(redacted("Bearer 日本語日本語日本")).toBe("[redacted:bearer-token]"); + }); +}); + +describe("redactJsonLine", () => { + it("redacts values, and leaves an untouched line byte-identical", () => { + const clean = JSON.stringify({ type: "tool_use", input: { q: "kites" } }); + expect(redactJsonLine(clean)).toBe(clean); + + const dirty = JSON.stringify({ type: "tool_use", input: { token: GITHUB } }); + expect(JSON.parse(redactJsonLine(dirty)).input.token).toBe("[redacted:github-token]"); + }); + + it("redacts a secret-NAMED field even when the value has no recognisable shape", () => { + const line = JSON.stringify({ type: "t", api_secret: "totally-ordinary-looking" }); + expect(JSON.parse(redactJsonLine(line)).api_secret).toBe("[redacted:secret-assignment]"); + }); + + it("applies the field name to every element of an array value", () => { + const line = JSON.stringify({ type: "t", api_secret: ["aaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbb"] }); + expect(JSON.parse(redactJsonLine(line)).api_secret).toEqual([ + "[redacted:secret-assignment]", + "[redacted:secret-assignment]", + ]); + }); + + it("keeps two keys distinct when redacting collapses them to the same name", () => { + const line = JSON.stringify({ [GITHUB]: 1, [GITHUB_OTHER]: 2 }); + expect(Object.keys(JSON.parse(redactJsonLine(line)))).toEqual([ + "[redacted:github-token]", + "[redacted:github-token]#2", + ]); + }); +}); + +describe("the daemon's redaction switch", () => { + it("defaults to ON when there is no config at all", () => { + const home = mkdtempSync(join(tmpdir(), "fp-config-")); + expect(redactionEnabled(join(home, "custom-agents"))).toBe(true); + }); + + it("defaults to ON when the config is unreadable or malformed", () => { + const home = mkdtempSync(join(tmpdir(), "fp-config-")); + writeFileSync(join(home, "config.json"), "{ not json"); + // Failing open here would mean one typo silently ships credentials. + expect(redactionEnabled(join(home, "custom-agents"))).toBe(true); + }); + + it("honours an explicit off", () => { + const home = mkdtempSync(join(tmpdir(), "fp-config-")); + writeFileSync(join(home, "config.json"), JSON.stringify({ collector: { redact: "off" } })); + expect(redactionEnabled(join(home, "custom-agents"))).toBe(false); + }); +}); + +describe("end to end", () => { + let spool: Spool; + beforeEach(() => { + spool = useSpool(); + }); + afterEach(async () => { + await spool.cleanup(); + }); + + it("redacts before the bytes reach disk", async () => { + runtime.event.toolUse({ + sessionId: "s", + toolName: "shell", + toolCallId: "c1", + input: { command: "curl -H 'Authorization: Bearer abcdefghijklmnop'" }, + }); + await runtime.writer.flushNow(); + const raw = spool.lines().join(""); + expect(raw).not.toContain("abcdefghijklmnop"); + expect(raw).toContain("[redacted:bearer-token]"); + }); +}); diff --git a/sdk/typescript/test/runtimes.test.ts b/sdk/typescript/test/runtimes.test.ts new file mode 100644 index 000000000..f5a106e8a --- /dev/null +++ b/sdk/typescript/test/runtimes.test.ts @@ -0,0 +1,101 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; + +import { appImportsReachCommonJs, entryIsCommonJs, isCommonJsMain, isNextServer } from "../src/node-require.js"; + +/** + * Runtime differences the SDK core has to read correctly. The end-to-end proof + * for each is in `integration/runtimes.*.test.ts` and `integration/nextjs.test.ts`; + * these pin the decisions. + */ + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +describe("isCommonJsMain", () => { + it("reads Node's undefined and Deno's null alike: an ES-module entry", () => { + // Deno sets `require.main` to null for an ES-module entry. Treating that as + // CommonJS patched the frameworks' CommonJS copies in every ES-module Deno + // app, while the app ran the ES-module copies — nothing was recorded. + expect(isCommonJsMain(undefined)).toBe(false); + expect(isCommonJsMain(null)).toBe(false); + }); + + it("reads a module object as a CommonJS entry", () => { + expect(isCommonJsMain({ filename: "/app/index.cjs", id: "." })).toBe(true); + }); +}); + +describe("appImportsReachCommonJs", () => { + const saved = process.env.NEXT_RUNTIME; + afterEach(() => { + if (saved === undefined) delete process.env.NEXT_RUNTIME; + else process.env.NEXT_RUNTIME = saved; + }); + + it("follows the entry point outside Next.js", () => { + delete process.env.NEXT_RUNTIME; + expect(isNextServer()).toBe(false); + expect(appImportsReachCommonJs()).toBe(entryIsCommonJs()); + }); + + it("is false in a Next.js server, whatever the (CommonJS) launcher is", () => { + process.env.NEXT_RUNTIME = "nodejs"; + expect(isNextServer()).toBe(true); + expect(appImportsReachCommonJs()).toBe(false); + }); +}); + +/** + * The same decision from a real CommonJS entry, the shape `next start` has: + * Next's launcher is CommonJS, and it `import()`s every server-external + * package — so inside a Next server the ES-module copy is the one to patch. + */ +describe("requireModuleCopies under a CommonJS launcher", () => { + let app: string; + + beforeAll(() => { + app = mkdtempSync(join(tmpdir(), "failproofai-launcher-")); + const pkg = join(app, "node_modules", "dualfw"); + mkdirSync(join(pkg, "esm"), { recursive: true }); + mkdirSync(join(pkg, "cjs"), { recursive: true }); + writeFileSync( + join(pkg, "package.json"), + JSON.stringify({ name: "dualfw", exports: { ".": { import: "./esm/index.js", require: "./cjs/index.cjs" } } }), + ); + writeFileSync(join(pkg, "esm", "index.js"), 'export const marker = "esm";\n'); + writeFileSync(join(pkg, "esm", "package.json"), JSON.stringify({ type: "module" })); + writeFileSync(join(pkg, "cjs", "index.cjs"), 'exports.marker = "cjs";\n'); + const compat = join(root, "dist", "cjs", "integrations", "compat.js"); + writeFileSync( + join(app, "launcher.cjs"), + `const compat = require(${JSON.stringify(compat)});\n` + + `compat.requireModuleCopies("dualfw", "npm i dualfw").then((copies) => ` + + `console.log(JSON.stringify(copies.map((m) => m.marker))));\n`, + ); + }); + + afterAll(() => rmSync(app, { recursive: true, force: true })); + + const run = (env: Record) => { + const child = spawnSync(process.execPath, [join(app, "launcher.cjs")], { + cwd: app, + encoding: "utf8", + env: { ...process.env, NEXT_RUNTIME: "", ...env }, + }); + expect(child.stderr).toBe(""); + return JSON.parse(child.stdout.trim()) as string[]; + }; + + it("patches the CommonJS copy for a plain CommonJS app", () => { + expect(run({})).toEqual(["cjs"]); + }); + + it("patches the ES-module copy inside a Next.js server", () => { + expect(run({ NEXT_RUNTIME: "nodejs" })).toEqual(["esm"]); + }); +}); diff --git a/sdk/typescript/test/sandbox.test.ts b/sdk/typescript/test/sandbox.test.ts new file mode 100644 index 000000000..6c6f8378b --- /dev/null +++ b/sdk/typescript/test/sandbox.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it } from "vitest"; + +import { EvalResult, Score } from "../src/evaluator/authoring.js"; +import { sessionTranscriptFromWire } from "../src/evaluator/protocol.js"; +import { + EvaluationSandboxUnavailable, + EvaluationTimeout, + UnsafeEvaluatorSource, + compileCondition, + compileEvaluator, + sourceChecksum, +} from "../src/evaluator/source.js"; +import { transcriptWire } from "./helpers.js"; + +/** + * The `worker_threads` sandbox, exercised for real. + * + * It is not the correctness boundary — `expression.ts` is — but it is the only + * thing that bounds CPU, heap and wall clock, and the only thing that can KILL + * a running evaluation. Every one of these cases is a way a permitted + * expression could otherwise take the worker down with it. + */ + +const session = sessionTranscriptFromWire( + transcriptWire([ + { type: "tool_use", payload: { tool_name: "search" } }, + { type: "tool_result", payload: { error: "timeout" } }, + { type: "tool_result", payload: { output: "ok" } }, + ]), +); + +describe("running a managed evaluation", () => { + it("returns a rebuilt EvalResult from the worker", async () => { + const evaluate = compileEvaluator( + "EvalResult({ score: Score(1 - session.eventsOfType('tool_result')" + + ".filter(e => e.payload.error != null).length / session.count('tool_result')), " + + "reasoning: `saw ${session.eventCount} events` })", + { evalKey: "tool_success_rate" }, + ); + const result = await evaluate(session); + expect(result).toBeInstanceOf(EvalResult); + expect(result.score).toBeInstanceOf(Score); + expect(result.score!.value).toBe(0.5); + expect(result.reasoning).toBe("saw 3 events"); + // The rebuilt result is a real one, so the authoring bounds apply to it. + expect(result.resultItems("tool_success_rate")).toHaveLength(1); + }); + + it("runs a managed condition in the same sandbox", async () => { + expect(await compileCondition("session.count('tool_use') > 0")(session)).toBe(true); + const structured = await compileCondition("ConditionResult(false, 'no_tools')")(session); + expect(structured).toMatchObject({ applicable: false, reasonCode: "no_tools" }); + }); + + it("carries a metric and an assertion across the boundary", async () => { + const evaluate = compileEvaluator( + "EvalResult({ metrics: { tool_calls: Metric(session.count('tool_use'), { unit: 'count' }) }, " + + "assertions: { used_a_tool: Assertion(session.count('tool_use') > 0) } })", + { evalKey: "tool_calls" }, + ); + const result = await evaluate(session); + const items = result.resultItems("tool_calls"); + expect(items.map((item) => [item.resultKey, item.resultKind, item.numericValue ?? item.boolValue])).toEqual([ + ["tool_calls", "metric", 1], + ["used_a_tool", "assertion", true], + ]); + }); +}); + +describe("failures come back as data, not as a crash", () => { + it("preserves the error's name across the worker boundary", async () => { + const evaluate = compileEvaluator("EvalResult({ score: Score(session.nope.deeper) })"); + await expect(evaluate(session)).rejects.toThrow(TypeError); + }); + + it("reports a result that is not an EvalResult", async () => { + await expect(compileEvaluator("42")(session)).rejects.toThrow(/must return an EvalResult/); + }); + + it("reports a condition that is not a boolean or a ConditionResult", async () => { + await expect(compileCondition("42")(session)).rejects.toThrow( + /must return a boolean or a ConditionResult/, + ); + }); + + it("rejects unsafe source BEFORE a worker is started", () => { + // Compilation is synchronous and happens in the parent, so a poison + // definition costs nothing and dead-letters as one failed run. + expect(() => compileEvaluator("(x => x).constructor")).toThrow(UnsafeEvaluatorSource); + }); +}); + +describe("resource bounds", () => { + it("kills an evaluation that overruns its wall clock", async () => { + // Under the step ceiling this would run for a very long time; the worker is + // terminated either way, and both arrive as an EvaluationTimeout. + const evaluate = compileEvaluator( + "EvalResult({ score: Score(Array.from({ length: 1 }).length) })", + { timeoutSeconds: 0.001 }, + ); + await expect(evaluate(session)).rejects.toThrow(EvaluationTimeout); + }, 20_000); + + it("bounds an unbounded recursion inside the worker", async () => { + const evaluate = compileEvaluator("(f => f(f))(f => f(f))", { timeoutSeconds: 10 }); + await expect(evaluate(session)).rejects.toThrow(EvaluationTimeout); + }, 30_000); + + it("refuses to evaluate at all when the sandbox cannot be started", async () => { + // Fails CLOSED. An evaluation that cannot be terminated is not one we are + // willing to start, so an unusable sandbox is a refusal — never a fallback + // to running tenant source in the worker's own thread. + const previous = process.env.FAILPROOFAI_SDK_SANDBOX_WORKER; + process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = "/nonexistent/failproofai-sandbox-worker.js"; + try { + const evaluate = compileEvaluator("EvalResult({ score: Score(1) })"); + await expect(evaluate(session)).rejects.toThrow(); + // And specifically NOT by quietly returning a result. + await expect(evaluate(session)).rejects.not.toBeInstanceOf(EvalResult); + } finally { + process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = previous; + } + }); + + it("names the override when it cannot locate the worker at all", () => { + // `EvaluationSandboxUnavailable` carries the remedy, because the situation + // it describes — a bundled install whose published layout is gone — is one + // the operator can only fix if they are told which variable to set. + const error = new EvaluationSandboxUnavailable("x"); + expect(error.name).toBe("EvaluationSandboxUnavailable"); + }); + + it("clamps a server-supplied timeout to the ceiling", async () => { + // A large `timeout_seconds` must not be able to remove the bound. The + // evaluation itself is instant; what matters is that it still completes, + // i.e. the clamp did not turn a huge number into an invalid one. + const evaluate = compileEvaluator("EvalResult({ score: Score(1) })", { + timeoutSeconds: 86_400, + }); + await expect(evaluate(session)).resolves.toBeInstanceOf(EvalResult); + }); +}); + +describe("isolation", () => { + it("gives the worker no environment to read", async () => { + // The language cannot reach `process` at all, so this is defence in depth — + // it means a future gap could not be escalated into credential theft. + const previous = process.env.FAILPROOFAI_EVALUATOR_TOKEN; + process.env.FAILPROOFAI_EVALUATOR_TOKEN = "a-cross-tenant-credential"; + try { + expect(() => compileEvaluator("process.env")).toThrow(UnsafeEvaluatorSource); + expect(() => compileEvaluator("Object.keys(process.env)")).toThrow(UnsafeEvaluatorSource); + } finally { + if (previous === undefined) delete process.env.FAILPROOFAI_EVALUATOR_TOKEN; + else process.env.FAILPROOFAI_EVALUATOR_TOKEN = previous; + } + }); + + it("does not let one evaluation see another's state", async () => { + const first = compileEvaluator("EvalResult({ score: Score(0.25) })"); + const second = compileEvaluator("EvalResult({ score: Score(0.75) })"); + const [a, b] = await Promise.all([first(session), second(session)]); + expect(a.score!.value).toBe(0.25); + expect(b.score!.value).toBe(0.75); + }); + + it("runs more evaluations than the concurrency cap without losing any", async () => { + // The semaphore bounds how many sandboxes exist at once; the extras queue + // rather than pile up memory, and every one still returns. + const results = await Promise.all( + Array.from({ length: 8 }, (_, index) => + compileEvaluator(`EvalResult({ score: Score(${(index / 10).toFixed(1)}) })`)(session), + ), + ); + expect(results.map((result) => result.score!.value)).toEqual([ + 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, + ]); + }, 30_000); +}); + +describe("sourceChecksum", () => { + it("covers the condition and the evaluator together, separated unambiguously", () => { + // A separator that could appear in either source would let two different + // pairs hash the same, which is what the checksum exists to prevent. + expect(sourceChecksum("a", "b")).not.toBe(sourceChecksum("ab", "")); + expect(sourceChecksum(null, "b")).toBe(sourceChecksum("", "b")); + expect(sourceChecksum("a", "b")).toMatch(/^sha256:[0-9a-f]{64}$/); + }); +}); diff --git a/sdk/typescript/test/scopes.test.ts b/sdk/typescript/test/scopes.test.ts new file mode 100644 index 000000000..751a0ceca --- /dev/null +++ b/sdk/typescript/test/scopes.test.ts @@ -0,0 +1,271 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { current, propagate } from "../src/context.js"; +import { agent, session, toolCall } from "../src/scopes.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); +}); +afterEach(async () => { + await spool.cleanup(); +}); + +const typesOf = (events: Array>): string[] => + events.map((event) => String(event.type)); + +describe("session", () => { + it("binds identity and emits nothing on its own", async () => { + const id = await session(async (sessionId) => { + expect(current().sessionId).toBe(sessionId); + return sessionId; + }); + expect(id).toMatch(/^[0-9a-f]{32}$/); + expect(await flushed(spool)).toEqual([]); + }); + + it("inherits an already-bound session rather than splitting the run in two", async () => { + await session({ sessionId: "outer" }, async () => { + await session(async (inner) => { + expect(inner).toBe("outer"); + }); + }); + }); + + it("unbinds on the way out", async () => { + await session({ sessionId: "s" }, () => undefined); + expect(current().sessionId).toBeNull(); + }); +}); + +describe("agent", () => { + it("brackets the block with agent_start and agent_end", async () => { + await agent("planner", { goal: "g" }, () => undefined); + const events = await flushed(spool); + expect(typesOf(events)).toEqual(["agent_start", "agent_end"]); + expect(events[0]!.goal).toBe("g"); + expect(events[1]!.outcome).toBe("success"); + }); + + it("emits error BEFORE agent_end, and marks the outcome 'failed' not 'failure'", async () => { + await expect( + agent("planner", async () => { + throw new TypeError("boom"); + }), + ).rejects.toThrow("boom"); + + const events = await flushed(spool); + expect(typesOf(events)).toEqual(["agent_start", "error", "agent_end"]); + expect(events[1]!.error_type).toBe("TypeError"); + expect(events[1]!.message).toBe("boom"); + expect(typeof events[1]!.traceback).toBe("string"); + // The server only counts error|failed|timeout|rejected as a failure. + expect(events[2]!.outcome).toBe("failed"); + }); + + it("names the error by its class when a subclass leaves `name` as 'Error'", async () => { + // openai's BadRequestError (and many SDKs' errors) never set `name`, so + // `error.name` reads "Error" and the class was lost from the Errors surface. + class BadRequestError extends Error {} + await expect( + agent("planner", async () => { + await toolCall("t", () => { + throw new BadRequestError("model does not exist"); + }); + }), + ).rejects.toThrow("model does not exist"); + const events = await flushed(spool); + expect(events.find((e) => e.type === "tool_result")!.error).toBe("BadRequestError: model does not exist"); + expect(events.find((e) => e.type === "error")!.error_type).toBe("BadRequestError"); + }); + + it("treats an AbortError as cancellation: no error event, outcome 'cancelled'", async () => { + const abort = new Error("stopped"); + abort.name = "AbortError"; + await expect( + agent("planner", async () => { + throw abort; + }), + ).rejects.toThrow("stopped"); + + const events = await flushed(spool); + expect(typesOf(events)).toEqual(["agent_start", "agent_end"]); + expect(events[1]!.outcome).toBe("cancelled"); + }); + + it("nests, with parent_id taken from the enclosing agent", async () => { + await agent("supervisor", async () => { + await agent("worker", () => undefined); + }); + const starts = (await flushed(spool)).filter((event) => event.type === "agent_start"); + expect(starts[0]!.parent_id).toBeUndefined(); + expect(starts[1]!.agent_id).toBe("worker"); + expect(starts[1]!.parent_id).toBe("supervisor"); + }); + + it("does NOT inherit a parent across an explicit new session", async () => { + // The span tree is keyed by session, so a cross-session parent would render + // as a dangling reference or get grafted onto an unrelated agent. + await agent("server", { sessionId: "boot" }, async () => { + await agent("handler", { sessionId: "request-1" }, () => undefined); + }); + const handler = (await flushed(spool)).find( + (event) => event.type === "agent_start" && event.agent_id === "handler", + )!; + expect(handler.session_id).toBe("request-1"); + expect(handler.parent_id).toBeUndefined(); + }); + + it("honours an explicit parentId of null as a forced root", async () => { + await agent("outer", async () => { + await agent("inner", { parentId: null }, () => undefined); + }); + const inner = (await flushed(spool)).find( + (event) => event.type === "agent_start" && event.agent_id === "inner", + )!; + expect(inner.parent_id).toBeUndefined(); + }); + + it("stays synchronous for a synchronous body", () => { + const value = agent("sync", () => 42); + expect(value).toBe(42); + }); + + it("leaves no frame behind when agent_start itself throws", () => { + expect(() => agent("bad", { type: "reserved" }, () => undefined)).toThrow(/Reserved field/); + expect(current().sessionId).toBeNull(); + expect(current().depth).toBe(0); + }); +}); + +describe("toolCall", () => { + it("records the body's resolved value as the output", async () => { + await session({ sessionId: "s" }, async () => { + const hits = await toolCall("search", { input: { q: "x" } }, async () => ["a", "b"]); + expect(hits).toEqual(["a", "b"]); + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect(result.output).toEqual(["a", "b"]); + expect(result.tool_name).toBe("search"); + }); + + it("prefers an explicitly assigned output over the return value", async () => { + await session({ sessionId: "s" }, async () => { + await toolCall("search", async (call) => { + call.output = { chosen: true }; + return "ignored"; + }); + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect(result.output).toEqual({ chosen: true }); + }); + + it("records a failure on the leaf and emits NO error event", async () => { + await session({ sessionId: "s" }, async () => { + await expect( + toolCall("search", async () => { + throw new RangeError("nope"); + }), + ).rejects.toThrow("nope"); + }); + const events = await flushed(spool); + expect(typesOf(events)).toEqual(["tool_use", "tool_result"]); + expect(events[1]!.error).toBe("RangeError: nope"); + }); + + it("closes a cancelled tool call with no error string", async () => { + const abort = new Error("cancelled"); + abort.name = "AbortError"; + await session({ sessionId: "s" }, async () => { + await expect( + toolCall("search", async () => { + throw abort; + }), + ).rejects.toThrow(); + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect("error" in result).toBe(false); + }); + + it("resolves identity once, at entry, so a nested scope cannot move the result", async () => { + await agent("outer", async () => { + await toolCall("search", async () => { + await agent("inner", () => undefined); + }); + }); + const result = (await flushed(spool)).find((event) => event.type === "tool_result")!; + expect(result.agent_id).toBe("outer"); + }); +}); + +describe("using-style scopes", () => { + it("emits the same events as the callback form", async () => { + { + using span = agent.open("planner", { goal: "g" }); + expect(span.agentId).toBe("planner"); + using call = toolCall.open("search", { input: { q: "x" } }); + call.call.output = "done"; + } + const events = await flushed(spool); + expect(typesOf(events)).toEqual(["agent_start", "tool_use", "tool_result", "agent_end"]); + expect(events[2]!.output).toBe("done"); + expect(events[3]!.outcome).toBe("success"); + expect(current().sessionId).toBeNull(); + }); + + it("records a failure the block caught, via fail()", async () => { + { + using span = agent.open("planner"); + span.fail(new Error("handled")); + } + const events = await flushed(spool); + expect(events.at(-1)!.outcome).toBe("failed"); + }); +}); + +describe("propagate", () => { + it("carries identity into a callback invoked outside the scope", async () => { + let stored: (() => void) | null = null; + await session({ sessionId: "s1" }, async () => { + await agent("planner", () => { + stored = propagate(() => { + expect(current().sessionId).toBe("s1"); + expect(current().agentId).toBe("planner"); + }); + }); + }); + expect(current().sessionId).toBeNull(); + stored!(); + }); + + it("binds the same identity on every invocation, not the previous call's leftovers", async () => { + const seen: Array = []; + const wrapped = await session({ sessionId: "s1" }, () => + propagate(() => { + seen.push(current().sessionId); + }), + ); + wrapped(); + wrapped(); + expect(seen).toEqual(["s1", "s1"]); + }); +}); + +describe("concurrency", () => { + it("keeps two concurrent runs on separate identities", async () => { + await Promise.all([ + agent("a", { sessionId: "sa" }, async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(current().sessionId).toBe("sa"); + }), + agent("b", { sessionId: "sb" }, async () => { + expect(current().sessionId).toBe("sb"); + }), + ]); + const starts = (await flushed(spool)).filter((event) => event.type === "agent_start"); + expect(new Set(starts.map((event) => event.session_id))).toEqual(new Set(["sa", "sb"])); + }); +}); diff --git a/sdk/typescript/test/setup.ts b/sdk/typescript/test/setup.ts new file mode 100644 index 000000000..abc08c534 --- /dev/null +++ b/sdk/typescript/test/setup.ts @@ -0,0 +1,19 @@ +import { afterAll } from "vitest"; + +import { runtime } from "../src/runtime.js"; + +/** + * Close the process-wide writer when a test FILE finishes. + * + * Vitest gives each file a fresh module registry but one process, so every file + * that touches `runtime.ts` builds another writer with another flush timer and + * another `exit` listener. Left open they accumulate across the run, Node warns + * about the listener count, and the last file's assertions race timers the + * first file started. + * + * `close()` is the API the writer already exposes for exactly this. + */ +afterAll(async () => { + await runtime.writer.flushNow(); + runtime.writer.close(); +}); diff --git a/sdk/typescript/test/skill-snippets.test.ts b/sdk/typescript/test/skill-snippets.test.ts new file mode 100644 index 000000000..73ae67767 --- /dev/null +++ b/sdk/typescript/test/skill-snippets.test.ts @@ -0,0 +1,73 @@ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; + +import ts from "typescript"; +import { describe, expect, it } from "vitest"; + +import * as failproofai from "../src/index.js"; +import * as evaluator from "../src/evaluator/index.js"; + +/** + * The `failproofai-sdk` skill teaches this package too, and its TypeScript blocks + * are instructions an agent copies into someone's real agent loop — code under + * test, like the Python blocks `sdk/python/tests/test_skill_snippets.py` guards. + * + * Two checks, both cheap and both aimed at drift rather than style: every + * ```ts block parses, and every SDK name a block calls — `failproofai.x`, + * `failproofai.event.x`, a named import from `@failproofai/sdk/evaluator` — + * still exists. A rename in `src/` otherwise leaves the skill teaching a call + * that throws `is not a function` in the customer's process. + */ + +const SKILL = join(__dirname, "..", "..", "python", "skill"); + +function markdownFiles(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const path = join(dir, name); + if (statSync(path).isDirectory()) return markdownFiles(path); + return name.endsWith(".md") ? [path] : []; + }); +} + +function tsBlocks(path: string): string[] { + const text = readFileSync(path, "utf8"); + return [...text.matchAll(/```(?:ts|typescript)[^\n]*\n([\s\S]*?)```/g)].map((m) => m[1]!); +} + +const FILES = markdownFiles(SKILL).filter((path) => tsBlocks(path).length > 0); +const BLOCKS = FILES.flatMap((path) => + tsBlocks(path).map((code, index) => ({ id: `${relative(SKILL, path)} #${index}`, code })), +); + +describe("the failproofai-sdk skill's TypeScript snippets", () => { + it("has snippets to check (a path typo would make every test below vacuous)", () => { + expect(BLOCKS.length).toBeGreaterThanOrEqual(8); + }); + + it.each(BLOCKS)("$id parses", ({ code }) => { + const out = ts.transpileModule(code, { + reportDiagnostics: true, + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }); + const errors = (out.diagnostics ?? []).map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n")); + expect(errors).toEqual([]); + }); + + it.each(BLOCKS)("$id calls only names the SDK exports", ({ code }) => { + const missing: string[] = []; + for (const [, name] of code.matchAll(/\bfailproofai\.(?!event\.)(\w+)/g)) { + if (!(name! in failproofai)) missing.push(`failproofai.${name}`); + } + for (const [, name] of code.matchAll(/\bfailproofai\.event\.(\w+)/g)) { + if (typeof (failproofai.event as unknown as Record)[name!] !== "function") { + missing.push(`failproofai.event.${name}`); + } + } + for (const [, names] of code.matchAll(/import\s*\{([^}]*)\}\s*from\s*"@failproofai\/sdk\/evaluator"/g)) { + for (const name of names!.split(",").map((n) => n.trim()).filter(Boolean)) { + if (!(name in evaluator)) missing.push(`@failproofai/sdk/evaluator: ${name}`); + } + } + expect(missing).toEqual([]); + }); +}); diff --git a/sdk/typescript/test/spool-contract.test.ts b/sdk/typescript/test/spool-contract.test.ts new file mode 100644 index 000000000..9b8456918 --- /dev/null +++ b/sdk/typescript/test/spool-contract.test.ts @@ -0,0 +1,124 @@ +import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + expandUser, + failproofaiCustomAgentsDir, + getBaseDir, + legacyAgenteyeDir, + setBaseDir, +} from "../src/resolver.js"; + +/** + * Four implementations resolve where this SDK's spool lives, and they must + * agree or the SDK writes where no daemon reads — with NO error on either side, + * which is the whole reason this file exists: + * + * * this package's `resolver.ts` + * * the Python SDK's `failproofai_sdk/_resolver.py` + * * the daemon's `crates/fpai-collect/src/config.rs` + * * the CLI's `src/hooks/fp-home.ts` + * + * These assertions read the OTHER THREE from disk rather than restating them. + * A test that spelled the path out a fifth time would pass while the daemon + * watched somewhere else. + */ + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); + +function read(relative: string): string { + return readFileSync(join(repoRoot, relative), "utf8"); +} + +afterEach(() => { + setBaseDir(null); +}); + +describe("agreement with the other implementations", () => { + it("matches the daemon's custom_agents_events_dir", () => { + const rust = read("crates/fpai-collect/src/config.rs"); + // `home.join("custom-agents").join("events")` + expect(rust).toMatch(/\.join\("custom-agents"\)\s*\.join\("events"\)/); + expect(join(failproofaiCustomAgentsDir(), "events")).toBe( + join(homedir(), ".failproofai", "custom-agents", "events"), + ); + }); + + it("matches the CLI's customAgentsDir", () => { + const hooks = read("src/hooks/fp-home.ts"); + expect(hooks).toContain('atHome(home, "custom-agents")'); + expect(hooks).toMatch(/process\.env\.FAILPROOFAI_HOME\s*\|\|\s*resolve\(homedir\(\), "\.failproofai"\)/); + }); + + it("matches the Python SDK's resolver", () => { + const python = read("sdk/python/failproofai_sdk/_resolver.py"); + expect(python).toContain('base / "custom-agents"'); + expect(python).toContain('os.environ.get("FAILPROOFAI_HOME")'); + }); + + it("keeps the daemon watching the legacy root, so an older SDK is not stranded", () => { + const rust = read("crates/fpai-collect/src/config.rs"); + expect(rust).toContain("agenteye_events_dir()"); + expect(legacyAgenteyeDir()).toBe(join(homedir(), ".agenteye")); + }); +}); + +describe("resolution order", () => { + it("prefers an explicit base directory over everything", () => { + setBaseDir("/tmp/somewhere-explicit"); + expect(getBaseDir()).toBe("/tmp/somewhere-explicit"); + }); + + it("moves with FAILPROOFAI_HOME but never leaves it", () => { + const previous = process.env.FAILPROOFAI_HOME; + process.env.FAILPROOFAI_HOME = "/opt/fp"; + try { + // The `custom-agents` segment is appended unconditionally: the variable + // MOVES the umbrella, it cannot take the spool outside it. + expect(failproofaiCustomAgentsDir()).toBe(join("/opt/fp", "custom-agents")); + } finally { + if (previous === undefined) delete process.env.FAILPROOFAI_HOME; + else process.env.FAILPROOFAI_HOME = previous; + } + }); + + it("is NOT redirected by AGENTEYE_HOME", () => { + const previous = process.env.AGENTEYE_HOME; + process.env.AGENTEYE_HOME = "/tmp/agenteye-elsewhere"; + try { + // A redirect with no confirmation and no error means batches land in a + // directory nothing reads, and an unread spool is indistinguishable from + // an idle one. The Python SDK removed this for the same reason. + expect(getBaseDir()).not.toContain("agenteye-elsewhere"); + } finally { + if (previous === undefined) delete process.env.AGENTEYE_HOME; + else process.env.AGENTEYE_HOME = previous; + } + }); +}); + +describe("tilde expansion", () => { + it("expands a leading ~, so the documented migration bridge does not create one", () => { + // `configure({ baseDir: "~/.agenteye" })` is the documented explicit bridge. + // Without expansion, the writer's recursive mkdir cheerfully creates a + // directory literally named `~` under the process's cwd and spools into it: + // nothing on the machine watches that path, so 100% of the telemetry is + // lost. + setBaseDir("~/.agenteye"); + expect(getBaseDir()).toBe(join(homedir(), ".agenteye")); + expect(getBaseDir().startsWith("~")).toBe(false); + }); + + it("leaves a tilde that is not a home reference alone", () => { + expect(expandUser("/var/lib/~weird")).toBe("/var/lib/~weird"); + }); + + it("resolves a relative base directory against the cwd", () => { + setBaseDir("relative/spool"); + expect(getBaseDir()).toBe(resolve("relative/spool")); + }); +}); diff --git a/sdk/typescript/test/tracker-bounds.test.ts b/sdk/typescript/test/tracker-bounds.test.ts new file mode 100644 index 000000000..5b5018dc2 --- /dev/null +++ b/sdk/typescript/test/tracker-bounds.test.ts @@ -0,0 +1,191 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { runExitClosers } from "../src/exit.js"; +import * as core from "../src/integrations/core.js"; +import { runtime } from "../src/runtime.js"; +import { PAUSED_SESSION_TTL_MS, _stats, adapter, langchainHandler } from "../src/integrations/langchain.js"; +import { flushed, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +/** + * What a LONG-RUNNING process keeps in memory, and what that does to events. + * + * Every other adapter test runs a handful of runs and looks at the trace. These + * run tens of thousands, because the failures here only exist at volume: a + * table that is never pruned on normal completion fills to its cap, and a + * FIFO cap then evicts the entries of runs that are still LIVE — a model call + * that has been running for ten seconds on a busy server loses its + * `model_response`, with one warning, while every test with three runs in it + * passes. + */ + +type Handler = Record unknown>; + +let spool: Spool; +let h: Handler; + +beforeEach(() => { + spool = useSpool(); + core.resetFailures(); + h = langchainHandler() as Handler; +}); +afterEach(async () => { + vi.useRealTimers(); + adapter.uninstall(); + await spool.cleanup(); +}); + +const start = (id: string, parent: string | undefined, name: string, meta: Record = {}, tags: string[] = []) => + h.handleChainStart!({ name }, {}, id, parent, tags, meta, undefined, name); +const end = (id: string) => h.handleChainEnd!({}, id); + +/** One ordinary request: a graph root and one node, both completed. */ +async function request(i: number): Promise { + await start(`r${i}`, undefined, "g", { failproofai_sdk_session_id: `S${i}` }); + await start(`c${i}`, `r${i}`, "n", { langgraph_node: "n", langgraph_checkpoint_ns: `n:${i}` }, ["graph:step:1"]); + await end(`c${i}`); + await end(`r${i}`); +} + +describe("RunTracker", () => { + it("forgets a link once the run it belongs to has closed", () => { + const tracker = new core.RunTracker("t"); + tracker.startAgent("root", { agentId: "a", sessionId: "s" }); + tracker.emit("toolUse", "tool", { parentKey: "root", toolName: "x", toolCallId: "1" }); + expect(tracker.stats().links).toBe(1); + tracker.emit("toolResult", "tool", { toolName: "x", toolCallId: "1" }); + expect(tracker.stats().links).toBe(0); + tracker.endAgent("root"); + expect(tracker.stats()).toEqual({ runs: 0, links: 0 }); + }); + + it("answers isOpen without copying every open agent", () => { + const tracker = new core.RunTracker("t"); + tracker.startAgent("root", { agentId: "a", sessionId: "s" }); + expect(tracker.isOpen("root")).toBe(true); + expect(tracker.isOpen("other")).toBe(false); + tracker.forget("root"); + expect(tracker.isOpen("root")).toBe(false); + }); + + it("closes open agents at exit, but leaves one paused on a human for its resumer", async () => { + // A LangGraph interrupt or a suspended Mastra workflow is resumed by + // whichever process takes the answer; ending it here would end a run that + // is not over. Everything else a dying process holds is abandoned. + const tracker = new core.RunTracker("t"); + tracker.startAgent("busy", { agentId: "busy", sessionId: "s1" }); + tracker.startAgent("waiting", { agentId: "waiting", sessionId: "s2" }); + tracker.emit("agentPause", "waiting", { pauseId: "p1" }); + tracker.startAgent("resumed", { agentId: "resumed", sessionId: "s3" }); + tracker.emit("agentPause", "resumed", { pauseId: "p2" }); + tracker.emit("agentResume", "resumed", { pauseId: "p2" }); + tracker.closeAtExit(); + const ends = (await flushed(spool)).filter((e) => e.type === "agent_end"); + expect(ends.map((e) => [e.agent_id, e.outcome])).toEqual([ + ["resumed", "failed"], + ["busy", "failed"], + ]); + expect(tracker.isOpen("waiting")).toBe(true); + }); + + it("closes an adapter's open tool, hook and model call at exit, then its agent with an error", async () => { + // Live on LangGraph, Mastra, LlamaIndex and the AI SDK: SIGTERM mid-tool + // closed the agent and left the tool_use, the node's hook_triggered and + // the model_request open — spans the dashboard shows as running forever. + const tracker = new core.RunTracker("t"); + tracker.startAgent("root", { agentId: "svc", sessionId: "exit-s" }); + tracker.emit("hookTriggered", "node", { parentKey: "root", hookName: "tools", hookId: "h1" }); + tracker.emit("modelRequest", "llm", { parentKey: "root", requestId: "r1", model: "m" }); + tracker.emit("toolUse", "tool", { parentKey: "node", toolName: "slow", toolCallId: "c1" }); + tracker.emit("toolUse", "done", { parentKey: "node", toolName: "fast", toolCallId: "c0" }); + tracker.emit("toolResult", "done", { toolName: "fast", toolCallId: "c0" }); + runExitClosers(143); // both phases, as the writer's exit hook runs them + // This session only: the exit closes whatever else this test process left open too. + const events = (await flushed(spool)).filter((e) => e.session_id === "exit-s").slice(6); // start, hook, model, two tool_use, one tool_result + expect(events.map((e) => [e.type, e.tool_call_id ?? e.hook_id ?? e.request_id ?? null])).toEqual([ + ["tool_result", "c1"], + ["model_response", "r1"], + ["hook_completed", "h1"], + ["error", null], + ["agent_end", null], + ]); + expect(events[0]!.error).toMatch(/^ProcessExit: the process exited \(code 143\)/); + expect(events[1]!.stop_reason).toBe("error"); + expect(events[2]!.outcome).toBe("failed"); + expect(events[3]!.error_type).toBe("ProcessExit"); + expect(events[4]!.outcome).toBe("failed"); + }); +}); + +describe("langchain under load", () => { + it("keeps a long-running model call attributable while thousands of runs complete", async () => { + await start("rootA", undefined, "graph", { failproofai_sdk_session_id: "A" }); + await start("nodeA", "rootA", "agent", { langgraph_node: "agent", langgraph_checkpoint_ns: "agent:1" }, ["graph:step:1"]); + await h.handleChatModelStart!({ name: "ChatX" }, [[]], "modelA", "nodeA", {}, [], { langgraph_node: "agent" }, "ChatX"); + + for (let i = 0; i < 12_000; i++) { + await request(i); + // A real server yields to the writer's flush timer constantly; this tight + // loop never does, and would otherwise hit the writer's OWN queue cap. + if (i % 1_000 === 0) await runtime.writer.flushNow(); + } + + await h.handleLLMEnd!( + { generations: [[{ text: "hi", message: { content: "hi", usage_metadata: { input_tokens: 3, output_tokens: 1 } } }]] }, + "modelA", + ); + await end("nodeA"); + await end("rootA"); + + const types = (await flushed(spool)).filter((e) => e.session_id === "A").map((e) => e.type); + expect(types).toEqual([ + "agent_start", + "hook_triggered", + "model_request", + "model_response", + "hook_completed", + "agent_end", + ]); + }); + + it("holds nothing for a request once it has completed", async () => { + for (let i = 0; i < 5_000; i++) await request(i); + expect(_stats()).toEqual({ runs: 0, sessions: 0, tracker: { runs: 0, links: 0 } }); + }); + + // A resume arriving here after the TTL takes the cross-worker path, which the + // integration suite proves with two real processes (`remote-resume`). + it("forgets a paused run after the TTL", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const interrupt = (id: string) => + Object.assign(new Error("interrupt"), { name: "GraphInterrupt", is_bubble_up: true, interrupts: [{ id, value: "approve?" }] }); + + // Paused here, resumed (if ever) by another worker — the ordinary shape. + for (let i = 0; i < 50; i++) { + await start(`p${i}`, undefined, "g", { thread_id: `t${i}` }); + await start(`pn${i}`, `p${i}`, "ask", { langgraph_node: "ask", langgraph_checkpoint_ns: `ask:${i}` }, ["graph:step:1"]); + await h.handleChainError!(interrupt(`i${i}`), `pn${i}`); + await h.handleChainError!(interrupt(`i${i}`), `p${i}`); + } + expect(_stats().tracker.runs).toBe(50); + + vi.setSystemTime(Date.now() + PAUSED_SESSION_TTL_MS + 1); + await request(0); + expect(_stats()).toEqual({ runs: 0, sessions: 0, tracker: { runs: 0, links: 0 } }); + }); + + it("never lets paused runs crowd live ones out of the tracker", async () => { + const interrupt = (id: string) => + Object.assign(new Error("interrupt"), { name: "GraphInterrupt", is_bubble_up: true, interrupts: [{ id, value: "approve?" }] }); + for (let i = 0; i < 3_000; i++) { + await start(`p${i}`, undefined, "g", { thread_id: `t${i}` }); + await start(`pn${i}`, `p${i}`, "ask", { langgraph_node: "ask", langgraph_checkpoint_ns: `ask:${i}` }, ["graph:step:1"]); + await h.handleChainError!(interrupt(`i${i}`), `pn${i}`); + await h.handleChainError!(interrupt(`i${i}`), `p${i}`); + } + // Bounded by the session cap, not by the tracker's — the tracker's room is + // for runs that are still executing. + expect(_stats().tracker.runs).toBeLessThanOrEqual(1_000); + expect(_stats().sessions).toBeLessThanOrEqual(1_000); + }); +}); diff --git a/sdk/typescript/test/wire-format.test.ts b/sdk/typescript/test/wire-format.test.ts new file mode 100644 index 000000000..fe7fbc5be --- /dev/null +++ b/sdk/typescript/test/wire-format.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { setEnvironment } from "../src/environment.js"; +import * as schema from "../src/schema.js"; + +/** + * The wire format, frozen. + * + * These assertions exist to FAIL when somebody reorders a field or renames one, + * because the other end of this pipe is a Rust daemon, an ingest endpoint whose + * dedup key hashes the canonical payload, and a Python SDK writing into the + * same directories. None of those three will tell us when we drift; this file + * is the only thing that will. + * + * Key ORDER is asserted, not just membership. The identity block comes first, + * then `environment`, then the declared optionals that were supplied, then the + * caller's extras — the same order `failproofai_sdk/_schema.py` produces. + */ + +const IDENTITY = { + timestamp: "2026-01-01T00:00:00.000000Z", + sessionId: "s1", + agentId: "main", +} as const; + +describe("event wire format", () => { + beforeEach(() => { + setEnvironment("test-env"); + }); + afterEach(() => { + setEnvironment(null); + }); + + it("puts the identity block first, then environment, then optionals, then extras", () => { + const event = schema.toolUseEvent({ + ...IDENTITY, + toolName: "search", + toolCallId: "c1", + input: { q: "kites" }, + extraFields: { fw_run_id: "r1" }, + }); + expect(Object.keys(event)).toEqual([ + "timestamp", + "session_id", + "agent_id", + "type", + "tool_name", + "tool_call_id", + "environment", + "input", + "fw_run_id", + ]); + expect(event).toEqual({ + timestamp: "2026-01-01T00:00:00.000000Z", + session_id: "s1", + agent_id: "main", + type: "tool_use", + tool_name: "search", + tool_call_id: "c1", + environment: "test-env", + input: { q: "kites" }, + fw_run_id: "r1", + }); + }); + + it("omits an optional that was not supplied rather than writing null", () => { + const event = schema.toolResultEvent({ ...IDENTITY, toolName: "s", toolCallId: "c1" }); + expect(Object.keys(event)).toEqual([ + "timestamp", + "session_id", + "agent_id", + "type", + "tool_name", + "tool_call_id", + "environment", + ]); + expect("output" in event).toBe(false); + expect("duration_ms" in event).toBe(false); + }); + + it("treats null and undefined identically for optionals", () => { + const withNull = schema.agentEndEvent({ ...IDENTITY, outcome: null, summary: null }); + const withUndefined = schema.agentEndEvent({ ...IDENTITY }); + expect(withNull).toEqual(withUndefined); + }); + + it("appends request_id LAST on model events, so older events are byte-identical", () => { + const withoutRequestId = schema.modelResponseEvent({ ...IDENTITY, model: "m", role: "assistant" }); + const withRequestId = schema.modelResponseEvent({ + ...IDENTITY, + model: "m", + role: "assistant", + requestId: "r1", + }); + expect(Object.keys(withRequestId)).toEqual([...Object.keys(withoutRequestId), "request_id"]); + expect(JSON.stringify(withRequestId).startsWith(JSON.stringify(withoutRequestId).slice(0, -1))).toBe( + true, + ); + }); + + it("lets extras override nothing that is declared — they are merged last by design", () => { + // This is the hazard `guardExtras` and the reserved-name check exist for: + // the schema itself does NOT protect the declared field. + const event = schema.toolUseEvent({ + ...IDENTITY, + toolName: "declared", + toolCallId: "c1", + extraFields: { tool_name: "overwritten" }, + }); + expect(event.tool_name).toBe("overwritten"); + }); + + it("covers all 15 event types with the type discriminator the server reads", () => { + const types = [ + schema.toolUseEvent({ ...IDENTITY, toolName: "t", toolCallId: "c" }), + schema.toolResultEvent({ ...IDENTITY, toolName: "t", toolCallId: "c" }), + schema.modelRequestEvent({ ...IDENTITY }), + schema.modelResponseEvent({ ...IDENTITY }), + schema.agentStartEvent({ ...IDENTITY }), + schema.agentEndEvent({ ...IDENTITY }), + schema.agentPauseEvent({ ...IDENTITY, pauseId: "p" }), + schema.agentResumeEvent({ ...IDENTITY, pauseId: "p" }), + schema.hookTriggeredEvent({ ...IDENTITY, hookName: "h", hookId: "h1" }), + schema.hookCompletedEvent({ ...IDENTITY, hookName: "h", hookId: "h1" }), + schema.errorEvent({ ...IDENTITY, errorType: "E", message: "m" }), + schema.humanWaitEvent({ ...IDENTITY, inputId: "i" }), + schema.humanInputEvent({ ...IDENTITY, inputId: "i" }), + schema.humanPauseEvent({ ...IDENTITY }), + schema.humanInterruptEvent({ ...IDENTITY }), + ].map((event) => event.type); + + expect(types).toEqual([ + "tool_use", + "tool_result", + "model_request", + "model_response", + "agent_start", + "agent_end", + "agent_pause", + "agent_resume", + "hook_triggered", + "hook_completed", + "error", + "human_wait", + "human_input", + "human_pause", + "human_interrupt", + ]); + }); + + it("names every declared field, so integrations/core.ts cannot hold a stale copy", () => { + // `DECLARED_FIELD_NAMES` drives `FORBIDDEN_EXTRAS`. A field added above and + // missing here is a field an adapter could silently overwrite. + const declared = new Set(); + const identity = { ...IDENTITY }; + const builders = [ + schema.toolUseEvent({ ...identity, toolName: "t", toolCallId: "c", input: {} }), + schema.toolResultEvent({ + ...identity, + toolName: "t", + toolCallId: "c", + output: 1, + error: "e", + durationMs: 1, + }), + schema.modelRequestEvent({ + ...identity, + model: "m", + messages: [], + system: "s", + tools: [], + requestId: "r", + }), + schema.modelResponseEvent({ + ...identity, + model: "m", + stopReason: "s", + inputTokens: 1, + outputTokens: 1, + content: "c", + role: "assistant", + requestId: "r", + }), + schema.agentStartEvent({ ...identity, goal: "g", parentId: "p" }), + schema.agentEndEvent({ ...identity, outcome: "success", summary: "s" }), + schema.agentPauseEvent({ ...identity, pauseId: "p", reason: "r", userId: "u" }), + schema.agentResumeEvent({ ...identity, pauseId: "p", durationMs: 1, reason: "r", userId: "u" }), + schema.hookTriggeredEvent({ + ...identity, + hookName: "h", + hookId: "h", + triggerEvent: "t", + input: 1, + }), + schema.hookCompletedEvent({ + ...identity, + hookName: "h", + hookId: "h", + outcome: "o", + output: 1, + error: "e", + durationMs: 1, + }), + schema.errorEvent({ ...identity, errorType: "E", message: "m", traceback: "t" }), + schema.humanWaitEvent({ ...identity, inputId: "i", prompt: "p", options: ["a"], reason: "r" }), + schema.humanInputEvent({ ...identity, inputId: "i", response: "r", durationMs: 1 }), + schema.humanPauseEvent({ ...identity, reason: "r", userId: "u" }), + schema.humanInterruptEvent({ ...identity, reason: "r", userId: "u", atStep: "s" }), + ]; + for (const event of builders) for (const key of Object.keys(event)) declared.add(key); + + expect([...declared].sort()).toEqual([...schema.DECLARED_FIELD_NAMES].sort()); + }); +}); diff --git a/sdk/typescript/test/writer.test.ts b/sdk/typescript/test/writer.test.ts new file mode 100644 index 000000000..c0a4d5a62 --- /dev/null +++ b/sdk/typescript/test/writer.test.ts @@ -0,0 +1,407 @@ +import { statSync } from "node:fs"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { setLogger } from "../src/logger.js"; +import { runtime } from "../src/runtime.js"; +import { + EventWriter, + approxSize, + capFields, + encodeEntry, + roll, + sanitize, + scrubSurrogates, + validatedInterval, +} from "../src/writer.js"; +import { indexUrl, runNode, useSpool } from "./helpers.js"; +import type { Spool } from "./helpers.js"; + +let spool: Spool; + +beforeEach(() => { + spool = useSpool(); +}); +afterEach(async () => { + await spool.cleanup(); + setLogger(null); +}); + +describe("flush interval validation", () => { + it("refuses a value the timer cannot run on, at the boundary", () => { + for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => validatedInterval(bad)).toThrow(/finite number greater than zero/); + } + expect(validatedInterval(0.25)).toBe(0.25); + }); + + it("leaves the writer on its old interval when a new one is rejected", () => { + const writer = new EventWriter(0.5); + try { + expect(() => writer.setFlushInterval(-1)).toThrow(); + expect(writer.getFlushInterval()).toBe(0.5); + } finally { + writer.close(); + } + }); +}); + +describe("durability", () => { + it("publishes .jsonl files, never leaving a .tmp behind", async () => { + runtime.event.agentStart({ sessionId: "s" }); + await runtime.writer.flushNow(); + expect(spool.files()).toHaveLength(1); + expect(spool.files()[0]).toMatch(/^event-.*-\d+-\d+\.jsonl$/); + }); + + it("names each batch uniquely, so two in the same millisecond cannot overwrite", async () => { + runtime.event.agentStart({ sessionId: "s" }); + await runtime.writer.flushNow(); + runtime.event.agentStart({ sessionId: "s" }); + await runtime.writer.flushNow(); + const files = spool.files(); + expect(new Set(files).size).toBe(2); + // The pid is in the stem so unrelated processes sharing one spool root + // cannot collide either. + for (const name of files) expect(name).toContain(`-${process.pid}-`); + }); + + it("writes 0600 batches inside a 0700 directory", async () => { + runtime.event.agentStart({ sessionId: "s" }); + await runtime.writer.flushNow(); + const eventsDir = join(spool.dir, "events"); + expect(statSync(eventsDir).mode & 0o777).toBe(0o700); + expect(statSync(join(eventsDir, spool.files()[0]!)).mode & 0o777).toBe(0o600); + }); + + it("flushNow() writes an event emitted while another flush was already writing", async () => { + // The interval timer (or a second caller) can be mid-write when you call + // `flush()`. That flush drained the queue BEFORE this event arrived, so + // returning it — what flushNow() used to do — resolved with the newest + // event still in memory, breaking "resolves once your events are on disk" + // for exactly the emit-flush-exit script it exists for. + runtime.event.agentStart({ sessionId: "s" }); + const alreadyWriting = runtime.writer.flushNow(); + runtime.event.agentEnd({ sessionId: "s" }); + await runtime.writer.flushNow(); + expect(spool.events().map((e) => e.type)).toEqual(["agent_start", "agent_end"]); + await alreadyWriting; + }); + + it("ends every batch file with a newline, as the collector's line reader expects", async () => { + runtime.event.agentStart({ sessionId: "s" }); + await runtime.writer.flushNow(); + const raw = spool.lines(); + expect(raw).toHaveLength(1); + expect(JSON.parse(raw[0]!)).toMatchObject({ type: "agent_start" }); + }); +}); + +describe("encoding isolation", () => { + it("drops ONE unencodable event rather than the batch around it", async () => { + const circular: Record = { name: "loop" }; + circular.self = circular; + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + + runtime.event.agentStart({ sessionId: "s", fw_before: 1 }); + runtime.event.agentStart({ sessionId: "s", fw_cycle: circular }); + runtime.event.agentStart({ sessionId: "s", fw_after: 1 }); + await runtime.writer.flushNow(); + + const events = spool.events(); + expect(events).toHaveLength(3); + // The cyclic one is still published — sanitized, not dropped. + expect(events[1]!.fw_cycle).toEqual({ name: "loop", self: "" }); + }); + + it("sanitizes a cycle rather than failing the event", () => { + const node: Record = { a: 1 }; + node.self = node; + expect(sanitize(node, new Set())).toEqual({ a: 1, self: "" }); + }); + + it("does not mistake a DAG for a cycle", () => { + const shared = { value: 1 }; + expect(sanitize({ left: shared, right: shared }, new Set())).toEqual({ + left: { value: 1 }, + right: { value: 1 }, + }); + }); + + it("makes a lone surrogate inert, which ingest would otherwise skip at 200 OK", () => { + expect(scrubSurrogates("ok\ud800bad")).toBe("ok\\ud800bad"); + // A well-formed pair is left alone. + expect(scrubSurrogates("emoji \u{1F600}")).toBe("emoji \u{1F600}"); + }); + + it("scrubs a lone surrogate in a KEY as well as a value", () => { + // An unscrubbed key reaches the wire as a JSON lone-surrogate escape, ingest + // answers 200 with `{"accepted":0,"skipped":1}`, and the uploader parks that + // batch and poisons it after three retries — so one bad key loses every + // event batched with it. A filesystem path, the realistic source, is most + // naturally a key. + const encoded = encodeEntry({ type: "t", ["path\udcff"]: "x", value: "v\udcff" })!; + // No RAW surrogate survives anywhere in the line... + expect(/[\uD800-\uDFFF]/.test(encoded)).toBe(false); + // ...and both the key and the value carry the byte visibly instead. + const parsed = JSON.parse(encoded) as Record; + expect(Object.keys(parsed)).toContain(String.raw`path\udcff`); + expect(parsed.value).toBe(String.raw`v\udcff`); + }); + + it("writes a non-finite number as null rather than invalid JSON", () => { + const encoded = encodeEntry({ type: "t", value: Number.NaN, other: Number.POSITIVE_INFINITY })!; + expect(JSON.parse(encoded)).toEqual({ type: "t", value: null, other: null }); + }); + + it("encodes the values JSON.stringify refuses", () => { + const encoded = encodeEntry({ + type: "t", + big: 1n, + set: new Set([1, 2]), + map: new Map([["a", 1]]), + when: new Date("2026-01-01T00:00:00Z"), + })!; + expect(JSON.parse(encoded)).toEqual({ + type: "t", + big: "1", + set: [1, 2], + map: { a: 1 }, + when: "2026-01-01T00:00:00.000Z", + }); + }); + + it("never lets a throwing getter escape the encoder", () => { + const hostile = { + type: "t", + get boom(): never { + throw new Error("from the caller's own code"); + }, + }; + setLogger({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }); + expect(() => encodeEntry(hostile)).not.toThrow(); + }); +}); + +describe("size bounds", () => { + it("caps an oversized event's fields so the collector can still deliver the batch", () => { + const huge = "x".repeat(5 * 1024 * 1024); + const encoded = encodeEntry({ type: "tool_result", output: huge })!; + expect(Buffer.byteLength(encoded, "utf8")).toBeLessThan(4 * 1024 * 1024); + expect(JSON.parse(encoded).output).toContain("…[truncated]"); + }); + + it("never splits a surrogate pair when it truncates", () => { + const capped = capFields("\u{1F600}".repeat(10), 5) as string; + expect(capped.endsWith("…[truncated]")).toBe(true); + expect(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/.test(capped)).toBe(false); + }); + + it("rolls lines into batches that each stay under the limit", () => { + const lines = Array.from({ length: 5 }, () => ({ text: "x".repeat(40), bytes: 40 })); + const chunks = roll(lines, 100); + expect(chunks.map((chunk) => chunk.length)).toEqual([2, 2, 1]); + }); + + it("emits an over-limit line alone rather than dropping it", () => { + const chunks = roll( + [ + { text: "a", bytes: 1 }, + { text: "b", bytes: 500 }, + ], + 100, + ); + expect(chunks).toHaveLength(2); + expect(chunks[1]![0]!.bytes).toBe(500); + }); + + it("sizes a value by walking nodes, not characters", () => { + expect(approxSize("abcd")).toBe(4); + expect(approxSize({ a: "bb" })).toBe(3); + expect(approxSize([1, 2, 3])).toBe(24); + }); +}); + +describe("queue bounds", () => { + it("discards the OLDEST events when the queue stops draining, and says so once", () => { + const warn = vi.fn(); + setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }); + const writer = new EventWriter(3600); + try { + for (let i = 0; i < 10_050; i += 1) writer.submit({ type: "t", index: i }); + const stats = writer.stats(); + expect(stats.queued).toBeLessThanOrEqual(10_000); + expect(stats.dropped).toBeGreaterThan(0); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("queue is full")); + } finally { + writer.close(); + } + }); + + it("bounds by BYTES as well as by count, because a count is not a memory bound", () => { + const writer = new EventWriter(3600); + try { + const chunk = "x".repeat(2 * 1024 * 1024); + for (let i = 0; i < 64; i += 1) writer.submit({ type: "t", blob: chunk }); + // 64 x 2 MiB is 128 MiB, well past the 64 MiB ceiling, and far under the + // 10,000-event count cap — so only the byte bound can have stopped it. + expect(writer.stats().queuedBytes).toBeLessThanOrEqual(64 * 1024 * 1024); + expect(writer.stats().dropped).toBeGreaterThan(0); + } finally { + writer.close(); + } + }); +}); + +describe("process lifetime", () => { + it("lets a script that merely imports it exit, and flushes on the way out", async () => { + // The flush interval MUST be unref'd. Without that, importing this SDK + // stops every script that uses it from ever exiting — the most visible bug + // a telemetry library can ship, and one no in-process assertion can catch, + // because the test runner keeps the loop alive by itself. + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + fp.event.agentStart({ sessionId: "exit-test" }); + console.log("emitted"); + `); + expect(child.code).toBe(0); + expect(child.stdout.trim()).toBe("emitted"); + // The exit hook is the only thing that can have written this: the interval + // was set to an hour. + expect(spool.events().map((event) => event.session_id)).toEqual(["exit-test"]); + }); + + it("closes the runs a SIGTERM abandons, so none renders as running forever", async () => { + // The documented shutdown recipe, killed mid-tool. It used to leave an + // agent_start with no agent_end and a tool_use with no tool_result, and + // exit 0 — every deploy stranded the runs it interrupted. + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + for (const signal of ["SIGINT", "SIGTERM"]) { + process.once(signal, () => { fp.flushSync(); process.exit(0); }); + } + setInterval(() => {}, 1000); // a service's server keeps the loop alive + setTimeout(() => process.kill(process.pid, "SIGTERM"), 50); + await fp.agent("svc", { sessionId: "term-test" }, async () => { + await fp.agent("writer", async () => { + await fp.toolCall("slow", { toolCallId: "t1" }, () => new Promise(() => {})); + }); + }); + `); + expect(child.code).toBe(0); + const events = spool.events(); + expect(events.map((e) => [e.agent_id, e.type])).toEqual([ + ["svc", "agent_start"], + ["writer", "agent_start"], + ["writer", "tool_use"], + ["writer", "tool_result"], + ["writer", "error"], + ["writer", "agent_end"], + ["svc", "error"], + ["svc", "agent_end"], + ]); + const result = events.find((e) => e.type === "tool_result")!; + expect(result.tool_call_id).toBe("t1"); + expect(result.error).toMatch(/^ProcessExit: the process exited \(code 0\) while tool "slow"/); + for (const end of events.filter((e) => e.type === "agent_end")) expect(end.outcome).toBe("failed"); + expect(events.find((e) => e.type === "error")!.error_type).toBe("ProcessExit"); + }); + + it("closes a hand-written model call left open by a SIGTERM", async () => { + // The no-framework recipe: modelRequest before the provider call, + // modelResponse after. Killed in between, the request had no response and + // rendered as running forever. + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + process.once("SIGTERM", () => { fp.flushSync(); process.exit(143); }); + setInterval(() => {}, 1000); + setTimeout(() => process.kill(process.pid, "SIGTERM"), 50); + await fp.agent("planner", { sessionId: "model-term" }, async () => { + fp.event.modelRequest({ model: "m", requestId: "r1" }); + await new Promise(() => {}); + }); + `); + expect(child.code).toBe(143); + const events = spool.events(); + expect(events.map((e) => e.type)).toEqual(["agent_start", "model_request", "model_response", "error", "agent_end"]); + const response = events[2]!; + expect(response.request_id).toBe("r1"); + expect(response.stop_reason).toBe("error"); + expect(response.error).toMatch(/^ProcessExit: the process exited \(code 143\) while the model call/); + // Timed like the tools and hooks closed beside it. + expect(typeof response.duration_ms).toBe("number"); + }); + + it("names the uncaught exception a crash exited on, in what it closes", async () => { + // A timer throwing mid-run used to leave only "the process exited (code 1)" + // in the trace — the exception itself was nowhere. + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + setInterval(() => {}, 1000); + class QuotaError extends Error {} + setTimeout(() => { throw new QuotaError("boom from a timer"); }, 50); + await fp.agent("svc", { sessionId: "crash" }, async () => { + await fp.toolCall("slow", { toolCallId: "t1" }, () => new Promise(() => {})); + }); + `); + expect(child.code).toBe(1); + expect(child.stderr).toContain("boom from a timer"); // still crashes as it would have + const events = spool.events(); + expect(events.map((e) => e.type)).toEqual(["agent_start", "tool_use", "tool_result", "error", "agent_end"]); + expect(events[2]!.error).toMatch(/\(code 1\) on an uncaught QuotaError: boom from a timer while tool "slow"/); + expect(events[3]!.message).toMatch(/on an uncaught QuotaError: boom from a timer/); + }); + + it("closes a nested run most-recent-first: a sub-agent ends before the tool that started it", async () => { + // Live: a planner's delegate tool ran a writer sub-agent, killed during the + // writer's model call. Closing all tools before all agents showed the + // delegate tool finishing while the writer it started was still running. + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + process.once("SIGTERM", () => { fp.flushSync(); process.exit(143); }); + setInterval(() => {}, 1000); + setTimeout(() => process.kill(process.pid, "SIGTERM"), 50); + await fp.agent("planner", { sessionId: "nested-term" }, async () => { + await fp.toolCall("delegate_writer", { toolCallId: "d1" }, async () => { + await fp.agent("writer", async () => { + fp.event.modelRequest({ model: "m", requestId: "w1" }); + await new Promise(() => {}); + }); + }); + }); + `); + expect(child.code).toBe(143); + const closing = spool.events().slice(4); // planner start, tool_use, writer start, model_request + expect(closing.map((e) => [e.agent_id, e.type])).toEqual([ + ["writer", "model_response"], + ["writer", "error"], + ["writer", "agent_end"], + ["planner", "tool_result"], + ["planner", "error"], + ["planner", "agent_end"], + ]); + expect(closing.find((e) => e.type === "error")!.traceback).toBeUndefined(); + }); + + it("closes nothing on a flushSync() while the process carries on", async () => { + const child = await runNode(` + const fp = await import(${JSON.stringify(indexUrl())}); + fp.configure({ baseDir: ${JSON.stringify(spool.dir)}, flushInterval: 3600 }); + await fp.agent("svc", { sessionId: "flush-test" }, async () => { + fp.flushSync(); + await new Promise((r) => setTimeout(r, 10)); + }); + `); + expect(child.code).toBe(0); + expect(spool.events().map((e) => [e.type, e.outcome ?? null])).toEqual([ + ["agent_start", null], + ["agent_end", "success"], + ]); + }); +}); diff --git a/sdk/typescript/tsconfig.build.json b/sdk/typescript/tsconfig.build.json new file mode 100644 index 000000000..b6df0dcbf --- /dev/null +++ b/sdk/typescript/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist/esm", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": false, + // `@internal` exports (test hooks such as `_stats`, `_internals`, + // `attach`) stay out of the published declarations: a name in a `.d.ts` + // is a name a customer can come to depend on. + "stripInternal": true + }, + "include": ["src/**/*.ts"] +} diff --git a/sdk/typescript/tsconfig.cjs.json b/sdk/typescript/tsconfig.cjs.json new file mode 100644 index 000000000..476031b20 --- /dev/null +++ b/sdk/typescript/tsconfig.cjs.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig.json", + // Declarations are emitted here too, and `exports[*].require.types` points + // at them. Under dist/cjs's `"type": "commonjs"` they ARE CommonJS + // declarations; handing `require` the ESM ones instead is what made every + // CommonJS project on `module: node16` fail with TS1479. + "compilerOptions": { + "rootDir": "src", + "outDir": "dist/cjs", + "module": "CommonJS", + "moduleResolution": "Node10", + "noEmit": false, + // `@internal` exports (test hooks such as `_stats`, `_internals`, + // `attach`) stay out of the published declarations: a name in a `.d.ts` + // is a name a customer can come to depend on. + "stripInternal": true + }, + "include": ["src/**/*.ts"] +} diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.json new file mode 100644 index 000000000..8e15be7f8 --- /dev/null +++ b/sdk/typescript/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023", "ESNext.Disposable"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "types": ["node"], + + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useUnknownInCatchVariables": true, + "verbatimModuleSyntax": false, + + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "isolatedModules": true + }, + "include": ["src/**/*.ts", "test/**/*.ts", "integration/*.ts", "scripts/**/*.mjs", "*.ts", "*.mts"] +} diff --git a/sdk/typescript/vitest.config.ts b/sdk/typescript/vitest.config.ts new file mode 100644 index 000000000..56761cfc7 --- /dev/null +++ b/sdk/typescript/vitest.config.ts @@ -0,0 +1,33 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + // Vite searches UPWARD from the project root for a PostCSS config, and this + // package sits inside a monorepo whose root has one — which requires + // `@tailwindcss/postcss`, a root devDependency that is deliberately not in + // this package's isolated `node_modules`. Vite then dies before a single test + // runs. + // + // It only fails where the isolation is real: locally the root's + // `node_modules` is present and Node's resolution walks up into it, so the + // search succeeds and nothing looks wrong. In CI, where this package is + // installed on its own, it is fatal. + // + // An inline (empty) config turns the search off. This package has no CSS at + // all, so there is nothing to configure — the only thing that search can do + // here is find somebody else's tooling. + css: { postcss: {} }, + test: { + include: ["test/**/*.test.ts"], + environment: "node", + globalSetup: ["./test/global-setup.ts"], + setupFiles: ["./test/setup.ts"], + // The writer, the sandbox and the adapters all reach for process-wide state + // (an exit hook, a patched prototype, a spool directory). Running files one + // at a time keeps a test from observing another file's patch, and keeps the + // sandbox suite from competing with itself for the concurrency semaphore. + pool: "forks", + fileParallelism: false, + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}); diff --git a/sdk/typescript/vitest.integration.config.ts b/sdk/typescript/vitest.integration.config.ts new file mode 100644 index 000000000..72fd11aa2 --- /dev/null +++ b/sdk/typescript/vitest.integration.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from "vitest/config"; + +/** + * The real-framework suite (`integration/`). Separate from `vitest.config.ts` + * because it installs real frameworks from the registry and spawns real Node + * processes against the PACKED artifact — minutes, not seconds, and it needs + * the network the unit suite deliberately does not. + */ +export default defineConfig({ + // Same reason as vitest.config.ts: stop Vite finding the monorepo root's + // PostCSS config, which needs a package this isolated install does not have. + css: { postcss: {} }, + test: { + include: ["integration/**/*.test.ts"], + environment: "node", + globalSetup: ["./integration/global-setup.ts"], + // Every case is its own child process with its own spool, so files are + // independent and can run side by side. + fileParallelism: true, + testTimeout: 120_000, + hookTimeout: 900_000, + }, +}); diff --git a/tsconfig.json b/tsconfig.json index bb2ab23dd..12984d781 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -38,6 +38,7 @@ ], "exclude": [ "node_modules", - "dist" + "dist", + "sdk/typescript" ] }