Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .changeset/five-pens-dance.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
"braintrust": minor
---

feat(mastra): auto-instrument Mastra via its native `ObservabilityExporter`

Replaces the chunk-AST instrumentation with a Braintrust `ObservabilityExporter` that the loader auto-installs into every `new Mastra(...)`. Survives Mastra's content-hashed chunk renames release-to-release because the loader only touches the stable `dist/mastra/index.{js,cjs}` entry point.

Two integration paths:

- **Auto** (default with `node --import braintrust/hook.mjs`): no user code change, the loader wraps `Mastra` to call `defaultInstance.registerExporter(...)` after construction. Requires the user to enable observability via `new Mastra({ observability: new Observability({ ... }) })`.
- **Manual**: `import { BraintrustObservabilityExporter } from "braintrust";` and pass it via `new Mastra({ observability: new Observability({ configs: { default: { exporters: [new BraintrustObservabilityExporter()] } } }) })`.

Requires `@mastra/core >= 1.20.0` for the auto path (the version that added `Mastra.prototype.registerExporter`); older versions silently no-op. Manual integration works on any Mastra version that accepts an `ObservabilityExporter`.
6 changes: 6 additions & 0 deletions e2e/config/pr-comment-scenarios.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@
"metadataScenario": "genkit-instrumentation",
"variants": [{ "variantKey": "genkit-v1-33-0", "label": "v1.33.0" }]
},
{
"scenarioDirName": "mastra-instrumentation",
"label": "Mastra Instrumentation",
"metadataScenario": "mastra-instrumentation",
"variants": [{ "variantKey": "mastra-v1260", "label": "v1.26.0" }]
},
{
"scenarioDirName": "groq-instrumentation",
"label": "Groq Instrumentation",
Expand Down
7 changes: 7 additions & 0 deletions e2e/helpers/normalize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ const TEMP_HELPER_PATH_REGEX = /\/e2e\/\.bt-tmp\/[^/\s)]+\/helpers\/?/g;
const PROVIDER_HELPER_CALLER_REGEX = /^<repo>\/e2e\/helpers\/.+-scenario\.mjs$/;
const ANTHROPIC_MESSAGE_STREAM_PATH_REGEX =
/([/\\]node_modules[/\\]\.pnpm[/\\]@anthropic-ai\+sdk@[^/\\\s)]+[/\\]node_modules[/\\]@anthropic-ai[/\\]sdk[/\\])(?:src[/\\]lib[/\\]MessageStream\.ts|lib[/\\]MessageStream\.js)/g;
// tsup's `splitting: true` for our own `dist/` emits content-hashed chunk
// files (e.g. `<repo>/js/dist/chunk-7DWPOXBX.mjs`) whose names change any
// time the bundle graph changes. Normalize them to a stable placeholder so
// stack traces in error snapshots don't churn on unrelated bundle splits.
const SDK_CHUNK_PATH_REGEX =
/(<repo>\/js\/dist\/)chunk-[A-Z0-9]+(\.(?:c?js|cjs|mjs))/g;
const ANTHROPIC_PNPM_VERSION_REGEX =
/([/\\]\.pnpm[/\\]@anthropic-ai\+sdk@)[^/\\\s)]+/g;

Expand Down Expand Up @@ -153,6 +159,7 @@ function normalizeStackLikeString(value: string): string {
normalized = normalized.replace(REPO_PATH_REGEX, (_, suffix: string) => {
return `<repo>${suffix.replace(/\\/g, "/")}`;
});
normalized = normalized.replace(SDK_CHUNK_PATH_REGEX, "$1index$2");
normalized = normalized.replace(
/(<repo>(?:\/(?:e2e|js)\/[^:\s)\n]+)):\d+:\d+/g,
"$1:0:0",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,266 @@
[
{
"metadata": {
"scenario": "mastra-instrumentation"
},
"metric_keys": [],
"name": "mastra-instrumentation-root",
"type": "task"
},
{
"metadata": {
"operation": "generate"
},
"metric_keys": [],
"name": "mastra-agent-generate-operation",
"type": null
},
{
"input": "What is the weather in Paris?",
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "agent run: 'weather-agent'",
"output": {
"files": [],
"text": "The forecast is sunny."
},
"type": "task"
},
{
"input": {
"messages": [
{
"content": "Answer weather questions with the provided mock forecast.",
"role": "system"
},
{
"content": [
{
"providerOptions": {
"mastra": {
"createdAt": 0
}
},
"text": "What is the weather in Paris?",
"type": "text"
}
],
"role": "user"
}
]
},
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent",
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"name": "llm: 'mock-model-id'",
"output": {
"files": [],
"reasoning": [],
"sources": [],
"text": "The forecast is sunny.",
"warnings": []
},
"type": "llm"
},
{
"input": {},
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"name": "step: 0",
"output": {
"text": "The forecast is sunny.",
"toolCalls": []
},
"type": "llm"
},
{
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "chunk: 'text'",
"output": {
"text": "The forecast is sunny."
},
"type": "llm"
},
{
"metadata": {
"operation": "stream"
},
"metric_keys": [],
"name": "mastra-agent-stream-operation",
"type": null
},
{
"input": "Stream the Paris forecast.",
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "agent run: 'weather-agent'",
"output": {
"files": [],
"text": "The forecast is sunny."
},
"type": "task"
},
{
"input": {
"messages": [
{
"content": "Answer weather questions with the provided mock forecast.",
"role": "system"
},
{
"content": [
{
"providerOptions": {
"mastra": {
"createdAt": 0
}
},
"text": "Stream the Paris forecast.",
"type": "text"
}
],
"role": "user"
}
]
},
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent",
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"name": "llm: 'mock-model-id'",
"output": {
"files": [],
"reasoning": [],
"sources": [],
"text": "The forecast is sunny.",
"warnings": []
},
"type": "llm"
},
{
"input": {},
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"name": "step: 0",
"output": {
"text": "The forecast is sunny.",
"toolCalls": []
},
"type": "llm"
},
{
"metadata": {
"entity_id": "weather-agent",
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "chunk: 'text'",
"output": {
"text": "The forecast is sunny."
},
"type": "llm"
},
{
"metadata": {
"operation": "tool"
},
"metric_keys": [],
"name": "mastra-tool-operation",
"type": null
},
{
"metadata": {
"operation": "workflow"
},
"metric_keys": [],
"name": "mastra-workflow-operation",
"type": null
},
{
"input": {
"city": "Berlin"
},
"metadata": {
"entity_id": "travel-flow",
"entity_name": "travel-flow",
"entity_type": "workflow_run"
},
"metric_keys": [],
"name": "workflow run: 'travel-flow'",
"output": {
"forecast": "Sunny in Berlin"
},
"type": "task"
},
{
"input": {
"city": "Berlin"
},
"metadata": {
"entity_id": "lookup-step",
"entity_name": "travel-flow",
"entity_type": "workflow_step"
},
"metric_keys": [],
"name": "workflow step: 'lookup-step'",
"output": {
"forecast": "Sunny in Berlin"
},
"type": "function"
},
{
"metadata": {
"scenario": "mastra-instrumentation"
},
"metric_keys": [],
"name": "mastra-instrumentation-root",
"type": "task"
}
]
Loading
Loading