diff --git a/README.md b/README.md index da049b3..5a3a9dc 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ It reads the transcripts your harness leaves on disk, reconstructs the run as sp - [What it finds](#what-it-finds) - [Supported harnesses](#supported-harnesses) - [CLI reference](#cli-reference) +- [Policy-mining evidence](#policy-mining-evidence) - [Upload to the Intelligence Platform](#upload-to-the-intelligence-platform) - [External engines (bring your own)](#external-engines-bring-your-own) - [Library (SDK)](#library-sdk) @@ -85,6 +86,7 @@ traces list --harness claude-code --last 20 # discover sessions traces analyze --harness codex --last 1 # $0 deterministic report traces analyze --all --since 2026-06-18 --out report.md traces convert --harness claude-code --last 1 --otlp spans.jsonl # OTLP only +traces evidence --harness codex --last 20 --out policy-evidence.jsonl traces watch --all # live observer; notify on stuck loops traces upload --since 1h --dry-run # redact + dedup + preview, no network traces upload --since 24h # upload last day to the Intelligence Platform @@ -99,13 +101,31 @@ traces upload --since 24h # upload last day to the Inte | `--cwd ` | Filter by working directory | | `--since ` | `upload`: window — `30m`/`2h`/`7d` or ISO (default 24h); `analyze`: ISO cutoff | | `--out ` | Write the report to a file | -| `--otlp ` | OTLP artifact path (also the dry-run upload preview) | +| `--otlp ` | OTLP artifact path (also evidence provenance / dry-run upload preview) | | `--llm` / `--budget ` | Enable agentic analysts (needs `OPENAI_API_KEY`) / cap their spend | | `--interval ` / `--window ` | `watch`: poll seconds (default 5) / active-session window minutes (default 30) | | `--min-loop ` | Identical repeated calls before flagging a loop (default 3) | | `--no-content` | `upload`: send metadata only — strip all prompt/response text | | `--dry-run` / `--yes` | `upload`: preview without sending / skip the confirm prompt | +## Policy-mining evidence + +`traces` does **not** emit benchmark campaign cells. It emits normalized coding-agent session evidence that another system can mine. + +```bash +traces evidence --all --since 24h --out policy-evidence.jsonl --otlp spans.otlp.jsonl +``` + +Each JSONL row is one session: + +- session provenance: harness, session id, cwd, path, mtime +- repo labels: `tangle.subject.key`, `git.repository`, branch, commit +- behavior metrics: span counts, LLM turns, tool calls, errored tool calls, tokens, models, tool histogram +- mining signals: stuck loops and tool error rate +- provenance marker: `notCampaignCell: true` + +That boundary matters. `agent-lab` campaign `cells.jsonl` says "arm X beat arm Y on task Z." `traces evidence` says "this real agent session had this repo/model/tool/failure shape." A downstream policy compiler can cluster these rows, propose candidate policies, then validate those policies in a separate eval campaign. + ## Upload to the Intelligence Platform `upload` **redacts locally before anything leaves the machine**, dedups against already-uploaded sessions, and tags each with metadata (harness, cwd, git branch, host). @@ -163,6 +183,8 @@ The CLI is a thin consumer of these exports. |---|---|---| | `analyzeSpans` | `(spans, { registry?, ai?, budgetUsd? }) → AnalyzeResult` | run analysts — built-in, or **your own** via `registry` | | `watchSessions` | `(ObserverOptions) → Promise` | live observer; `onLoop` / `onReport` / `signal` / `adapters` | +| `buildPolicyEvidenceRecord` | `(ref, spans, opts?) → PolicyEvidenceRecord` | summarize one session for downstream policy mining | +| `collectPolicyEvidence` | `(ScanOptions) → PolicyEvidenceRecord[]` | scan harness sessions and emit policy-evidence rows | | `scanSessions` | `(ScanOptions) → AsyncIterable` | the shared locate→parse iterator | | `collectSessions` | `(CollectOptions) → SessionBatch[]` | redacted per-session batches for your own pipeline | | `redactSpans` | `(spans, rules?) → { spans, report }` | PII/secret redaction (`TRACES_REDACTION_RULES`) | diff --git a/package.json b/package.json index fd71d85..9957952 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ }, "dependencies": { "@ax-llm/ax": "^19.0.45", - "@tangle-network/agent-eval": "0.95.1" + "@tangle-network/agent-eval": "^0.99.0" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3c688f..4efec2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^19.0.45 version: 19.0.45(zod@4.4.3) '@tangle-network/agent-eval': - specifier: 0.95.1 - version: 0.95.1(typescript@5.9.3) + specifier: ^0.99.0 + version: 0.99.0(typescript@5.9.3) devDependencies: '@types/node': specifier: ^22.0.0 @@ -699,8 +699,8 @@ packages: '@scure/bip39@2.2.0': resolution: {integrity: sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==} - '@tangle-network/agent-eval@0.95.1': - resolution: {integrity: sha512-hpIes551lbAL756P9zyo/3sCbhNTDjAlakvK5ql4aMfwo4ICLoJVXv4v4LUSmXe4uhm2sFURGD/Cs5/hXxvlcw==} + '@tangle-network/agent-eval@0.99.0': + resolution: {integrity: sha512-jcCuqDfIhgE2SnVfAu/fwUaYJaX+CpJ6Va9CPI5KtjLsKevtt/ZV2+OOpg84JWbrrYaTS+4DVps7VPN4HV/vaA==} engines: {node: '>=20'} hasBin: true @@ -1578,7 +1578,7 @@ snapshots: '@noble/hashes': 2.2.0 '@scure/base': 2.2.0 - '@tangle-network/agent-eval@0.95.1(typescript@5.9.3)': + '@tangle-network/agent-eval@0.99.0(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) diff --git a/src/attributes.ts b/src/attributes.ts index 56b861b..bc2791a 100644 --- a/src/attributes.ts +++ b/src/attributes.ts @@ -19,6 +19,16 @@ export const ATTR = { INGEST_SOURCE: 'tangle.ingest_source', HARNESS: 'tangle.harness', CWD: 'tangle.cwd', + /** THE per-session grouping key the spine derives subjects from + * (`deriveSubjectKey` reads `tangle.subject.key` first). Resolves to the git + * remote (host/owner/repo) when readable, else the cwd path basename. */ + SUBJECT_KEY: 'tangle.subject.key', + /** Normalized git remote (e.g. `github.com/tangle-network/agent-dev-container`). */ + GIT_REPOSITORY: 'git.repository', + /** Current branch at conversion time. */ + GIT_BRANCH_NAME: 'git.branch', + /** HEAD short sha at conversion time. */ + GIT_COMMIT: 'git.commit', GIT_BRANCH: 'tangle.git_branch', HOST: 'tangle.host', /** Basename of the session file. (Renamed from the ambiguous `tangle.source`, diff --git a/src/cli.ts b/src/cli.ts index 60da40a..bc4c822 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,6 +5,7 @@ * traces list [--harness claude-code] [--last 20] [--all] * traces analyze [--harness claude-code] [--last 1] [--out report.md] [--llm] * traces convert [--harness claude-code] [--last 1] --otlp spans.jsonl + * traces evidence [--harness claude-code] [--last 20] --out policy-evidence.jsonl * traces watch [--all] [--interval 5] [--window 30] [--min-loop 3] * * `analyze` runs the agent-eval analyst suite (deterministic + the shipped @@ -17,6 +18,7 @@ import { stat, writeFile } from 'node:fs/promises' import { analyzeAdoption } from './adoption.js' import { analyzeSpans } from './analyze.js' +import { buildPolicyEvidenceRecord, serializePolicyEvidence, writePolicyEvidenceFile } from './evidence.js' import { commandAnalyzer, commandRedactor, haloAnalyzer, runExternalAnalyzers } from './external.js' import type { OtlpSpan } from './otlp.js' import { writeOtlpFile } from './otlp.js' @@ -24,6 +26,7 @@ import { watchSessions } from './observer.js' import { runPipelines } from './pipelines.js' import { knownHarnesses, resolveAdapter, selectAdapters } from './registry.js' import { analyzeReactions } from './reactions.js' +import { parseSession } from './session-source.js' import { renderAdoption, renderPipelines, renderReactions, renderReport } from './report.js' import { parseSince } from './time.js' import type { HarnessTraceAdapter, SessionRef } from './types.js' @@ -171,7 +174,7 @@ async function collectSpans(args: Args): Promise<{ spans: OtlpSpan[]; harness: s cwd: args.cwd ?? null, mtimeMs: st.mtimeMs, } - return { spans: await adapter.parse(ref), harness: adapter.harness, sessionCount: 1, cwds: ref.cwd ? [ref.cwd] : [] } + return { spans: await parseSession(adapter, ref), harness: adapter.harness, sessionCount: 1, cwds: ref.cwd ? [ref.cwd] : [] } } const groups = await discover({ ...args, last: args.last || 1 }) const spans: OtlpSpan[] = [] @@ -181,7 +184,7 @@ async function collectSpans(args: Args): Promise<{ spans: OtlpSpan[]; harness: s for (const { adapter, refs } of groups) { if (refs.length > 0) harnesses.push(adapter.harness) for (const ref of refs) { - spans.push(...(await adapter.parse(ref))) + spans.push(...(await parseSession(adapter, ref))) if (ref.cwd) cwds.push(ref.cwd) sessionCount += 1 } @@ -196,6 +199,49 @@ async function cmdConvert(args: Args): Promise { console.log(`wrote ${spans.length} spans → ${path}`) } +async function collectSessionRows(args: Args): Promise> { + if (args.session) { + const adapter = resolveAdapter(args.harness) + if (!adapter) throw new Error(`unknown harness "${args.harness}"`) + const st = await stat(args.session) + const ref: SessionRef = { + harness: adapter.harness, + sessionId: args.session, + path: args.session, + cwd: args.cwd ?? null, + mtimeMs: st.mtimeMs, + } + return [{ ref, spans: await parseSession(adapter, ref) }] + } + const groups = await discover({ ...args, last: args.last || 20 }) + const rows: Array<{ ref: SessionRef; spans: OtlpSpan[] }> = [] + for (const { adapter, refs } of groups) { + for (const ref of refs) rows.push({ ref, spans: await parseSession(adapter, ref) }) + } + return rows +} + +async function cmdEvidence(args: Args): Promise { + const rows = (await collectSessionRows(args)).filter((row) => row.spans.length > 0) + if (rows.length === 0) throw new Error('no spans found for the given selection') + const otlpPath = args.otlp ? await writeOtlpFile(rows.flatMap((row) => row.spans), args.otlp) : undefined + const generatedAt = new Date().toISOString() + const records = await Promise.all(rows.map((row) => + buildPolicyEvidenceRecord(row.ref, row.spans, { + generatedAt, + minLoopOccurrences: args.minLoop, + maxLoopExamples: 25, + otlpPath, + }), + )) + if (args.out) { + const path = await writePolicyEvidenceFile(records, args.out) + console.log(`policy evidence → ${path} (${records.length} session rows${otlpPath ? `, OTLP: ${otlpPath}` : ''})`) + } else { + process.stdout.write(serializePolicyEvidence(records)) + } +} + async function cmdAnalyze(args: Args): Promise { const { spans, harness, sessionCount, cwds } = await collectSpans(args) if (spans.length === 0) throw new Error('no spans found for the given selection') @@ -324,6 +370,7 @@ Commands: list List discovered sessions analyze Run analyst suite + loop/waste pipelines, write a markdown report convert Emit OTLP-JSONL only (HALO: use analyze --analyzer halo) + evidence Emit compact session-evidence JSONL for downstream policy miners watch Online observer: tail active sessions, notify on stuck loops (read-only) upload Redact + upload sessions in a time window to the Tangle Intelligence Platform @@ -335,7 +382,7 @@ Options: --cwd Filter sessions by working directory --since upload: window — 30m / 2h / 7d or an ISO date (default 24h); analyze: ISO cutoff --out Write report to a file - --otlp OTLP artifact path (also the dry-run upload preview path) + --otlp OTLP artifact path (also evidence provenance / dry-run upload preview) --llm Enable agentic RLM analysts (needs OPENAI_API_KEY / OPENAI_BASE_URL) --model --llm model id (e.g. a router model like glm-5.2); default is agent-eval's --budget USD cap for agentic analysts @@ -358,6 +405,7 @@ async function main(): Promise { case 'list': await cmdList(args); break case 'analyze': await cmdAnalyze(args); break case 'convert': await cmdConvert(args); break + case 'evidence': await cmdEvidence(args); break case 'watch': await cmdWatch(args); break case 'upload': await cmdUpload(args); break default: usage() diff --git a/src/evidence.ts b/src/evidence.ts new file mode 100644 index 0000000..38e8134 --- /dev/null +++ b/src/evidence.ts @@ -0,0 +1,208 @@ +import { mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { ATTR } from './attributes.js' +import type { OtlpSpan } from './otlp.js' +import { runPipelines } from './pipelines.js' +import { type ScanOptions, scanSessions } from './session-source.js' +import type { SessionRef } from './types.js' + +export interface PolicyEvidenceToolSummary { + readonly name: string + readonly calls: number + readonly errors: number +} + +export interface PolicyEvidenceLoopSummary { + readonly toolName: string + readonly occurrences: number +} + +export interface PolicyEvidenceRecord { + readonly schemaVersion: 1 + readonly kind: 'traces.policy_evidence.session' + readonly generatedAt: string + readonly session: { + readonly harness: string + readonly sessionId: string + readonly path: string + readonly cwd: string | null + readonly mtimeMs: number + } + readonly repo: { + readonly subjectKey?: string + readonly repository?: string + readonly branch?: string + readonly commit?: string + readonly cwd?: string + } + readonly metrics: { + readonly spanCount: number + readonly llmTurnCount: number + readonly toolCallCount: number + readonly erroredToolCallCount: number + readonly inputTokens: number + readonly outputTokens: number + readonly models: readonly string[] + readonly tools: readonly PolicyEvidenceToolSummary[] + readonly firstSpanAt: string | null + readonly lastSpanAt: string | null + } + readonly signals: { + readonly stuckLoopCount: number + readonly affectedRunRatio: number + readonly stuckLoops: readonly PolicyEvidenceLoopSummary[] + readonly stuckLoopsOmitted: number + readonly toolErrorRate: number + } + readonly provenance: { + readonly source: 'traces' + readonly evidenceKind: 'session-summary' + readonly otlpPath?: string + readonly notCampaignCell: true + readonly note: string + } +} + +export interface BuildPolicyEvidenceOptions { + readonly generatedAt?: string + readonly minLoopOccurrences?: number + readonly maxLoopExamples?: number + readonly otlpPath?: string +} + +export interface CollectPolicyEvidenceOptions extends ScanOptions, BuildPolicyEvidenceOptions {} + +function stringAttr(span: OtlpSpan, key: string): string | undefined { + const value = span.attributes[key] + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +function numberAttr(span: OtlpSpan, key: string): number { + const value = span.attributes[key] + return typeof value === 'number' && Number.isFinite(value) ? value : 0 +} + +function spanKind(span: OtlpSpan): string | undefined { + return stringAttr(span, 'openinference.span.kind') +} + +function repoFromSpans(spans: readonly OtlpSpan[]): PolicyEvidenceRecord['repo'] { + const attrs: { + subjectKey?: string + repository?: string + branch?: string + commit?: string + cwd?: string + } = {} + for (const span of spans) { + attrs.subjectKey ??= stringAttr(span, ATTR.SUBJECT_KEY) + attrs.repository ??= stringAttr(span, ATTR.GIT_REPOSITORY) + attrs.branch ??= stringAttr(span, ATTR.GIT_BRANCH_NAME) + attrs.commit ??= stringAttr(span, ATTR.GIT_COMMIT) + attrs.cwd ??= stringAttr(span, ATTR.CWD) + if (attrs.subjectKey && attrs.repository && attrs.branch && attrs.commit && attrs.cwd) break + } + return attrs +} + +function timeBounds(spans: readonly OtlpSpan[]): { firstSpanAt: string | null; lastSpanAt: string | null } { + const times = spans + .flatMap((span) => [span.start_time, span.end_time]) + .filter((value) => value && value !== 'now') + .sort() + return { + firstSpanAt: times[0] ?? null, + lastSpanAt: times[times.length - 1] ?? null, + } +} + +function summarizeTools(spans: readonly OtlpSpan[]): PolicyEvidenceToolSummary[] { + const byTool = new Map() + for (const span of spans) { + if (spanKind(span) !== 'TOOL') continue + const name = stringAttr(span, 'tool.name') ?? span.name.replace(/^tool\./, '') + const current = byTool.get(name) ?? { calls: 0, errors: 0 } + current.calls += 1 + if (span.status.code === 'ERROR') current.errors += 1 + byTool.set(name, current) + } + return [...byTool.entries()] + .map(([name, row]) => ({ name, calls: row.calls, errors: row.errors })) + .sort((a, b) => b.calls - a.calls || a.name.localeCompare(b.name)) +} + +export async function buildPolicyEvidenceRecord( + ref: SessionRef, + spans: readonly OtlpSpan[], + opts: BuildPolicyEvidenceOptions = {}, +): Promise { + const llmSpans = spans.filter((span) => spanKind(span) === 'LLM') + const toolSpans = spans.filter((span) => spanKind(span) === 'TOOL') + const erroredToolCallCount = toolSpans.filter((span) => span.status.code === 'ERROR').length + const pipelines = await runPipelines(spans, { minLoopOccurrences: opts.minLoopOccurrences }) + const loopLimit = opts.maxLoopExamples ?? 25 + const loopFindings = pipelines.stuckLoops.findings + const { firstSpanAt, lastSpanAt } = timeBounds(spans) + return { + schemaVersion: 1, + kind: 'traces.policy_evidence.session', + generatedAt: opts.generatedAt ?? new Date().toISOString(), + session: { + harness: ref.harness, + sessionId: ref.sessionId, + path: ref.path, + cwd: ref.cwd, + mtimeMs: ref.mtimeMs, + }, + repo: repoFromSpans(spans), + metrics: { + spanCount: spans.length, + llmTurnCount: llmSpans.length, + toolCallCount: toolSpans.length, + erroredToolCallCount, + inputTokens: llmSpans.reduce((sum, span) => sum + numberAttr(span, 'llm.input_tokens'), 0), + outputTokens: llmSpans.reduce((sum, span) => sum + numberAttr(span, 'llm.output_tokens'), 0), + models: [...new Set(llmSpans.map((span) => stringAttr(span, 'llm.model_name')).filter((value): value is string => Boolean(value)))].sort(), + tools: summarizeTools(spans), + firstSpanAt, + lastSpanAt, + }, + signals: { + stuckLoopCount: loopFindings.length, + affectedRunRatio: pipelines.stuckLoops.affectedRunRatio, + stuckLoops: loopFindings.slice(0, loopLimit).map((finding) => ({ + toolName: finding.toolName, + occurrences: finding.occurrences, + })), + stuckLoopsOmitted: Math.max(0, loopFindings.length - loopLimit), + toolErrorRate: toolSpans.length === 0 ? 0 : erroredToolCallCount / toolSpans.length, + }, + provenance: { + source: 'traces', + evidenceKind: 'session-summary', + ...(opts.otlpPath ? { otlpPath: opts.otlpPath } : {}), + notCampaignCell: true, + note: 'This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.', + }, + } +} + +export async function collectPolicyEvidence(opts: CollectPolicyEvidenceOptions): Promise { + const records: PolicyEvidenceRecord[] = [] + for await (const session of scanSessions(opts)) { + records.push(await buildPolicyEvidenceRecord(session.ref, session.spans, opts)) + } + return records +} + +export function serializePolicyEvidence(records: readonly PolicyEvidenceRecord[]): string { + if (records.length === 0) return '' + return `${records.map((record) => JSON.stringify(record)).join('\n')}\n` +} + +export async function writePolicyEvidenceFile(records: readonly PolicyEvidenceRecord[], outPath?: string): Promise { + const path = outPath ?? join(await mkdtemp(join(tmpdir(), 'traces-evidence-')), 'policy-evidence.jsonl') + await writeFile(path, serializePolicyEvidence(records), 'utf8') + return path +} diff --git a/src/index.ts b/src/index.ts index 91d4ff6..dea9ee8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,8 @@ export * from './otlp.js' // OtlpSpan, span(), serializeSpans(), writeOtlpFile() export * from './attributes.js' // ATTR keys, INGEST_SOURCE_CLI, DEFAULT_HARNESS export * from './time.js' // parseIsoToEpochMs(), parseSince() export { knownHarnesses, listAdapters, resolveAdapter, selectAdapters } from './registry.js' -export * from './session-source.js' // scanSessions() — shared locate→parse iterator +export * from './session-source.js' // scanSessions() / parseSession() — locate→parse→stamp +export * from './repo.js' // resolveRepoAttrs() — per-session repo/git resource labels export { ClaudeAdapter } from './adapters/claude.js' export { CodexAdapter } from './adapters/codex.js' export { OpencodeAdapter } from './adapters/opencode.js' @@ -48,6 +49,7 @@ export * from './reactions.js' // analyzeReactions() — human-reaction analyst export * from './adoption.js' // analyzeAdoption() — skill + subagent metrics export * from './runtime-store.js' // toRuntimeStore() — feed agent-eval pipelines export * from './analyze.js' // analyzeSpans({ registry? }) — run YOUR analysts +export * from './evidence.js' // policy-evidence JSONL for downstream miners // ── External engines (NOT bundled — shell out to tools you install) ──────── export * from './external.js' // haloAnalyzer / commandAnalyzer; commandRedactor diff --git a/src/otlp.ts b/src/otlp.ts index 3719e49..6a62cd2 100644 --- a/src/otlp.ts +++ b/src/otlp.ts @@ -102,6 +102,11 @@ export function toOpenInferenceSpan(s: OtlpSpan): Record { const resourceAttrs: Record = {} if (a['service.name'] != null) resourceAttrs['service.name'] = a['service.name'] if (a['agent.name'] != null) resourceAttrs['agent.name'] = a['agent.name'] + // Per-session repo/git grouping labels (see src/repo.ts). `tangle.subject.key` + // is THE spine grouping key; the rest are stored-but-not-parsed provenance. + for (const k of ['tangle.subject.key', 'git.repository', 'git.branch', 'git.commit', 'tangle.cwd']) { + if (a[k] != null) resourceAttrs[k] = a[k] + } return { trace_id: s.trace_id, span_id: s.span_id, diff --git a/src/repo.ts b/src/repo.ts new file mode 100644 index 0000000..6ea2297 --- /dev/null +++ b/src/repo.ts @@ -0,0 +1,162 @@ +/** + * Per-session repo / git RESOURCE labels. + * + * Today every session groups under one `service.name` bucket (the harness, e.g. + * "claude-code"), so all repos collapse together on the Tangle spine. The spine + * groups by `deriveSubjectKey`, which reads `tangle.subject.key` first — so we + * resolve a per-REPO subject key here and stamp it as a resource attribute. + * + * Derivation (fail-safe; historical sessions may point at a DELETED cwd, e.g. an + * old worktree): + * - cwd exists AND has a readable `.git` → read the git REMOTE url, normalize + * to `host/owner/repo` (e.g. github.com/tangle-network/agent-dev-container) + * for `tangle.subject.key` + `git.repository`, plus the current branch and + * HEAD short sha. `tangle.cwd` carries the path. + * - cwd is null/gone, or no readable `.git` → fall back to the cwd path + * basename for `tangle.subject.key` (the project dir name still groups + * per-project even when the dir is deleted), set `tangle.cwd`, and OMIT the + * `git.*` keys (no fabrication). + * + * Never throws. A missing cwd yields `{}` so the session keeps today's behavior. + */ + +import { ATTR } from './attributes.js' + +/** Resource-attribute keys this resolver may stamp. */ +export type RepoAttrs = Partial< + Record< + | typeof ATTR.SUBJECT_KEY + | typeof ATTR.GIT_REPOSITORY + | typeof ATTR.GIT_BRANCH_NAME + | typeof ATTR.GIT_COMMIT + | typeof ATTR.CWD, + string + > +> + +/** + * Normalize a git remote url to `host/owner/repo`, dropping scheme, auth, the + * `.git` suffix, and any trailing slash. Handles both `https://` and the + * `git@host:owner/repo.git` SCP-style forms. Returns null when unparseable. + */ +export function normalizeRemote(url: string): string | null { + const raw = url.trim() + if (!raw) return null + + // URL form: scheme://[user[:pass]@]host[:port]/owner/repo[.git] + const url2 = raw.match(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/(.+)$/) + if (url2) { + let rest = url2[1]! + const at = rest.lastIndexOf('@') + if (at !== -1) rest = rest.slice(at + 1) + rest = rest.replace(/\.git$/, '').replace(/\/+$/g, '') + // Strip a :port on the host segment. + const slash = rest.indexOf('/') + if (slash === -1) return null + const host = rest.slice(0, slash).replace(/:\d+$/, '') + const path = rest.slice(slash + 1).replace(/^\/+/, '') + return host && path ? `${host}/${path}` : null + } + + // SCP-style (no scheme): git@github.com:owner/repo.git + const scp = raw.match(/^[^@/]+@([^:/]+):(.+)$/) + if (scp) { + const path = scp[2]!.replace(/\.git$/, '').replace(/^\/+|\/+$/g, '') + return path ? `${scp[1]}/${path}` : null + } + + return null +} + +async function readGit(cwd: string): Promise<{ remote: string | null; branch: string | null; commit: string | null }> { + const { stat } = await import('node:fs/promises') + // A readable `.git` (dir for a normal repo, file for a worktree/submodule). + await stat(`${cwd}/.git`) // throws if absent → caller falls back + + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const run = promisify(execFile) + + const git = async (args: string[]): Promise => { + try { + const { stdout } = await run('git', ['-C', cwd, ...args], { timeout: 5000 }) + const v = stdout.trim() + return v.length > 0 ? v : null + } catch { + return null + } + } + + // Prefer `origin`, else the first configured remote. + let remote = await git(['remote', 'get-url', 'origin']) + if (!remote) { + const first = await git(['remote']) + const name = first?.split('\n')[0]?.trim() + if (name) remote = await git(['remote', 'get-url', name]) + } + const branch = await git(['rev-parse', '--abbrev-ref', 'HEAD']) + const commit = await git(['rev-parse', '--short', 'HEAD']) + return { + remote, + branch: branch === 'HEAD' ? null : branch, // detached HEAD → no branch name + commit, + } +} + +/** + * Resolve per-session repo/git resource attributes from a session's cwd. + * Fail-safe and never throws; see the module doc for the derivation contract. + */ +export async function resolveRepoAttrs(cwd: string | null | undefined): Promise { + if (!cwd) return {} + + const attrs: RepoAttrs = { [ATTR.CWD]: cwd } + + // basename fallback (also used when git can't resolve a remote / dir is gone). + const basename = (() => { + const trimmed = cwd.replace(/\/+$/g, '') + const idx = trimmed.lastIndexOf('/') + const name = idx === -1 ? trimmed : trimmed.slice(idx + 1) + return name || trimmed + })() + + try { + const { remote, branch, commit } = await readGit(cwd) + const normalized = remote ? normalizeRemote(remote) : null + if (normalized) { + attrs[ATTR.SUBJECT_KEY] = normalized + attrs[ATTR.GIT_REPOSITORY] = normalized + if (branch) attrs[ATTR.GIT_BRANCH_NAME] = branch + if (commit) attrs[ATTR.GIT_COMMIT] = commit + return attrs + } + // Readable `.git` but no usable remote → still group by path basename, + // and surface branch/commit when we have them (no remote to fabricate). + if (branch) attrs[ATTR.GIT_BRANCH_NAME] = branch + if (commit) attrs[ATTR.GIT_COMMIT] = commit + } catch { + // cwd/.git gone (deleted worktree) or git unavailable → path-basename group, + // no git.* fabrication. + } + + attrs[ATTR.SUBJECT_KEY] = basename + return attrs +} + +/** + * Stamp resolved repo attrs onto every span's attributes (so they land in the + * OTLP resource attributes via `toOpenInferenceSpan`). Additive — existing + * `service.name` / `agent.name` are untouched. Mutates in place and returns the + * same array for ergonomic chaining. A no-op when `attrs` is empty. + */ +export function stampRepoAttrs(spans: readonly import('./otlp.js').OtlpSpan[], attrs: RepoAttrs): readonly import('./otlp.js').OtlpSpan[] { + const keys = Object.keys(attrs) + if (keys.length === 0) return spans + for (const s of spans) { + for (const k of keys) { + // Never clobber a value an adapter already set deliberately. + if (s.attributes[k] === undefined) s.attributes[k] = (attrs as Record)[k] + } + } + return spans +} diff --git a/src/session-source.ts b/src/session-source.ts index 024877d..367a36a 100644 --- a/src/session-source.ts +++ b/src/session-source.ts @@ -8,7 +8,21 @@ import type { OtlpSpan } from './otlp.js' import { type AdapterSelection, selectAdapters } from './registry.js' -import type { SessionRef } from './types.js' +import { resolveRepoAttrs, stampRepoAttrs } from './repo.js' +import type { HarnessTraceAdapter, SessionRef } from './types.js' + +/** + * Parse one session to spans and stamp per-session repo/git resource attrs + * (`tangle.subject.key` etc.) derived from the ref's cwd. Every OTLP-producing + * path funnels through here so the spine can group by repo. Repo resolution is + * computed ONCE per session; it is fail-safe and never throws. + */ +export async function parseSession(adapter: HarnessTraceAdapter, ref: SessionRef): Promise { + const spans = await adapter.parse(ref) + const repoAttrs = await resolveRepoAttrs(ref.cwd) + stampRepoAttrs(spans, repoAttrs) + return spans +} export interface ScanOptions extends AdapterSelection { /** Filter by working directory (exact/prefix). */ @@ -47,7 +61,7 @@ export async function* scanSessions(opts: ScanOptions): AsyncGenerator + span({ + traceId: 'sess-policy', + spanId: `tool-${i}`, + parentSpanId: 'llm-1', + name: 'tool.bash', + kind: 'TOOL', + startTime: `2026-06-26T00:00:0${i + 1}.000Z`, + service: 'codex', + tool: 'bash', + content: JSON.stringify({ cmd: 'pnpm test' }), + status: 'ERROR', + extra: repo, + })), + ] +} + +describe('policy evidence export', () => { + it('summarizes a session as miner-ready evidence, not a campaign cell', async () => { + const record = await buildPolicyEvidenceRecord(ref, policySpans(), { + generatedAt: '2026-06-26T00:00:10.000Z', + otlpPath: '/tmp/spans.otlp.jsonl', + }) + + expect(record.kind).toBe('traces.policy_evidence.session') + expect(record.session.sessionId).toBe('sess-policy') + expect(record.repo.subjectKey).toBe('github.com/tangle-network/agent-lab') + expect(record.repo.branch).toBe('research/x') + expect(record.metrics.spanCount).toBe(5) + expect(record.metrics.llmTurnCount).toBe(1) + expect(record.metrics.toolCallCount).toBe(3) + expect(record.metrics.erroredToolCallCount).toBe(3) + expect(record.metrics.inputTokens).toBe(100) + expect(record.metrics.outputTokens).toBe(20) + expect(record.metrics.models).toEqual(['glm-5.2']) + expect(record.metrics.tools).toEqual([{ name: 'bash', calls: 3, errors: 3 }]) + expect(record.signals.stuckLoopCount).toBe(1) + expect(record.signals.stuckLoops[0]).toEqual({ toolName: 'bash', occurrences: 3 }) + expect(record.signals.stuckLoopsOmitted).toBe(0) + expect(record.signals.toolErrorRate).toBe(1) + expect(record.provenance.notCampaignCell).toBe(true) + expect(record.provenance.otlpPath).toBe('/tmp/spans.otlp.jsonl') + + const [line] = serializePolicyEvidence([record]).trim().split('\n') + expect(JSON.parse(line!).provenance.notCampaignCell).toBe(true) + }) + + it('collects policy evidence through the public scan path', async () => { + const adapter: HarnessTraceAdapter = { + harness: 'synthetic', + async locate() { + return [ref] + }, + async parse() { + return policySpans() + }, + } + + const [record] = await collectPolicyEvidence({ + adapters: [adapter], + generatedAt: '2026-06-26T00:00:10.000Z', + }) + expect(record?.session.harness).toBe('synthetic') + expect(record?.metrics.toolCallCount).toBe(3) + }) +}) diff --git a/tests/repo.test.ts b/tests/repo.test.ts new file mode 100644 index 0000000..f81b3fd --- /dev/null +++ b/tests/repo.test.ts @@ -0,0 +1,157 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, describe, expect, it } from 'vitest' +import { ATTR } from '../src/attributes.js' +import { serializeSpans, span, toOpenInferenceSpan } from '../src/otlp.js' +import { normalizeRemote, resolveRepoAttrs, stampRepoAttrs } from '../src/repo.js' +import { parseSession } from '../src/session-source.js' +import type { HarnessTraceAdapter, OtlpSpan, SessionRef } from '../src/index.js' + +const run = promisify(execFile) +const created: string[] = [] + +afterAll(async () => { + for (const d of created) await rm(d, { recursive: true, force: true }) +}) + +async function gitRepo(remote: string): Promise { + const dir = await mkdtemp(join(tmpdir(), 'traces-repo-')) + created.push(dir) + await run('git', ['-C', dir, 'init', '-q', '-b', 'main']) + await run('git', ['-C', dir, 'remote', 'add', 'origin', remote]) + await run('git', ['-C', dir, 'config', 'user.email', 't@t.t']) + await run('git', ['-C', dir, 'config', 'user.name', 'T']) + await run('git', ['-C', dir, 'commit', '-q', '--allow-empty', '-m', 'init']) + return dir +} + +/** Minimal adapter: one root span per ref, parents nothing — repo attrs are + * stamped by parseSession, NOT by the adapter. */ +function adapter(): HarnessTraceAdapter { + return { + harness: 'synthetic', + async locate() { + return [] + }, + async parse(r: SessionRef): Promise { + return [ + span({ + traceId: r.sessionId, + spanId: `root:${r.sessionId}`, + name: 'session', + kind: 'AGENT', + startTime: '2026-01-01T00:00:00.000Z', + service: 'claude-code', + agent: 'claude-code', + }), + ] + }, + } +} + +const ref = (id: string, cwd: string | null): SessionRef => ({ + harness: 'synthetic', + sessionId: id, + path: `/tmp/${id}`, + cwd, + mtimeMs: 0, +}) + +describe('normalizeRemote', () => { + it('normalizes https + scp git urls to host/owner/repo', () => { + expect(normalizeRemote('https://github.com/tangle-network/agent-dev-container.git')).toBe( + 'github.com/tangle-network/agent-dev-container', + ) + expect(normalizeRemote('git@github.com:tangle-network/agent-dev-container.git')).toBe( + 'github.com/tangle-network/agent-dev-container', + ) + expect(normalizeRemote('https://user:tok@gitlab.com:443/a/b')).toBe('gitlab.com/a/b') + }) +}) + +describe('resolveRepoAttrs', () => { + it('null/undefined cwd → {} (keeps today behavior)', async () => { + expect(await resolveRepoAttrs(null)).toEqual({}) + expect(await resolveRepoAttrs(undefined)).toEqual({}) + }) + + it('git cwd → remote-derived subject key + git.* labels', async () => { + const dir = await gitRepo('git@github.com:tangle-network/agent-dev-container.git') + const a = await resolveRepoAttrs(dir) + expect(a[ATTR.SUBJECT_KEY]).toBe('github.com/tangle-network/agent-dev-container') + expect(a[ATTR.GIT_REPOSITORY]).toBe('github.com/tangle-network/agent-dev-container') + expect(a[ATTR.GIT_BRANCH_NAME]).toBe('main') + expect(typeof a[ATTR.GIT_COMMIT]).toBe('string') + expect(a[ATTR.GIT_COMMIT]!.length).toBeGreaterThan(0) + expect(a[ATTR.CWD]).toBe(dir) + }) + + it('DELETED cwd (gone worktree) → path-basename subject key, NO git.* fabrication', async () => { + const gone = '/tmp/this-dir-does-not-exist-12345/my-old-worktree' + const a = await resolveRepoAttrs(gone) + expect(a[ATTR.SUBJECT_KEY]).toBe('my-old-worktree') + expect(a[ATTR.CWD]).toBe(gone) + expect(ATTR.GIT_REPOSITORY in a).toBe(false) + expect(ATTR.GIT_BRANCH_NAME in a).toBe(false) + expect(ATTR.GIT_COMMIT in a).toBe(false) + }) +}) + +describe('per-session repo grouping in OTLP resource attrs', () => { + it('two sessions with different cwds get DIFFERENT tangle.subject.key in resource attributes', async () => { + const repoA = await gitRepo('git@github.com:tangle-network/agent-dev-container.git') + const repoB = await gitRepo('https://github.com/tangle-network/traces.git') + const ad = adapter() + + const spansA = await parseSession(ad, ref('sessA', repoA)) + const spansB = await parseSession(ad, ref('sessB', repoB)) + + // Resource attributes (what the spine groups on) must differ per repo. + const resourceKeyOf = (spans: OtlpSpan[]): unknown => { + const line = serializeSpans(spans).trim().split('\n')[0]! + return (JSON.parse(line).resource.attributes as Record)[ATTR.SUBJECT_KEY] + } + const keyA = resourceKeyOf(spansA) + const keyB = resourceKeyOf(spansB) + expect(keyA).toBe('github.com/tangle-network/agent-dev-container') + expect(keyB).toBe('github.com/tangle-network/traces') + expect(keyA).not.toBe(keyB) + + // service.name + agent.name still present (additive, not replaced). + const resA = JSON.parse(serializeSpans(spansA).trim().split('\n')[0]!).resource.attributes + expect(resA['service.name']).toBe('claude-code') + expect(resA['agent.name']).toBe('claude-code') + expect(resA['git.repository']).toBe('github.com/tangle-network/agent-dev-container') + }) + + it('deleted-cwd session falls back to the path basename in resource attrs, no git.* keys', async () => { + const spans = await parseSession(adapter(), ref('sessGone', '/tmp/nope-9999/legacy-worktree')) + const res = JSON.parse(serializeSpans(spans).trim().split('\n')[0]!).resource.attributes as Record + expect(res[ATTR.SUBJECT_KEY]).toBe('legacy-worktree') + expect('git.repository' in res).toBe(false) + expect('git.commit' in res).toBe(false) + }) + + it('toOpenInferenceSpan copies subject/git/cwd keys into resource attributes', () => { + const s = stampRepoAttrs( + [span({ traceId: 't', spanId: 's', name: 'x', kind: 'AGENT', startTime: 'now', service: 'codex', agent: 'codex' })], + { + [ATTR.SUBJECT_KEY]: 'github.com/o/r', + [ATTR.GIT_REPOSITORY]: 'github.com/o/r', + [ATTR.GIT_BRANCH_NAME]: 'feat/x', + [ATTR.GIT_COMMIT]: 'abc1234', + [ATTR.CWD]: '/work/r', + }, + )[0]! + const res = (toOpenInferenceSpan(s).resource as { attributes: Record }).attributes + expect(res['tangle.subject.key']).toBe('github.com/o/r') + expect(res['git.repository']).toBe('github.com/o/r') + expect(res['git.branch']).toBe('feat/x') + expect(res['git.commit']).toBe('abc1234') + expect(res['tangle.cwd']).toBe('/work/r') + expect(res['service.name']).toBe('codex') + }) +})