diff --git a/.github/workflows/mutation-affected.yml b/.github/workflows/mutation-affected.yml index cd46ff6126..9ade741720 100644 --- a/.github/workflows/mutation-affected.yml +++ b/.github/workflows/mutation-affected.yml @@ -2,43 +2,33 @@ name: Mutation Affected # PR-side half of the decision-kernel mutation lane (issue #1415). # -# Graduation, not a flag day: this job runs on every PR that touches a kernel -# module, but `scripts/mutation/run.ts` only exits non-zero once the committed -# baseline reports `gating: true` — earned by two consecutive stable weekly -# sweeps (mutation-weekly.yml). Until then it is a report with the same numbers, -# so the gate's first failing day is not also its first running day. +# Advisory only — the lane reports and never gates (#1457). `scripts/mutation/run.ts` +# exits non-zero on a harness failure, never on a score. # -# Cost control: before graduation an affected run is a report nobody acts on, so -# `select` returns an empty matrix and no mutants run — except when the diff -# touches the lane's own tooling, the one case where a pre-graduation run buys -# something (the gate has to be proven before it can bite). +# Cost control: the kernel report that pays is the weekly sweep, so this lane +# spends mutants on one thing only — proving the harness still runs end to end +# when the harness itself changes. `select` therefore returns an empty matrix +# unless the diff touches the lane's own sources. Selecting on derived kernel +# ownership instead would run the full ten-shard sweep on 24 of the last 40 +# merged PRs. # -# Scope is the AFFECTED modules only, and affectedness is DERIVED from the import -# graph (scripts/mutation/ownership.ts): a kernel source, or any test that reaches -# one. Reaching a kernel is a superset of killing its mutants, so the `select` job -# frequently returns several modules — sharded like the weekly sweep so the PR's -# wall clock is one module, not their sum. The weekly run stays the full sweep. +# On a harness diff the matrix is the canary (`LANE_CANARY`, the registry's +# cheapest real sweep — lane sources own no kernel, so derivation alone would +# select nothing) plus any kernel that same diff reaches through the import +# graph (scripts/mutation/ownership.ts): a kernel source, or a test that reaches +# one. Selected modules are sharded like the weekly sweep, so the PR's wall +# clock is one module rather than their sum. on: pull_request: paths: - # Kernel sources, every src/package test (ownership is derived, so any - # test may own a kernel — `select` decides, not this filter), and the - # lane's own tooling. scripts/mutation/workflow.test.ts asserts this - # covers the registry. - - 'packages/kernel/src/errors.ts' - - 'src/daemon/ref-frame.ts' - - 'src/commands/interaction/runtime/settle.ts' - - 'src/utils/scroll-edge-state.ts' - - 'packages/selectors/src/**' - - 'packages/ad-script/src/internal/target-annotation-serde.ts' - - 'src/snapshot/snapshot-occlusion.ts' - - 'src/**/*.test.ts' - - 'packages/*/src/**/*.test.ts' + # Exactly `LANE_TOOLING` in scripts/mutation/run.ts plus this file — the + # only diffs that can produce a non-empty matrix, asserted both ways by + # scripts/mutation/workflow.test.ts. A kernel or test path here would only + # start a job that selects nothing. - 'scripts/mutation/**' - 'scripts/lib/**' - 'stryker.config.json' - - 'mutation-baselines/**' - '.github/workflows/mutation-affected.yml' permissions: @@ -64,7 +54,7 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Ratchet self-test + - name: Harness self-test uses: ./.github/actions/run-gate with: { gate: mutation-model } @@ -138,7 +128,7 @@ jobs: .tmp/mutation/lane-envelope.json if-no-files-found: warn - ratchet: + report: name: Affected decision-kernel mutants needs: [select, mutants] if: always() && needs.select.result == 'success' @@ -162,7 +152,8 @@ jobs: # No shards means no affected kernel: run.ts reports "nothing to mutate" # and still writes the envelope, so the lane is never silently absent. - - name: Ratchet the affected modules + # Either invocation writes the score table to $GITHUB_STEP_SUMMARY. + - name: Score the affected modules run: | if [ -d .tmp/mutation/shards ]; then expected=$(echo '${{ needs.select.outputs.modules }}' | jq length) @@ -177,7 +168,7 @@ jobs: if: failure() run: | pnpm gate mutation --affected --fail-envelope \ - "affected ratchet failed before producing a verdict (run ${{ github.run_id }})" || true + "affected report failed before producing a score table (run ${{ github.run_id }})" || true - name: Upload mutation report if: always() diff --git a/.github/workflows/mutation-weekly.yml b/.github/workflows/mutation-weekly.yml index bb0fa1572a..3b475f82cf 100644 --- a/.github/workflows/mutation-weekly.yml +++ b/.github/workflows/mutation-weekly.yml @@ -8,13 +8,11 @@ name: Mutation Weekly # Sharded one job per module: the whole sweep is ~2,150 mutants, and the selector # module alone is ~1,280 of them, so a single job would sit near the 30-minute # acceptance budget on an ubuntu runner. The shards' JSON reports are merged into -# one verdict by the ratchet job (`--report-dir`), so the ratchet still sees a -# full sweep. +# one score table by the report job (`--report-dir`). # -# The lane reports; it does not commit. The proposed baseline rides in the -# artifact and applying it is a reviewed `pnpm mutation:baseline` commit — a score -# can never lower itself. Gating (after two consecutive stable weekly runs) is -# enforced on PRs by mutation-affected.yml. +# The lane reports and never gates (#1457): the per-kernel table lands in the job +# summary and the artifact, and a low score is an input for a human-authored +# test-strengthening PR (#1474, #1475 were written that way). on: schedule: @@ -90,8 +88,8 @@ jobs: .tmp/mutation/lane-envelope.json if-no-files-found: warn - ratchet: - name: Mutation ratchet + report: + name: Mutation score report runs-on: ubuntu-latest needs: shard if: always() @@ -103,7 +101,7 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Ratchet self-test + - name: Harness self-test uses: ./.github/actions/run-gate with: { gate: mutation-model } @@ -113,9 +111,13 @@ jobs: pattern: mutation-shard-* path: .tmp/mutation/shards - # run.ts writes the markdown verdict to $GITHUB_STEP_SUMMARY when the + # run.ts writes the per-kernel score table to $GITHUB_STEP_SUMMARY when the # runner exports it, so the summary and the artifact carry the same numbers. - - name: Ratchet the merged sweep and propose the next baseline + # It publishes the table before judging the shard set, so even the one + # failure it can raise — an incomplete shard set — still reports the kernels + # that completed. A red lane here always means the sweep did not happen, + # never a low score. + - name: Score the merged sweep uses: ./.github/actions/run-gate with: gate: mutation-check @@ -124,20 +126,14 @@ jobs: .tmp/mutation/shards --expect-shards 10 - --update - - name: Stage the proposed baseline artifact - run: | - cp mutation-baselines/decision-kernels.json .tmp/mutation/proposed-baseline.json - git checkout -- mutation-baselines/decision-kernels.json - - # The self-test and the artifact download both run before the ratchet, so a - # failure there would otherwise leave the aggregate lane with no envelope. + # The self-test and the artifact download both run before the scoring step, + # so a failure there would otherwise leave the aggregate lane with no envelope. - name: Record a failed lane envelope if: failure() run: | pnpm gate mutation --fail-envelope \ - "weekly ratchet job failed before producing a verdict (run ${{ github.run_id }})" || true + "weekly report job failed before producing a score table (run ${{ github.run_id }})" || true # Freshness/drift telemetry (#1430): the envelope states commit, Stryker # version, config hash, duration and result, so a lane going dark or a tool @@ -167,5 +163,4 @@ jobs: path: | .tmp/mutation/shards .tmp/mutation/lane-envelope.json - .tmp/mutation/proposed-baseline.json if-no-files-found: warn diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 110e05a75e..ef771b1b35 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -10,7 +10,6 @@ "node_modules/**", "**/*.md", "scripts/maestro-conformance/corpus/**", - "fallow-baselines/**", - "mutation-baselines/**" + "fallow-baselines/**" ] } diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 9d2f90abca..18564246f3 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -44,7 +44,7 @@ The mapping it encodes, for when you need to run a gate directly or reason about | Daemon RPC wire surface (the declarations listed in `test/wire-compat/surface.ts` — JSON-RPC envelope, request/response/error/artifact/progress framing, `/health` payload, HTTP auth headers) | `pnpm exec vitest run --project unit-core test/wire-compat` holds the ledger to its source and prints the digest to paste; `pnpm check:daemon-wire-compat` compares it against the last released tag and requires a `DAEMON_RPC_PROTOCOL_VERSION` bump or a `compatibleChanges` ack for the drift. Read ADR 0006 to decide which; `test/wire-compat/README.md` walks both | | Anything in `src/`, `test/` | `pnpm format` (`skills/` is Markdown-only guidance: oxfmt ignores `**/*.md`, and the affected-check selector classifies it docs-only) | | Workspace package source (`packages/*/src/**`) | Root format/lint/typecheck plus layering (R11 package-boundaries); Vitest resolves affected tests through the module graph; package manifests/tsconfigs fail open to the full set | -| A decision kernel or its tests (`packages/kernel/src/errors.ts`, `src/daemon/ref-frame.ts`, `src/commands/interaction/runtime/settle.ts`, `src/utils/scroll-edge-state.ts`, `packages/selectors/src/`) | `pnpm mutation:affected --base origin/main` (minutes; GitHub runs it per PR — see the mutation ratchet section) | +| A decision kernel or its tests (`packages/kernel/src/errors.ts`, `src/daemon/ref-frame.ts`, `src/commands/interaction/runtime/settle.ts`, `src/utils/scroll-edge-state.ts`, `packages/selectors/src/`) | `pnpm mutation:run --modules ` (minutes; optional — the lane reports, it never gates, see the mutation section) | Two traps worth naming: @@ -243,7 +243,7 @@ The three facts the manifest cannot derive live together in `scripts/gate/declar coverage wrapper, one reporting-only `test:*` script, and the Android replay owner hidden inside a third-party action's `script:` input. -## Mutation ratchet over decision kernels +## Mutation report over decision kernels Mutation score is the mechanical answer to "is this test load-bearing or decorative". A full-suite sweep is unaffordable, so the scope is an enumerated list of pure decision kernels — modules where a @@ -252,46 +252,47 @@ surviving mutant means a silently wrong agent-facing decision. The registry are asserted against it, and PR-affected selection maps changed files through it. Modules that spawn subprocesses or wait real time stay out by construction. +Mutation runs **report-only** on the seven decision kernels — a weekly full sweep plus a per-PR +affected sweep. It never gates: `scripts/mutation/run.ts` exits non-zero on a harness failure (a +missing report, an incomplete shard set, a bad argument) and never on a score. A low score is an +input for a human-authored test-strengthening PR, which is exactly how #1474 and #1475 were written. +The ratchet, baseline file and graduation rule this lane used to carry were deleted in #1457: in +three weeks nobody applied a baseline, so the gate half never operated while the report half was +paying. + ```sh -pnpm mutation:test # ratchet self-test (fast, no Stryker) +pnpm mutation:test # harness self-test (fast, no Stryker) pnpm mutation:run --modules selectors # one module locally (~7 min for selectors) -pnpm mutation:check # ratchet an existing .tmp/mutation/mutation.json -pnpm mutation:baseline # full sweep, then record it (reviewed commit) +pnpm mutation:check # score an existing .tmp/mutation/mutation.json ``` - **Weekly full sweep** (`.github/workflows/mutation-weekly.yml`) runs `shardMatrix()` from the registry: one job per module, except modules that declare a `shards` count and are sliced with - `--shard i/n` (selectors is ~1,280 mutants, well past the 30-minute budget in one job). The ratchet - merges the shard reports (`--report-dir`) for one verdict and requires the full set - (`--expect-shards`), so a dead shard fails the lane instead of scoring its module as 0. Results are - reported as a job summary plus an artifact. It never commits: the proposed baseline rides in the - artifact, and applying it is a reviewed `pnpm mutation:baseline` commit, so a score cannot lower - itself. -- **PR lane** (`.github/workflows/mutation-affected.yml`) derives the affected shard matrix - (`--list-affected`) and merges the shards into one verdict. Before graduation the matrix is empty — - a report nobody acts on is not worth the runner minutes — unless the diff touches the lane's own - tooling, the one pre-graduation run that buys something: the gate has to be proven before it bites. - Lane sources own no kernel, so that exception adds `LANE_CANARY` (`kernel-errors`, the registry's - cheapest real sweep) to whatever the diff derives; otherwise it would select zero mutants and prove - nothing. `scripts/mutation/selection.test.ts` drives both halves of the rule through the real CLI - against a throwaway worktree commit. -- **Ratchet**: scores may only rise. `mutation-baselines/decision-kernels.json` records the - high-water score per module plus the Stryker version and config content hash that produced it, so a - score change caused by a tool/config change is reported as provenance drift, never as a - test-strength regression. -- **Graduation, not a flag day**: gating is off until two consecutive comparable weekly sweeps pass - (`stableRuns`/`requiredStableRuns` in the baseline); the PR job starts selecting modules — and - failing on them — once the committed baseline says `gating: true`. A regression or provenance drift - resets the counter. + `--shard i/n` (selectors is ~1,280 mutants, well past the 30-minute budget in one job). The report + job merges the shard reports (`--report-dir`) into one per-kernel table — kernel, score, killed, + survived, total, timeouts, plus the surviving mutants — and requires the full set + (`--expect-shards`), so a dead shard fails the lane instead of publishing its module as 0%. The + table lands in the job summary and the artifact. +- **PR lane** (`.github/workflows/mutation-affected.yml`) exists to prove the harness still runs end + to end when the harness changes, so its matrix (`--list-affected`) is empty unless the diff touches + the lane's own sources: the weekly sweep is the kernel report, and selecting on derived kernel + ownership would run the full ten-shard sweep on 24 of the last 40 merged PRs for a report nobody + gates on. The workflow triggers on exactly those sources (`LANE_TOOLING` in `run.ts`, asserted both + ways by `workflow.test.ts`), so a PR that could only select `[]` never starts the job. Lane sources + own no kernel, so a harness diff adds `LANE_CANARY` (`kernel-errors`, the registry's cheapest real + sweep) to whatever kernels that same diff derives; otherwise it would select zero mutants and prove + nothing. It reports the same table. `scripts/mutation/selection.test.ts` drives both halves of the + rule through the real CLI against a throwaway worktree commit. +- **Provenance**: every report and lane envelope carries the Stryker version and the config content + hash that produced it, so scores measured across a tool or config change are not read as + test-strength change. - **Test scope** is derived from Vitest's module graph (`vitest related` over the mutated files), the same delegation `pnpm check:affected` uses; see `scripts/mutation/test-scope.ts` for the three groups it drops and why dropping them cannot hide a surviving mutant. - **Test ownership is derived, never listed** (`scripts/mutation/ownership.ts`): a test owns every kernel its imports reach, so `src/__tests__/daemon-error.test.ts` selects `kernel-errors` through - `src/daemon.ts` without naming it. A listed set of test files would silently omit exactly those - indirect tests and rot as tests are added — weakening one would skip the ratchet. Reaching a kernel - is a superset of killing its mutants, so the PR lane over-selects on purpose and shards the - selected modules; a false positive costs runner minutes, a false negative costs the gate. Non-kernel + `src/daemon.ts` without naming it. Reaching a kernel is a superset of killing its mutants, so the + derivation over-selects on purpose; it applies to the modules a lane-tooling diff selects. Non-kernel *sources* are not owned: they can only move a score through those tests, and the weekly sweep re-measures the whole surface. - **Lane envelope** (`scripts/lib/lane-envelope.ts`, issue #1430): every run writes diff --git a/mutation-baselines/decision-kernels.json b/mutation-baselines/decision-kernels.json deleted file mode 100644 index 402b34ce74..0000000000 --- a/mutation-baselines/decision-kernels.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "schemaVersion": 1, - "requiredStableRuns": 2, - "stableRuns": 0, - "gating": false, - "modules": { - "kernel-errors": { - "score": 55.19, - "killed": 101, - "survived": 82, - "total": 183, - "strykerVersion": "9.6.1", - "configHash": "sha256:806d9f2e657f", - "updatedAt": "2026-07-27T14:29:40.750Z" - }, - "daemon-ref-frame": { - "score": 100, - "killed": 55, - "survived": 0, - "total": 55, - "strykerVersion": "9.6.1", - "configHash": "sha256:806d9f2e657f", - "updatedAt": "2026-07-27T14:29:40.750Z" - }, - "interaction-settle": { - "score": 68.95, - "killed": 151, - "survived": 68, - "total": 219, - "strykerVersion": "9.6.1", - "configHash": "sha256:806d9f2e657f", - "updatedAt": "2026-07-27T14:29:40.750Z" - }, - "scroll-edge-state": { - "score": 28.75, - "killed": 92, - "survived": 228, - "total": 320, - "strykerVersion": "9.6.1", - "configHash": "sha256:806d9f2e657f", - "updatedAt": "2026-07-27T14:29:40.750Z" - }, - "selectors": { - "score": 70.56, - "killed": 901, - "survived": 376, - "total": 1277, - "strykerVersion": "9.6.1", - "configHash": "sha256:806d9f2e657f", - "updatedAt": "2026-07-27T14:29:40.750Z" - }, - "target-annotation-serde": { - "score": 94.03, - "killed": 315, - "survived": 20, - "total": 335, - "strykerVersion": "9.6.1", - "configHash": "sha256:8b633048f754", - "updatedAt": "2026-08-02T07:28:56.278Z" - }, - "snapshot-occlusion": { - "score": 89.74, - "killed": 175, - "survived": 20, - "total": 195, - "strykerVersion": "9.6.1", - "configHash": "sha256:8b633048f754", - "updatedAt": "2026-08-02T07:28:56.278Z" - } - } -} diff --git a/package.json b/package.json index d8ce471883..4154935e46 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,6 @@ "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", - "mutation:baseline": "node --experimental-strip-types scripts/mutation/run.ts --update", "mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run", "mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected", "mutation:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/mutation/*.test.ts", diff --git a/scripts/check-affected/checks.ts b/scripts/check-affected/checks.ts index d9f8648289..b40c62e072 100644 --- a/scripts/check-affected/checks.ts +++ b/scripts/check-affected/checks.ts @@ -128,8 +128,8 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [ gate('fuzz-parsers', 'Parser fuzz invariants', 'fuzz:parsers', false), gate('mutation', 'Mutation sweep', 'mutation:run', false), gate('mutation-affected', 'Affected mutation shard selection', 'mutation:affected', false), - gate('mutation-check', 'Mutation ratchet against the baseline', 'mutation:check', false), - gate('mutation-model', 'Mutation ratchet self-test', 'mutation:test'), + gate('mutation-check', 'Mutation score for an existing report', 'mutation:check', false), + gate('mutation-model', 'Mutation harness self-test', 'mutation:test'), gate( 'concurrency-torture', 'Session/lease/lock torture sweep', diff --git a/scripts/mutation/config.test.ts b/scripts/mutation/config.test.ts index ff37d06b32..c6896a5735 100644 --- a/scripts/mutation/config.test.ts +++ b/scripts/mutation/config.test.ts @@ -25,12 +25,12 @@ test('stryker mutate globs mirror the kernel-module registry', () => { ); }); -test('stryker owns no pass/fail threshold — the ratchet does', () => { +test('stryker owns no pass/fail threshold — the lane never gates on a score', () => { const thresholds = readConfig().thresholds as { break?: number | null } | undefined; assert.equal( thresholds?.break ?? null, null, - "A Stryker `break` threshold would fail runs on an absolute score; gating is the ratchet's job (scripts/mutation/ratchet.ts).", + 'A Stryker `break` threshold would fail runs on an absolute score; this lane reports scores and never gates on them (#1457).', ); }); diff --git a/scripts/mutation/envelope.test.ts b/scripts/mutation/envelope.test.ts index 8f28c0b359..ea026942d5 100644 --- a/scripts/mutation/envelope.test.ts +++ b/scripts/mutation/envelope.test.ts @@ -41,7 +41,7 @@ test('the envelope carries schema, commit, tool/config provenance, duration and assert.deepEqual(envelope.data, { scope: 'full-sweep' }); }); -test('a failed ratchet is recorded as a failed lane run', () => { +test('a lane that produced no report is recorded as a failed run', () => { const envelope = laneEnvelope({ lane: 'mutation-decision-kernels', commit: 'b'.repeat(40), @@ -92,7 +92,7 @@ test('a crashed run still writes an envelope naming the stage it died in', () => // Shard artifacts carry the envelope next to the report, so merging "every JSON // under the shard directory" fed the envelope to the report parser and crashed -// the ratchet job after the mutants had already run. +// the reporting job after the mutants had already run. test('merging shard reports ignores the envelope sitting beside them', () => { const shards = path.join(repoRoot, '.tmp/mutation/envelope-test-shards/shard-a'); fs.mkdirSync(shards, { recursive: true }); @@ -170,9 +170,9 @@ function readEnvelope(): Envelope { } // A merged shard set is only a sweep if every requested module actually reported. -// summarizeReport scores an absent module as 0, and while the lane is non-gating a -// 0 only *reports* a regression — so a dead matrix shard would otherwise be -// aggregated into a passing "complete" envelope claiming the sweep happened. +// summarizeReport scores an absent module as 0, so a dead matrix shard would +// otherwise be published as a 0% kernel in a passing "complete" envelope +// claiming the sweep happened. test('an incomplete shard set fails instead of scoring the missing module as zero', () => { const shards = path.join(repoRoot, '.tmp/mutation/partial-shards/shard-kernel-errors'); fs.mkdirSync(shards, { recursive: true }); @@ -191,14 +191,46 @@ test('an incomplete shard set fails instead of scoring the missing module as zer assert.notEqual(result.exitCode, 0, 'a missing shard must fail the aggregate'); assert.match(result.stderr, /Incomplete shard set/); assert.match(result.stderr, /daemon-ref-frame/); + // The kernels that did complete are still worth reading on a failed-shard day, + // so the table is published before the shard set is judged. + assert.match(result.stdout, /\| `kernel-errors` — [^|]+\| 100% \|/); const envelope = readEnvelope(); assert.equal(envelope.result, 'fail'); - assert.equal(envelope.data.stage, 'ratchet'); + assert.equal(envelope.data.stage, 'score'); fs.rmSync(path.join(repoRoot, '.tmp/mutation/partial-shards'), { recursive: true, force: true }); }); -// Argument parsing and the provenance/baseline reads used to sit outside the -// envelope boundary, so the lane could exit without declaring itself at all. +// The weekly job's real shape: `--expect-shards 10` over a directory missing +// shards. The count check used to run inside the merge, before anything was +// scored, so one dead shard discarded the nine that had reported. +test('a short shard set publishes the shards that did report, then fails', () => { + const shards = path.join(repoRoot, '.tmp/mutation/short-shards/shard-kernel-errors'); + fs.mkdirSync(shards, { recursive: true }); + fs.writeFileSync( + path.join(shards, 'mutation.json'), + JSON.stringify({ + files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } }, + }), + ); + const result = runMutation([ + '--report-dir', + '.tmp/mutation/short-shards', + '--expect-shards', + '10', + '--modules', + 'kernel-errors', + ]); + assert.notEqual(result.exitCode, 0, 'a short shard set must fail the aggregate'); + assert.match(result.stderr, /1 report\(s\), expected 10/); + assert.match(result.stdout, /\| `kernel-errors` — [^|]+\| 100% \|/); + const envelope = readEnvelope(); + assert.equal(envelope.result, 'fail'); + assert.equal(envelope.data.stage, 'score'); + fs.rmSync(path.join(repoRoot, '.tmp/mutation/short-shards'), { recursive: true, force: true }); +}); + +// Argument parsing and the provenance read used to sit outside the envelope +// boundary, so the lane could exit without declaring itself at all. test('a malformed invocation still writes an envelope', () => { fs.rmSync(path.join(repoRoot, '.tmp/mutation/lane-envelope.json'), { force: true }); const result = runMutation(['--modules', 'not-a-kernel']); @@ -221,7 +253,7 @@ test('--fail-envelope declares a step that failed before the sweep', () => { assert.equal(envelope.data.error, 'self-test failed'); }); -// The workflows run it from `if: failure()`, which also fires when the ratchet +// The workflows run it from `if: failure()`, which also fires when the sweep // itself failed — a generic reason must never displace the specific one. test('--fail-envelope keeps a failure the run already reported', () => { fs.rmSync(path.join(repoRoot, '.tmp/mutation/lane-envelope.json'), { force: true }); @@ -230,20 +262,18 @@ test('--fail-envelope keeps a failure the run already reported', () => { assert.equal(readEnvelope().data.error, 'the real failure'); }); -// A pass is not a verdict worth preserving: the weekly job copies the proposed -// baseline and restores the committed one *after* the ratchet passed, so a failure -// there would otherwise publish the failed scheduled job as a passing lane. +// A pass is not a result worth preserving: the weekly job uploads its artifact +// *after* the report was rendered, so a failure there would otherwise publish +// the failed scheduled job as a passing lane. test('--fail-envelope downgrades a passing envelope when a later step fails', () => { const shards = path.join(repoRoot, '.tmp/mutation/pass-then-fail/shard-kernel-errors'); fs.mkdirSync(shards, { recursive: true }); - // A perfect shard so the ratchet passes: the score can only rise from the - // committed kernel-errors baseline. + // A shard whose only mutant survived — a 0% score. The lane reports scores and + // never gates on them (#1457), so this run is still a pass: the sweep happened. fs.writeFileSync( path.join(shards, 'mutation.json'), JSON.stringify({ - files: { - 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }, { status: 'Killed' }] }, - }, + files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Survived' }] } }, }), ); const passing = runMutation([ @@ -252,13 +282,13 @@ test('--fail-envelope downgrades a passing envelope when a later step fails', () '--modules', 'kernel-errors', ]); - assert.equal(passing.exitCode, 0, passing.stderr); + assert.equal(passing.exitCode, 0, 'a low score must never fail the lane'); assert.equal(readEnvelope().result, 'pass'); - runMutation(['--fail-envelope', 'the baseline copy step failed']); + runMutation(['--fail-envelope', 'the artifact upload step failed']); const envelope = readEnvelope(); assert.equal(envelope.result, 'fail', 'a failed job must not publish a passing envelope'); - assert.equal(envelope.data.error, 'the baseline copy step failed'); + assert.equal(envelope.data.error, 'the artifact upload step failed'); fs.rmSync(path.join(repoRoot, '.tmp/mutation/pass-then-fail'), { recursive: true, force: true }); }); diff --git a/scripts/mutation/modules.ts b/scripts/mutation/modules.ts index 3a3184e669..e337fa33ce 100644 --- a/scripts/mutation/modules.ts +++ b/scripts/mutation/modules.ts @@ -3,7 +3,7 @@ // Mutation score is the only mechanical answer to "is this test load-bearing or // decorative", but a full-suite sweep is unaffordable. This registry is the // single source of truth for what Stryker mutates: `stryker.config.json`'s -// `mutate` globs are asserted against it, and PR-affected gating maps changed +// `mutate` globs are asserted against it, and PR-affected selection maps changed // files onto modules through it. // // Membership rule: pure decision kernels only — a surviving mutant here means a @@ -112,10 +112,10 @@ export function mutateGlobs(ids: readonly ModuleId[] = ALL_MODULE_IDS): string[] } /** - * The module a lane-tooling change proves itself against before graduation. - * `kernel-errors` is the cheapest real sweep in the registry (one file, ~183 - * mutants), so a change to the ratchet, the config, or the baseline runs actual - * mutants end to end without paying for the full sweep. + * The module a lane-tooling change proves itself against. `kernel-errors` is the + * cheapest real sweep in the registry (one file, ~183 mutants), so a change to + * the harness or the config runs actual mutants end to end without paying for + * the full sweep. */ export const LANE_CANARY: ModuleId = 'kernel-errors'; diff --git a/scripts/mutation/ownership.ts b/scripts/mutation/ownership.ts index ce5baba5f0..ecd3d7379b 100644 --- a/scripts/mutation/ownership.ts +++ b/scripts/mutation/ownership.ts @@ -1,8 +1,8 @@ // Which kernel module a changed file belongs to, DERIVED — never hand-listed. // -// A mutation score is a statement about the tests that kill the mutants, so the -// PR lane must re-measure a kernel whenever one of *those* tests changes. An -// enumerated list of test files cannot state that: it silently omits tests that +// A mutation score is a statement about the tests that kill the mutants, so a +// selection is only honest if it follows *those* tests. An enumerated list of +// test files cannot state that: it silently omits tests that // exercise a kernel indirectly (`src/__tests__/daemon-error.test.ts` reaches // `normalizeError` through `src/daemon.ts`), and nothing fails when a new test // is added. So ownership is computed from the static import graph instead: a test @@ -11,13 +11,12 @@ // The derivation is deliberately a superset — reaching a kernel is cheaper to // prove than killing its mutants, so an unrelated diff can select a module and // pay for a report. False positives cost runner minutes; a false negative would -// let a weakened test slip past the ratchet, which is the thing the lane exists -// to catch. +// leave a kernel whose tests changed unmeasured. // // Non-test source changes outside the registry are NOT owned: they can only move // a score through the tests that reach the kernel, and the weekly full sweep is -// what re-measures the whole surface. The PR lane's claim is narrower on purpose -// — kernel sources plus the tests that exercise them. +// what re-measures the whole surface. The derived claim is narrower on purpose — +// kernel sources plus the tests that exercise them. import fs from 'node:fs'; import path from 'node:path'; diff --git a/scripts/mutation/ratchet.test.ts b/scripts/mutation/ratchet.test.ts deleted file mode 100644 index 4c9d9e3de0..0000000000 --- a/scripts/mutation/ratchet.test.ts +++ /dev/null @@ -1,240 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { test } from 'node:test'; -import { ALL_MODULE_IDS } from './modules.ts'; -import { renderReport } from './report.ts'; -import { - applyRun, - BASELINE_SCHEMA_VERSION, - DEFAULT_REQUIRED_STABLE_RUNS, - emptyBaseline, - evaluateRatchet, - type Baseline, - type ModuleBaseline, - type Provenance, -} from './ratchet.ts'; -import { summarizeReport, type StrykerReport } from './score.ts'; - -const PROVENANCE: Provenance = { strykerVersion: '9.6.1', configHash: 'sha256:abcdef123456' }; -const NOW = '2026-07-27T00:00:00.000Z'; - -function mutants(statuses: readonly string[]): StrykerReport { - return { - files: { - 'packages/kernel/src/errors.ts': { - mutants: statuses.map((status, index) => ({ - status, - mutatorName: 'ConditionalExpression', - location: { start: { line: index + 1 } }, - })), - }, - }, - }; -} - -function baselineWith(entry: Partial): Baseline { - return { - ...emptyBaseline(), - modules: { - 'kernel-errors': { - score: 75, - killed: 3, - survived: 1, - total: 4, - strykerVersion: PROVENANCE.strykerVersion, - configHash: PROVENANCE.configHash, - updatedAt: NOW, - ...entry, - }, - }, - }; -} - -test('scores count timeouts as killed and uncovered mutants as survived', () => { - const [score] = summarizeReport(mutants(['Killed', 'Timeout', 'Survived', 'NoCoverage']), [ - 'kernel-errors', - ]); - assert.deepEqual( - { score: score?.score, killed: score?.killed, survived: score?.survived, total: score?.total }, - { score: 50, killed: 2, survived: 2, total: 4 }, - ); -}); - -test('statuses outside the score (Ignored, CompileError) leave the denominator', () => { - const [score] = summarizeReport(mutants(['Killed', 'Ignored', 'CompileError']), [ - 'kernel-errors', - ]); - assert.equal(score?.total, 1); - assert.equal(score?.score, 100); -}); - -test('a lowered score is a regression, and gating makes it fail with its survivors', () => { - const baseline = { ...baselineWith({ score: 100 }), stableRuns: 2, gating: true }; - const scores = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - const result = evaluateRatchet(scores, baseline, PROVENANCE); - - assert.equal(result.failed, true); - assert.deepEqual( - result.regressions.map((verdict) => verdict.module), - ['kernel-errors'], - ); - assert.match(result.regressions[0]?.detail ?? '', /fell 100% -> 50%/); - - const markdown = renderReport(result, baseline, PROVENANCE); - assert.match(markdown, /Surviving mutants:/); - assert.match(markdown, /`packages\/kernel\/src\/errors\.ts:2` ConditionalExpression/); - assert.match(markdown, /scores may only rise/); -}); - -test('a lowered score is reported but does not fail while the lane is non-gating', () => { - const scores = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - const baseline = baselineWith({ score: 100 }); - const result = evaluateRatchet(scores, baseline, PROVENANCE); - - assert.equal(result.regressions.length, 1); - assert.equal(result.failed, false); - assert.match(renderReport(result, baseline, PROVENANCE), /still non-gating/); -}); - -test('a regression never rewrites the baseline high-water mark', () => { - const baseline = baselineWith({ score: 100 }); - const scores = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - const next = applyRun(baseline, scores, evaluateRatchet(scores, baseline, PROVENANCE), { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.equal(next.modules['kernel-errors']?.score, 100); - assert.equal(next.stableRuns, 0); -}); - -test('a risen score is recorded with its provenance', () => { - const baseline = baselineWith({ score: 50 }); - const scores = summarizeReport(mutants(['Killed', 'Killed']), ['kernel-errors']); - const result = evaluateRatchet(scores, baseline, PROVENANCE); - assert.equal(result.verdicts[0]?.status, 'improved'); - - const next = applyRun(baseline, scores, result, { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.deepEqual(next.modules['kernel-errors'], { - score: 100, - killed: 2, - survived: 0, - total: 2, - strykerVersion: PROVENANCE.strykerVersion, - configHash: PROVENANCE.configHash, - updatedAt: NOW, - }); -}); - -test('gating graduates after the required consecutive stable full sweeps', () => { - const scores = summarizeReport(mutants(['Killed', 'Killed']), ['kernel-errors']); - let baseline = baselineWith({ score: 100 }); - assert.equal(baseline.gating, false); - - for (let run = 1; run <= baseline.requiredStableRuns; run += 1) { - const result = evaluateRatchet(scores, baseline, PROVENANCE); - baseline = applyRun(baseline, scores, result, { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.equal(baseline.stableRuns, run); - } - assert.equal(baseline.gating, true); - - // A later regression resets graduation, so gating has to be re-earned. - const dropped = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - const regressed = evaluateRatchet(dropped, baseline, PROVENANCE); - assert.equal(regressed.failed, true); - const after = applyRun(baseline, dropped, regressed, { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.equal(after.stableRuns, 0); - assert.equal(after.gating, false); -}); - -test('an affected PR run never advances graduation', () => { - const scores = summarizeReport(mutants(['Killed', 'Killed']), ['kernel-errors']); - const baseline = baselineWith({ score: 100 }); - const next = applyRun(baseline, scores, evaluateRatchet(scores, baseline, PROVENANCE), { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: false, - }); - assert.equal(next.stableRuns, 0); -}); - -test('a tool or config change is provenance drift, not a test-strength regression', () => { - const scores = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - for (const drift of [{ strykerVersion: '9.5.0' }, { configHash: 'sha256:000000000000' }]) { - const baseline = { ...baselineWith({ score: 100, ...drift }), stableRuns: 2, gating: true }; - const result = evaluateRatchet(scores, baseline, PROVENANCE); - assert.equal(result.regressions.length, 0); - assert.equal(result.failed, false); - assert.equal(result.comparable, false); - assert.match(result.drifted[0]?.detail ?? '', /not attributable to test strength/); - - // Drift rebases onto the new tool/config and costs the graduation streak. - const next = applyRun(baseline, scores, result, { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.equal(next.modules['kernel-errors']?.score, 50); - assert.equal(next.modules['kernel-errors']?.configHash, PROVENANCE.configHash); - assert.equal(next.stableRuns, 0); - } -}); - -test('a module with no baseline yet is new, recorded, and blocks graduation', () => { - const scores = summarizeReport(mutants(['Killed', 'Survived']), ['kernel-errors']); - const baseline = emptyBaseline(); - const result = evaluateRatchet(scores, baseline, PROVENANCE); - assert.equal(result.verdicts[0]?.status, 'new'); - assert.equal(result.comparable, false); - - const next = applyRun(baseline, scores, result, { - provenance: PROVENANCE, - now: NOW, - countsTowardGraduation: true, - }); - assert.equal(next.modules['kernel-errors']?.score, 50); - assert.equal(next.stableRuns, 0); -}); - -test('the report names the fix command and the graduation state', () => { - const scores = summarizeReport(mutants(['Killed', 'Killed']), ['kernel-errors']); - const baseline = { ...baselineWith({ score: 100 }), stableRuns: 1 }; - const markdown = renderReport( - evaluateRatchet(scores, baseline, PROVENANCE), - baseline, - PROVENANCE, - ); - assert.match(markdown, /gating \*\*off\*\* \(1\/2 stable weekly runs\)/); - assert.match(markdown, /Stryker `9\.6\.1` · config `sha256:abcdef123456`/); -}); - -test('the committed baseline is the shape the lane graduates from', () => { - const baseline = JSON.parse( - fs.readFileSync( - path.resolve(import.meta.dirname, '../../mutation-baselines/decision-kernels.json'), - 'utf8', - ), - ) as Baseline; - assert.equal(baseline.schemaVersion, BASELINE_SCHEMA_VERSION); - // Issue #1415's N: gating is earned by two consecutive stable weekly sweeps. - assert.equal(baseline.requiredStableRuns, DEFAULT_REQUIRED_STABLE_RUNS); - assert.deepEqual(Object.keys(baseline.modules), [...ALL_MODULE_IDS]); - for (const [id, module] of Object.entries(baseline.modules)) { - assert.ok(module.total > 0, `${id} has no mutants`); - assert.ok(module.strykerVersion.length > 0, `${id} has no Stryker version`); - assert.match(module.configHash, /^sha256:[0-9a-f]{12}$/); - } -}); diff --git a/scripts/mutation/ratchet.ts b/scripts/mutation/ratchet.ts deleted file mode 100644 index 63700d1df6..0000000000 --- a/scripts/mutation/ratchet.ts +++ /dev/null @@ -1,208 +0,0 @@ -// The ratchet itself: scores may only rise, gating is earned, and a score that -// moved for tool/config reasons is never mistaken for test-strength change. -// -// Graduation (issue #1415): the lane starts non-gating. Each comparable weekly -// full sweep with no regression increments `stableRuns`; once it reaches -// `requiredStableRuns` the baseline flips to `gating: true`, after which a -// regression fails the run — weekly as a full sweep, PRs for AFFECTED modules -// only. A regression or a non-comparable run resets the counter to zero. - -import { roundScore, type ModuleScore, type SurvivingMutant } from './score.ts'; -import type { ModuleId } from './modules.ts'; - -export const BASELINE_SCHEMA_VERSION = 1; -export const DEFAULT_REQUIRED_STABLE_RUNS = 2; - -export type ModuleBaseline = { - score: number; - killed: number; - survived: number; - total: number; - // Provenance travels with every module baseline so a tool or config change is - // distinguishable from a test-strength change. - strykerVersion: string; - configHash: string; - updatedAt: string; -}; - -export type Baseline = { - schemaVersion: number; - requiredStableRuns: number; - stableRuns: number; - gating: boolean; - modules: Record; -}; - -export type Provenance = { readonly strykerVersion: string; readonly configHash: string }; - -export type VerdictStatus = 'new' | 'improved' | 'held' | 'regressed' | 'provenance-drift'; - -export type ModuleVerdict = { - readonly module: ModuleId; - readonly score: number; - readonly killed: number; - readonly total: number; - readonly baselineScore: number | undefined; - readonly delta: number | undefined; - readonly status: VerdictStatus; - readonly surviving: readonly SurvivingMutant[]; - readonly detail: string; -}; - -export type RatchetResult = { - readonly verdicts: readonly ModuleVerdict[]; - readonly regressions: readonly ModuleVerdict[]; - readonly drifted: readonly ModuleVerdict[]; - /** A run is comparable when every module has a same-provenance baseline. */ - readonly comparable: boolean; - readonly gating: boolean; - /** True when the caller must exit non-zero. */ - readonly failed: boolean; -}; - -export function emptyBaseline(requiredStableRuns: number = DEFAULT_REQUIRED_STABLE_RUNS): Baseline { - return { - schemaVersion: BASELINE_SCHEMA_VERSION, - requiredStableRuns, - stableRuns: 0, - gating: false, - modules: {}, - }; -} - -function driftDetail(previous: ModuleBaseline, provenance: Provenance): string { - return ( - `baseline recorded by stryker ${previous.strykerVersion} / config ${previous.configHash}, ` + - `this run used stryker ${provenance.strykerVersion} / config ${provenance.configHash}; ` + - 'the score change is not attributable to test strength — re-record with `pnpm mutation:baseline`' - ); -} - -function verdictFor( - score: ModuleScore, - previous: ModuleBaseline | undefined, - provenance: Provenance, -): ModuleVerdict { - const base = { - module: score.module, - score: score.score, - killed: score.killed, - total: score.total, - surviving: score.surviving, - }; - if (!previous) { - return { - ...base, - baselineScore: undefined, - delta: undefined, - status: 'new', - detail: `no baseline recorded yet (measured ${score.score}%)`, - }; - } - const delta = roundScore(score.score - previous.score); - if ( - previous.strykerVersion !== provenance.strykerVersion || - previous.configHash !== provenance.configHash - ) { - return { - ...base, - baselineScore: previous.score, - delta, - status: 'provenance-drift', - detail: driftDetail(previous, provenance), - }; - } - if (delta < 0) { - return { - ...base, - baselineScore: previous.score, - delta, - status: 'regressed', - detail: - `mutation score fell ${previous.score}% -> ${score.score}% ` + - `(${score.survived} surviving mutants)`, - }; - } - return { - ...base, - baselineScore: previous.score, - delta, - status: delta > 0 ? 'improved' : 'held', - detail: - delta > 0 - ? `mutation score rose ${previous.score}% -> ${score.score}%` - : `mutation score held at ${score.score}%`, - }; -} - -export function evaluateRatchet( - scores: readonly ModuleScore[], - baseline: Baseline, - provenance: Provenance, -): RatchetResult { - const verdicts = scores.map((score) => - verdictFor(score, baseline.modules[score.module], provenance), - ); - const regressions = verdicts.filter((verdict) => verdict.status === 'regressed'); - const drifted = verdicts.filter((verdict) => verdict.status === 'provenance-drift'); - const comparable = drifted.length === 0 && verdicts.every((v) => v.status !== 'new'); - return { - verdicts, - regressions, - drifted, - comparable, - gating: baseline.gating, - failed: baseline.gating && regressions.length > 0, - }; -} - -export type ApplyOptions = { - readonly provenance: Provenance; - readonly now: string; - /** Only the weekly full sweep may advance graduation; affected runs may not. */ - readonly countsTowardGraduation: boolean; -}; - -/** - * Fold a run into the baseline: keep the high-water score per module (a - * regression never rewrites it, so the ratchet keeps failing until the tests are - * restored), rebase modules whose provenance drifted onto the new tool/config, - * then advance or reset graduation. - */ -export function applyRun( - baseline: Baseline, - scores: readonly ModuleScore[], - result: RatchetResult, - options: ApplyOptions, -): Baseline { - const modules: Record = { ...baseline.modules }; - for (const score of scores) { - const previous = modules[score.module]; - const status = result.verdicts.find((verdict) => verdict.module === score.module)?.status; - const rebase = !previous || status === 'provenance-drift'; - if (!rebase && score.score < previous.score) continue; - modules[score.module] = { - score: rebase ? score.score : Math.max(previous.score, score.score), - killed: score.killed, - survived: score.survived, - total: score.total, - strykerVersion: options.provenance.strykerVersion, - configHash: options.provenance.configHash, - updatedAt: options.now, - }; - } - - const stable = result.comparable && result.regressions.length === 0; - const stableRuns = !options.countsTowardGraduation - ? baseline.stableRuns - : stable - ? baseline.stableRuns + 1 - : 0; - return { - ...baseline, - schemaVersion: BASELINE_SCHEMA_VERSION, - stableRuns, - gating: stableRuns >= baseline.requiredStableRuns, - modules, - }; -} diff --git a/scripts/mutation/report.test.ts b/scripts/mutation/report.test.ts new file mode 100644 index 0000000000..74202f95cb --- /dev/null +++ b/scripts/mutation/report.test.ts @@ -0,0 +1,80 @@ +// The report is the lane's whole product (#1457): it never gates, so a table +// that stops carrying the numbers is the lane failing silently rather than +// loudly. Both #1474 and #1475 were written from this table. +// +// Only the renderer is exercised here; every test that runs the CLI lives in +// envelope.test.ts, because a second file spawning `run.ts` would race it over +// the repo-relative lane envelope. + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { renderReport } from './report.ts'; +import { summarizeReport } from './score.ts'; + +const PROVENANCE = { strykerVersion: '9.6.1', configHash: 'sha256:abcdef123456' }; + +test('the table carries score, killed, survived, total and timeouts per kernel', () => { + const scores = summarizeReport( + { + files: { + 'packages/kernel/src/errors.ts': { + mutants: [ + { status: 'Killed' }, + { status: 'Timeout' }, + { + status: 'Survived', + mutatorName: 'ConditionalExpression', + location: { start: { line: 7 } }, + }, + { + status: 'NoCoverage', + mutatorName: 'BooleanLiteral', + location: { start: { line: 9 } }, + }, + ], + }, + }, + }, + ['kernel-errors'], + ); + const markdown = renderReport(scores, PROVENANCE); + // Timeouts count as killed, so a score propped up by slow mutants rather than + // assertions is only visible if the column is rendered. + assert.match(markdown, /\| `kernel-errors` — [^|]+\| 50% \| 2 \| 2 \| 4 \| 1 \|/); + // The surviving mutants are what a test-strengthening PR works from. + assert.match(markdown, /- `packages\/kernel\/src\/errors\.ts:7` ConditionalExpression/); + assert.match(markdown, /- `packages\/kernel\/src\/errors\.ts:9` BooleanLiteral/); +}); + +// The denominator is the report's arithmetic: counting tool-side noise would +// silently deflate every score the lane publishes. Ported from the deleted +// ratchet.test.ts, which was where this lived. +test('statuses outside the score (Ignored, CompileError, RuntimeError) leave the denominator', () => { + const [score] = summarizeReport( + { + files: { + 'packages/kernel/src/errors.ts': { + mutants: [ + { status: 'Killed' }, + { status: 'Ignored' }, + { status: 'CompileError' }, + { status: 'RuntimeError' }, + ], + }, + }, + }, + ['kernel-errors'], + ); + assert.equal(score?.total, 1); + assert.equal(score?.score, 100); +}); + +test('a kernel with no surviving mutants lists no detail section', () => { + const scores = summarizeReport( + { files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } } }, + ['kernel-errors'], + ); + const markdown = renderReport(scores, PROVENANCE); + assert.match(markdown, /\| `kernel-errors` — [^|]+\| 100% \| 1 \| 0 \| 1 \| 0 \|/); + assert.doesNotMatch(markdown, /^### /m); +}); diff --git a/scripts/mutation/report.ts b/scripts/mutation/report.ts index 31d761fcfa..7da43dc0ca 100644 --- a/scripts/mutation/report.ts +++ b/scripts/mutation/report.ts @@ -1,36 +1,36 @@ // Markdown rendering for the mutation lane: GitHub job summary and terminal // output share one renderer, so the artifact and the console never disagree. +// +// The lane reports and never gates (#1457), so the report is the whole product: +// a per-kernel score table plus the surviving mutants a test-strengthening PR +// would have to kill. import { moduleById } from './modules.ts'; -import type { Baseline, ModuleVerdict, Provenance, RatchetResult } from './ratchet.ts'; +import type { ModuleScore } from './score.ts'; const DEFAULT_MAX_SURVIVING_LISTED = 20; -function formatDelta(delta: number | undefined): string { - if (delta === undefined) return '—'; - return delta > 0 ? `+${delta}` : String(delta); -} +export type Provenance = { readonly strykerVersion: string; readonly configHash: string }; -function renderRow(verdict: ModuleVerdict): string { - const module = moduleById(verdict.module); - const baseline = verdict.baselineScore === undefined ? '—' : `${verdict.baselineScore}%`; +function renderRow(score: ModuleScore): string { + const module = moduleById(score.module); return ( - `| \`${verdict.module}\` — ${module.label} | ${verdict.score}% | ${baseline} | ` + - `${formatDelta(verdict.delta)} | ${verdict.killed}/${verdict.total} | ` + - `${verdict.surviving.length} | ${verdict.status} |` + `| \`${score.module}\` — ${module.label} | ${score.score}% | ${score.killed} | ` + + `${score.survived} | ${score.total} | ${score.timeout} |` ); } -function renderDetail(verdict: ModuleVerdict, maxListed: number): string[] { - const lines = ['', `### \`${verdict.module}\` — ${verdict.status}`, '', verdict.detail]; - if (verdict.surviving.length > 0) { - lines.push('', 'Surviving mutants:', ''); - for (const mutant of verdict.surviving.slice(0, maxListed)) { - lines.push(`- \`${mutant.file}:${mutant.line}\` ${mutant.mutator}`); - } - if (verdict.surviving.length > maxListed) { - lines.push(`- …and ${verdict.surviving.length - maxListed} more`); - } +function renderDetail(score: ModuleScore, maxListed: number): string[] { + const lines = [ + '', + `### \`${score.module}\` — ${score.survived} surviving mutant(s) at ${score.score}%`, + '', + ]; + for (const mutant of score.surviving.slice(0, maxListed)) { + lines.push(`- \`${mutant.file}:${mutant.line}\` ${mutant.mutator}`); + } + if (score.surviving.length > maxListed) { + lines.push(`- …and ${score.surviving.length - maxListed} more`); } return lines; } @@ -41,8 +41,7 @@ export type RenderOptions = { }; export function renderReport( - result: RatchetResult, - baseline: Baseline, + scores: readonly ModuleScore[], provenance: Provenance, options: RenderOptions = {}, ): string { @@ -51,31 +50,21 @@ export function renderReport( `## ${options.title ?? 'Mutation score — decision kernels'}`, '', `Stryker \`${provenance.strykerVersion}\` · config \`${provenance.configHash}\` · ` + - `gating **${baseline.gating ? 'on' : 'off'}** ` + - `(${baseline.stableRuns}/${baseline.requiredStableRuns} stable weekly runs)`, + 'report only — this lane never fails a build.', '', - '| Module | Score | Baseline | Δ | Killed/Total | Surviving | Status |', - '| --- | --- | --- | --- | --- | --- | --- |', - ...result.verdicts.map(renderRow), + '| Kernel | Score | Killed | Survived | Total | Timeout |', + '| --- | --- | --- | --- | --- | --- |', + ...scores.map(renderRow), ]; - for (const verdict of [...result.regressions, ...result.drifted]) { - lines.push(...renderDetail(verdict, maxListed)); + for (const score of scores) { + if (score.surviving.length > 0) lines.push(...renderDetail(score, maxListed)); } - if (result.failed) { - lines.push( - '', - 'Mutation ratchet failed: scores may only rise. Kill the surviving mutants listed above, ' + - 'then re-run `pnpm mutation:run` (full sweep) or `pnpm mutation:check --report ` ' + - 'against an existing Stryker report.', - ); - } else if (result.regressions.length > 0) { - lines.push( - '', - 'Scores regressed while the lane is still non-gating — no failure recorded, but the ' + - 'surviving mutants above are the tests to strengthen before gating turns on.', - ); - } + lines.push( + '', + 'A low score names tests worth strengthening (#1474, #1475 were written from this table); ' + + 'it is an input for a human-authored PR, not a verdict.', + ); return `${lines.join('\n')}\n`; } diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index db608ae0ed..a4888dfcb0 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -1,15 +1,15 @@ // Entrypoint for the decision-kernel mutation lane (issue #1415). // -// pnpm mutation:run full sweep + ratchet check -// pnpm mutation:baseline full sweep, then record the scores -// pnpm mutation:check --report ratchet an existing Stryker report +// pnpm mutation:run full sweep, rendered as a report +// pnpm mutation:check --report score an existing Stryker report // pnpm mutation:affected --base origin/main // PR lane: mutate only the kernel // modules the diff touches // -// The weekly workflow runs the full sweep and writes the rendered report to the -// job summary plus an artifact; the PR lane runs the affected subset and only -// fails once the baseline has graduated to `gating: true`. +// The lane reports and never gates (#1457): a score can only ever be printed, +// written to the job summary and uploaded as an artifact. The only non-zero exit +// is a harness failure — a missing report, an incomplete shard set, a bad +// argument — never a low score. import crypto from 'node:crypto'; import fs from 'node:fs'; @@ -29,15 +29,7 @@ import { type ShardSpec, } from './modules.ts'; import { derivedAffectedModules } from './ownership.ts'; -import { renderReport } from './report.ts'; -import { - applyRun, - emptyBaseline, - evaluateRatchet, - type Baseline, - type Provenance, - type RatchetResult, -} from './ratchet.ts'; +import { renderReport, type Provenance } from './report.ts'; import { mergeReports, summarizeReport, type ModuleScore, type StrykerReport } from './score.ts'; import { expandMutateFiles, @@ -49,7 +41,6 @@ import { const repoRoot = runCmdSync('git', ['rev-parse', '--show-toplevel']).stdout.trim(); const CONFIG_PATH = 'stryker.config.json'; -const BASELINE_PATH = 'mutation-baselines/decision-kernels.json'; const DEFAULT_REPORT_PATH = '.tmp/mutation/mutation.json'; const TEST_SCOPE_PATH = '.tmp/mutation/test-scope.json'; const ENVELOPE_PATH = '.tmp/mutation/lane-envelope.json'; @@ -66,12 +57,10 @@ const USAGE = `Usage: pnpm mutation:run [options] Fail unless --report-dir holds exactly n shard reports --shard Mutate only the i-th of n balanced slices of the module's sources (the big modules exceed one job's budget) - --update Record the run into the baseline (ratchet + graduation) --summary Also write the markdown report to --no-run Alias for --report with the default report path --list-affected Print the PR lane's shard matrix as JSON and exit (empty - until the baseline graduates, unless the diff touches the - lane's own tooling) + unless the diff touches the lane's own tooling) --fail-envelope Write a failed lane envelope for a step that ran before (or instead of) the sweep, e.g. a self-test failure @@ -83,7 +72,6 @@ type Args = { base: string; report: string | undefined; reportDir: string | undefined; - update: boolean; summary: string | undefined; listAffected: boolean; failEnvelope: string | undefined; @@ -127,7 +115,6 @@ function parseMutationArgs(argv: readonly string[]): Args { base: { type: 'string', default: 'origin/main' }, report: { type: 'string' }, 'report-dir': { type: 'string' }, - update: { type: 'boolean', default: false }, summary: { type: 'string' }, 'no-run': { type: 'boolean', default: false }, 'list-affected': { type: 'boolean', default: false }, @@ -141,7 +128,6 @@ function parseMutationArgs(argv: readonly string[]): Args { base: values.base ?? 'origin/main', report: values.report ?? (values['no-run'] ? DEFAULT_REPORT_PATH : undefined), reportDir: values['report-dir'], - update: Boolean(values.update), summary: values.summary, listAffected: Boolean(values['list-affected']), failEnvelope: values['fail-envelope'], @@ -166,18 +152,6 @@ function readProvenance(root: string = repoRoot): Provenance { }; } -function readBaseline(root: string = repoRoot): Baseline { - const file = path.join(root, BASELINE_PATH); - if (!fs.existsSync(file)) return emptyBaseline(); - return JSON.parse(fs.readFileSync(file, 'utf8')) as Baseline; -} - -function writeBaseline(baseline: Baseline, root: string = repoRoot): void { - const file = path.join(root, BASELINE_PATH); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, `${JSON.stringify(baseline, null, 2)}\n`); -} - function changedFiles(base: string): string[] { const result = runCmdSync('git', ['diff', '--name-only', '--merge-base', base, 'HEAD'], { cwd: repoRoot, @@ -242,8 +216,8 @@ async function runStryker( onStdoutChunk: (chunk) => void process.stdout.write(chunk), onStderrChunk: (chunk) => void process.stderr.write(chunk), }); - // Stryker exits non-zero on a low score too; the ratchet — not Stryker's own - // thresholds — owns the verdict, so only a missing report is fatal here. + // Stryker exits non-zero on a low score too; this lane reports scores rather + // than judging them, so only a missing report is fatal here. if (!fs.existsSync(absolute)) { throw new Error( `Stryker produced no report at ${reportPath} (exit ${result.exitCode}). See output above.`, @@ -267,7 +241,7 @@ function emit(markdown: string, summaryPath: string | undefined): void { * failure the freshness monitor must see, so the stage rides in the envelope * rather than only in the job log. */ -type Stage = 'setup' | 'select' | 'stryker' | 'report' | 'ratchet' | 'complete'; +type Stage = 'setup' | 'select' | 'stryker' | 'report' | 'score' | 'complete'; /** * Provenance for a lane that died before it could read any: `unknown` is a @@ -278,17 +252,15 @@ const UNKNOWN_PROVENANCE: Provenance = { strykerVersion: 'unknown', configHash: type LaneState = { stage: Stage; provenance: Provenance; - baseline: Baseline; modules: readonly ModuleId[]; affected: boolean; scores: readonly ModuleScore[]; - result: RatchetResult | undefined; error: string | undefined; /** * `--fail-envelope` keeps an existing *failure* (its reason is the specific - * one) but replaces an existing pass: a post-verdict step can fail after the - * ratchet passed, and publishing that job as passing is the bug the envelope - * exists to prevent. + * one) but replaces an existing pass: a step can fail after the report was + * rendered, and publishing that job as passing is the bug the envelope exists + * to prevent. */ recoveryOnly: boolean; }; @@ -303,6 +275,19 @@ function existingResult(file: string): string | undefined { } } +/** + * A module's measurement for the envelope, or explicit nulls when the lane died + * before measuring it — an absent field would read as "no mutants", a score. + */ +function envelopeModule(id: ModuleId, scores: readonly ModuleScore[]) { + const score = scores.find((entry) => entry.module === id); + if (!score) { + return { id, score: null, killed: null, survived: null, total: null, timeout: null }; + } + const { module: _module, surviving: _surviving, ...measured } = score; + return { id, ...measured }; +} + /** * Scheduled-lane artifact envelope (#1430). Without it a downloaded report cannot * say which commit or Stryker/config version produced it, how long the sweep took, @@ -329,24 +314,14 @@ function writeEnvelope(state: LaneState, startedAtMs: number): void { tool: { stryker: state.provenance.strykerVersion }, configHash: state.provenance.configHash, startedAtMs, - result: state.stage === 'complete' && !state.result?.failed ? 'pass' : 'fail', + // The lane never judges a score, so the result states whether the lane + // produced a report at all. + result: state.stage === 'complete' ? 'pass' : 'fail', data: { scope: state.affected ? 'affected' : 'full-sweep', stage: state.stage, error: state.error ?? null, - modules: state.modules.map((id) => { - const score = state.scores.find((entry) => entry.module === id); - return { - id, - score: score?.score ?? null, - killed: score?.killed ?? null, - total: score?.total ?? null, - status: state.result?.verdicts.find((verdict) => verdict.module === id)?.status ?? null, - }; - }), - gating: state.baseline.gating, - stableRuns: state.baseline.stableRuns, - requiredStableRuns: state.baseline.requiredStableRuns, + modules: state.modules.map((id) => envelopeModule(id, state.scores)), }, }); fs.mkdirSync(path.dirname(target), { recursive: true }); @@ -362,7 +337,12 @@ function readReport(file: string): StrykerReport { return JSON.parse(fs.readFileSync(absolute, 'utf8')) as StrykerReport; } -function readShardedReports(dir: string, expected: number | undefined): StrykerReport { +/** + * Merges whatever shard reports are present and says how many there were. The + * expected-count verdict is the caller's, so a short set is still scored and + * published before it fails. + */ +function readShardedReports(dir: string): { report: StrykerReport; count: number } { const root = path.isAbsolute(dir) ? dir : path.join(repoRoot, dir); // Shard artifacts also carry the lane envelope and the derived test scope, so // the report is selected by name rather than by "every .json here". @@ -371,17 +351,8 @@ function readShardedReports(dir: string, expected: number | undefined): StrykerR .map((file) => path.join(root, file)) .sort(); if (files.length === 0) throw new Error(`No Stryker JSON reports found under ${dir}`); - // Sub-sharded modules make "every module has mutants" too weak on its own: the - // surviving slices would still cover the module, so the expected shard count is - // asserted as well. - if (expected !== undefined && files.length !== expected) { - throw new Error( - `Incomplete shard set from ${dir}: ${files.length} report(s), expected ${expected}. ` + - 'A shard job failed or its artifact is absent; the aggregate is not a sweep.', - ); - } process.stdout.write(`mutation: merging ${files.length} shard report(s) from ${dir}\n`); - return mergeReports(files.map(readReport)); + return { report: mergeReports(files.map(readReport)), count: files.length }; } async function produceReport( @@ -393,28 +364,23 @@ async function produceReport( return readReport(reportPath ?? DEFAULT_REPORT_PATH); } -function recordRun(args: Args, state: LaneState, result: RatchetResult): void { - const next = applyRun(state.baseline, state.scores, result, { - provenance: state.provenance, - now: new Date().toISOString(), - // Only the full sweep proves stability; an affected subset says nothing - // about the modules it skipped. - countsTowardGraduation: !args.affected && state.modules.length === ALL_MODULE_IDS.length, - }); - writeBaseline(next); - process.stdout.write( - `\nBaseline updated (${BASELINE_PATH}): ${next.stableRuns}/${next.requiredStableRuns} ` + - `stable runs, gating ${next.gating ? 'on' : 'off'}.\n`, - ); -} - /** - * A merged shard set must cover every requested module. `summarizeReport` scores - * a module with no mutants as 0, and while the lane is non-gating a 0 is only - * *reported* as a regression — so a matrix shard that died would otherwise be - * aggregated into a `complete`/`pass` envelope claiming the sweep happened. + * A merged shard set is only a sweep if every expected shard reported and every + * requested module has mutants — sub-sharded modules make the second check too + * weak alone, since a surviving slice still covers its module. `summarizeReport` + * scores an absent module as 0, so without this a dead shard would be published + * as a `complete`/`pass` envelope claiming the sweep happened. + * + * Both verdicts run after the table is published: the kernels that did complete + * are the part of the run still worth reading on a failed-shard day. */ -function assertShardsCoverModules(state: LaneState, dir: string): void { +function assertCompleteSweep(state: LaneState, dir: string, shards: number, args: Args): void { + if (args.expectShards !== undefined && shards !== args.expectShards) { + throw new Error( + `Incomplete shard set from ${dir}: ${shards} report(s), expected ${args.expectShards}. ` + + 'A shard job failed or its artifact is absent; the aggregate is not a sweep.', + ); + } const missing = state.scores.filter((score) => score.total === 0).map((score) => score.module); if (missing.length === 0) return; throw new Error( @@ -423,24 +389,24 @@ function assertShardsCoverModules(state: LaneState, dir: string): void { ); } -/** Reads shard reports, an existing report, or runs Stryker — and scores them. */ -async function scoreModules(args: Args, state: LaneState): Promise { +/** + * Reads shard reports, an existing report, or runs Stryker — and scores them. + * Returns the shard count when the source was a shard directory. + */ +async function scoreModules(args: Args, state: LaneState): Promise { state.stage = args.reportDir || args.report ? 'report' : 'stryker'; - const report = args.reportDir - ? readShardedReports(args.reportDir, args.expectShards) - : await produceReport(state.modules, args.report, args.shard); + const shards = args.reportDir ? readShardedReports(args.reportDir) : undefined; + const report = shards?.report ?? (await produceReport(state.modules, args.report, args.shard)); - state.stage = 'ratchet'; + state.stage = 'score'; state.scores = summarizeReport(report, state.modules); - if (args.reportDir) assertShardsCoverModules(state, args.reportDir); + return shards?.count; } async function sweep(args: Args, state: LaneState): Promise { state.stage = 'select'; - // Test attribution is derived from the import graph, not a listed set of test - // files: see scripts/mutation/ownership.ts. - // Same selection the PR matrix uses, graduation rule included, so the ratchet - // job can never run mutants the `select` job decided not to spend. + // Same selection the PR matrix uses, so the reporting job can never score + // mutants the `select` job decided not to spend. if (args.affected) { state.modules = [...new Set(affectedMatrix(args.base).map((entry) => entry.module))]; } @@ -450,49 +416,52 @@ async function sweep(args: Args, state: LaneState): Promise { return 0; } - await scoreModules(args, state); - const result = evaluateRatchet(state.scores, state.baseline, state.provenance); - state.result = result; + const shards = await scoreModules(args, state); const title = args.affected ? 'Mutation score — affected decision kernels' : 'Mutation score — decision kernels'; - emit(renderReport(result, state.baseline, state.provenance, { title }), args.summary); + emit(renderReport(state.scores, state.provenance, { title }), args.summary); - if (args.update) recordRun(args, state, result); + if (args.reportDir) assertCompleteSweep(state, args.reportDir, shards ?? 0, args); state.stage = 'complete'; - return result.failed ? 1 : 0; + return 0; } -/** Sources of the lane itself: a change here must prove itself on real mutants. */ -const LANE_TOOLING = ['scripts/mutation/', 'scripts/lib/', 'stryker.config.json', 'mutation-']; +/** + * Sources of the lane itself: a change here must prove itself on real mutants. + * These are also the only paths that can produce a non-empty matrix, so + * `mutation-affected.yml` triggers on exactly them — asserted by + * `workflow.test.ts`, since a filter that missed one would let a harness change + * merge unproven, and any wider filter only buys a no-op job. + */ +export const LANE_TOOLING = ['scripts/mutation/', 'scripts/lib/', 'stryker.config.json']; /** - * The PR lane's matrix. Before graduation the affected run is a report nobody - * acts on, so it costs runner minutes for no verdict: it stays empty until the - * baseline reaches `gating: true`. The exception is a diff that changes the lane - * itself — that is the one case where the pre-graduation run buys something, - * because the gate has to be proven before it can bite. + * The PR lane's matrix: empty unless the diff touches the harness, otherwise the + * canary plus whatever kernels that same diff derives. The kernel report that + * pays is the weekly sweep, so the PR lane spends mutants on one thing only — + * proving the harness still runs end to end when the harness changes. Selecting + * on derived kernel ownership alone would run the full ten-shard sweep on 24 of + * the last 40 merged PRs, a per-PR full sweep for a report nobody gates on. */ export function affectedMatrixFor( changed: readonly string[], - gating: boolean, root: string = repoRoot, ): ShardSpec[] { const touchesLane = changed.some((file) => LANE_TOOLING.some((prefix) => normalizePath(file).startsWith(prefix)), ); - if (!gating && !touchesLane) return []; + if (!touchesLane) return []; const modules = new Set(derivedAffectedModules(changed, root)); // Lane sources own no kernel, so a tooling-only diff derives nothing: without - // the canary the "prove the gate" exception would select zero mutants and - // prove nothing. - if (touchesLane) modules.add(LANE_CANARY); + // the canary the lane would select zero mutants and prove nothing. + modules.add(LANE_CANARY); return shardMatrix(ALL_MODULE_IDS.filter((id) => modules.has(id))); } function affectedMatrix(base: string): ShardSpec[] { - return affectedMatrixFor(changedFiles(base), readBaseline().gating); + return affectedMatrixFor(changedFiles(base)); } /** @@ -504,7 +473,6 @@ async function run(argv: readonly string[], state: LaneState): Promise { const args = parseMutationArgs(argv); state.affected = args.affected; state.provenance = readProvenance(); - state.baseline = readBaseline(); state.modules = args.modules; if (args.failEnvelope) { // A step that ran before the sweep failed (the weekly self-test, setup): the @@ -528,11 +496,9 @@ async function main(argv = process.argv.slice(2)): Promise { const state: LaneState = { stage: 'setup', provenance: UNKNOWN_PROVENANCE, - baseline: emptyBaseline(), modules: [], affected: false, scores: [], - result: undefined, error: undefined, recoveryOnly: false, }; diff --git a/scripts/mutation/score.ts b/scripts/mutation/score.ts index 514f5ee4b7..57bb7efbbe 100644 --- a/scripts/mutation/score.ts +++ b/scripts/mutation/score.ts @@ -25,6 +25,9 @@ export type ModuleScore = { readonly killed: number; readonly survived: number; readonly total: number; + /** Timeouts, already counted in `killed` — reported so a score propped up by + * slow mutants rather than assertions is visible. */ + readonly timeout: number; readonly surviving: readonly SurvivingMutant[]; }; @@ -37,8 +40,8 @@ const SURVIVED_STATUSES = new Set(['Survived', 'NoCoverage']); /** * Merge sharded Stryker reports into one. The weekly sweep runs one shard per - * kernel module so no single job approaches its time budget; the ratchet still - * evaluates a single full-sweep report. + * kernel module so no single job approaches its time budget; the report is still + * rendered from a single full-sweep view. */ export function mergeReports(reports: readonly StrykerReport[]): StrykerReport { const files: Record = {}; @@ -52,7 +55,7 @@ export function mergeReports(reports: readonly StrykerReport[]): StrykerReport { return { files }; } -export function roundScore(value: number): number { +function roundScore(value: number): number { return Math.round(value * 100) / 100; } @@ -60,33 +63,35 @@ function compareMutants(a: SurvivingMutant, b: SurvivingMutant): number { return a.file.localeCompare(b.file) || a.line - b.line || a.mutator.localeCompare(b.mutator); } +type Bucket = { killed: number; survived: number; timeout: number; surviving: SurvivingMutant[] }; + +function tally(bucket: Bucket, file: string, mutant: StrykerMutant): void { + if (KILLED_STATUSES.has(mutant.status)) { + bucket.killed += 1; + if (mutant.status === 'Timeout') bucket.timeout += 1; + return; + } + if (!SURVIVED_STATUSES.has(mutant.status)) return; + bucket.survived += 1; + bucket.surviving.push({ + file: normalizePath(file), + line: mutant.location?.start?.line ?? 0, + mutator: mutant.mutatorName ?? 'unknown', + }); +} + export function summarizeReport( report: StrykerReport, ids: readonly ModuleId[] = ALL_MODULE_IDS, ): ModuleScore[] { - const buckets = new Map< - ModuleId, - { killed: number; survived: number; surviving: SurvivingMutant[] } - >(); - for (const id of ids) buckets.set(id, { killed: 0, survived: 0, surviving: [] }); + const buckets = new Map(); + for (const id of ids) buckets.set(id, { killed: 0, survived: 0, timeout: 0, surviving: [] }); for (const [file, entry] of Object.entries(report.files)) { const id = moduleForFile(file); - if (!id) continue; - const bucket = buckets.get(id); + const bucket = id ? buckets.get(id) : undefined; if (!bucket) continue; - for (const mutant of entry.mutants) { - if (KILLED_STATUSES.has(mutant.status)) { - bucket.killed += 1; - } else if (SURVIVED_STATUSES.has(mutant.status)) { - bucket.survived += 1; - bucket.surviving.push({ - file: normalizePath(file), - line: mutant.location?.start?.line ?? 0, - mutator: mutant.mutatorName ?? 'unknown', - }); - } - } + for (const mutant of entry.mutants) tally(bucket, file, mutant); } return [...buckets].map(([module, bucket]) => { @@ -97,6 +102,7 @@ export function summarizeReport( killed: bucket.killed, survived: bucket.survived, total, + timeout: bucket.timeout, surviving: bucket.surviving.sort(compareMutants), }; }); diff --git a/scripts/mutation/selection.test.ts b/scripts/mutation/selection.test.ts index d6546f0082..137b102528 100644 --- a/scripts/mutation/selection.test.ts +++ b/scripts/mutation/selection.test.ts @@ -1,7 +1,7 @@ // Selection is what the PR lane spends money on, so both halves of the rule are -// asserted end to end through the real CLI: nothing runs before graduation, and -// the "prove the gate" exception for a lane-tooling diff selects real mutants -// rather than an empty matrix that proves nothing. +// asserted end to end through the real CLI: a kernel diff selects nothing (the +// weekly sweep is the kernel report), and a lane-tooling diff selects real +// mutants rather than an empty matrix that proves nothing. import assert from 'node:assert/strict'; import fs from 'node:fs'; @@ -70,23 +70,21 @@ function listAffected(cwd: string): ShardSpec[] { return JSON.parse(result.stdout.trim().split('\n').at(-1)!) as ShardSpec[]; } -test('a lane-tooling diff selects real mutants even before graduation', () => { - const dir = worktreeWithCommit('tooling', ['scripts/mutation/ratchet.ts']); +test('a lane-tooling diff selects real mutants', () => { + const dir = worktreeWithCommit('tooling', ['scripts/mutation/run.ts']); // The lane's own sources own no kernel, so derivation alone yields nothing: - // without the canary this exception would run zero mutants. + // without the canary a harness change would run zero mutants. assert.deepEqual(listAffected(dir), shardMatrix([LANE_CANARY])); }); -test('a kernel diff selects nothing until the baseline graduates', () => { +// The weekly sweep is the kernel report; selecting on derived ownership would +// run the full ten-shard sweep on most PRs for a report nobody gates on. +test('a kernel diff selects nothing — only a harness diff spends mutants', () => { const dir = worktreeWithCommit('kernel', ['src/utils/scroll-edge-state.ts']); assert.deepEqual(listAffected(dir), []); - // …and the same diff selects that module once gating is on. - assert.deepEqual( - affectedMatrixFor(['src/utils/scroll-edge-state.ts'], true), - shardMatrix(['scroll-edge-state']), - ); + assert.deepEqual(affectedMatrixFor(['src/utils/scroll-edge-state.ts']), []); }); -test('a docs-only diff selects nothing even once gating is on', () => { - assert.deepEqual(affectedMatrixFor(['docs/agents/testing.md'], true), []); +test('a docs-only diff selects nothing', () => { + assert.deepEqual(affectedMatrixFor(['docs/agents/testing.md']), []); }); diff --git a/scripts/mutation/test-scope.ts b/scripts/mutation/test-scope.ts index a18e251804..caa87ccb38 100644 --- a/scripts/mutation/test-scope.ts +++ b/scripts/mutation/test-scope.ts @@ -21,7 +21,7 @@ // assert over the repo's own registries rather than any decision kernel // and own their CI job. // -// Nothing here weakens the ratchet: a mutant only an excluded test could kill +// Nothing here inflates the score: a mutant only an excluded test could kill // shows up as a survivor — visible work, never a silent pass. import fs from 'node:fs'; diff --git a/scripts/mutation/workflow.test.ts b/scripts/mutation/workflow.test.ts index 28b3d50e3d..0a0e10e268 100644 --- a/scripts/mutation/workflow.test.ts +++ b/scripts/mutation/workflow.test.ts @@ -1,13 +1,15 @@ -// The workflows' YAML cannot read the kernel registry, so these assertions keep -// the two in step: a module added to KERNEL_MODULES that no weekly shard runs -// would silently drop out of the sweep, and one no PR path filter selects would -// silently stop gating once the ratchet graduates. +// The workflows' YAML cannot read the kernel registry or the lane's own source +// list, so these assertions keep them in step: a module added to KERNEL_MODULES +// that no weekly shard runs would silently drop out of the sweep, and a PR path +// filter out of step with LANE_TOOLING either lets a harness change merge +// unproven or starts a job that selects nothing. import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'node:test'; -import { KERNEL_MODULES, shardMatrix } from './modules.ts'; +import { shardMatrix } from './modules.ts'; +import { LANE_TOOLING } from './run.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); @@ -33,15 +35,15 @@ test('the weekly sweep shards exactly the registry matrix', () => { ); }); -test('the weekly sweep merges the shards into one ratcheted verdict', () => { +test('the weekly sweep merges the shards into one score table', () => { const yaml = workflow('mutation-weekly.yml'); assert.match(yaml, /gate: mutation-check[\s\S]*--report-dir/); assert.match(yaml, /GITHUB_STEP_SUMMARY|\$GITHUB_STEP_SUMMARY/); - // A dead shard must not be merged into a verdict that looks like a sweep. + // A dead shard must not be merged into a table that looks like a sweep. assert.match( yaml, new RegExp(`--expect-shards\\s+${shardMatrix().length}\\b`), - 'the weekly ratchet does not require the full shard set', + 'the weekly report does not require the full shard set', ); }); @@ -55,39 +57,25 @@ test('no mutation shard is allowed to exceed the 30-minute budget', () => { } }); -test('every kernel path a PR can touch selects the affected mutation job', () => { +// Only a harness diff can produce a non-empty matrix, so the trigger is asserted +// in both directions against LANE_TOOLING: a missing path lets a harness change +// merge without ever running a mutant, and an extra one starts a select job that +// can only answer `[]`. +test('the affected lane triggers on exactly the lane sources that can select mutants', () => { // Quote style is the formatter's business (oxfmt formats the workflow tree), so // accept either spelling of the same scalar rather than pinning this gate to it. const paths = [ ...workflow('mutation-affected.yml').matchAll(/^ {6}- (?['"])(?[^'"]+)\k$/gm), ].map((match) => match.groups!.glob); - for (const module of KERNEL_MODULES) { - for (const owned of module.owns) { - const selected = paths.some( - (glob) => - glob === owned || - glob === `${owned}**` || - (glob.endsWith('/**') && owned.startsWith(glob.slice(0, -2))), - ); - assert.ok(selected, `no path filter selects ${owned} (module ${module.id})`); - } - } - // Ownership is derived, so any test in src/ or a workspace package's src/ - // can own a kernel; the filter must let all of them through and leave the - // decision to the `select` job. A narrower filter is exactly the omission - // the derivation exists to prevent. - assert.ok( - paths.includes('src/**/*.test.ts'), - 'the PR lane must trigger on every src test, since test ownership is derived', - ); - assert.ok( - paths.includes('packages/*/src/**/*.test.ts'), - 'the PR lane must trigger on every packages/*/src test too — target-annotation-serde is owned by one', + const expected = [ + ...LANE_TOOLING.map((prefix) => (prefix.endsWith('/') ? `${prefix}**` : prefix)), + // The workflow reruns itself so a trigger edit is proven by the lane it edits. + '.github/workflows/mutation-affected.yml', + ]; + assert.deepEqual( + [...paths].sort(), + [...expected].sort(), + 'the PR path filter drifted from LANE_TOOLING in scripts/mutation/run.ts', ); assert.match(workflow('mutation-affected.yml'), /gate: mutation-affected[\s\S]*--list-affected/); - // The lane's own sources fail open into it too: a ratchet or baseline edit must - // prove itself against real mutants, not against a stale report. - for (const own of ['scripts/mutation/**', 'stryker.config.json', 'mutation-baselines/**']) { - assert.ok(paths.includes(own), `missing path filter ${own}`); - } }); diff --git a/stryker.config.json b/stryker.config.json index 133cb0116a..1ab0498448 100644 --- a/stryker.config.json +++ b/stryker.config.json @@ -1,6 +1,6 @@ { "$schema": "./node_modules/@stryker-mutator/core/schema/stryker-schema.json", - "_comment": "Decision-kernel mutation lane (issue #1415). The mutate list below MUST mirror KERNEL_MODULES in scripts/mutation/modules.ts — scripts/mutation/config.test.ts asserts it. Ratchet verdicts come from scripts/mutation/run.ts, never from Stryker's own thresholds. Any edit here changes the config content hash, which invalidates every recorded baseline as non-comparable until re-recorded with `pnpm mutation:baseline`.", + "_comment": "Decision-kernel mutation lane (issue #1415). The mutate list below MUST mirror KERNEL_MODULES in scripts/mutation/modules.ts — scripts/mutation/config.test.ts asserts it. The lane reports scores and never gates on them (#1457), so no `break` threshold may be set here either. Any edit changes the config content hash the report and lane envelope carry, so scores measured before and after are not comparable.", "packageManager": "pnpm", "tsconfigFile_comment": "Points at a path that does not exist on purpose. Stryker's sandbox rewrites tsconfig `extends`/`references` through the classic TypeScript API, which the repo's TypeScript 7 native package no longer exposes (`ts.parseConfigFileTextToJson is not a function`). Our tsconfig has neither extends nor references, so there is nothing to rewrite — skipping the preprocessor is exact, not a workaround for a real dependency. The behaviour relied on is that the preprocessor no-ops when the file is absent: see packages/core/src/sandbox/ts-config-preprocessor.ts (https://github.com/stryker-mutator/stryker-js/blob/master/packages/core/src/sandbox/ts-config-preprocessor.ts) and https://stryker-mutator.io/docs/stryker-js/configuration/#tsconfigfile-string. Re-check it on every Stryker upgrade: if a future version errors on a missing tsconfig instead, point this at a real, extends-free tsconfig.", "tsconfigFile": "tsconfig.stryker-absent.json",