diff --git a/.changeset/gap-report-output-fixes.md b/.changeset/gap-report-output-fixes.md new file mode 100644 index 0000000..b9dfc21 --- /dev/null +++ b/.changeset/gap-report-output-fixes.md @@ -0,0 +1,10 @@ +--- +'@tanstack/oracles': patch +--- + +Fix four parts of the `oracle-gaps` output. + +- The baseline's replay command now runs the failing test. Before, its `-t` pattern was the test's `>`-joined name in JSON quotes: Vitest 4 matches `-t` against names joined by a space, so a test in a `describe` ran nothing, and a name with regular-expression syntax, `$`, or a backtick broke the pattern or the shell. The pattern is now the suites' and the test's names, escaped, joined to match Vitest 4 and Vitest 5, anchored, and in single quotes. A file that fails with no failing test (it did not load) replays whole, with no `-t`, where before it printed `-t undefined`. A control character in a test's name or failure, such as a newline, prints as its `\uXXXX` escape, in the baseline and in messages, so each stays one line. +- A gap line stays one line: the code of an operator mutant and the intent of an authored mutant have their whitespace collapsed and are cut at 60 characters with `...`, counted by code point so that a cut never splits an emoji. An authored intent prints as a code span. `results.json` keeps the full text. +- A `not-infected` message prints each operand as a code span of a typed value: strings quoted, `-0` as `-0`, bigints with `n`, and `undefined`, `null`, `NaN`, and a value that is not a primitive kept apart. A string longer than 40 characters (code points) prints its first 37 and a digest. Before, `1` and `"1"`, `0` and `-0`, and two long strings with one prefix printed alike. +- A `not-observed` directive also tells the author to add a history where the mutated code computes something different, because the run cannot tell a value nothing observed from a forced branch that computed the original's result. diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 616ebbc..8e5d1cf 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -282,9 +282,9 @@ function spin() { // operand that is not a primitive might run code when compared, so it counts // as a change without a second comparison. let infected = new Set() -// The first pair of operands each comparison saw in the running test, as -// text (no mutant active: the unmutated runs classify with them). An operand -// that is not a primitive is recorded as its type. +// The first pair of operands each comparison saw in the running test (no +// mutant active: the unmutated runs classify with them), each as `shown` +// records it. let operands = new Map() const backgroundInfected = new Set() // The outcomes of the `if`s that ran outside any test, such as at module @@ -294,8 +294,25 @@ const backgroundOutcomes = new Map() const infect = (id) => (inTest ? infected : backgroundInfected).add(id) const primitive = (value) => value === null || (typeof value !== `object` && typeof value !== `function`) -const shown = (value) => - primitive(value) ? String(value).slice(0, 40) : typeof value +// An operand as its kind and its text, which classify.mjs prints +// (`operandText`): `-0` for negative zero, a string past 40 characters as +// its first 37 and a digest of the whole, so two different values never +// record alike, and a value that is not a primitive as its type alone. +const shown = (value) => { + if (!primitive(value)) return [typeof value] + // Characters are code points, so a cut never splits an astral-plane one. + const chars = typeof value === `string` ? real.from(value) : undefined + if (chars !== undefined && chars.length > 40) + return [ + `string`, + chars.slice(0, 37).join(``), + createHash(`sha1`).update(value).digest(`hex`).slice(0, 8), + ] + return [ + value === null ? `null` : typeof value, + real.is(value, -0) ? `-0` : String(value), + ] +} // Temporary profiling (ORACLE_GAPS_PROFILE): how many fast-check histories // each test ran, and in how many of them each site was hit, to measure what diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index aa91ff7..20aa61f 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -43,7 +43,7 @@ import { renderReport, renderStdout, } from './report.mjs' -import { baselineLines, gapLine, progressLine } from './stream.mjs' +import { baselineLines, gapLine, progressLine, testNamesOf } from './stream.mjs' import { analyze, isRunnerCopy } from './analysis.mjs' import { Refusal } from './ending.mjs' import { @@ -637,6 +637,8 @@ export default async (env) => { return { key: `${module.moduleId} > ${testCase.fullName}`, fullName: testCase.fullName, + // The names a baseline's replay selects the test by (stream.mjs). + names: testNamesOf(testCase), state: testResult.state, // fast-check attaches the error its predicate threw as the cause. message: (testResult.errors ?? []) @@ -1345,9 +1347,7 @@ export default async (env) => { [...oracleFiles].map((file) => ({ file: relative(file), failure: first.failed.get(file), - fullName: first.units - .find((u) => u.file === file) - ?.tests.find((t) => t.state === `failed`)?.fullName, + tests: first.units.find((u) => u.file === file)?.tests ?? [], })), ) for (const line of baseline0) log(line) diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index beb4d83..4104ec1 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -8,6 +8,7 @@ // timed out, and its capture rows (capture-setup.mjs): the sites each test // reached, the outcomes each `if` took, the hashes of what it asserted on // and observed, its registered route, and its law verdicts. +import { codeSpan } from './stream.mjs' // Which kind of test produced a failure: the route the framework registered // for it through `__oracleRoute` (capture-setup.mjs), never its name. A kill @@ -228,14 +229,32 @@ function infectionOf(mutant, targets, baseline) { return { infected: infected.length > 0, infectedBy: test } } -// The operand pairs that the owner tests' comparison recorded for a mutant: -// distinct, in test order, up to three. A nondeterministic test's do not count. +/** + * An operand as the capture recorded it (capture-setup.mjs `shown`), as a + * typed text: a string quoted, with a digest after a long one's cut, a bigint + * with `n`, a number as JavaScript prints it but `-0`, and a value that is + * not a primitive as its type in parentheses. Two values the capture records + * differently never print alike. + */ +export function operandText([kind, text, digest]) { + if (text === undefined) return `(${kind})` + if (kind === `string`) + return digest === undefined + ? JSON.stringify(text) + : `${JSON.stringify(`${text}...`)}#${digest}` + return kind === `bigint` ? `${text}n` : text +} + +// The operand pairs that the owner tests' comparison recorded for a mutant, +// as code spans: distinct, in test order, up to three. A nondeterministic +// test's do not count. function operandsOf(mutant, targets, baseline) { const pairs = [] for (const k of targets.oracleKeys) if (!baseline.nondeterministic.has(k)) for (const [id, pair] of baseline.firstRows.get(k)?.operands ?? []) - if (id === mutant.number) pairs.push(pair.join(` and `)) + if (id === mutant.number) + pairs.push(pair.map((o) => codeSpan(operandText(o))).join(` and `)) return [...new Set(pairs)].slice(0, 3) } diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 83849d3..821f915 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -66,14 +66,48 @@ * 7. Every survivor's message is one directive from its class's fixed set, * built from the run's data (the test that infected it, the operands * seen, the outcome an `if` took). A message never offers an exit: - * no equivalence, approval, limit, or question. A mutant with the + * no equivalence, approval, limit, choice, or question. A mutant with the * `boundary: -0` hint adds "Add a boundary history with -0 and 0." to its * message. The summary's message is read as an author reads it. + * A `not-observed` survivor was infected, and no recorded observation + * changed. The records cannot tell a changed value that nothing observed + * from a forced branch that computed the same result as the original on + * the generated inputs (infected, not propagated: klona's M6), so its + * directive names both next steps: observe the value, and add a history + * where the mutated code computes something different (maintainer + * finding F22, 2026-11-10). Both are steps, so the message offers no + * choice. + * Each operand a `not-infected` message names is printed as a code span + * of a typed text: two operands that the model treats as different + * values never print the same (maintainer finding, 2026-11-10). The + * model's values are primitives by SameValue (so `-0` and `0`, `1` and + * `"1"`, `1n` and `1`, `undefined`, `null`, and `NaN` all differ, and two + * long strings that share a prefix differ), and a non-primitive operand + * as its type alone, because the capture records only its type. Each + * printed text is the model's own rendering of its operand: a string as + * JSON, past 40 code points its first 37, `...`, `#`, and eight hex digits + * of its SHA-1; a bigint with `n`; `-0`; another primitive as JavaScript + * prints it; a non-primitive as `(object)` or `(function)`. * 8. The compressed stdout summary (report.mjs `renderStdout`) lists the * totals, one legend line per class present, and every gap once, one line * each, grouped by function; it names the full report's path. The * streamed gap lines (stream.mjs `gapLine`), printed in the order the - * mutants are judged, are the same lines, each once. + * mutants are judged, are the same lines, each once. A gap line is one + * line however long its code or intent is: each of the mutant's original + * and replacement code, and an authored mutant's intent, is shown with its + * whitespace collapsed to single spaces and, past 60 characters (code + * points, so an astral-plane character is never split), cut to its first + * 57 and `...`. An intent is the author's text, so it prints as a code + * span, which the exit law does not read. The results keep the full text. + * 9. Before any mutant, each oracle file's baseline is printed: `pass`, or + * `fail` with the command that replays its first failing test. That + * command, read as a POSIX shell reads it, is `vitest run `, with + * a `-t` pattern when a test failed: Vitest then runs exactly that test + * of the file, whether it matches the pattern against the names of the + * test's suites and its own joined by a space (Vitest 4) or by ` > ` + * (Vitest 5). A file that failed with no failing test (it did not load) + * replays whole, with no `-t`. Every baseline line is one line, even when + * a test's name holds a newline or a tab. * 6. A survivor or an unreached mutant that ordinary tests reached gets * their outcome for information, where an assertion failure counts on the * test's route; it never changes the class. @@ -83,20 +117,42 @@ * key, with its kind and line, every unowned mutant as a warning, and counts * that agree. * + * Authored mutants (written by someone other than the oracle's author, + * authored.mjs) follow laws 1 to 4 over every test of their owner files, + * because their copy is not instrumented and records no reach: no owner is + * `unowned`, and a `find` that does not apply is `does-not-apply`. A + * survivor is `not-judged` when a deterministic owner test's observation + * changed, and `not-observed` otherwise. + * * Authority: the maintainer's gap-check decisions (design runs v11, v14, * and v19), as for oracle-gaps.report; ORC-006's outcome kinds; the frozen - * design's oracle over the checker's pure core (v19 P10, F4); and the - * maintainer's few-seconds budget for an incremental test run (2026-11-05). + * design's oracle over the checker's pure core (v19 P10, F4); the + * maintainer's few-seconds budget for an incremental test run (2026-11-05); + * the maintainer's findings on the report's output (F22 and the leftovers of + * #12, 2026-11-10); and, for law 9, Vitest's runner, which reads `-t` as a + * regular expression and matches it against `getTaskFullName`, the suites' + * names and the test's joined by a space (@vitest/runner 4.1). * - * Limits: authored mutants are not generated here (the end-to-end oracle - * has them). The records' shapes are those capture-setup.mjs writes; that a - * real run writes them correctly is the end-to-end oracle's. + * Limits: the records' shapes are those capture-setup.mjs writes; that a + * real run writes them correctly is the end-to-end oracle's. The driver + * encodes operands as the capture does (`shown`); that the capture's + * encoding is lossless is not judged here, and two long strings are told + * apart by a 32-bit digest. No symbol operand is generated: two symbols of + * one description print alike. Only the runner's single-file path + * (`unit()`) records a test's names, and only it feeds the baseline; units + * from the batch path carry none. Test names hold no backtick and no ` > `, the + * names within a file are distinct, and the replay's file argument is judged + * as text, not as Vitest's substring match over every file. An authored + * mutant's intent holds no word the exit law forbids, because the stdout + * prints it as text, and its author is not shortened. */ +import { createHash } from 'node:crypto' import fc from 'fast-check' import { declareOracle } from '../src/oracle-declaration.ts' import { oracleProperty, pass, violation } from '../src/oracle-property.ts' import { baselineOf, + judgeAuthored, judgeMutant, judgeOrdinary, needsOrdinary, @@ -106,18 +162,109 @@ import { readSummary } from './gap-report-model-oracle.mjs' import * as report from './report.mjs' const MUTANTS = 4 +const AUTHORED = 2 +// Each test's effect under each mutant: the operator mutants first, then +// the authored ones. +const SLOTS = MUTANTS + AUTHORED // ## History grammar // // A history is a package's test files, each an oracle file or an ordinary -// one, and up to four mutants of one source, each with its owner files, its -// line, operator, and statement anchor (with backticks, which the summary -// must fence). Each test has a registered route or none, may be -// nondeterministic, and has, for each mutant, whether it reached it, which -// outcomes an `if` there took, and what it did under it: nothing different, -// a changed asserted or observed value, a law verdict (on the route then active, or with none -// active), an assertion failure, a crash, a timeout message, a timed-out -// run, a setup error, or an unhandled error. +// one, up to four mutants of one source, each with its owner files, its +// line, operator, statement anchor (with backticks, which the summary must +// fence), and code, and up to two authored mutants, each with its owners, +// its intent and author, and whether its `find` applies. Each test has a +// registered route or none, may be nondeterministic, and has, for each +// mutant, whether it reached it, which outcomes an `if` there took, the +// operands a comparison there saw, and what it did under it: nothing +// different, a changed asserted or observed value, a law verdict (on the +// route then active, or with none active), an assertion failure, a crash, a +// timeout message, a timed-out run, a setup error, or an unhandled error. +// Each test also has a name and enclosing suites, whose names hold regular +// expression and shell syntax, and each oracle file a baseline: it passes, +// a test of it fails with no mutant active, or it does not load. +// +// The code and intents straddle the gap line's 60-character cut (exactly 60, +// 61, and longer), span lines, and hold backticks; one holds an astral-plane +// character (two UTF-16 units) where the cut falls, so a cut by UTF-16 units +// would split it. An intent holds words the exit law forbids, which the gap +// line must keep inside its code span. +const sixty = `value.total > limit.maximum && value.total !== limit.exact` +const codes = [ + [`x > 4`, `true`], + [`out.push('a')`, `;`], + [`${sixty}()`, `false`], + [`${sixty}(1)`, `false`], + [ + `items\n .filter((item) => item.ok)\n .map((item) => item.id)\n .join(',')`, + `undefined`, + ], + [`\`\${a}\` + b`, `\`\${a}\` - b`], + [`${`a`.repeat(56)}😀😀 && b.rest()`, `false`], +] +const intents = [ + `pushes a different label`, + `remembers removed ids it never held, and returns them after a reset of the cache`, + `skips the write\n on a retry`, + `pushes a label that is sixty characters long, as lines read.`, + `pushes a label that is sixty-one characters long, as lines do`, + `${`b`.repeat(56)}😀😀 pushes past the cut`, + `returns null or undefined, whichever comes first?`, +] +// The operands a comparison sees: small numbers, so that ties (which +// separate `>` from `>=`) happen, and typed values that a printer which +// drops the type, the sign of zero, or a long string's tail would conflate. +const longText = `${`the shared prefix of two long operands `.repeat(2)}` +const typedOperands = [ + -0, + `1`, + `0`, + ``, + `NaN`, + `undefined`, + `null`, + `1n`, + `a and b`, + `x; y`, + `or?`, + undefined, + null, + NaN, + 1n, + 0n, + true, + `true`, + `${longText}one`, + `${longText}two`, + // An astral-plane character where a 37-character cut falls. + `${`c`.repeat(36)}😀😀 and more text past the cut`, + {}, + [], +] +const operand = fc.oneof( + { weight: 3, arbitrary: fc.nat({ max: 2 }) }, + { weight: 2, arbitrary: fc.constantFrom(...typedOperands) }, +) +const suiteNames = [ + `t0`, + `s (1)`, + `a.b`, + `$x`, + `it's`, + `back\\slash`, + `two\nlines`, +] +const labels = [ + ``, + ``, + `(a|b) `, + `$HOME `, + `it's `, + `q"t `, + `[t0]+ `, + `^x$ `, + `tab\there `, +] const routes = [ undefined, `fixed seed`, @@ -158,13 +305,19 @@ const testRecord = fc.record({ minLength: MUTANTS, maxLength: MUTANTS, }), - under: fc.array(effect, { minLength: MUTANTS, maxLength: MUTANTS }), - // The two operands of each comparison the test reached. Small, so that - // ties (which separate `>` from `>=`) happen. - pair: fc.array(fc.tuple(fc.nat({ max: 2 }), fc.nat({ max: 2 })), { + under: fc.array(effect, { minLength: SLOTS, maxLength: SLOTS }), + // The two operands of each comparison the test reached. + pair: fc.array(fc.tuple(operand, operand), { minLength: MUTANTS, maxLength: MUTANTS, }), + // The test's enclosing suites and the label before its name, and whether + // it fails with no mutant active when its file's baseline is `test`. + suites: fc.array(fc.constantFrom(...suiteNames), { maxLength: 2 }), + label: fc.constantFrom(...labels), + // A tail after its index, so that one test's name can end with another's. + tail: fc.constantFrom(``, ``, ` t0`), + fails: fc.boolean(), }) const records = fc.record({ files: fc.array( @@ -182,6 +335,9 @@ const records = fc.record({ minLength: MUTANTS, maxLength: MUTANTS, }), + // The file with no mutant active: it passes, a test fails, or it does + // not load. + baseline: fc.constantFrom(`pass`, `pass`, `test`, `load`), }), { minLength: 1, maxLength: 3 }, ), @@ -201,6 +357,8 @@ const records = fc.record({ `field-drop`, ), original: fc.constantFrom(`>`, `>=`, `===`, `&&`, `||`, `??`), + // The code any other operator replaces, and its replacement. + code: fc.constantFrom(...codes), evidence: fc.constantFrom( undefined, undefined, @@ -218,10 +376,35 @@ const records = fc.record({ }), { minLength: 1, maxLength: MUTANTS }, ), + authored: fc.array( + fc.record({ + owners: fc.array(fc.nat({ max: 2 }), { maxLength: 2 }), + line: fc.nat({ max: 1 }), + applies: fc.constantFrom(true, true, true, false), + intent: fc.constantFrom(...intents), + author: fc.constantFrom(`agent a`, `the reviewer`), + }), + { maxLength: AUTHORED }, + ), }) const fileName = (index) => `/pkg/tests/f${index}.test.ts` -const keyOf = (fi, ti) => `${fileName(fi)} > t${ti}` +// A test's names, as the runner reports them: its suites' names, then its +// own. Its own name holds its index, so the names within a file are +// distinct. Its full name joins them with ` > `. +const namesOf = (h, fi, ti) => { + const t = h.files[fi].tests[ti] + return [...t.suites, `${t.label}t${ti}${t.tail}`] +} +const nameOf = (h, fi, ti) => namesOf(h, fi, ti).join(` > `) +const keyOf = (h, fi, ti) => `${fileName(fi)} > ${nameOf(h, fi, ti)}` +// A test's name as a message prints it: one line, each control character, +// line separator, or paragraph separator as its \\uXXXX escape. +const shownName = (h, fi, ti) => + nameOf(h, fi, ti).replace( + /[\u0000-\u001f\u007f\u2028\u2029]/g, + (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, `0`)}`, + ) // The framework's route vocabulary (oracle-route.ts in PR #1): three // reproducible routes, the random campaign, and one route for the property's // own checks. Any other name, the retired `constructed history`, @@ -240,43 +423,27 @@ const kindOf = (route) => ? `framework` : `unknown` +// The swap a comparison or logical mutant states (its replacement): the +// history's data, which the driver gives the mutant and the gap line shows. +const swapFor = { '>': `>=`, '>=': `>`, '===': `!==`, '&&': `||` } + // ## Model // // The model states the law above over the records, without the CLI's // functions: it reads each test's effect under the mutant directly. -function modelOutcome(h, mi) { - const m = h.mutants[mi] - const owners = new Set( - m.owners.filter((fi) => fi < h.files.length && h.files[fi].oracle), - ) - const reachingIn = (fi) => - h.files[fi].background[mi] || h.files[fi].tests.some((t) => t.reaches[mi]) - const ordinary = h.files - .map((f, fi) => fi) - .filter((fi) => !h.files[fi].oracle && reachingIn(fi)) - const withOrdinary = (result) => - (result.outcome === `survived` || result.outcome === `unreached`) && - ordinary.length > 0 - ? { ...result, testOutcome: ordinaryOutcome(h, mi, ordinary) } - : result - if (owners.size === 0) return { outcome: `unowned` } - const targets = [...owners].filter(reachingIn) - if (targets.length === 0) - return withOrdinary({ - outcome: `unreached`, - mention: [`line ${m.line + 1}`], - }) - const tests = targets.flatMap((fi) => - h.files[fi].tests.map((t, ti) => ({ t, fi, ti })), - ) - const under = ({ t }) => t.under[mi].kind + +// Laws 2 to 4 over the tests that judge a mutant, read from each test's +// effect in the mutant's slot: the failure's outcome, if a failure decides, +// and the tests that ran (a file that failed to load, or whose run timed +// out, ran none). +function failureOf(h, slot, tests) { + const under = ({ t }) => t.under[slot].kind const timedOut = tests.some((x) => under(x) === `runTimeout`) const setup = tests.some((x) => under(x) === `setupError`) - // A file that failed to load, or whose run timed out, ran no tests. const ran = tests.filter( (x) => !h.files[x.fi].tests.some((t) => - [`setupError`, `runTimeout`].includes(t.under[mi].kind), + [`setupError`, `runTimeout`].includes(t.under[slot].kind), ), ) const failing = ran.filter( @@ -289,41 +456,75 @@ function modelOutcome(h, mi) { .filter( (x) => under(x) === `verdict` && - kindOf(x.t.under[mi].firedAt ?? x.t.route) === kind, + kindOf(x.t.under[slot].firedAt ?? x.t.route) === kind, ) - .map((x) => x.t.under[mi].law) + .map((x) => x.t.under[slot].law) const kills = verdictsOn(`reproducible`) const behavioral = kills.filter((law) => law !== `reach agreement`) + const decided = (failure) => ({ failure, ran }) if (behavioral.length > 0) - return { + return decided({ outcome: `killed`, laws: [...new Set(behavioral)].sort(), lawsMayBePart: true, - } - if (timedOut) return { outcome: `timeout` } + }) + if (timedOut) return decided({ outcome: `timeout` }) if (kills.length > 0) - return { outcome: `agreement-only-kill`, laws: [`reach agreement`] } + return decided({ + outcome: `agreement-only-kill`, + laws: [`reach agreement`], + }) if (verdictsOn(`random`).length > 0) - return withOrdinary({ outcome: `random-only-kill` }) - if (verdictsOn(`unknown`).length > 0) return { outcome: `unregistered-kill` } + return decided({ outcome: `random-only-kill` }) + if (verdictsOn(`unknown`).length > 0) + return decided({ outcome: `unregistered-kill` }) if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return { outcome: `timeout` } + return decided({ outcome: `timeout` }) if ( failing.some( (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, ) ) - return { outcome: `crashed` } - if (failing.length > 0) return { outcome: `framework-error` } + return decided({ outcome: `crashed` }) + if (failing.length > 0) return decided({ outcome: `framework-error` }) const unhandled = ran.some((x) => under(x) === `unhandled`) - if (setup || unhandled) return { outcome: `setup-error` } + if (setup || unhandled) return decided({ outcome: `setup-error` }) + return decided(undefined) +} + +function modelOutcome(h, mi) { + const m = h.mutants[mi] + const owners = new Set( + m.owners.filter((fi) => fi < h.files.length && h.files[fi].oracle), + ) + const reachingIn = (fi) => + h.files[fi].background[mi] || h.files[fi].tests.some((t) => t.reaches[mi]) + const ordinary = h.files + .map((f, fi) => fi) + .filter((fi) => !h.files[fi].oracle && reachingIn(fi)) + const withOrdinary = (result) => + ordinary.length > 0 + ? { ...result, testOutcome: ordinaryOutcome(h, mi, ordinary) } + : result + if (owners.size === 0) return { outcome: `unowned` } + const targets = [...owners].filter(reachingIn) + if (targets.length === 0) + return withOrdinary({ + outcome: `unreached`, + mention: [`line ${m.line + 1}`], + }) + const tests = targets.flatMap((fi) => + h.files[fi].tests.map((t, ti) => ({ t, fi, ti })), + ) + const { failure, ran } = failureOf(h, mi, tests) + if (failure) return failure // A survivor. Only reaching, deterministic owner tests judge it; code that // ran when a file loaded the module reaches every test of that file. const compared = ran.filter( (x) => (x.t.reaches[mi] || h.files[x.fi].background[mi]) && !x.t.flaky, ) const changed = compared.find((x) => - [`changed`, `observedChanged`].includes(under(x)), + [`changed`, `observedChanged`].includes(x.t.under[mi].kind), ) const sized = (cls, extra = {}) => withOrdinary({ @@ -335,7 +536,9 @@ function modelOutcome(h, mi) { }) if (changed) return sized(`not-judged`, { - mention: [`Make a law in \`t${changed.ti}\` reject the changed value.`], + mention: [ + `Make a law in \`${shownName(h, changed.fi, changed.ti)}\` reject the changed value.`, + ], }) if (compared.length === 0) return sized(`indeterminate`) if (m.evidence === `dead code`) @@ -354,9 +557,7 @@ function modelOutcome(h, mi) { } if (isCompare(m)) { if (!infectedIn(h, mi, owners)) - return sized(`not-infected`, { - mention: [`Operands seen: ${operandsSeen(h, mi, owners)}.`], - }) + return sized(`not-infected`, { operands: operandsSeen(h, mi, owners) }) return sized(`not-observed`, { mention: [observedMention(h, mi, owners)] }) } // The other operators' infection is not measured, so the directive says @@ -366,6 +567,44 @@ function modelOutcome(h, mi) { }) } +// An authored mutant's outcome: laws 1 to 4 over every test of its owner +// files (its copy records no reach), and a survivor is not-judged when a +// deterministic owner test's observation changed, not-observed otherwise. +function modelAuthored(h, ai) { + const a = h.authored[ai] + const slot = MUTANTS + ai + if (!a.applies) return { outcome: `does-not-apply` } + const owners = [ + ...new Set( + a.owners.filter((fi) => fi < h.files.length && h.files[fi].oracle), + ), + ].sort((x, y) => x - y) + if (owners.length === 0) return { outcome: `unowned` } + const tests = owners.flatMap((fi) => + h.files[fi].tests.map((t, ti) => ({ t, fi, ti })), + ) + const { failure, ran } = failureOf(h, slot, tests) + if (failure) return failure + const changed = ran.find( + (x) => + !x.t.flaky && + [`changed`, `observedChanged`].includes(x.t.under[slot].kind), + ) + return changed + ? { + outcome: `survived`, + class: `not-judged`, + mention: [ + `Make a law in \`${shownName(h, changed.fi, changed.ti)}\` reject the changed value.`, + ], + } + : { + outcome: `survived`, + class: `not-observed`, + mention: [observedText(undefined)], + } +} + // The directive for an operator whose infection is not measured: the input // that makes the mutant's value differ, built from the operator's semantics. function distinguishingDirective(m) { @@ -383,16 +622,18 @@ function distinguishingDirective(m) { return `Add an owner history where one operand of \`${m.original}\` on line ${line} is falsy and the other is truthy, so \`&&\` and \`||\` give different results, and assert on the result.` } // The observation a survivor with an infection names: its infecting test, or -// the owner tests when only loaded code infected it. -const observedMention = (h, mi, owners) => { - const infecting = infectingTest(h, mi, owners) - return infecting === undefined - ? `Observe the value this code changes in the owner tests, and assert on it.` - : `Observe the value this code changes in \`${infecting}\`, and assert on it.` -} +// the owner tests when only loaded code infected it (or the mutant is +// authored). The second step covers a forced branch that computed the same +// result as the original (F22). +const observedText = (test) => + `Observe the value this code changes in ${test === undefined ? `the owner tests` : `\`${test}\``}, and assert on it. Where the mutated code computes the same result as the original, add a history where it computes something different.` +const observedMention = (h, mi, owners) => + observedText(infectingTest(h, mi, owners)) // Comparisons and their swaps, as the mutant states them. A comparison is -// infected when its operators disagree on the operands a test reached. +// infected when its operators disagree on the operands a test reached, or +// when an operand is not a primitive (comparing it might run its code, so +// the capture counts it as a change: capture-setup.mjs). const comparisons = new Set([`>`, `>=`, `===`]) const swapOf = { '>': `>=`, '>=': `>`, '===': `!==` } const holds = { @@ -401,6 +642,14 @@ const holds = { '===': (a, b) => a === b, '!==': (a, b) => a !== b, } +const isPrimitive = (value) => + value === null || (typeof value !== `object` && typeof value !== `function`) +// The model's values (law 7): a primitive by SameValue, a non-primitive as +// its type alone. +const sameValue = (a, b) => + isPrimitive(a) && isPrimitive(b) + ? Object.is(a, b) + : !isPrimitive(a) && !isPrimitive(b) && typeof a === typeof b const isIf = (m) => m.operator === `if-true` || m.operator === `if-false` const isCompare = (m) => m.operator === `binary-swap` && comparisons.has(m.original) @@ -414,6 +663,7 @@ function infects(h, mi, t) { if (m.operator === `if-false`) return t.seen[mi] !== `false` if (isCompare(m)) { const [a, b] = t.pair[mi] + if (!isPrimitive(a) || !isPrimitive(b)) return true return holds[m.original](a, b) !== holds[swapOf[m.original]](a, b) } return true @@ -424,7 +674,7 @@ function reachingTests(h, mi, owners) { .sort((a, b) => a - b) .flatMap((fi) => h.files[fi].tests.flatMap((t, ti) => - t.reaches[mi] && !t.flaky ? [{ t, ti }] : [], + t.reaches[mi] && !t.flaky ? [{ t, fi, ti }] : [], ), ) } @@ -440,18 +690,18 @@ function infectedIn(h, mi, owners) { // The first deterministic test that infected the mutant, if any test did. function infectingTest(h, mi, owners) { const hit = reachingTests(h, mi, owners).find(({ t }) => infects(h, mi, t)) - return hit ? `t${hit.ti}` : undefined + return hit ? shownName(h, hit.fi, hit.ti) : undefined } -// The operand pairs the owners' tests reached, distinct, in order, up to three. +// The operand pairs the owners' tests reached, distinct as values, in order, +// up to three. function operandsSeen(h, mi, owners) { - const pairs = [ - ...new Set( - reachingTests(h, mi, owners) - .filter(({ t }) => isCompare(h.mutants[mi])) - .map(({ t }) => t.pair[mi].join(` and `)), - ), - ].slice(0, 3) - return pairs.length > 0 ? pairs.join(`; `) : `none recorded` + const pairs = [] + for (const { t } of reachingTests(h, mi, owners)) { + const [a, b] = t.pair[mi] + if (!pairs.some(([x, y]) => sameValue(x, a) && sameValue(y, b))) + pairs.push([a, b]) + } + return pairs.slice(0, 3) } // The ordinary tests' outcome: their assertion failures count on their @@ -501,8 +751,58 @@ function ordinaryOutcome(h, mi, files) { return `survived` } +// A text as a gap line shows it (law 8): whitespace collapsed, and past 60 +// characters its first 57 and `...`. +// It counts characters by code point, so an astral-plane character is one +// character and is never split. +const shortened = (text) => { + const flat = [...String(text).replace(/\s+/g, ` `)] + return flat.length > 60 ? `${flat.slice(0, 57).join(``)}...` : flat.join(``) +} +// An operand's text as the model renders it (law 7): a string quoted as +// JSON, past 40 characters (code points) its first 37 and `...`, then `#` +// and the first eight hex digits of the SHA-1 of the whole; a bigint with +// `n`; negative zero as `-0`; any other primitive as JavaScript prints it; +// and a value that is not a primitive as its type in parentheses. +function operandOf(value) { + if (!isPrimitive(value)) return `(${typeof value})` + if (typeof value === `string`) { + const chars = [...value] + return chars.length > 40 + ? `${JSON.stringify(`${chars.slice(0, 37).join(``)}...`)}#${createHash(`sha1`).update(value).digest(`hex`).slice(0, 8)}` + : JSON.stringify(value) + } + if (typeof value === `bigint`) return `${value}n` + return Object.is(value, -0) ? `-0` : String(value) +} + +// Law 9: the tests of a file that `vitest run -t ` runs. +// Vitest reads the pattern as a regular expression and matches it against +// each test's names joined by a space (Vitest 4) or by ` > ` (Vitest 5). +const selectedBy = (h, fi, pattern, join) => + h.files[fi].tests.flatMap((t, ti) => + new RegExp(pattern).test(namesOf(h, fi, ti).join(join)) ? [ti] : [], + ) +// The tests of a file whose names, joined, are test ti's. +const sameNamed = (h, fi, ti, join) => + h.files[fi].tests.flatMap((_, tj) => + namesOf(h, fi, tj).join(join) === namesOf(h, fi, ti).join(join) ? [tj] : [], + ) +// Each oracle file's baseline: whether it fails, and the test its replay +// must run alone (undefined: the whole file, as for a file that did not +// load). A file whose baseline is `test` fails at its first failing test, +// and at its first test when the grammar marked none. +const baselineOf_ = (h) => + h.files.flatMap((f, fi) => { + if (!f.oracle) return [] + if (f.baseline === `pass`) return [{ fi, fails: false }] + if (f.baseline === `load`) return [{ fi, fails: true, test: undefined }] + const first = f.tests.findIndex((t) => t.fails) + return [{ fi, fails: true, test: first < 0 ? 0 : first }] + }) + function model(h) { - const outcomes = h.mutants.map((m, mi) => modelOutcome(h, mi)) + const operatorOutcomes = h.mutants.map((m, mi) => modelOutcome(h, mi)) const kindOfResult = (o) => o.class ?? o.outcome const keyOf = (m, mi) => { const base = `src/x.ts#f${m.line}:${m.operator}:${m.anchor}` @@ -516,42 +816,61 @@ function model(h) { ).length return before === 0 ? base : `${base}~${before + 1}` } - const listed = h.mutants.map((m, mi) => ({ - m, - key: keyOf(m, mi), - kind: kindOfResult(outcomes[mi]), - })) - const gapsOf = listed.filter( - (x) => !x.m.probe && ![`killed`, `unowned`].includes(x.kind), - ) // Each mutant's key, line, id, and function, for the messages and the - // stdout summary (the driver names mutant mi `m${mi}` in `f${line}`). - outcomes.forEach((o, mi) => { - o.key = listed[mi].key - o.line = h.mutants[mi].line + 1 + // stdout summary (the driver names mutant mi `m${mi}` in `f${line}`, and + // authored mutant ai `authored:a${ai}`), and the text its gap line shows. + operatorOutcomes.forEach((o, mi) => { + const m = h.mutants[mi] + o.key = keyOf(m, mi) + o.line = String(m.line + 1) o.id = `m${mi}` - o.function = `f${h.mutants[mi].line}` - o.probe = h.mutants[mi].probe + o.function = `f${m.line}` + o.probe = m.probe + o.code = + m.operator === `binary-swap` + ? [m.original, swapFor[m.original]] + : [...m.code] }) + const authoredOutcomes = h.authored.map((a, ai) => ({ + ...modelAuthored(h, ai), + key: `src/x.ts#authored:a${ai}`, + // A `find` that does not apply has no line: the gap line prints `?`, and + // the full report the absent value. + line: a.applies ? String(a.line + 1) : `?`, + entryLine: a.applies ? String(a.line + 1) : `undefined`, + id: `authored:a${ai}`, + probe: false, + authored: { intent: a.intent, author: a.author }, + })) + const outcomes = [...operatorOutcomes, ...authoredOutcomes] + const gapsOf = outcomes.filter( + (o) => !o.probe && ![`killed`, `unowned`].includes(kindOfResult(o)), + ) return { outcomes, summary: { counts: { - mutants: h.mutants.length, - authored: 0, - killed: listed.filter((x) => !x.m.probe && x.kind === `killed`).length, + mutants: outcomes.length, + authored: authoredOutcomes.length, + killed: outcomes.filter((o) => !o.probe && o.outcome === `killed`) + .length, gaps: gapsOf.length, - probe: listed.filter((x) => x.m.probe).length, + probe: outcomes.filter((o) => o.probe).length, }, - gaps: gapsOf.map((x) => x.key).sort(), - unowned: listed - .filter((x) => !x.m.probe && x.kind === `unowned`) - .map((x) => x.key) + gaps: gapsOf.map((o) => o.key).sort(), + unowned: outcomes + .filter((o) => !o.probe && o.outcome === `unowned`) + .map((o) => o.key) .sort(), entries: gapsOf - .map((x) => ({ key: x.key, kind: x.kind, line: String(x.m.line + 1) })) + .map((o) => ({ + key: o.key, + kind: kindOfResult(o), + line: o.entryLine ?? o.line, + })) .sort((a, b) => (a.key < b.key ? -1 : 1)), }, + baseline: baselineOf_(h), } } @@ -575,7 +894,7 @@ const directives = { 'not-infected': /^Add a history whose operands are equal, where `[^`]+` and `[^`]+` disagree\. Operands seen: .+\.$/, 'not-observed': - /^Observe the value this code changes in (`[^`]+`|the owner tests), and assert on it\.$/, + /^Observe the value this code changes in (`[^`]+`|the owner tests), and assert on it\. Where the mutated code computes the same result as the original, add a history where it computes something different\.$/, 'not-judged': /^Make a law in `[^`]+` reject the changed value\.$/, indeterminate: /^(Make `[^`]+` deterministic: control its clock and its random seed, so its failures judge this code|Add an owner test that runs line \d+ and asserts on its result)\.$/, @@ -599,6 +918,66 @@ const directives = { 'does-not-apply': /^Update the `find` text of this authored mutant so that it occurs exactly once in the source\.$/, } +// A text without its code spans, read as Markdown reads them (a run of +// backticks opens a span, and the next run of exactly that length closes +// it), so that the run data a span holds is not read as an exit. +function withoutSpans(text) { + let out = `` + let rest = text + for (;;) { + const at = rest.indexOf(`\``) + if (at < 0) return out + rest + out += rest.slice(0, at) + const span = readSpan(rest.slice(at)) + // A run of backticks that nothing closes is text. + if (!span) return out + rest.slice(at) + rest = rest.slice(at + span.length) + } +} +// The code span that opens a text: its content, with one space of padding +// on each side dropped, and its length; undefined when none opens it. +function readSpan(text) { + const fence = text.match(/^`+/)?.[0] + if (!fence) return undefined + const rest = text.slice(fence.length) + const close = [...rest.matchAll(/`+/g)].find( + (run) => run[0].length === fence.length, + ) + if (!close) return undefined + const content = rest.slice(0, close.index) + return { + content: + content.startsWith(` `) && content.endsWith(` `) && content.trim() + ? content.slice(1, -1) + : content, + length: fence.length * 2 + close.index, + } +} +// The operand pairs a not-infected message names, as printed texts: +// `Operands seen: ` then `none recorded`, or code-span pairs joined by +// ` and `, the pairs by `; `. Undefined when the text is not of that form. +function printedOperands(message) { + const at = message.indexOf(`Operands seen: `) + if (at < 0) return undefined + let rest = message + .slice(at + `Operands seen: `.length) + .replace(` ${boundaryLine}`, ``) + if (rest === `none recorded.`) return [] + const pairs = [] + for (;;) { + const a = readSpan(rest) + if (!a || !rest.slice(a.length).startsWith(` and `)) return undefined + rest = rest.slice(a.length + ` and `.length) + const b = readSpan(rest) + if (!b) return undefined + pairs.push([a.content, b.content]) + rest = rest.slice(b.length) + if (rest === `.`) return pairs + if (!rest.startsWith(`; `)) return undefined + rest = rest.slice(2) + } +} + // Each gap's message as the report's author reads it: the line after its key. function messagesOf(summary) { const lines = summary.split(`\n`) @@ -635,8 +1014,29 @@ const captured = { '>=': (a, b) => a >= b, } const capturedSwap = { '>': `>=`, '>=': `>`, '===': `!==` } +const primitive = (value) => + value === null || (typeof value !== `object` && typeof value !== `function`) +// An operand as the capture records it (capture-setup.mjs `shown`): a +// primitive as its kind and its text (`-0` for negative zero), a string past +// 40 characters as its first 37 and a digest of the whole, and any other +// value as its type alone. +function shownByCapture(value) { + if (!primitive(value)) return [typeof value] + const chars = typeof value === `string` ? [...value] : undefined + if (chars !== undefined && chars.length > 40) + return [ + `string`, + chars.slice(0, 37).join(``), + createHash(`sha1`).update(value).digest(`hex`).slice(0, 8), + ] + return [ + value === null ? `null` : typeof value, + Object.is(value, -0) ? `-0` : String(value), + ] +} // What the capture records for a reached test: the mutants whose value it -// changed (`infected`), and the operands of each reached comparison. +// changed (`infected`), and the operands of each reached comparison. An +// operand that is not a primitive counts as a change. function capturedRows(h, t) { const infected = [] const operands = [] @@ -649,9 +1049,13 @@ function capturedRows(h, t) { if (t.seen[mi] !== `false`) infected.push(mi) } else if (m.operator === `binary-swap` && capturedSwap[m.original]) { const [a, b] = t.pair[mi] - operands.push([mi, [a, b]]) + operands.push([mi, [shownByCapture(a), shownByCapture(b)]]) const swapped = capturedSwap[m.original] - if (captured[m.original](a, b) !== captured[swapped](a, b)) + if ( + !primitive(a) || + !primitive(b) || + captured[m.original](a, b) !== captured[swapped](a, b) + ) infected.push(mi) } else infected.push(mi) }) @@ -661,8 +1065,8 @@ function rowsOf(h, pass) { const rows = new Map() h.files.forEach((f, fi) => { f.tests.forEach((t, ti) => { - rows.set(keyOf(fi, ti), { - key: keyOf(fi, ti), + rows.set(keyOf(h, fi, ti), { + key: keyOf(h, fi, ti), hash: t.flaky && pass === 2 ? `h-again` : `h`, observed: `o`, hits: t.reaches.flatMap((r, mi) => @@ -691,9 +1095,10 @@ function rowsOf(h, pass) { }) return rows } -function unitOf(h, fi, mi) { +// A file's units under the mutant in effect slot `slot`. +function unitOf(h, fi, slot) { const f = h.files[fi] - const under = (t) => t.under[mi] + const under = (t) => t.under[slot] if (f.tests.some((t) => under(t).kind === `runTimeout`)) return { file: fileName(fi), timedOut: true, seconds: 1 } const loadFailed = f.tests.some((t) => under(t).kind === `setupError`) @@ -709,8 +1114,8 @@ function unitOf(h, fi, mi) { tests: tests.map((t, ti) => { const kind = under(t).kind return { - key: keyOf(fi, ti), - fullName: `t${ti}`, + key: keyOf(h, fi, ti), + fullName: nameOf(h, fi, ti), state: messages[kind] ? `failed` : `passed`, message: messages[kind] ?? ``, } @@ -718,9 +1123,9 @@ function unitOf(h, fi, mi) { rows: tests.map((t, ti) => { const kind = under(t).kind return [ - keyOf(fi, ti), + keyOf(h, fi, ti), { - key: keyOf(fi, ti), + key: keyOf(h, fi, ti), hash: kind === `changed` ? `h-changed` : `h`, observed: kind === `observedChanged` ? `o-changed` : `o`, hits: [], @@ -742,14 +1147,52 @@ function unitOf(h, fi, mi) { }), } } +// A test as Vitest's reporter gives it to the runner (TestCase): its name, +// and its parent suites up to its module. +function testCaseOf(h, fi, ti) { + const names = namesOf(h, fi, ti) + let parent = { type: `module`, moduleId: fileName(fi) } + for (const name of names.slice(0, -1)) + parent = { type: `suite`, name, parent } + return { type: `test`, name: names.at(-1), parent } +} +// Each oracle file's baseline, as the runner's first pass gives it to the +// printer: its relative path, its first failure, and its tests, each with +// its state and the names the runner read from its TestCase. +function baselineFiles(h, printer) { + return h.files.flatMap((f, fi) => { + if (!f.oracle) return [] + const file = `tests/f${fi}.test.ts` + if (f.baseline === `load`) + return [{ file, failure: `Error: cannot load the file`, tests: [] }] + // A file whose baseline is `test` fails its marked tests, or its first + // test when none is marked. + const marked = f.tests.some((t) => t.fails) + const fails = (t, ti) => + f.baseline === `test` && (marked ? t.fails : ti === 0) + const tests = f.tests.map((t, ti) => ({ + fullName: nameOf(h, fi, ti), + state: fails(t, ti) ? `failed` : `passed`, + names: printer?.testNamesOf?.(testCaseOf(h, fi, ti)), + })) + const first = tests.find((x) => x.state === `failed`) + return [ + { + file, + failure: first ? `${first.fullName}: Error: violation` : undefined, + tests, + }, + ] + }) +} async function driver(h, observe) { const oracleFiles = new Set( h.files.flatMap((f, fi) => (f.oracle ? [fileName(fi)] : [])), ) const baseline = baselineOf(rowsOf(h, 1), rowsOf(h, 2)) - // The swap a comparison or logical mutant states (its replacement). - const swapFor = { '>': `>=`, '>=': `>`, '===': `!==`, '&&': `||` } + // An operator mutant's change: a comparison or logical swap states its + // operator and the swap, any other operator the code it replaces. const mutants = h.mutants.map((m, mi) => ({ id: `m${mi}`, number: mi, @@ -760,7 +1203,7 @@ async function driver(h, observe) { anchor: m.anchor, ...(m.operator === `binary-swap` ? { original: m.original, replacement: swapFor[m.original] } - : {}), + : { original: m.code[0], replacement: m.code[1] }), ...(m.evidence ? { evidence: m.evidence } : {}), ...(m.probe ? { probeGuard: true } : {}), })) @@ -790,6 +1233,41 @@ async function driver(h, observe) { return result }) report.assignKeys(results) + // Authored mutants, as the runner judges them: one that does not apply + // and one with no owner are decided before any run; the rest run every + // test of each owner file, in the owners' order. + for (const [ai, a] of h.authored.entries()) { + const result = { + id: `authored:a${ai}`, + authored: true, + authoredId: `a${ai}`, + source: `src/x.ts`, + key: report.authoredKey(`src/x.ts`, `a${ai}`), + operator: `authored`, + function: `authored mutants`, + intent: a.intent, + author: a.author, + original: `out.push('a')`, + replacement: `out.push('b')`, + line: a.applies ? a.line + 1 : undefined, + } + results.push(result) + const owners = [ + ...new Set( + a.owners.filter((fi) => fi < h.files.length && h.files[fi].oracle), + ), + ].sort((x, y) => x - y) + if (!a.applies) result.outcome = `does-not-apply` + else if (owners.length === 0) result.outcome = `unowned` + else + Object.assign( + result, + judgeAuthored( + owners.map((fi) => unitOf(h, fi, MUTANTS + ai)), + baseline, + ), + ) + } const summary = report.renderReport({ results, run: { id: `r`, inputs: `records` }, @@ -818,15 +1296,95 @@ async function driver(h, observe) { ? [...report.gapsOf(results)].reverse().map((r) => printer.gapLine(r)) : undefined, killsSection: summary.slice(summary.indexOf(`## Kills by law`)), + // What results.json keeps of each mutant's change, after the printing. + kept: results.map((r) => + r.authored ? r.intent : [String(r.original), String(r.replacement)], + ), }) + // The baseline the printer shows before any mutant (law 9). + observe(`baseline`, printer?.baselineLines(baselineFiles(h, printer)) ?? []) for (const r of results) { - states.add(r.class ?? r.outcome) + states.add( + r.authored ? `authored ${r.class ?? r.outcome}` : (r.class ?? r.outcome), + ) if (r.testOutcome === `killed`) states.add(`ordinary killed`) if (r.class === `not-infected` && r.evidence) states.add(`not-infected with a boundary hint`) } if (h.mutants.some((m) => m.anchor.includes(`\``) && !m.probe)) states.add(`a key with backticks`) + // The gap line's cut (law 8): a gap whose code or intent is longer than + // 60 characters once collapsed, or spans lines. + for (const r of report.gapsOf(results)) { + const texts = r.authored ? [r.intent] : [r.original, r.replacement] + if (texts.some((text) => String(text).replace(/\s+/g, ` `).length > 60)) + states.add(`a gap line cut at 60 characters`) + if (texts.some((text) => /\n/.test(String(text)))) + states.add(`a gap line of code on several lines`) + // A cut by UTF-16 units would split the character at the cut. + const units = (text) => String(text).replace(/\s+/g, ` `) + if (texts.some((text) => /[\ud800-\udbff]/.test(units(text).slice(56, 57)))) + states.add(`a gap line cut at an astral character`) + if (r.authored && exits.test(r.intent)) + states.add(`an authored intent with words the exit law forbids`) + } + // Operands a printer that drops types would conflate (law 7): two + // distinct values with one String() text, or two long strings that share + // their first 40 characters, in one not-infected message. + for (const [mi, r] of results.entries()) + if (r.class === `not-infected`) { + const seen = h.files.flatMap((f) => + f.tests.flatMap((t) => (t.reaches[mi] ? t.pair[mi] : [])), + ) + const conflated = seen.some((a, i) => + seen.some( + (b, j) => + j > i && + primitive(a) && + primitive(b) && + !Object.is(a, b) && + String(a).slice(0, 40) === String(b).slice(0, 40), + ), + ) + if (conflated) states.add(`not-infected operands that String conflates`) + if ( + seen.some((v) => typeof v === `bigint` || v === null || v === undefined) + ) + states.add(`a not-infected message with typed operands`) + if ( + seen.some( + (v) => + typeof v === `string` && /[\ud800-\udbff]/.test(v.slice(36, 37)), + ) + ) + states.add(`a not-infected operand cut at an astral character`) + } + // Law 9: a failing test in a suite whose name holds regular-expression + // syntax, a test whose full name another test's contains, and a file + // that did not load. + for (const [fi, f] of h.files.entries()) { + if (!f.oracle || f.baseline === `pass`) continue + if (f.baseline === `load`) states.add(`a baseline file that did not load`) + else { + states.add(`a baseline failing test`) + const full = f.tests.map((_, ti) => namesOf(h, fi, ti).join(` `)) + if (full.some((x, i) => full.some((y, j) => j !== i && y.includes(x)))) + states.add(`a baseline test whose name another test's contains`) + if ( + f.tests.some( + (t) => + t.suites.length > 0 && /[.*+?^${}()|[\]\\]/.test(t.suites.join(``)), + ) + ) + states.add(`a baseline test in a suite named with regex syntax`) + if ( + f.tests.some((t, ti) => + /[\u0000-\u001f]/.test(namesOf(h, fi, ti).join(``)), + ) + ) + states.add(`a baseline test whose name holds a newline or a tab`) + } + } for (const state of states) observe(`reach`, state) } @@ -861,6 +1419,9 @@ function check(h, observations, expected) { const gapsOfModel = expected.outcomes.filter( (o) => !o.probe && !['killed', 'unowned'].includes(o.class ?? o.outcome), ) + // Each operand text the messages printed, with the model value it stood + // for (law 7). + const printedAs = new Map() for (const owed of gapsOfModel) { const kind = owed.class ?? owed.outcome const message = got.messages?.[owed.key] @@ -885,7 +1446,7 @@ function check(h, observations, expected) { `messages`, { actual: message, expected: directives[kind] }, ) - if (exits.test(message.replace(/`[^`]*`/g, ``))) + if (exits.test(withoutSpans(message))) return violation( `classified`, `gap ${owed.key} (${kind}) has an exit in its message: ${JSON.stringify(message)}`, @@ -898,6 +1459,37 @@ function check(h, observations, expected) { `gap ${owed.key} (${kind}) message ${JSON.stringify(message)} does not name ${JSON.stringify(text)}`, `messages`, ) + // A not-infected message names the operand pairs the model saw, in + // order, and no printed text stands for two values the model tells + // apart. + if (owed.operands) { + const pairs = printedOperands(message) + if (pairs?.length !== owed.operands.length) + return violation( + `classified`, + `gap ${owed.key} names ${pairs?.length ?? `no readable`} operand pairs, owed ${owed.operands.length}: ${JSON.stringify(message)}`, + `operands`, + ) + for (const [i, pair] of pairs.entries()) + for (const side of [0, 1]) { + const text = pair[side] + const value = owed.operands[i][side] + if (text !== operandOf(value)) + return violation( + `classified`, + `gap ${owed.key} prints the operand ${shownValue(value)} as ${JSON.stringify(text)}, owed ${JSON.stringify(operandOf(value))}`, + `operands`, + ) + const earlier = printedAs.get(text) + if (earlier !== undefined && !sameValue(earlier.value, value)) + return violation( + `classified`, + `the operand text ${JSON.stringify(text)} stands for two different values: ${earlier.shown} and ${shownValue(value)}`, + `operands`, + ) + printedAs.set(text, { value, shown: shownValue(value) }) + } + } } // The compressed stdout summary: the totals, one legend line per class // present, and every gap once, under its function. @@ -906,11 +1498,16 @@ function check(h, observations, expected) { return violation(`classified`, `no compressed stdout summary`, `stdout`) const stdoutGaps = stdout .split(`\n`) - .filter((line) => /^ \S+ L\d+ m\d+ /.test(line)) + .filter((line) => /^ \S+ L(\d+|\?) (m\d+|authored:a\d+) /.test(line)) const kinds = new Set(gapsOfModel.map((o) => o.class ?? o.outcome)) - const functions = new Set(gapsOfModel.map((o) => o.function)) + // Gaps are grouped by function, and the authored ones in one group. + const groupOf = (o) => + o.authored ? `authored mutants (src/x.ts)` : `fn ${o.function} (src/x.ts)` + const functions = new Set(gapsOfModel.map(groupOf)) const legend = stdout.split(`\n`).filter((line) => /^ [a-z-]+: /.test(line)) - const headings = stdout.split(`\n`).filter((line) => line.startsWith(`fn `)) + const headings = stdout + .split(`\n`) + .filter((line) => line.startsWith(`fn `) || line.startsWith(`authored `)) const stdoutProblem = !stdout.includes(`, ${gapsOfModel.length} gaps,`) ? `the totals do not state ${gapsOfModel.length} gaps` : !stdout.includes(`out/summary.md`) @@ -930,7 +1527,9 @@ function check(h, observations, expected) { ).length !== 1, ) ? `a gap is not printed once under its class, line, and id` - : exits.test(stdout.replace(/`[^`]*`/g, ``)) + : // A gap with no line prints `L?` (stream.mjs), a placeholder, + // not a question. + exits.test(withoutSpans(stdout).replace(/ L\? /g, ` L `)) ? `the stdout summary has an exit` : undefined if (stdoutProblem) @@ -942,12 +1541,12 @@ function check(h, observations, expected) { const unownedFns = new Map() for (const o of expected.outcomes) if (!o.probe && o.outcome === `unowned`) - unownedFns.set(o.function, (unownedFns.get(o.function) ?? 0) + 1) + unownedFns.set(groupOf(o), (unownedFns.get(groupOf(o)) ?? 0) + 1) const unownedLines = stdout .split(`\n`) - .filter((line) => line.startsWith(` unowned fn `)) - const unownedText = (fn, n) => - ` unowned fn ${fn} (src/x.ts): ${n} ${n === 1 ? `mutant` : `mutants`}` + .filter((line) => line.startsWith(` unowned `)) + const unownedText = (group, n) => + ` unowned ${group}: ${n} ${n === 1 ? `mutant` : `mutants`}` if ( unownedLines.length !== unownedFns.size || [...unownedFns].some( @@ -969,7 +1568,7 @@ function check(h, observations, expected) { `messages`, { actual: got.killsSection }, ) - if (exits.test(got.killsSection.replace(/`[^`]*`/g, ``))) + if (exits.test(withoutSpans(got.killsSection))) return violation( `classified`, `the kills-by-law section offers an exit`, @@ -988,6 +1587,48 @@ function check(h, observations, expected) { `stream`, { actual: streamed, expected: printed }, ) + // Law 8: each gap line is one line, and shows its change shortened: an + // operator mutant's original and replacement as two code spans, an + // authored mutant's intent as a code span and its author. The results keep the full text. + for (const owed of gapsOfModel) { + const head = `${owed.class ?? owed.outcome} L${owed.line} ${owed.id} ` + const line = streamed.find((l) => l.startsWith(head)) + const rest = line?.slice(head.length) + let shown + if (owed.authored) { + // The intent is the author's text, so it sits in a code span. + const intent = rest === undefined ? undefined : readSpan(rest) + shown = + intent?.content === shortened(owed.authored.intent) && + rest.slice(intent.length) === ` (authored by ${owed.authored.author})` + } else { + const a = rest === undefined ? undefined : readSpan(rest) + const arrow = a && rest.slice(a.length).startsWith(` → `) + const b = arrow ? readSpan(rest.slice(a.length + 3)) : undefined + shown = + b !== undefined && + a.length + 3 + b.length === rest.length && + a.content === shortened(owed.code[0]) && + b.content === shortened(owed.code[1]) + } + if (!shown || /\n/.test(line)) + return violation( + `classified`, + `the gap line of ${owed.id} is ${JSON.stringify(line)}, which does not show its change on one line, shortened to 60 characters`, + `gap line`, + { actual: line, expected: owed.authored ?? owed.code }, + ) + } + const owedKept = expected.outcomes.map((o) => + o.authored ? o.authored.intent : o.code.map(String), + ) + if (JSON.stringify(got.kept) !== JSON.stringify(owedKept)) + return violation( + `classified`, + `the results do not keep each mutant's full change`, + `gap line`, + { actual: got.kept, expected: owedKept }, + ) if (JSON.stringify(got.summary) !== JSON.stringify(expected.summary)) return violation( `classified`, @@ -995,12 +1636,138 @@ function check(h, observations, expected) { `report completeness`, { actual: got.summary, expected: expected.summary }, ) - return pass + return baselineBreach(h, observations, expected) ?? pass } +// Law 9: each oracle file's baseline line, and for a failing file the +// replay command, read as a POSIX shell reads it, which runs exactly the +// owed test of the file under either Vitest's join of its names. +function baselineBreach(h, observations, expected) { + const lines = observations.find((o) => o.checkpoint === `baseline`)?.value + if (!Array.isArray(lines)) + return violation(`baseline`, `no baseline was printed`, `baseline`) + const breach = (difference) => + violation(`baseline`, difference, `baseline`, { + actual: lines, + expected: expected.baseline, + }) + // Each printed line is one line, whatever a test's name holds. + const broken = lines.find((line) => /[\r\n\u2028\u2029]/.test(line)) + if (broken !== undefined) + return breach(`a baseline line spans lines: ${JSON.stringify(broken)}`) + for (const owed of expected.baseline) { + const file = `tests/f${owed.fi}.test.ts` + const at = lines.findIndex( + (line) => line === `pass ${file}` || line.startsWith(`fail ${file}: `), + ) + if (at < 0) return breach(`no baseline line for ${file}`) + if (!owed.fails) { + if (lines[at] !== `pass ${file}`) + return breach(`${file} passes, and its line is ${lines[at]}`) + continue + } + if (!lines[at].startsWith(`fail ${file}: `)) + return breach(`${file} fails, and its line is ${lines[at]}`) + const command = lines[at + 1]?.match(/^ replay: (.*)$/)?.[1] + const words = command === undefined ? undefined : shellWords(command) + if (!words) + return breach( + `the replay of ${file} is ${JSON.stringify(command)}, which a POSIX shell does not read as plain words`, + ) + const [vitest, run, target, flag, pattern, ...more] = words + if (vitest !== `vitest` || run !== `run` || target !== file) + return breach(`the replay of ${file} is not \`vitest run ${file}\``) + if (owed.test === undefined) { + if (flag !== undefined) + return breach( + `${file} did not load, and its replay selects tests: ${command}`, + ) + continue + } + if (flag !== `-t` || pattern === undefined || more.length > 0) + return breach( + `the replay of ${file} has no single -t pattern: ${command}`, + ) + for (const join of [` `, ` > `]) { + let selected + try { + selected = selectedBy(h, owed.fi, pattern, join) + } catch (error) { + return breach( + `the replay pattern ${pattern} is not a regular expression: ${error.message}`, + ) + } + // The owed test, and any test whose joined names equal its own, which + // no pattern can tell apart from it. + const owedSet = sameNamed(h, owed.fi, owed.test, join) + if (JSON.stringify(selected) !== JSON.stringify(owedSet)) + return breach( + `the replay pattern ${JSON.stringify(pattern)} of ${file} runs tests ${JSON.stringify(selected)} when Vitest joins names with ${JSON.stringify(join)}, owed ${JSON.stringify(owedSet)}`, + ) + } + } + return undefined +} + +// A command line as a POSIX shell splits it into words (POSIX.1-2017, Shell +// Command Language 2.2 and 2.6), when nothing in it expands, globs, or +// redirects: plain characters, single-quoted text, backslash escapes, and +// double-quoted text with neither `$` nor a backtick (whose meaning there +// can depend on what follows, unspecified for some). Undefined otherwise. +function shellWords(line) { + const words = [] + let word + for (let i = 0; i < line.length;) { + const c = line[i] + if (c === ` `) { + if (word !== undefined) words.push(word) + word = undefined + i += 1 + continue + } + word ??= `` + if (c === `'`) { + const end = line.indexOf(`'`, i + 1) + if (end < 0) return undefined + word += line.slice(i + 1, end) + i = end + 1 + } else if (c === `\\`) { + if (i + 1 >= line.length) return undefined + word += line[i + 1] + i += 2 + } else if (c === `"`) { + let j = i + 1 + for (; j < line.length && line[j] !== `"`; j += 1) { + if (line[j] === `$` || line[j] === `\``) return undefined + if (line[j] === `\\` && /[$`"\\]/.test(line[j + 1] ?? ``)) j += 1 + word += line[j] + } + if (j >= line.length) return undefined + i = j + 1 + } else if (/[\w./,:@%+=-]/.test(c)) { + word += c + i += 1 + } else return undefined + } + if (word !== undefined) words.push(word) + return words +} + +// A model value, as a violation names it. +const shownValue = (value) => + typeof value === `bigint` + ? `${value}n` + : Object.is(value, -0) + ? `-0` + : typeof value === `string` + ? JSON.stringify(value) + : isPrimitive(value) + ? String(value) + : `a value of type ${typeof value}` + declareOracle({ - check: [check], - model: [model, modelOutcome, ordinaryOutcome], + check: [check, baselineBreach], + model: [model, modelOutcome, modelAuthored, ordinaryOutcome], grammar: [records], }) @@ -1126,12 +1893,45 @@ const required = [ `dead-code`, `ordinary killed`, `a key with backticks`, + `authored not-observed`, + `authored not-judged`, + `authored killed`, + `authored does-not-apply`, + `authored unowned`, + `a gap line cut at 60 characters`, + `a gap line of code on several lines`, + `not-infected operands that String conflates`, + `a baseline failing test`, + `a baseline test in a suite named with regex syntax`, + `a baseline test whose name another test's contains`, + `a baseline file that did not load`, + `a gap line cut at an astral character`, + `an authored intent with words the exit law forbids`, + `a not-infected operand cut at an astral character`, + `a baseline test whose name holds a newline or a tab`, + `a not-infected message with typed operands`, ] // One constructed history per outcome class: one mutant judged by one // oracle file's test, with the effect that makes the class, and an ordinary // file whose assertion fails beside a survivor. const four = (value) => Array.from({ length: MUTANTS }, () => value) +const slots = (value) => Array.from({ length: SLOTS }, () => value) +// A test of the grammar: on a fixed seed, deterministic, reaching every +// mutant with no effect, at the top of its file, passing its baseline. +const plainTest = (effect = {}, test = {}) => ({ + route: `fixed seed`, + flaky: false, + reaches: four(true), + seen: four(`both`), + under: slots({ kind: `same`, law: `L`, firedAt: undefined, ...effect }), + pair: four([0, 1]), + suites: [], + label: ``, + tail: ``, + fails: false, + ...test, +}) const single = ( name, enters, @@ -1142,6 +1942,9 @@ const single = ( ordinary = false, background = four(false), backgroundInfects = four(false), + more = [], + baseline = `pass`, + authored = [], } = {}, ) => ({ name, @@ -1150,45 +1953,19 @@ const single = ( files: [ { oracle: true, - tests: [ - { - route: `fixed seed`, - flaky: false, - reaches: four(true), - seen: four(`both`), - pair: four([0, 1]), - under: four({ - kind: `same`, - law: `L`, - firedAt: undefined, - ...effect, - }), - ...test, - }, - ], + tests: [plainTest(effect, test), ...more], background, backgroundInfects, + baseline, }, ...(ordinary ? [ { oracle: false, - tests: [ - { - route: `fixed seed`, - flaky: false, - reaches: four(true), - seen: four(`both`), - pair: four([0, 1]), - under: four({ - kind: `assertion`, - law: `L`, - firedAt: undefined, - }), - }, - ], + tests: [plainTest({ kind: `assertion` })], background: four(false), backgroundInfects: four(false), + baseline: `pass`, }, ] : []), @@ -1199,12 +1976,21 @@ const single = ( line: 0, operator: `delete-call`, original: `>`, + code: codes[1], evidence: undefined, anchor: `out . push ( 'a' ) ;`, probe: false, ...mutant, }, ], + authored: authored.map((a) => ({ + owners: [0], + line: 0, + applies: true, + intent: intents[0], + author: `agent a`, + ...a, + })), }), }) const constructed = [ @@ -1244,7 +2030,7 @@ const constructed = [ tests: [ { ...base.files[0].tests[0], - under: four({ + under: slots({ kind: `runTimeout`, law: `L`, firedAt: undefined, @@ -1348,18 +2134,194 @@ const constructed = [ mutant: { anchor: `x \`\` y` }, effect: { kind: `changed` }, }), + // Authored mutants: judged by every test of their owners. + single(`an authored mutant no observation shows`, [`authored not-observed`], { + authored: [{ intent: intents[1] }], + }), + single(`an authored mutant that changes a value`, [`authored not-judged`], { + effect: { kind: `changed` }, + authored: [{}], + }), + single(`an authored mutant a verdict rejects`, [`authored killed`], { + effect: { kind: `verdict` }, + authored: [{}], + }), + single( + `an authored mutant that does not apply`, + [`authored does-not-apply`], + { + authored: [{ applies: false }], + }, + ), + single(`an authored mutant with no owner`, [`authored unowned`], { + authored: [{ owners: [] }], + }), + // Law 8: code past the cut and across lines, and an intent past it. + single( + `a gap whose code is long and spans lines`, + [`a gap line cut at 60 characters`, `a gap line of code on several lines`], + { mutant: { code: codes[4] } }, + ), + single(`a gap whose code is exactly 60 characters`, [`not-distinguished`], { + mutant: { code: codes[2] }, + }), + single( + `a gap whose code is 61 characters`, + [`a gap line cut at 60 characters`], + { + mutant: { code: codes[3] }, + }, + ), + // Law 7: operands whose String() texts agree, in one not-infected + // message: the string "0" and the number 0, and two long strings that + // share their first 40 characters. `>` and `>=` agree on each pair. + single( + `operands that String conflates`, + [`not-infected`, `not-infected operands that String conflates`], + { + mutant: { operator: `binary-swap`, original: `>` }, + test: { pair: four([`0`, 1]) }, + more: [ + plainTest({}, { pair: four([0, 1]) }), + plainTest({}, { pair: four([`${longText}one`, `true`]) }), + ], + }, + ), + single( + `negative zero and zero`, + [`not-infected operands that String conflates`], + { + mutant: { operator: `binary-swap`, original: `>` }, + test: { pair: four([-0, 1]) }, + more: [plainTest({}, { pair: four([0, 1]) })], + }, + ), + single( + `long operands that share a prefix`, + [`not-infected operands that String conflates`], + { + mutant: { operator: `binary-swap`, original: `>` }, + test: { pair: four([`${longText}one`, `true`]) }, + more: [plainTest({}, { pair: four([`${longText}two`, `true`]) })], + }, + ), + // Law 9: a failing test in a suite named with regular-expression syntax, + // beside a test whose full name contains the failing one's, and a file + // that does not load. + single( + `a failing baseline test in a suite with regex syntax`, + [ + `a baseline failing test`, + `a baseline test in a suite named with regex syntax`, + ], + { + baseline: `test`, + test: { suites: [`s (1)`], label: `(a|b) `, fails: true }, + more: [plainTest({}, { suites: [`$x`, `a.b`], label: `it's ` })], + }, + ), + single( + `a failing baseline test whose name another test's contains`, + [`a baseline test whose name another test's contains`], + { + baseline: `test`, + test: { suites: [`t0`], fails: true }, + more: [plainTest({}, { suites: [`t0`, `t0`] })], + }, + ), + // Another test's name ends with the failing test's. + single( + `a failing baseline test whose name ends another test's`, + [`a baseline test whose name another test's contains`], + { + baseline: `test`, + test: { fails: true }, + more: [plainTest({}, { tail: ` t0` })], + }, + ), + // The review round's states: a cut at an astral-plane character in code, + // an intent, and an operand; an intent with exit words; and test names + // with a newline and a tab. + single( + `a gap whose code is cut at an astral character`, + [`a gap line cut at an astral character`], + { mutant: { code: codes[6] } }, + ), + single( + `an authored intent cut at an astral character, beside exit words`, + [ + `a gap line cut at an astral character`, + `an authored intent with words the exit law forbids`, + ], + { + authored: [{ intent: intents[5] }, { intent: intents[6] }], + }, + ), + // Each operand kind a not-infected message can print, three to a message, + // each beside 2, where `>` and `>=` agree. + ...[ + [1n, -0, undefined], + [null, NaN, true], + ].map((values) => + single( + `operands of the kinds ${values.map(String).join(`, `)}`, + [`not-infected`, `a not-infected message with typed operands`], + { + mutant: { operator: `binary-swap`, original: `>` }, + test: { pair: four([values[0], 2]) }, + more: values.slice(1).map((v) => plainTest({}, { pair: four([v, 2]) })), + }, + ), + ), + single( + `an operand cut at an astral character`, + [`not-infected`, `a not-infected operand cut at an astral character`], + { + mutant: { operator: `binary-swap`, original: `>` }, + test: { pair: four([typedOperands[20], `true`]) }, + }, + ), + single( + `a failing baseline test whose names hold a newline and a tab`, + [ + `a baseline failing test`, + `a baseline test whose name holds a newline or a tab`, + ], + { + baseline: `test`, + test: { suites: [`two\nlines`], label: `tab\there `, fails: true }, + more: [plainTest({}, { suites: [`two\nlines`] })], + }, + ), + single( + `a baseline file that does not load`, + [`a baseline file that did not load`], + { + baseline: `load`, + }, + ), ] oracleProperty({ id: `oracle-gaps.report-classify`, law: { - statement: `From the records of the runs that judged it, each mutant gets the outcome the law assigns: kills only by law verdicts on reproducible routes, agreement-only, random-only, and unregistered kills, timeouts, crashes, framework and setup errors as gaps, survivors split into not-judged, indeterminate, dead-code, branch-never-taken, not-infected, and not-observed, and ordinary tests for information only. Every gap has one directive message from its kind's fixed set, with no exit. The summary lists every gap and unowned mutant once with consistent counts; the stdout summary and the streamed gap lines cover every gap once.`, - authority: `The maintainer's gap-check decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), as for oracle-gaps.report; the oracle rubric's outcome kinds (ORC-006); the frozen design's oracle over the checker's pure core (v19 P10, F4); the maintainer's few-seconds budget for an incremental test run (2026-11-05).`, - limits: `No authored mutants. That real runs produce these records is oracle-gaps.report's.`, + statement: `From the records of the runs that judged it, each mutant, operator or authored, gets the outcome the law assigns: kills only by law verdicts on reproducible routes, agreement-only, random-only, and unregistered kills, timeouts, crashes, framework and setup errors as gaps, survivors split into not-judged, indeterminate, dead-code, branch-never-taken, not-infected, not-observed, and not-distinguished, and ordinary tests for information only. Every gap has one directive message from its kind's fixed set, with no exit; a not-observed directive also directs a history where the mutated code computes something different, and a not-infected message never prints two different operand values alike. The summary lists every gap and unowned mutant once with consistent counts; the stdout summary and the streamed gap lines cover every gap once, each on one line with its code or intent cut at 60 characters, and the results keep the full text. Each failing oracle file's baseline replay command, read by a POSIX shell, runs exactly its first failing test under Vitest 4's and Vitest 5's name joins, and the whole file when no test failed.`, + authority: `The maintainer's gap-check decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), as for oracle-gaps.report; the oracle rubric's outcome kinds (ORC-006); the frozen design's oracle over the checker's pure core (v19 P10, F4); the maintainer's few-seconds budget for an incremental test run (2026-11-05); the maintainer's findings F22 and the leftovers of #12 (2026-11-10); Vitest's runner (@vitest/runner 4.1: -t is a regular expression matched against getTaskFullName, the suites' names and the test's joined by a space); POSIX.1-2017 Shell Command Language 2.2 and 2.6 for the replay's quoting.`, + limits: `That real runs produce these records is oracle-gaps.report's, and so is a real replay. The driver encodes operands as capture-setup.mjs \`shown\` does, by a copy of it; long strings are told apart by a 32-bit digest; no symbol operand is generated, and two symbols of one description print alike. Only the runner's single-file path records a test's names, and only it feeds the baseline. Test names hold no backtick and no \` > \`, and names within a file are distinct; the replay's file argument is judged as text, not as Vitest's substring filter over every file. Authored intents hold no word that the exit law forbids, and an author is not shortened.`, }, histories: { arbitrary: records, states: required, + // An authored mutant's other outcomes, measured. + measured: [ + `timeout`, + `agreement-only-kill`, + `random-only-kill`, + `unregistered-kill`, + `crashed`, + `framework-error`, + `setup-error`, + ].map((kind) => `authored ${kind}`), runs: 300, fixedSeeds: [20261105], constructed, diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 4fedf9d..2385123 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -2970,6 +2970,10 @@ function judge(plan, tests, sites, active) { * tagged `runtime`, so the static oracle, which runs only the analysis, does * not expect it. */ +// The suite of the failing owner test (`brokenOwner`): regular-expression +// syntax, a dollar sign, and a quote. +const BROKEN_SUITE = `broken (b.*) $x it's` + function refusal(plan) { // A Vitest that does not load, and a config the runner cannot honor, refuse // the run rather than leave every mutant unreached (maintainer decision, @@ -3009,11 +3013,13 @@ function refusal(plan) { // so the run refuses as for a failing owner, naming the setup's error. if (plan.fault === `setupThrows`) return { cause: `the fixture setup broke`, runtime: true } - // A failing owner is named with its test's error, so the author can fix it. + // A failing owner is named with its test's error, so the author can fix it, + // and its baseline's replay runs that test alone. if (plan.fault === `brokenOwner`) return { cause: `b.test.ts`, detail: `this owner fails with no mutant active`, + replayed: [`${BROKEN_SUITE} fixed seed 999`], runtime: true, } return undefined @@ -3420,12 +3426,20 @@ function oracleTestFile(plan, file) { `})`, ) }) + // The failing owner test sits in a suite whose name holds regular + // expression and shell syntax, beside a test whose name extends it and a + // top-level test of its own name, so the baseline's replay command must + // escape, anchor, and quote its pattern to run it alone (law 3). if (file === `b` && plan.fault === `brokenOwner`) out.push( - `test('fixed seed 999', () => {`, - ` g.__oracleRoute?.('fixed seed')`, - ` throw new Error('this owner fails with no mutant active')`, + `describe(${JSON.stringify(BROKEN_SUITE)}, () => {`, + ` test('fixed seed 999', () => {`, + ` g.__oracleRoute?.('fixed seed')`, + ` throw new Error('this owner fails with no mutant active')`, + ` })`, + ` test('fixed seed 999 again', () => {})`, `})`, + `test('fixed seed 999', () => {})`, ) // A file with no tests still loads, so the CLI can read what it owns. if (!plan.tests.some((t) => t.file === file)) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0272717..4abd09a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -66,7 +66,11 @@ * a gap. An owns entry that names nothing, an owns list that is not a * literal, and an owner whose tests fail with no mutant active each make * the run fail and name the cause, because each would otherwise hide gaps; - * a failing owner is named with a failing test and its error. The CLI's + * a failing owner is named with a failing test and its error, and the + * baseline's replay command, pasted into a POSIX shell in the package, + * runs that test alone, though its suite's name holds regular-expression + * and shell syntax and other tests' names extend it or repeat its own + * (the classify oracle's law 9 over generated names). The CLI's * capture must not change what a test observes: an owner that passes * alone passes under the CLI, even when it counts the reads of a getter * on a value it asserts on, or spies on the builtins a value is scanned @@ -472,6 +476,11 @@ function runCli(dir, files, how = {}) { env: cleanEnv(), }) const run = readRun(dir, out, child.status, child.stderr, how.cli ?? here) + // The command the baseline printed to replay a failing owner (law 3). + const replay = `${child.stderr ?? ``}` + .split(`\n`) + .find((line) => line.startsWith(` replay: `)) + ?.slice(` replay: `.length) // What the run wrote beside its output directory and outside the package's // own state (`.oracle-gaps`), as a path relative to the package. const strayWrites = filesUnder(dir).filter( @@ -480,7 +489,42 @@ function runCli(dir, files, how = {}) { !KNOWN_STRAY_WRITES.test(file) && !OWN_WRITES.some((home) => file === home || file.startsWith(`${home}/`)), ) - return { ...run, strayWrites } + return { ...run, strayWrites, replay } +} + +/** + * Runs a baseline's replay command as an author pastes it into a POSIX + * shell in the package, with this repository's Vitest on the path, and + * names the tests it ran (passed or failed) by their full names, the suites' + * names and the test's joined by a space. + */ +function replayed(dir, command) { + if (command === undefined) return undefined + const scratch = mkdtempSync(join(realpathSync(tmpdir()), `gap-replay-`)) + try { + const report = join(scratch, `report.json`) + spawnSync( + `sh`, + [`-c`, `${command} --reporter=json --outputFile='${report}'`], + { + cwd: dir, + encoding: `utf8`, + timeout: 120_000, + env: { + ...cleanEnv(), + PATH: `${join(here, `..`, `node_modules`, `.bin`)}:${process.env.PATH}`, + }, + }, + ) + if (!existsSync(report)) return [] + return JSON.parse(readFileSync(report, `utf8`)) + .testResults.flatMap((file) => file.assertionResults) + .filter((test) => test.status === `passed` || test.status === `failed`) + .map((test) => test.fullName) + .sort() + } finally { + rmSync(scratch, { recursive: true, force: true }) + } } /** Relative paths of the files under a directory, not inside node_modules. */ @@ -989,10 +1033,13 @@ async function driver(plan, observe) { budget: plan.budget, cli: plan.cliPath === `spaced` ? copy : here, } - const cli = runCli(dir, files, how) + const { replay, ...cli } = runCli(dir, files, how) // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) + // A failing owner's replay, run as the author would run it (law 3). + if (plan.fault === `brokenOwner`) + observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() // Each kind of declaration fault is its own state, so each refusal has @@ -1372,6 +1419,20 @@ function check(plan, observations, expected) { `the refusal does not give the failing test's error, ${expected.refused.detail}: ${cli.stderr}`, `ownership`, ) + // The baseline's replay of a failing owner runs exactly its failing test. + if (expected.refused.replayed) { + const replay = observations.find((o) => o.checkpoint === `replay`)?.value + if ( + JSON.stringify(replay?.ran) !== + JSON.stringify(expected.refused.replayed) + ) + return violation( + `replay`, + `the replay command ${JSON.stringify(replay?.command)} ran ${JSON.stringify(replay?.ran)}, owed only ${JSON.stringify(expected.refused.replayed)}`, + `ownership`, + { actual: replay?.ran, expected: expected.refused.replayed }, + ) + } // Law 12: the refusal is one block with its fix, and the output // directory the run was given holds it. // Every e2e run names its output directory (`--out`), so even a diff --git a/packages/oracles/oracle-gaps/report.mjs b/packages/oracles/oracle-gaps/report.mjs index 2e757dc..6b023bf 100644 --- a/packages/oracles/oracle-gaps/report.mjs +++ b/packages/oracles/oracle-gaps/report.mjs @@ -19,7 +19,7 @@ // innermost statement that holds the site, joined by single spaces and kept in // full, with `~n` for the nth repeat of one anchor in a function. An authored // mutant's key is its source and id (`source#authored:id`). -import { change, codeSpan, gapLine, kindOf } from './stream.mjs' +import { change, codeSpan, escapeControls, gapLine, kindOf } from './stream.mjs' /** Outcomes that are gaps. Only a law verdict on a reproducible route kills. */ export const gapOutcomes = new Set([ @@ -80,9 +80,11 @@ const BOUNDARY = `boundary: -0` const BOUNDARY_SENTENCE = `Add a boundary history with -0 and 0.` // The name of the test a key names: its path after the file. +// A test's name prints on one line: a control character as its escape. const testName = (key) => - key ? key.split(` > `).slice(1).join(` > `) : `(background)` -const first = (names) => names?.[0] ?? `(none)` + key ? escapeControls(key.split(` > `).slice(1).join(` > `)) : `(background)` +const first = (names) => + names?.[0] === undefined ? `(none)` : escapeControls(names[0]) // The error a crash threw, as one short line that fits a code span. const errorText = (text = `(no message)`) => text.replace(/`/g, `'`).split(`\n`)[0].slice(0, 120) @@ -96,10 +98,10 @@ const directive = { `Add an owner history where the condition on line ${g.line} is ${g.wanted}, so the branch it guards runs.`, 'not-infected': (g) => `Add a history whose operands are equal, where \`${g.original}\` and \`${g.replacement}\` disagree. Operands seen: ${g.operands?.length ? g.operands.join(`; `) : `none recorded`}.`, + // Infected, and no observation changed: a value nothing observed, or a + // forced branch that computed the same result as the original (F22). 'not-observed': (g) => - g.infectedBy === undefined - ? `Observe the value this code changes in the owner tests, and assert on it.` - : `Observe the value this code changes in \`${testName(g.infectedBy)}\`, and assert on it.`, + `Observe the value this code changes in ${g.infectedBy === undefined ? `the owner tests` : `\`${testName(g.infectedBy)}\``}, and assert on it. Where the mutated code computes the same result as the original, add a history where it computes something different.`, 'not-distinguished': (g) => distinguishingDirective(g), 'not-judged': (g) => `Make a law in \`${testName(g.changedTests?.[0])}\` reject the changed value.`, diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index d1bdf62..c1e1516 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -16,10 +16,23 @@ export function codeSpan(text) { return `${fence} ${text} ${fence}` } -/** What a mutant changes: its code, or the authored edit. */ +// A text as one short line: whitespace (newlines included) collapsed, and +// past 60 characters cut to its first 57 and `...`, so one gap stays one +// line however long its code or intent is. Characters are code points, so a +// cut never splits an astral-plane character. results.json keeps the full +// text. +const oneLine = (text) => { + const flat = [...String(text).replace(/\s+/g, ` `)] + return flat.length > 60 ? `${flat.slice(0, 57).join(``)}...` : flat.join(``) +} + +/** What a mutant changes: its code, or the authored edit, each on one line. */ export function change(result) { - if (result.authored) return `${result.intent} (authored by ${result.author})` - return `${codeSpan(String(result.original).replace(/\s+/g, ` `))} → ${codeSpan(String(result.replacement))}` + // An intent is the author's own text, so it prints as code, never as the + // report's words. + if (result.authored) + return `${codeSpan(oneLine(result.intent))} (authored by ${result.author})` + return `${codeSpan(oneLine(result.original))} → ${codeSpan(oneLine(result.replacement))}` } /** @@ -37,19 +50,54 @@ export function progressLine({ judged, total, gaps }) { return `judged ${judged} of ${total} mutants: ${gaps} gaps so far, ${judged - gaps} not gaps` } +/** A test's names, from Vitest's TestCase: its suites', then its own. */ +export function testNamesOf(testCase) { + const names = [testCase.name] + for (let at = testCase.parent; at.type !== `module`; at = at.parent) + names.unshift(at.name) + return names +} + +// A word that a POSIX shell reads back as it is: plain, or in single quotes. +const shellWord = (word) => + /^[\w./-]+$/.test(word) ? word : `'${word.replaceAll(`'`, `'\\''`)}'` + +// A control character, or a line or paragraph separator, as its `\uXXXX` +// escape: the line stays one line, and the escape is regular-expression +// syntax for the same character. +export const escapeControls = (text) => + text.replace( + /[\u0000-\u001f\u007f\u2028\u2029]/g, + (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, `0`)}`, + ) + +// The `-t` pattern that runs exactly one test of a file: its names, with +// regular-expression syntax escaped, joined by what either Vitest puts +// between them (a space in Vitest 4, ` > ` in Vitest 5), anchored at both +// ends. The framework's own rerun command (oracle-property.ts) is built so. +const replayPattern = (names) => + `^${names.map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`)).join(`(?: | > )`)}$` + /** * The baseline before any mutant: each oracle file's result, and for a - * failing file its first failing law with the command that replays it. - * `files` names each oracle file, relative to the package, with its failure - * (undefined when it passed) and the failing test's full name. + * failing file its first failure with the command that replays it, which + * pastes into a POSIX shell. `files` names each oracle file, relative to + * the package, with its first failure (undefined when it passed) and its + * tests, each with its state and names (`testNamesOf`). A file with no + * failing test (one that did not load) replays whole. A control character + * in a name or a failure prints as its escape, so each line stays one line. */ export function baselineLines(files) { - return files.flatMap(({ file, failure, fullName }) => - failure === undefined - ? [`pass ${file}`] - : [ - `fail ${file}: ${failure}`, - ` replay: vitest run ${file} -t ${JSON.stringify(fullName)}`, - ], - ) + return files.flatMap(({ file, failure, tests }) => { + if (failure === undefined) return [`pass ${file}`] + const names = tests.find((t) => t.state === `failed`)?.names + const only = + names === undefined + ? `` + : ` -t ${shellWord(escapeControls(replayPattern(names)))}` + return [ + `fail ${file}: ${escapeControls(failure)}`, + ` replay: vitest run ${shellWord(file)}${only}`, + ] + }) }