From f60765a5c2595efb81e9124139f9d4b9e355e272 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:49:40 +0800 Subject: [PATCH 1/2] fix(sources): catalogue structured calculator evidence --- src/lib/sources/repository-providers.ts | 90 +++++++++++++++++++----- src/lib/sources/source-url-policy.ts | 1 + tests/source-catalogue-providers.test.ts | 59 ++++++++++++++-- 3 files changed, 127 insertions(+), 23 deletions(-) diff --git a/src/lib/sources/repository-providers.ts b/src/lib/sources/repository-providers.ts index 83d45386f..6062e3e67 100644 --- a/src/lib/sources/repository-providers.ts +++ b/src/lib/sources/repository-providers.ts @@ -6,7 +6,8 @@ import formsPdfManifest from "../../../data/forms-pdf-manifest.json"; import formsSnapshot from "../../../data/forms-page-snapshot.json"; import dsmClinicalContent from "../../data/dsm-clinical-content.json"; import therapiesSource from "../../data/therapies-source.json"; -import { calculators } from "../../components/calculators/calculator-fixtures"; +import { calculatorEvidence, type CalculatorEvidenceSource } from "../../components/calculators/calculator-evidence"; +import { allCalculatorFixtures } from "../../components/calculators/calculator-fixtures"; import { factsheets } from "../../components/factsheets/factsheets-data"; import { dictionaryComparisonPairs, @@ -515,20 +516,57 @@ const dsmProvider: ClinicalSourceProvider = { }, }; +function isAcademicCalculatorEvidence(source: CalculatorEvidenceSource) { + return source.type !== "internal_governance_record" && source.type !== "rights_statement"; +} + +function calculatorEvidenceType(source: CalculatorEvidenceSource): ClinicalSourceType { + if (source.type === "journal_article") return "primary_study"; + if (source.type === "government_information_paper" || source.type === "government_web_guidance") { + return "professional_reference"; + } + return "unknown"; +} + +function calculatorEvidenceLifecycle(source: CalculatorEvidenceSource) { + if (source.status === "not_for_active_use") return "excluded" as const; + if (source.status === "permission_review_required") return "inactive" as const; + return "active" as const; +} + +const calculatorEvidenceById = new Map(calculatorEvidence.sources.map((source) => [source.id, source])); + const calculatorProvider: ClinicalSourceProvider = { id: "calculators", - sourcePaths: ["src/components/calculators/calculator-fixtures.ts"], + sourcePaths: ["data/calculators/evidence.json", "src/components/calculators/calculator-fixtures.ts"], references: () => - calculators.map((calculator) => - reference( - { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "source" }, - { - title: `${calculator.abbrev} source`, - validationStatus: "unverified", - referenceText: calculator.source, - topics: [calculator.domain], - }, - ), + allCalculatorFixtures.flatMap((calculator) => + calculator.sourceIds.flatMap((sourceId) => { + const source = calculatorEvidenceById.get(sourceId); + if (!source || !isAcademicCalculatorEvidence(source)) return []; + return [ + reference( + { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "sourceIds" }, + { + sourceId: source.id, + title: source.title, + publisher: source.issuer, + canonicalUrl: source.url, + version: source.version, + reviewDate: strictSourceDate(source.lastReviewed), + expiryDate: strictSourceDate(source.nextReview), + jurisdiction: source.jurisdiction, + evidenceType: calculatorEvidenceType(source), + documentStatus: source.status === "reviewed" ? "current" : "unknown", + validationStatus: source.status === "reviewed" ? "locally_reviewed" : "unverified", + contentMode: "link_only", + lifecycleStatus: calculatorEvidenceLifecycle(source), + supersedes: source.supersedes ? [source.supersedes] : [], + topics: [calculator.domain], + }, + ), + ]; + }), ), }; @@ -644,12 +682,28 @@ export function repositorySourceCoverageIssues(inputs: RepositorySourceCoverageI if (!usedFormulationSources.has(sourceId)) issues.push(`Formulation source ${sourceId} has no mechanism usage`); } - for (const calculator of calculators) { - const expected = reference( - { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "source" }, - { referenceText: calculator.source }, - ); - if (!keys.has(coverageKey(expected))) issues.push(`Calculator ${calculator.id} source is not captured`); + const usedCalculatorEvidenceIds = new Set(allCalculatorFixtures.flatMap((calculator) => calculator.sourceIds)); + for (const calculator of allCalculatorFixtures) { + for (const sourceId of calculator.sourceIds) { + const source = calculatorEvidenceById.get(sourceId); + if (!source) { + issues.push(`Calculator ${calculator.id} references missing evidence source ${sourceId}`); + continue; + } + if (!isAcademicCalculatorEvidence(source)) continue; + const expected = reference( + { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "sourceIds" }, + { sourceId, canonicalUrl: source.url }, + ); + if (!keys.has(coverageKey(expected))) { + issues.push(`Calculator evidence source ${sourceId} is missing usage ${calculator.id}`); + } + } + } + for (const source of calculatorEvidence.sources) { + if (isAcademicCalculatorEvidence(source) && !usedCalculatorEvidenceIds.has(source.id)) { + issues.push(`Calculator evidence source ${source.id} has no calculator usage`); + } } for (const { medication, section, row } of medicationSourceRows()) { diff --git a/src/lib/sources/source-url-policy.ts b/src/lib/sources/source-url-policy.ts index 24ab90e56..f3fb76317 100644 --- a/src/lib/sources/source-url-policy.ts +++ b/src/lib/sources/source-url-policy.ts @@ -15,6 +15,7 @@ export const GOVERNED_SOURCE_HOSTS = [ "rph.health.wa.gov.au", "ruah.org.au", "smhs.health.wa.gov.au", + "www.abs.gov.au", "www.aihw.gov.au", "www.beyondblue.org.au", "www.cci.health.wa.gov.au", diff --git a/tests/source-catalogue-providers.test.ts b/tests/source-catalogue-providers.test.ts index e8eec9e4c..622f996c0 100644 --- a/tests/source-catalogue-providers.test.ts +++ b/tests/source-catalogue-providers.test.ts @@ -6,7 +6,8 @@ import formsPdfManifest from "../data/forms-pdf-manifest.json"; import formsSnapshot from "../data/forms-page-snapshot.json"; import dsmClinicalContent from "../src/data/dsm-clinical-content.json"; import therapies from "../src/data/therapies-source.json"; -import { calculators } from "@/components/calculators/calculator-fixtures"; +import { calculatorEvidence } from "@/components/calculators/calculator-evidence"; +import { allCalculatorFixtures } from "@/components/calculators/calculator-fixtures"; import { factsheets } from "@/components/factsheets/factsheets-data"; import { dictionarySources } from "@/lib/dictionary-data"; import { formulationMechanisms, formulationSourceLibrary } from "@/lib/formulation"; @@ -41,7 +42,7 @@ const expectedProviders = { medications: ["data/medications-snapshot.json"], services: ["data/services-snapshot.json"], dsm: ["src/data/dsm-clinical-content.json"], - calculators: ["src/components/calculators/calculator-fixtures.ts"], + calculators: ["data/calculators/evidence.json", "src/components/calculators/calculator-fixtures.ts"], acquisitions: ["src/data/source-acquisitions.json"], } as const; @@ -80,7 +81,7 @@ describe("repository source providers", () => { ); }); - it("pins the 50 currently governed structured source hosts without deriving trust at runtime", () => { + it("pins every emitted structured source host without deriving trust at runtime", () => { const emittedHosts = new Set( repositorySourceReferences() .map((reference) => reference.canonicalUrl) @@ -88,7 +89,6 @@ describe("repository source providers", () => { .map((value) => new URL(value).hostname), ); - expect(GOVERNED_SOURCE_HOSTS).toHaveLength(50); expect(new Set(GOVERNED_SOURCE_HOSTS)).toEqual(emittedHosts); }); @@ -221,7 +221,56 @@ describe("repository source providers", () => { new Set([dsmClinicalContent.source_repository]), ); - expect(provider("calculators").references()).toHaveLength(calculators.length); + expect(provider("calculators").references()).toHaveLength(8); + }); + + it("catalogues every academic calculator evidence source with its structured provenance", () => { + const references = provider("calculators").references(); + const academicEvidence = calculatorEvidence.sources.filter( + (source) => source.type !== "internal_governance_record" && source.type !== "rights_statement", + ); + + expect(academicEvidence.map((source) => source.id)).toEqual([ + "source:phq9", + "source:gad7", + "source:k10", + "source:cage", + "source:auditc", + "source:mdq", + "source:sadpersons", + "source:ybocs", + ]); + expect(new Set(references.map((reference) => reference.sourceId))).toEqual( + new Set(academicEvidence.map((source) => source.id)), + ); + expect(references).not.toContainEqual(expect.objectContaining({ sourceId: "source:governance" })); + expect(references).toContainEqual( + expect.objectContaining({ + sourceId: "source:phq9", + publisher: "Kroenke, Spitzer & Williams", + canonicalUrl: "https://pmc.ncbi.nlm.nih.gov/articles/PMC1495268/", + version: "2001 validation study", + evidenceType: "primary_study", + validationStatus: "locally_reviewed", + lifecycleStatus: "active", + usage: expect.objectContaining({ modeId: "calculators", recordId: "phq9", field: "sourceIds" }), + }), + ); + expect(references).toContainEqual(expect.objectContaining({ sourceId: "source:mdq", lifecycleStatus: "inactive" })); + expect(references).toContainEqual( + expect.objectContaining({ sourceId: "source:sadpersons", lifecycleStatus: "excluded" }), + ); + + for (const calculator of allCalculatorFixtures) { + const academicSourceIds = calculator.sourceIds.filter((sourceId) => sourceId !== "source:governance"); + expect( + new Set( + references + .filter((reference) => reference.usage.recordId === calculator.id) + .map((reference) => reference.sourceId), + ), + ).toEqual(new Set(academicSourceIds)); + } }); it("keeps specifier authoritative-source usage IDs stable across insertion and reordering", () => { From 61eeb970d5612cc78014c1d95cafa6435f61705a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 12:41:59 +0000 Subject: [PATCH 2/2] fix(sources): read an unknown calculator evidence status as inactive, not active The registry types status as an unrestricted string and the reader treated everything except two named values as active, so a missing, misspelled or newly introduced status would have presented ambiguous or quarantined evidence on /sources with no inactive or excluded warning. Two guards, because they catch it at different moments. The reader now maps status through an explicit table and falls closed to inactive for anything it does not recognise, so only "reviewed" can read as active. check:calculator-content enumerates the same three statuses and refuses the data outright, so introducing a status is a deliberate decision at the registry rather than a source silently demoted at every read. Tests cover both halves: the table admits exactly one active status, the fallback is inactive and not active, missing/misspelled/new statuses all resolve to inactive, and the shipped checker is run against a mutated copy of the real evidence file for each of those cases. 7 passed. check:calculator-content passes on the file as committed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0142trjgjRAP2GA9wzqoYUAE --- scripts/check-calculator-content.mjs | 12 ++ src/lib/sources/repository-providers.ts | 29 +++- ...ulator-evidence-status-fail-closed.test.ts | 135 ++++++++++++++++++ 3 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 tests/calculator-evidence-status-fail-closed.test.ts diff --git a/scripts/check-calculator-content.mjs b/scripts/check-calculator-content.mjs index 30dd306fa..b7224438e 100644 --- a/scripts/check-calculator-content.mjs +++ b/scripts/check-calculator-content.mjs @@ -34,6 +34,9 @@ const fixturesPath = resolve(root, "src/components/calculators/calculator-fixtur const isoDate = /^\d{4}-\d{2}-\d{2}$/; +/** Mirrors CALCULATOR_EVIDENCE_LIFECYCLE in src/lib/sources/repository-providers.ts. */ +const KNOWN_EVIDENCE_STATUSES = ["reviewed", "permission_review_required", "not_for_active_use"]; + function validDate(value) { return typeof value === "string" && isoDate.test(value) && Number.isFinite(Date.parse(value)); } @@ -90,6 +93,15 @@ function main() { errors.push(`${label}: nextReview must be after lastReviewed`); } if (!Object.prototype.hasOwnProperty.call(source, "supersedes")) errors.push(`${label}: missing supersedes key`); + // The reader maps status to a lifecycle state and falls closed to "inactive" on anything + // it does not recognise. Enumerate the allowed values here so a new or misspelled status + // is a loud failure at the data rather than a source quietly demoted at every read. + if (!KNOWN_EVIDENCE_STATUSES.includes(source.status)) { + errors.push( + `${label}: status ${JSON.stringify(source.status)} is not one of ${KNOWN_EVIDENCE_STATUSES.join(", ")}. ` + + `Add it to CALCULATOR_EVIDENCE_LIFECYCLE in src/lib/sources/repository-providers.ts with its lifecycle state.`, + ); + } } } diff --git a/src/lib/sources/repository-providers.ts b/src/lib/sources/repository-providers.ts index 6062e3e67..bcf3e610a 100644 --- a/src/lib/sources/repository-providers.ts +++ b/src/lib/sources/repository-providers.ts @@ -29,7 +29,12 @@ import { loadServicesSnapshot } from "@/lib/service-catalog"; import { authoritativeSources, loadSpecifiersContent, type AuthoritativeSource } from "@/lib/specifiers-content"; import { acquisitionSourceReferences } from "@/lib/sources/acquisition-ledger"; import { safeHttpsUrl } from "@/lib/sources/catalogue-core"; -import type { ClinicalSourceReferenceInput, ClinicalSourceType, SourceUsage } from "@/lib/sources/catalogue-types"; +import type { + ClinicalSourceReferenceInput, + ClinicalSourceType, + SourceLifecycleStatus, + SourceUsage, +} from "@/lib/sources/catalogue-types"; import { hasInvalidStructuredSourceDate, strictSourceDate } from "@/lib/sources/source-date-policy"; export type ClinicalSourceProvider = { @@ -528,10 +533,24 @@ function calculatorEvidenceType(source: CalculatorEvidenceSource): ClinicalSourc return "unknown"; } -function calculatorEvidenceLifecycle(source: CalculatorEvidenceSource) { - if (source.status === "not_for_active_use") return "excluded" as const; - if (source.status === "permission_review_required") return "inactive" as const; - return "active" as const; +/** + * The evidence registry types `status` as an unrestricted string, so a missing, misspelled or + * newly introduced value reaches here. Only a status this map names may present a source as + * active evidence; anything else falls to `inactive`, which keeps the source visible with its + * not-in-active-use warning rather than letting unvetted or quarantined evidence read as + * current. `npm run check:calculator-content` fails on any status not listed here, so a new + * status is a loud failure at the data rather than a silent demotion at the read. + */ +const CALCULATOR_EVIDENCE_LIFECYCLE: Record = { + reviewed: "active", + permission_review_required: "inactive", + not_for_active_use: "excluded", +}; + +export const KNOWN_CALCULATOR_EVIDENCE_STATUSES = Object.keys(CALCULATOR_EVIDENCE_LIFECYCLE); + +function calculatorEvidenceLifecycle(source: CalculatorEvidenceSource): SourceLifecycleStatus { + return CALCULATOR_EVIDENCE_LIFECYCLE[source.status] ?? "inactive"; } const calculatorEvidenceById = new Map(calculatorEvidence.sources.map((source) => [source.id, source])); diff --git a/tests/calculator-evidence-status-fail-closed.test.ts b/tests/calculator-evidence-status-fail-closed.test.ts new file mode 100644 index 000000000..400bcb22a --- /dev/null +++ b/tests/calculator-evidence-status-fail-closed.test.ts @@ -0,0 +1,135 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +/** + * A calculator evidence source carries `status` as an unrestricted string. If an unrecognised + * value read as `active`, `/sources` would present ambiguous or quarantined evidence with no + * inactive or excluded warning — the failure this pair of guards exists to make impossible. + * + * Two independent guards, tested separately because they fail at different moments: + * - the reader falls closed to `inactive` for anything it does not recognise + * - `check:calculator-content` refuses the data outright, so a new status is noticed rather + * than silently demoting a source at every read + */ + +const REPO_ROOT = join(__dirname, ".."); +const PROVIDER = join(REPO_ROOT, "src/lib/sources/repository-providers.ts"); +const CHECKER = join(REPO_ROOT, "scripts/check-calculator-content.mjs"); +const EVIDENCE = join(REPO_ROOT, "data/calculators/evidence.json"); + +/** + * The mapping is module-private and the module is `server-only`, so read the literal it is + * declared from. That keeps the test honest: it fails if the table is edited, rather than + * restating a copy that can drift. + */ +function lifecycleTable(): Record { + const src = readFileSync(PROVIDER, "utf8"); + const block = src.match(/const CALCULATOR_EVIDENCE_LIFECYCLE: Record = \{([^}]*)\}/); + expect(block, "CALCULATOR_EVIDENCE_LIFECYCLE must exist in repository-providers.ts").toBeTruthy(); + const table: Record = {}; + for (const [, key, value] of block![1].matchAll(/(\w+):\s*"(\w+)"/g)) table[key] = value; + return table; +} + +describe("calculator evidence status is read fail-closed", () => { + it("maps only the reviewed status to active", () => { + const table = lifecycleTable(); + expect(table).toEqual({ + reviewed: "active", + permission_review_required: "inactive", + not_for_active_use: "excluded", + }); + expect( + Object.entries(table) + .filter(([, v]) => v === "active") + .map(([k]) => k), + ).toEqual(["reviewed"]); + }); + + it("falls back to inactive rather than active, so an unknown status cannot present as current", () => { + const src = readFileSync(PROVIDER, "utf8"); + const fn = src.match(/function calculatorEvidenceLifecycle\([^)]*\)[^{]*\{([^}]*)\}/); + expect(fn, "calculatorEvidenceLifecycle must exist").toBeTruthy(); + // The behaviour under test: the default for an unrecognised key. + expect(fn![1]).toContain('?? "inactive"'); + expect(fn![1]).not.toContain('?? "active"'); + }); + + it("resolves a missing, misspelled and newly introduced status to inactive", () => { + const table = lifecycleTable(); + const resolve = (status: unknown) => table[status as string] ?? "inactive"; + expect(resolve(undefined)).toBe("inactive"); + expect(resolve("")).toBe("inactive"); + expect(resolve("Reviewed")).toBe("inactive"); + expect(resolve("reviewd")).toBe("inactive"); + expect(resolve("provisionally_reviewed")).toBe("inactive"); + // Control: the one status that may be active still is. + expect(resolve("reviewed")).toBe("active"); + }); +}); + +describe("check:calculator-content refuses an unknown evidence status", () => { + function runCheckerAgainst(mutate: (evidence: { sources: { status: string }[] }) => void) { + const dir = mkdtempSync(join(tmpdir(), "calc-evidence-")); + const evidence = JSON.parse(readFileSync(EVIDENCE, "utf8")); + mutate(evidence); + const copy = join(dir, "evidence.json"); + writeFileSync(copy, JSON.stringify(evidence, null, 2)); + // Run the real checker with the evidence file swapped, so this exercises the shipped + // script rather than a restatement of its rule. + const script = readFileSync(CHECKER, "utf8") + // The script derives its root from its own location, so pin both the root and the one + // input under test; every other input still comes from the real repository. + .replace( + 'const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");', + `const root = ${JSON.stringify(REPO_ROOT)};`, + ) + .replace( + 'const evidencePath = resolve(root, "data/calculators/evidence.json");', + `const evidencePath = ${JSON.stringify(copy)};`, + ); + const scriptPath = join(dir, "check.mjs"); + writeFileSync(scriptPath, script); + try { + const out = execFileSync("node", [scriptPath], { cwd: REPO_ROOT, encoding: "utf8" }); + return { code: 0, out }; + } catch (error) { + const e = error as { status: number; stdout?: string; stderr?: string }; + return { code: e.status, out: `${e.stdout ?? ""}${e.stderr ?? ""}` }; + } + } + + it("passes on the evidence file as committed", () => { + const { code, out } = runCheckerAgainst(() => {}); + expect(out).toContain("CALCULATOR_CONTENT_PASS"); + expect(code).toBe(0); + }); + + it("fails on a misspelled status instead of letting the reader demote it silently", () => { + const { code, out } = runCheckerAgainst((evidence) => { + evidence.sources[0].status = "reviewd"; + }); + expect(code).not.toBe(0); + expect(out).toContain('status "reviewd" is not one of'); + }); + + it("fails on a newly introduced status, so adding one is a deliberate decision", () => { + const { code, out } = runCheckerAgainst((evidence) => { + evidence.sources[0].status = "provisionally_reviewed"; + }); + expect(code).not.toBe(0); + expect(out).toContain("provisionally_reviewed"); + }); + + it("fails on a missing status", () => { + const { code, out } = runCheckerAgainst((evidence) => { + delete (evidence.sources[0] as Partial<{ status: string }>).status; + }); + expect(code).not.toBe(0); + expect(out).toContain("is not one of"); + }); +});