diff --git a/scripts/check-calculator-content.mjs b/scripts/check-calculator-content.mjs index 30dd306fa..b7224438e 100644 --- a/scripts/check-calculator-content.mjs +++ b/scripts/check-calculator-content.mjs @@ -34,6 +34,9 @@ const fixturesPath = resolve(root, "src/components/calculators/calculator-fixtur const isoDate = /^\d{4}-\d{2}-\d{2}$/; +/** Mirrors CALCULATOR_EVIDENCE_LIFECYCLE in src/lib/sources/repository-providers.ts. */ +const KNOWN_EVIDENCE_STATUSES = ["reviewed", "permission_review_required", "not_for_active_use"]; + function validDate(value) { return typeof value === "string" && isoDate.test(value) && Number.isFinite(Date.parse(value)); } @@ -90,6 +93,15 @@ function main() { errors.push(`${label}: nextReview must be after lastReviewed`); } if (!Object.prototype.hasOwnProperty.call(source, "supersedes")) errors.push(`${label}: missing supersedes key`); + // The reader maps status to a lifecycle state and falls closed to "inactive" on anything + // it does not recognise. Enumerate the allowed values here so a new or misspelled status + // is a loud failure at the data rather than a source quietly demoted at every read. + if (!KNOWN_EVIDENCE_STATUSES.includes(source.status)) { + errors.push( + `${label}: status ${JSON.stringify(source.status)} is not one of ${KNOWN_EVIDENCE_STATUSES.join(", ")}. ` + + `Add it to CALCULATOR_EVIDENCE_LIFECYCLE in src/lib/sources/repository-providers.ts with its lifecycle state.`, + ); + } } } diff --git a/src/lib/sources/repository-providers.ts b/src/lib/sources/repository-providers.ts index 83d45386f..bcf3e610a 100644 --- a/src/lib/sources/repository-providers.ts +++ b/src/lib/sources/repository-providers.ts @@ -6,7 +6,8 @@ import formsPdfManifest from "../../../data/forms-pdf-manifest.json"; import formsSnapshot from "../../../data/forms-page-snapshot.json"; import dsmClinicalContent from "../../data/dsm-clinical-content.json"; import therapiesSource from "../../data/therapies-source.json"; -import { calculators } from "../../components/calculators/calculator-fixtures"; +import { calculatorEvidence, type CalculatorEvidenceSource } from "../../components/calculators/calculator-evidence"; +import { allCalculatorFixtures } from "../../components/calculators/calculator-fixtures"; import { factsheets } from "../../components/factsheets/factsheets-data"; import { dictionaryComparisonPairs, @@ -28,7 +29,12 @@ import { loadServicesSnapshot } from "@/lib/service-catalog"; import { authoritativeSources, loadSpecifiersContent, type AuthoritativeSource } from "@/lib/specifiers-content"; import { acquisitionSourceReferences } from "@/lib/sources/acquisition-ledger"; import { safeHttpsUrl } from "@/lib/sources/catalogue-core"; -import type { ClinicalSourceReferenceInput, ClinicalSourceType, SourceUsage } from "@/lib/sources/catalogue-types"; +import type { + ClinicalSourceReferenceInput, + ClinicalSourceType, + SourceLifecycleStatus, + SourceUsage, +} from "@/lib/sources/catalogue-types"; import { hasInvalidStructuredSourceDate, strictSourceDate } from "@/lib/sources/source-date-policy"; export type ClinicalSourceProvider = { @@ -515,20 +521,71 @@ const dsmProvider: ClinicalSourceProvider = { }, }; +function isAcademicCalculatorEvidence(source: CalculatorEvidenceSource) { + return source.type !== "internal_governance_record" && source.type !== "rights_statement"; +} + +function calculatorEvidenceType(source: CalculatorEvidenceSource): ClinicalSourceType { + if (source.type === "journal_article") return "primary_study"; + if (source.type === "government_information_paper" || source.type === "government_web_guidance") { + return "professional_reference"; + } + return "unknown"; +} + +/** + * The evidence registry types `status` as an unrestricted string, so a missing, misspelled or + * newly introduced value reaches here. Only a status this map names may present a source as + * active evidence; anything else falls to `inactive`, which keeps the source visible with its + * not-in-active-use warning rather than letting unvetted or quarantined evidence read as + * current. `npm run check:calculator-content` fails on any status not listed here, so a new + * status is a loud failure at the data rather than a silent demotion at the read. + */ +const CALCULATOR_EVIDENCE_LIFECYCLE: Record = { + reviewed: "active", + permission_review_required: "inactive", + not_for_active_use: "excluded", +}; + +export const KNOWN_CALCULATOR_EVIDENCE_STATUSES = Object.keys(CALCULATOR_EVIDENCE_LIFECYCLE); + +function calculatorEvidenceLifecycle(source: CalculatorEvidenceSource): SourceLifecycleStatus { + return CALCULATOR_EVIDENCE_LIFECYCLE[source.status] ?? "inactive"; +} + +const calculatorEvidenceById = new Map(calculatorEvidence.sources.map((source) => [source.id, source])); + const calculatorProvider: ClinicalSourceProvider = { id: "calculators", - sourcePaths: ["src/components/calculators/calculator-fixtures.ts"], + sourcePaths: ["data/calculators/evidence.json", "src/components/calculators/calculator-fixtures.ts"], references: () => - calculators.map((calculator) => - reference( - { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "source" }, - { - title: `${calculator.abbrev} source`, - validationStatus: "unverified", - referenceText: calculator.source, - topics: [calculator.domain], - }, - ), + allCalculatorFixtures.flatMap((calculator) => + calculator.sourceIds.flatMap((sourceId) => { + const source = calculatorEvidenceById.get(sourceId); + if (!source || !isAcademicCalculatorEvidence(source)) return []; + return [ + reference( + { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "sourceIds" }, + { + sourceId: source.id, + title: source.title, + publisher: source.issuer, + canonicalUrl: source.url, + version: source.version, + reviewDate: strictSourceDate(source.lastReviewed), + expiryDate: strictSourceDate(source.nextReview), + jurisdiction: source.jurisdiction, + evidenceType: calculatorEvidenceType(source), + documentStatus: source.status === "reviewed" ? "current" : "unknown", + validationStatus: source.status === "reviewed" ? "locally_reviewed" : "unverified", + contentMode: "link_only", + lifecycleStatus: calculatorEvidenceLifecycle(source), + supersedes: source.supersedes ? [source.supersedes] : [], + topics: [calculator.domain], + }, + ), + ]; + }), ), }; @@ -644,12 +701,28 @@ export function repositorySourceCoverageIssues(inputs: RepositorySourceCoverageI if (!usedFormulationSources.has(sourceId)) issues.push(`Formulation source ${sourceId} has no mechanism usage`); } - for (const calculator of calculators) { - const expected = reference( - { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "source" }, - { referenceText: calculator.source }, - ); - if (!keys.has(coverageKey(expected))) issues.push(`Calculator ${calculator.id} source is not captured`); + const usedCalculatorEvidenceIds = new Set(allCalculatorFixtures.flatMap((calculator) => calculator.sourceIds)); + for (const calculator of allCalculatorFixtures) { + for (const sourceId of calculator.sourceIds) { + const source = calculatorEvidenceById.get(sourceId); + if (!source) { + issues.push(`Calculator ${calculator.id} references missing evidence source ${sourceId}`); + continue; + } + if (!isAcademicCalculatorEvidence(source)) continue; + const expected = reference( + { modeId: "calculators", recordId: calculator.id, recordLabel: calculator.name, field: "sourceIds" }, + { sourceId, canonicalUrl: source.url }, + ); + if (!keys.has(coverageKey(expected))) { + issues.push(`Calculator evidence source ${sourceId} is missing usage ${calculator.id}`); + } + } + } + for (const source of calculatorEvidence.sources) { + if (isAcademicCalculatorEvidence(source) && !usedCalculatorEvidenceIds.has(source.id)) { + issues.push(`Calculator evidence source ${source.id} has no calculator usage`); + } } for (const { medication, section, row } of medicationSourceRows()) { diff --git a/src/lib/sources/source-url-policy.ts b/src/lib/sources/source-url-policy.ts index 24ab90e56..f3fb76317 100644 --- a/src/lib/sources/source-url-policy.ts +++ b/src/lib/sources/source-url-policy.ts @@ -15,6 +15,7 @@ export const GOVERNED_SOURCE_HOSTS = [ "rph.health.wa.gov.au", "ruah.org.au", "smhs.health.wa.gov.au", + "www.abs.gov.au", "www.aihw.gov.au", "www.beyondblue.org.au", "www.cci.health.wa.gov.au", diff --git a/tests/calculator-evidence-status-fail-closed.test.ts b/tests/calculator-evidence-status-fail-closed.test.ts new file mode 100644 index 000000000..400bcb22a --- /dev/null +++ b/tests/calculator-evidence-status-fail-closed.test.ts @@ -0,0 +1,135 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +/** + * A calculator evidence source carries `status` as an unrestricted string. If an unrecognised + * value read as `active`, `/sources` would present ambiguous or quarantined evidence with no + * inactive or excluded warning — the failure this pair of guards exists to make impossible. + * + * Two independent guards, tested separately because they fail at different moments: + * - the reader falls closed to `inactive` for anything it does not recognise + * - `check:calculator-content` refuses the data outright, so a new status is noticed rather + * than silently demoting a source at every read + */ + +const REPO_ROOT = join(__dirname, ".."); +const PROVIDER = join(REPO_ROOT, "src/lib/sources/repository-providers.ts"); +const CHECKER = join(REPO_ROOT, "scripts/check-calculator-content.mjs"); +const EVIDENCE = join(REPO_ROOT, "data/calculators/evidence.json"); + +/** + * The mapping is module-private and the module is `server-only`, so read the literal it is + * declared from. That keeps the test honest: it fails if the table is edited, rather than + * restating a copy that can drift. + */ +function lifecycleTable(): Record { + const src = readFileSync(PROVIDER, "utf8"); + const block = src.match(/const CALCULATOR_EVIDENCE_LIFECYCLE: Record = \{([^}]*)\}/); + expect(block, "CALCULATOR_EVIDENCE_LIFECYCLE must exist in repository-providers.ts").toBeTruthy(); + const table: Record = {}; + for (const [, key, value] of block![1].matchAll(/(\w+):\s*"(\w+)"/g)) table[key] = value; + return table; +} + +describe("calculator evidence status is read fail-closed", () => { + it("maps only the reviewed status to active", () => { + const table = lifecycleTable(); + expect(table).toEqual({ + reviewed: "active", + permission_review_required: "inactive", + not_for_active_use: "excluded", + }); + expect( + Object.entries(table) + .filter(([, v]) => v === "active") + .map(([k]) => k), + ).toEqual(["reviewed"]); + }); + + it("falls back to inactive rather than active, so an unknown status cannot present as current", () => { + const src = readFileSync(PROVIDER, "utf8"); + const fn = src.match(/function calculatorEvidenceLifecycle\([^)]*\)[^{]*\{([^}]*)\}/); + expect(fn, "calculatorEvidenceLifecycle must exist").toBeTruthy(); + // The behaviour under test: the default for an unrecognised key. + expect(fn![1]).toContain('?? "inactive"'); + expect(fn![1]).not.toContain('?? "active"'); + }); + + it("resolves a missing, misspelled and newly introduced status to inactive", () => { + const table = lifecycleTable(); + const resolve = (status: unknown) => table[status as string] ?? "inactive"; + expect(resolve(undefined)).toBe("inactive"); + expect(resolve("")).toBe("inactive"); + expect(resolve("Reviewed")).toBe("inactive"); + expect(resolve("reviewd")).toBe("inactive"); + expect(resolve("provisionally_reviewed")).toBe("inactive"); + // Control: the one status that may be active still is. + expect(resolve("reviewed")).toBe("active"); + }); +}); + +describe("check:calculator-content refuses an unknown evidence status", () => { + function runCheckerAgainst(mutate: (evidence: { sources: { status: string }[] }) => void) { + const dir = mkdtempSync(join(tmpdir(), "calc-evidence-")); + const evidence = JSON.parse(readFileSync(EVIDENCE, "utf8")); + mutate(evidence); + const copy = join(dir, "evidence.json"); + writeFileSync(copy, JSON.stringify(evidence, null, 2)); + // Run the real checker with the evidence file swapped, so this exercises the shipped + // script rather than a restatement of its rule. + const script = readFileSync(CHECKER, "utf8") + // The script derives its root from its own location, so pin both the root and the one + // input under test; every other input still comes from the real repository. + .replace( + 'const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");', + `const root = ${JSON.stringify(REPO_ROOT)};`, + ) + .replace( + 'const evidencePath = resolve(root, "data/calculators/evidence.json");', + `const evidencePath = ${JSON.stringify(copy)};`, + ); + const scriptPath = join(dir, "check.mjs"); + writeFileSync(scriptPath, script); + try { + const out = execFileSync("node", [scriptPath], { cwd: REPO_ROOT, encoding: "utf8" }); + return { code: 0, out }; + } catch (error) { + const e = error as { status: number; stdout?: string; stderr?: string }; + return { code: e.status, out: `${e.stdout ?? ""}${e.stderr ?? ""}` }; + } + } + + it("passes on the evidence file as committed", () => { + const { code, out } = runCheckerAgainst(() => {}); + expect(out).toContain("CALCULATOR_CONTENT_PASS"); + expect(code).toBe(0); + }); + + it("fails on a misspelled status instead of letting the reader demote it silently", () => { + const { code, out } = runCheckerAgainst((evidence) => { + evidence.sources[0].status = "reviewd"; + }); + expect(code).not.toBe(0); + expect(out).toContain('status "reviewd" is not one of'); + }); + + it("fails on a newly introduced status, so adding one is a deliberate decision", () => { + const { code, out } = runCheckerAgainst((evidence) => { + evidence.sources[0].status = "provisionally_reviewed"; + }); + expect(code).not.toBe(0); + expect(out).toContain("provisionally_reviewed"); + }); + + it("fails on a missing status", () => { + const { code, out } = runCheckerAgainst((evidence) => { + delete (evidence.sources[0] as Partial<{ status: string }>).status; + }); + expect(code).not.toBe(0); + expect(out).toContain("is not one of"); + }); +}); diff --git a/tests/source-catalogue-providers.test.ts b/tests/source-catalogue-providers.test.ts index e8eec9e4c..622f996c0 100644 --- a/tests/source-catalogue-providers.test.ts +++ b/tests/source-catalogue-providers.test.ts @@ -6,7 +6,8 @@ import formsPdfManifest from "../data/forms-pdf-manifest.json"; import formsSnapshot from "../data/forms-page-snapshot.json"; import dsmClinicalContent from "../src/data/dsm-clinical-content.json"; import therapies from "../src/data/therapies-source.json"; -import { calculators } from "@/components/calculators/calculator-fixtures"; +import { calculatorEvidence } from "@/components/calculators/calculator-evidence"; +import { allCalculatorFixtures } from "@/components/calculators/calculator-fixtures"; import { factsheets } from "@/components/factsheets/factsheets-data"; import { dictionarySources } from "@/lib/dictionary-data"; import { formulationMechanisms, formulationSourceLibrary } from "@/lib/formulation"; @@ -41,7 +42,7 @@ const expectedProviders = { medications: ["data/medications-snapshot.json"], services: ["data/services-snapshot.json"], dsm: ["src/data/dsm-clinical-content.json"], - calculators: ["src/components/calculators/calculator-fixtures.ts"], + calculators: ["data/calculators/evidence.json", "src/components/calculators/calculator-fixtures.ts"], acquisitions: ["src/data/source-acquisitions.json"], } as const; @@ -80,7 +81,7 @@ describe("repository source providers", () => { ); }); - it("pins the 50 currently governed structured source hosts without deriving trust at runtime", () => { + it("pins every emitted structured source host without deriving trust at runtime", () => { const emittedHosts = new Set( repositorySourceReferences() .map((reference) => reference.canonicalUrl) @@ -88,7 +89,6 @@ describe("repository source providers", () => { .map((value) => new URL(value).hostname), ); - expect(GOVERNED_SOURCE_HOSTS).toHaveLength(50); expect(new Set(GOVERNED_SOURCE_HOSTS)).toEqual(emittedHosts); }); @@ -221,7 +221,56 @@ describe("repository source providers", () => { new Set([dsmClinicalContent.source_repository]), ); - expect(provider("calculators").references()).toHaveLength(calculators.length); + expect(provider("calculators").references()).toHaveLength(8); + }); + + it("catalogues every academic calculator evidence source with its structured provenance", () => { + const references = provider("calculators").references(); + const academicEvidence = calculatorEvidence.sources.filter( + (source) => source.type !== "internal_governance_record" && source.type !== "rights_statement", + ); + + expect(academicEvidence.map((source) => source.id)).toEqual([ + "source:phq9", + "source:gad7", + "source:k10", + "source:cage", + "source:auditc", + "source:mdq", + "source:sadpersons", + "source:ybocs", + ]); + expect(new Set(references.map((reference) => reference.sourceId))).toEqual( + new Set(academicEvidence.map((source) => source.id)), + ); + expect(references).not.toContainEqual(expect.objectContaining({ sourceId: "source:governance" })); + expect(references).toContainEqual( + expect.objectContaining({ + sourceId: "source:phq9", + publisher: "Kroenke, Spitzer & Williams", + canonicalUrl: "https://pmc.ncbi.nlm.nih.gov/articles/PMC1495268/", + version: "2001 validation study", + evidenceType: "primary_study", + validationStatus: "locally_reviewed", + lifecycleStatus: "active", + usage: expect.objectContaining({ modeId: "calculators", recordId: "phq9", field: "sourceIds" }), + }), + ); + expect(references).toContainEqual(expect.objectContaining({ sourceId: "source:mdq", lifecycleStatus: "inactive" })); + expect(references).toContainEqual( + expect.objectContaining({ sourceId: "source:sadpersons", lifecycleStatus: "excluded" }), + ); + + for (const calculator of allCalculatorFixtures) { + const academicSourceIds = calculator.sourceIds.filter((sourceId) => sourceId !== "source:governance"); + expect( + new Set( + references + .filter((reference) => reference.usage.recordId === calculator.id) + .map((reference) => reference.sourceId), + ), + ).toEqual(new Set(academicSourceIds)); + } }); it("keeps specifier authoritative-source usage IDs stable across insertion and reordering", () => {