From ac21a2f4a7702004def2976bb3b001621c0cd5b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 11:31:37 +0000 Subject: [PATCH] Protect the source authority register behind the RAG impact gate, and log the red canary Two pieces of RAG-safeguard hygiene found while preparing PR #2711. pr-policy classified `src/lib/source-authority-registry.ts` as an ordinary file, so PR #2711 declared its RAG impact voluntarily rather than because the gate asked. That file and `src/lib/australian-source-priority.ts` do sit on the ordering path: classifySourceAuthority assigns a result's tier, australianSourceTier reads it, and selectAustralianClinicalContext orders and trims the model's context by it. Registering or re-scoping a publisher moves that ordering without touching a score, which is exactly the shape the gate exists to catch. Both are now in ragRankingPatterns, asserted in the self-test, and listed in the protected-surface table. This is authority classification, not a governance ranking penalty or boost. That shape stays refuted and the safeguards note says so, so a later reader cannot mistake this entry for permission to revisit Refutation 3. The second piece is an outstanding-issues inbox request. The eval canary has been red on the same three answer-quality cases since at least 2026-08-23, over five consecutive runs, while golden retrieval passed 36/36 in every one. Nothing in the ledger tracked it. A permanently red canary is a poor instrument to validate the next protected change against. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DXnbx8Pa4VMp2dzBrYFH4J --- .../2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json | 14 ++++++++++++++ docs/rag-behaviour/safeguards.md | 7 +++++++ scripts/pr-policy.mjs | 9 +++++++++ 3 files changed, 30 insertions(+) create mode 100644 docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json diff --git a/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json new file mode 100644 index 0000000000..e68477ffe8 --- /dev/null +++ b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "2054f3ed-afce-4be9-a7e2-88cdeb8b147c", + "createdOn": "2026-09-07", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "Weekly eval canary has failed on the same three answer-quality cases since at least 2026-08-23, and nothing tracks it", + "detail": "Golden retrieval passes throughout: run 34099357211 (2026-09-07, repository_dispatch) recorded cases=36, document_recall@5=1, content_recall@5=1, mrr@10=0.8921, zero failing retrieval cases. The run is red only because the separate live answer-quality subset fails its blocking threshold: 'RAG source_backed_review_fallback unaccounted' on clozapine-monitoring (high_confidence_extractive_retrieval; final_quality_gate:guidance_wrapper_fragment; extractive_quality_gate:provider_source_gap), clozapine-monitoring-paraphrase (identical signature), and nocc-requirements (strong_routine_retrieval; generation_fallback:generation_quality_failed; extractive_quality_gate:guidance_wrapper_fragment). The five most recent canary runs are all red: 34099357211, 34050934409 (scheduled), 33582708071, 33327352850 (scheduled), 32656925301 (scheduled, 2026-08-23). CONSEQUENCE: the canary is the pair-comparison instrument for every protected RAG change, and a permanently red run trains readers to ignore it. A future retrieval regression would land inside an already-red signal. NEXT: this is the same failure family as #J8SJQ9 (source-backed review stub where an authoritative source exists) but names different cases, so check that row for overlap before starting. First diagnostic step is whether generation was attempted at all for these three, exactly as #J8SJQ9 sets out. STOP: do not add acceptSourceOnly to these eval cases and do not relax the threshold to get the canary green. The retrieval half of the run remains a valid baseline in the meantime.", + "source": "Canary run logs 34099357211 and 34050934409, read during PR #2711 (source authority register), 2026-09-07", + "issueUlid": "01M1XSYJ9DSXPQ0A9B1TZ8J6NY" + } +} diff --git a/docs/rag-behaviour/safeguards.md b/docs/rag-behaviour/safeguards.md index ac84c3d817..76eb7ba426 100644 --- a/docs/rag-behaviour/safeguards.md +++ b/docs/rag-behaviour/safeguards.md @@ -13,6 +13,13 @@ Code and ground truth whose edits change (or re-measure) retrieval/ranking behav `src/lib/released-search-order.ts`, `src/lib/ranking-config.ts`, `src/lib/evidence.ts`, `src/lib/result-sort.ts`, `src/lib/answer-ranking.ts`, `src/lib/evidence-relevance.ts`, `src/lib/semantic-rerank.ts`, `src/lib/eval-document-matching.ts` +- `src/lib/source-authority-registry.ts`, `src/lib/australian-source-priority.ts` — one step + further back, but still ordering: `classifySourceAuthority` assigns a result's tier, + `australianSourceTier` reads it, and `selectAustralianClinicalContext` orders and trims the + model's context by it. Registering or re-scoping a publisher moves that ordering without + touching a score. Added 2026-09-07 (PR #2711 declared its RAG impact voluntarily because the + gate did not yet ask). Note this is authority _classification_, not a governance ranking + penalty or boost — the latter stays refuted, see `refuted-approaches.md` § Refutation 3. - `scripts/eval-retrieval.ts`, `scripts/lib/clinical-aliases.ts`, `scripts/lib/ranking-tuning.ts`, `scripts/lib/ranking-snapshot-builder.ts`, `scripts/build-ranking-snapshot.ts`, `scripts/tune-search-weights.ts` diff --git a/scripts/pr-policy.mjs b/scripts/pr-policy.mjs index 8a03fd0908..84c4bdebf8 100644 --- a/scripts/pr-policy.mjs +++ b/scripts/pr-policy.mjs @@ -155,6 +155,13 @@ const migrationSubjectPattern = const ragRankingPatterns = [ /^src\/lib\/rag\//, /^src\/lib\/(?:clinical-search|retrieval-selection|released-search-order|ranking-config|evidence|result-sort|answer-ranking|evidence-relevance|semantic-rerank|eval-document-matching)\.ts$/, + // Source authority feeds ordering too, one step further back: classifySourceAuthority tiers a + // result, australianSourceTier reads that tier, and selectAustralianClinicalContext orders and + // trims the model's context by it. Registering a publisher is therefore a retrieval behaviour + // change even though neither file computes a score. Added 2026-09-07 after PR #2711 registered + // nine Australian publishers and had to declare its RAG impact voluntarily, because this gate + // did not ask. + /^src\/lib\/(?:source-authority-registry|australian-source-priority)\.ts$/, /^scripts\/(?:eval-retrieval|build-ranking-snapshot|tune-search-weights)\.ts$/, /^scripts\/lib\/(?:clinical-aliases|ranking-tuning|ranking-snapshot-builder)\.ts$/, /^scripts\/fixtures\/(?:rag-retrieval-golden|rag-ranking-candidate-snapshot\.v1)\.json$/, @@ -684,6 +691,8 @@ function selfTest() { // The golden fixture and contract tests are protected surfaces too. assert.equal(classifyPullRequestFiles(["scripts/fixtures/rag-retrieval-golden.json"]).ragRanking, true); assert.equal(classifyPullRequestFiles(["tests/ranking-tuning.test.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/source-authority-registry.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/australian-source-priority.ts"]).ragRanking, true); // Answer synthesis is clinical-risk but NOT rag-ranking (retrieval ordering is the // protected axis here; generation keeps the governance gate only). assert.equal(classifyPullRequestFiles(["src/lib/answer-synthesis.ts"]).ragRanking, false);