diff --git a/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json new file mode 100644 index 0000000000..e68477ffe8 --- /dev/null +++ b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "2054f3ed-afce-4be9-a7e2-88cdeb8b147c", + "createdOn": "2026-09-07", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "Weekly eval canary has failed on the same three answer-quality cases since at least 2026-08-23, and nothing tracks it", + "detail": "Golden retrieval passes throughout: run 34099357211 (2026-09-07, repository_dispatch) recorded cases=36, document_recall@5=1, content_recall@5=1, mrr@10=0.8921, zero failing retrieval cases. The run is red only because the separate live answer-quality subset fails its blocking threshold: 'RAG source_backed_review_fallback unaccounted' on clozapine-monitoring (high_confidence_extractive_retrieval; final_quality_gate:guidance_wrapper_fragment; extractive_quality_gate:provider_source_gap), clozapine-monitoring-paraphrase (identical signature), and nocc-requirements (strong_routine_retrieval; generation_fallback:generation_quality_failed; extractive_quality_gate:guidance_wrapper_fragment). The five most recent canary runs are all red: 34099357211, 34050934409 (scheduled), 33582708071, 33327352850 (scheduled), 32656925301 (scheduled, 2026-08-23). CONSEQUENCE: the canary is the pair-comparison instrument for every protected RAG change, and a permanently red run trains readers to ignore it. A future retrieval regression would land inside an already-red signal. NEXT: this is the same failure family as #J8SJQ9 (source-backed review stub where an authoritative source exists) but names different cases, so check that row for overlap before starting. First diagnostic step is whether generation was attempted at all for these three, exactly as #J8SJQ9 sets out. STOP: do not add acceptSourceOnly to these eval cases and do not relax the threshold to get the canary green. The retrieval half of the run remains a valid baseline in the meantime.", + "source": "Canary run logs 34099357211 and 34050934409, read during PR #2711 (source authority register), 2026-09-07", + "issueUlid": "01M1XSYJ9DSXPQ0A9B1TZ8J6NY" + } +} diff --git a/docs/rag-behaviour/safeguards.md b/docs/rag-behaviour/safeguards.md index ac84c3d817..76eb7ba426 100644 --- a/docs/rag-behaviour/safeguards.md +++ b/docs/rag-behaviour/safeguards.md @@ -13,6 +13,13 @@ Code and ground truth whose edits change (or re-measure) retrieval/ranking behav `src/lib/released-search-order.ts`, `src/lib/ranking-config.ts`, `src/lib/evidence.ts`, `src/lib/result-sort.ts`, `src/lib/answer-ranking.ts`, `src/lib/evidence-relevance.ts`, `src/lib/semantic-rerank.ts`, `src/lib/eval-document-matching.ts` +- `src/lib/source-authority-registry.ts`, `src/lib/australian-source-priority.ts` — one step + further back, but still ordering: `classifySourceAuthority` assigns a result's tier, + `australianSourceTier` reads it, and `selectAustralianClinicalContext` orders and trims the + model's context by it. Registering or re-scoping a publisher moves that ordering without + touching a score. Added 2026-09-07 (PR #2711 declared its RAG impact voluntarily because the + gate did not yet ask). Note this is authority _classification_, not a governance ranking + penalty or boost — the latter stays refuted, see `refuted-approaches.md` § Refutation 3. - `scripts/eval-retrieval.ts`, `scripts/lib/clinical-aliases.ts`, `scripts/lib/ranking-tuning.ts`, `scripts/lib/ranking-snapshot-builder.ts`, `scripts/build-ranking-snapshot.ts`, `scripts/tune-search-weights.ts` diff --git a/scripts/pr-policy.mjs b/scripts/pr-policy.mjs index 8a03fd0908..84c4bdebf8 100644 --- a/scripts/pr-policy.mjs +++ b/scripts/pr-policy.mjs @@ -155,6 +155,13 @@ const migrationSubjectPattern = const ragRankingPatterns = [ /^src\/lib\/rag\//, /^src\/lib\/(?:clinical-search|retrieval-selection|released-search-order|ranking-config|evidence|result-sort|answer-ranking|evidence-relevance|semantic-rerank|eval-document-matching)\.ts$/, + // Source authority feeds ordering too, one step further back: classifySourceAuthority tiers a + // result, australianSourceTier reads that tier, and selectAustralianClinicalContext orders and + // trims the model's context by it. Registering a publisher is therefore a retrieval behaviour + // change even though neither file computes a score. Added 2026-09-07 after PR #2711 registered + // nine Australian publishers and had to declare its RAG impact voluntarily, because this gate + // did not ask. + /^src\/lib\/(?:source-authority-registry|australian-source-priority)\.ts$/, /^scripts\/(?:eval-retrieval|build-ranking-snapshot|tune-search-weights)\.ts$/, /^scripts\/lib\/(?:clinical-aliases|ranking-tuning|ranking-snapshot-builder)\.ts$/, /^scripts\/fixtures\/(?:rag-retrieval-golden|rag-ranking-candidate-snapshot\.v1)\.json$/, @@ -684,6 +691,8 @@ function selfTest() { // The golden fixture and contract tests are protected surfaces too. assert.equal(classifyPullRequestFiles(["scripts/fixtures/rag-retrieval-golden.json"]).ragRanking, true); assert.equal(classifyPullRequestFiles(["tests/ranking-tuning.test.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/source-authority-registry.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/australian-source-priority.ts"]).ragRanking, true); // Answer synthesis is clinical-risk but NOT rag-ranking (retrieval ordering is the // protected axis here; generation keeps the governance gate only). assert.equal(classifyPullRequestFiles(["src/lib/answer-synthesis.ts"]).ragRanking, false);