feat: govern knowledge assets and source citations

2026-08-02 01:43:43 +08:00
parent cafb836cda
commit 699cc08050
144 changed files with 17205 additions and 453 deletions
@@ -0,0 +1,61 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  evaluateCase, ndcgAtK, parseArgs, recallAtK, reciprocalRank, summarize, thresholdFailures, validateDataset
+} from './evaluate-knowledge-quality.mjs';
+
+test('ranking metrics use fragment ids and preserve top-k order', () => {
+  assert.equal(recallAtK(['a', 'b', 'c'], ['b', 'd']), 0.5);
+  assert.equal(reciprocalRank(['a', 'b'], ['b']), 0.5);
+  assert.ok(ndcgAtK(['b', 'a'], ['b', 'c']) > 0.61);
+});
+
+test('forbidden fragments and sources are surfaced as leakage', () => {
+  const result = evaluateCase([
+    { fragmentId: 101, sourceName: 'Current SOP' },
+    { fragmentId: 202, sourceName: 'Other tenant policy' }
+  ], {
+    id: 'cross-tenant', relevantFragmentIds: [303], forbiddenFragmentIds: [202],
+    forbiddenSourceNames: ['Other tenant policy'], caseType: 'cross-tenant'
+  }, 5);
+  assert.equal(result.forbiddenLeak, true);
+  assert.equal(summarize([result]).forbiddenLeakageRate, 1);
+});
+
+test('no-answer cases measure false positives separately from answerable recall', () => {
+  const answerable = evaluateCase([{ fragmentId: 1, sourceName: 'SOP' }], {
+    id: 'answerable', relevantFragmentIds: [1], requiredSourceNames: ['SOP']
+  }, 5);
+  const noAnswer = evaluateCase([{ fragmentId: 9, sourceName: 'Unrelated' }], {
+    id: 'no-answer', expectedNoEvidence: true, relevantFragmentIds: [], caseType: 'no-answer'
+  }, 5);
+  const summary = summarize([answerable, noAnswer]);
+  assert.equal(summary.recallAtK, 1);
+  assert.equal(summary.noAnswerFalsePositiveRate, 1);
+  assert.equal(summary.noAnswerPrecision, 0);
+});
+
+test('threshold failures can fail a release gate', () => {
+  const failures = thresholdFailures({
+    recallAtK: 0.7, mrr: 0.5, ndcgAtK: 0.6, forbiddenLeakageRate: 0.1,
+    wrongSourceRate: 0.15, noAnswerFalsePositiveRate: 0.2
+  }, {
+    minRecall: 0.8, minMrr: 0.6, minNdcg: 0.7, maxLeakage: 0,
+    maxWrongSource: 0.1, maxNoAnswerFalsePositive: 0.1
+  });
+  assert.equal(failures.length, 6);
+});
+
+test('dataset contract distinguishes answerable and no-answer cases', () => {
+  assert.doesNotThrow(() => validateDataset({ schemaVersion: 1, cases: [
+    { id: 'a', query: 'q', relevantFragmentIds: [1] },
+    { id: 'b', query: 'q2', relevantFragmentIds: [], expectedNoEvidence: true }
+  ] }));
+  assert.throws(() => validateDataset({ schemaVersion: 1, cases: [
+    { id: 'bad', query: 'q', relevantFragmentIds: [] }
+  ] }), /answerable case/);
+  assert.equal(parseArgs(['--dataset', 'fixture.json', '--min-recall', '0.8']).minRecall, 0.8);
+  const thresholds = parseArgs(['--dataset', 'fixture.json', '--min-mrr', '0.7', '--max-wrong-source', '0.05']);
+  assert.equal(thresholds.minMrr, 0.7);
+  assert.equal(thresholds.maxWrongSource, 0.05);
+});