104 lines
4.2 KiB
JavaScript
104 lines
4.2 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
evaluateCase, ndcgAtK, normalizeSourceName, parseArgs, recallAtK, reciprocalRank, summarize,
|
|
thresholdFailures, validateDataset
|
|
} from './evaluate-knowledge-quality.mjs';
|
|
|
|
test('ranking metrics use fragment ids and preserve top-k order', () => {
|
|
assert.equal(recallAtK(['a', 'b', 'c'], ['b', 'd']), 0.5);
|
|
assert.equal(reciprocalRank(['a', 'b'], ['b']), 0.5);
|
|
assert.ok(ndcgAtK(['b', 'a'], ['b', 'c']) > 0.61);
|
|
assert.equal(ndcgAtK(['b', 'b', 'b'], ['b']), 1);
|
|
});
|
|
|
|
test('forbidden fragments and sources are surfaced as leakage', () => {
|
|
const result = evaluateCase([
|
|
{ fragmentId: 101, sourceName: 'Current SOP' },
|
|
{ fragmentId: 202, sourceName: 'Other tenant policy' }
|
|
], {
|
|
id: 'cross-tenant', relevantFragmentIds: [303], forbiddenFragmentIds: [202],
|
|
forbiddenSourceNames: ['Other tenant policy'], caseType: 'cross-tenant'
|
|
}, 5);
|
|
assert.equal(result.forbiddenLeak, true);
|
|
assert.equal(summarize([result]).forbiddenLeakageRate, 1);
|
|
});
|
|
|
|
test('no-answer cases measure false positives separately from answerable recall', () => {
|
|
const answerable = evaluateCase([{ fragmentId: 1, sourceName: 'SOP' }], {
|
|
id: 'answerable', relevantFragmentIds: [1], requiredSourceNames: ['SOP']
|
|
}, 5, false);
|
|
const noAnswer = evaluateCase([{ fragmentId: 9, sourceName: 'Unrelated' }], {
|
|
id: 'no-answer', expectedNoEvidence: true, relevantFragmentIds: [], caseType: 'no-answer'
|
|
}, 5);
|
|
const summary = summarize([answerable, noAnswer]);
|
|
assert.equal(summary.recallAtK, 1);
|
|
assert.equal(summary.noAnswerFalsePositiveRate, 1);
|
|
assert.equal(summary.noAnswerPrecision, 0);
|
|
});
|
|
|
|
test('candidate recall is independent from the final no-evidence gate and display prefixes', () => {
|
|
const result = evaluateCase([
|
|
{
|
|
fragmentId: 122126,
|
|
sourceName: '第 3 段：银城物业费用报销发票管理操作手册.pdf',
|
|
sourceAuthority: 'COMPANY_POLICY',
|
|
sourceKind: 'OPERATING_MANUAL',
|
|
selectedForEvidence: false
|
|
}
|
|
], {
|
|
id: 'finance',
|
|
requiredSourceNames: ['银城物业费用报销发票管理操作手册'],
|
|
requiredSourceAuthorities: ['COMPANY_POLICY'],
|
|
requiredSourceKinds: ['OPERATING_MANUAL'],
|
|
expectedNoEvidence: false
|
|
}, 20, true);
|
|
assert.equal(result.recallAt5, 1);
|
|
assert.equal(result.noEvidence, true);
|
|
assert.equal(normalizeSourceName('第 12 段：制度.PDF'), '制度');
|
|
});
|
|
|
|
test('same-name ungoverned copies do not satisfy formal source recall', () => {
|
|
const result = evaluateCase([
|
|
{
|
|
fragmentId: 102169,
|
|
sourceName: '银城物业费用报销发票管理操作手册.pdf',
|
|
sourceAuthority: 'UNKNOWN',
|
|
sourceKind: 'REFERENCE_MATERIAL'
|
|
}
|
|
], {
|
|
id: 'finance-authority',
|
|
requiredSourceNames: ['银城物业费用报销发票管理操作手册'],
|
|
requiredSourceAuthorities: ['COMPANY_POLICY'],
|
|
requiredSourceKinds: ['OPERATING_MANUAL']
|
|
}, 20, true);
|
|
assert.equal(result.recallAt20, 0);
|
|
assert.equal(result.wrongSource, true);
|
|
});
|
|
|
|
test('threshold failures can fail a release gate', () => {
|
|
const failures = thresholdFailures({
|
|
recallAtK: 0.7, mrr: 0.5, ndcgAtK: 0.6, forbiddenLeakageRate: 0.1,
|
|
wrongSourceRate: 0.15, noAnswerFalsePositiveRate: 0.2
|
|
}, {
|
|
minRecall: 0.8, minMrr: 0.6, minNdcg: 0.7, maxLeakage: 0,
|
|
maxWrongSource: 0.1, maxNoAnswerFalsePositive: 0.1
|
|
});
|
|
assert.equal(failures.length, 6);
|
|
});
|
|
|
|
test('dataset contract distinguishes answerable and no-answer cases', () => {
|
|
assert.doesNotThrow(() => validateDataset({ schemaVersion: 1, cases: [
|
|
{ id: 'a', query: 'q', relevantFragmentIds: [1] },
|
|
{ id: 'source', query: 'q-source', requiredSourceNames: ['正式手册'] },
|
|
{ id: 'b', query: 'q2', relevantFragmentIds: [], expectedNoEvidence: true }
|
|
] }));
|
|
assert.throws(() => validateDataset({ schemaVersion: 1, cases: [
|
|
{ id: 'bad', query: 'q', relevantFragmentIds: [] }
|
|
] }), /answerable case/);
|
|
assert.equal(parseArgs(['--dataset', 'fixture.json', '--min-recall', '0.8']).minRecall, 0.8);
|
|
const thresholds = parseArgs(['--dataset', 'fixture.json', '--min-mrr', '0.7', '--max-wrong-source', '0.05']);
|
|
assert.equal(thresholds.minMrr, 0.7);
|
|
assert.equal(thresholds.maxWrongSource, 0.05);
|
|
});
|