feat(aihr): harden finance RAG retrieval and evidence gates
This commit is contained in:
key
@@ -11,6 +11,29 @@ function hitSource(hit) {
|
||||
return typeof hit === 'object' && hit !== null ? String(hit.sourceName || hit.title || '') : '';
|
||||
}
|
||||
|
||||
export function normalizeSourceName(value) {
|
||||
return String(value || '')
|
||||
.replace(/^第\s*\d+\s*段[：:]\s*/u, '')
|
||||
.replace(/\.(?:pdf|docx?|pptx?|xlsx?|txt|md)$/iu, '')
|
||||
.replace(/\s+/gu, '')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function sourceMatches(actual, expected) {
|
||||
const left = normalizeSourceName(actual);
|
||||
const right = normalizeSourceName(expected);
|
||||
return Boolean(left && right && (left.includes(right) || right.includes(left)));
|
||||
}
|
||||
|
||||
function requiredSourceMatches(hit, source, authorities, kinds) {
|
||||
if (!sourceMatches(hitSource(hit), source)) return false;
|
||||
if (typeof hit !== 'object' || hit === null) return authorities.length === 0 && kinds.length === 0;
|
||||
const authority = String(hit.sourceAuthority || '').toUpperCase();
|
||||
const kind = String(hit.sourceKind || '').toUpperCase();
|
||||
return (authorities.length === 0 || authorities.includes(authority))
|
||||
&& (kinds.length === 0 || kinds.includes(kind));
|
||||
}
|
||||
|
||||
export function reciprocalRank(retrieved, relevant) {
|
||||
const relevantSet = new Set(relevant.map(String));
|
||||
const rank = retrieved.findIndex((hit) => relevantSet.has(hitId(hit)));
|
||||
@@ -27,38 +50,68 @@ export function recallAtK(retrieved, relevant) {
|
||||
export function ndcgAtK(retrieved, relevant) {
|
||||
const relevantSet = new Set(relevant.map(String));
|
||||
if (relevantSet.size === 0) return 0;
|
||||
const dcg = retrieved.reduce((sum, hit, index) =>
|
||||
sum + (relevantSet.has(hitId(hit)) ? 1 / Math.log2(index + 2) : 0), 0);
|
||||
const credited = new Set();
|
||||
const dcg = retrieved.reduce((sum, hit, index) => {
|
||||
const id = hitId(hit);
|
||||
if (!relevantSet.has(id) || credited.has(id)) return sum;
|
||||
credited.add(id);
|
||||
return sum + 1 / Math.log2(index + 2);
|
||||
}, 0);
|
||||
const idealLength = Math.min(relevantSet.size, retrieved.length);
|
||||
const ideal = Array.from({ length: idealLength }, (_, index) => 1 / Math.log2(index + 2))
|
||||
.reduce((sum, value) => sum + value, 0);
|
||||
return ideal === 0 ? 0 : dcg / ideal;
|
||||
}
|
||||
|
||||
export function evaluateCase(retrieved, testCase, k) {
|
||||
export function evaluateCase(retrieved, testCase, k, actualNoEvidence = retrieved.length === 0) {
|
||||
const ranked = retrieved.slice(0, k);
|
||||
const relevant = testCase.relevantFragmentIds || [];
|
||||
const forbiddenIds = new Set((testCase.forbiddenFragmentIds || []).map(String));
|
||||
const forbiddenSources = new Set((testCase.forbiddenSourceNames || []).map((value) => value.toLowerCase()));
|
||||
const requiredSources = new Set((testCase.requiredSourceNames || []).map((value) => value.toLowerCase()));
|
||||
const returnedSources = new Set(ranked.map(hitSource).filter(Boolean).map((value) => value.toLowerCase()));
|
||||
const forbiddenLeak = ranked.some((hit) => forbiddenIds.has(hitId(hit))
|
||||
|| forbiddenSources.has(hitSource(hit).toLowerCase()));
|
||||
const missingRequiredSource = requiredSources.size > 0
|
||||
&& ![...requiredSources].some((source) => returnedSources.has(source));
|
||||
const forbiddenSources = testCase.forbiddenSourceNames || [];
|
||||
const requiredSources = testCase.requiredSourceNames || [];
|
||||
const requiredAuthorities = (testCase.requiredSourceAuthorities || []).map((value) => String(value).toUpperCase());
|
||||
const requiredKinds = (testCase.requiredSourceKinds || []).map((value) => String(value).toUpperCase());
|
||||
const visibleEvidence = ranked.filter((hit) => typeof hit !== 'object' || hit === null
|
||||
|| hit.selectedForEvidence === undefined || hit.selectedForEvidence === true);
|
||||
const forbiddenLeak = visibleEvidence.some((hit) => forbiddenIds.has(hitId(hit))
|
||||
|| forbiddenSources.some((source) => sourceMatches(hitSource(hit), source)));
|
||||
const missingRequiredSource = requiredSources.length > 0
|
||||
&& !requiredSources.some((source) => ranked.some((hit) =>
|
||||
requiredSourceMatches(hit, source, requiredAuthorities, requiredKinds)));
|
||||
const sourceRanking = requiredSources.length > 0 && relevant.length === 0
|
||||
? ranked.map((hit, index) => requiredSources.some((source) =>
|
||||
requiredSourceMatches(hit, source, requiredAuthorities, requiredKinds))
|
||||
? normalizeSourceName(hitSource(hit)) : `__nonmatching_source_${index}`) : null;
|
||||
const normalizedRelevantSources = requiredSources.map(normalizeSourceName);
|
||||
const metricHits = sourceRanking || ranked;
|
||||
const metricRelevant = sourceRanking ? normalizedRelevantSources : relevant;
|
||||
const expectedNoEvidence = Boolean(testCase.expectedNoEvidence);
|
||||
const retrievalEvaluated = metricRelevant.length > 0;
|
||||
return {
|
||||
id: testCase.id,
|
||||
caseType: testCase.caseType || (expectedNoEvidence ? 'no-answer' : 'answerable'),
|
||||
answerable: !expectedNoEvidence,
|
||||
recallAtK: expectedNoEvidence ? null : recallAtK(ranked, relevant),
|
||||
reciprocalRank: expectedNoEvidence ? null : reciprocalRank(ranked, relevant),
|
||||
ndcgAtK: expectedNoEvidence ? null : ndcgAtK(ranked, relevant),
|
||||
answerable: retrievalEvaluated,
|
||||
retrievalEvaluated,
|
||||
expectedNoEvidence,
|
||||
recallAtK: retrievalEvaluated ? recallAtK(metricHits, metricRelevant) : null,
|
||||
recallAt5: retrievalEvaluated ? recallAtK(metricHits.slice(0, 5), metricRelevant) : null,
|
||||
recallAt10: retrievalEvaluated ? recallAtK(metricHits.slice(0, 10), metricRelevant) : null,
|
||||
recallAt20: retrievalEvaluated ? recallAtK(metricHits.slice(0, 20), metricRelevant) : null,
|
||||
reciprocalRank: retrievalEvaluated ? reciprocalRank(metricHits, metricRelevant) : null,
|
||||
ndcgAtK: retrievalEvaluated ? ndcgAtK(metricHits, metricRelevant) : null,
|
||||
forbiddenLeak,
|
||||
wrongSource: forbiddenLeak || missingRequiredSource,
|
||||
returned: ranked.length,
|
||||
noEvidence: ranked.length === 0,
|
||||
noAnswerFalsePositive: expectedNoEvidence && ranked.length > 0
|
||||
noEvidence: actualNoEvidence,
|
||||
noAnswerFalsePositive: expectedNoEvidence && !actualNoEvidence,
|
||||
topCandidates: ranked.slice(0, 5).map((hit, index) => ({
|
||||
rank: index + 1,
|
||||
fragmentId: typeof hit === 'object' && hit !== null ? hit.fragmentId : hit,
|
||||
sourceName: hitSource(hit),
|
||||
sourceAuthority: typeof hit === 'object' && hit !== null ? hit.sourceAuthority : null,
|
||||
sourceKind: typeof hit === 'object' && hit !== null ? hit.sourceKind : null,
|
||||
selectedForEvidence: typeof hit === 'object' && hit !== null ? hit.selectedForEvidence : undefined
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -71,8 +124,8 @@ function average(rows, key) {
|
||||
}
|
||||
|
||||
export function summarize(results) {
|
||||
const answerable = results.filter((row) => row.answerable);
|
||||
const noAnswer = results.filter((row) => !row.answerable);
|
||||
const answerable = results.filter((row) => row.retrievalEvaluated ?? row.answerable);
|
||||
const noAnswer = results.filter((row) => row.expectedNoEvidence ?? !row.answerable);
|
||||
const byCaseType = Object.fromEntries([...new Set(results.map((row) => row.caseType))].sort().map((caseType) => {
|
||||
const rows = results.filter((row) => row.caseType === caseType);
|
||||
return [caseType, { cases: rows.length, forbiddenLeakageRate: rate(rows, (row) => row.forbiddenLeak) }];
|
||||
@@ -82,6 +135,9 @@ export function summarize(results) {
|
||||
answerableCases: answerable.length,
|
||||
noAnswerCases: noAnswer.length,
|
||||
recallAtK: average(answerable, 'recallAtK'),
|
||||
recallAt5: average(answerable, 'recallAt5'),
|
||||
recallAt10: average(answerable, 'recallAt10'),
|
||||
recallAt20: average(answerable, 'recallAt20'),
|
||||
mrr: average(answerable, 'reciprocalRank'),
|
||||
ndcgAtK: average(answerable, 'ndcgAtK'),
|
||||
forbiddenLeakageRate: rate(results, (row) => row.forbiddenLeak),
|
||||
@@ -155,8 +211,10 @@ export function validateDataset(dataset) {
|
||||
for (const item of dataset.cases) {
|
||||
if (!item.id || ids.has(item.id) || !item.query) throw new Error(`invalid or duplicate case: ${item.id || '<missing>'}`);
|
||||
ids.add(item.id);
|
||||
if (!item.expectedNoEvidence && (!Array.isArray(item.relevantFragmentIds) || item.relevantFragmentIds.length === 0)) {
|
||||
throw new Error(`answerable case must define relevantFragmentIds: ${item.id}`);
|
||||
if (!item.expectedNoEvidence
|
||||
&& (!Array.isArray(item.relevantFragmentIds) || item.relevantFragmentIds.length === 0)
|
||||
&& (!Array.isArray(item.requiredSourceNames) || item.requiredSourceNames.length === 0)) {
|
||||
throw new Error(`answerable case must define relevantFragmentIds or requiredSourceNames: ${item.id}`);
|
||||
}
|
||||
if (item.expectedNoEvidence && (item.relevantFragmentIds || []).length > 0) {
|
||||
throw new Error(`no-answer case cannot define relevantFragmentIds: ${item.id}`);
|
||||
@@ -168,23 +226,42 @@ async function requestSearch(options, testCase, fetchImpl) {
|
||||
const headers = { 'content-type': 'application/json' };
|
||||
if (options.token) headers.Authorization = options.token.startsWith('Bearer ') ? options.token : `Bearer ${options.token}`;
|
||||
if (options.clientid) headers.clientid = options.clientid;
|
||||
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, '')}/api/knowledge/search`, {
|
||||
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, '')}/api/knowledge/query`, {
|
||||
method: 'POST', headers,
|
||||
body: JSON.stringify({
|
||||
queryText: testCase.query,
|
||||
category: testCase.category || undefined,
|
||||
position: testCase.position || undefined,
|
||||
source: 'knowledge_search',
|
||||
limit: options.k
|
||||
source: testCase.source || 'mobile_uni_agent',
|
||||
// Response display remains capped at 10; retrievalCandidates is the decoupled pre-generation pool.
|
||||
limit: Math.min(options.k, 10)
|
||||
})
|
||||
});
|
||||
const body = await response.json();
|
||||
if (!response.ok || body.code !== 200) {
|
||||
throw new Error(`case ${testCase.id} search failed with HTTP ${response.status} code ${body.code}`);
|
||||
throw new Error(`case ${testCase.id} search failed with HTTP ${response.status} code ${body.code}: ${body.msg || ''}`);
|
||||
}
|
||||
const snippets = body.data?.snippets || [];
|
||||
return snippets.filter((snippet) => snippet.fragmentId !== null && snippet.fragmentId !== undefined)
|
||||
.map((snippet) => ({ fragmentId: snippet.fragmentId, sourceName: snippet.title || '' }));
|
||||
const candidates = body.data?.retrievalCandidates;
|
||||
const fallback = body.data?.citations || body.data?.legacy?.snippets || [];
|
||||
const rows = Array.isArray(candidates) && candidates.length > 0
|
||||
? [...candidates].sort((left, right) => {
|
||||
const leftRank = Number.isInteger(left.candidateRank) ? left.candidateRank : Number.MAX_SAFE_INTEGER;
|
||||
const rightRank = Number.isInteger(right.candidateRank) ? right.candidateRank : Number.MAX_SAFE_INTEGER;
|
||||
return leftRank - rightRank;
|
||||
})
|
||||
: fallback;
|
||||
return {
|
||||
noEvidence: Boolean(body.data?.noEvidence),
|
||||
retrieved: rows.filter((item) => item.fragmentId !== null && item.fragmentId !== undefined)
|
||||
.map((item) => ({
|
||||
fragmentId: item.fragmentId,
|
||||
sourceName: item.title || '',
|
||||
sourceAuthority: item.sourceAuthority || null,
|
||||
sourceKind: item.sourceKind || null,
|
||||
candidateRank: item.candidateRank,
|
||||
selectedForEvidence: item.selectedForEvidence
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
export async function run(options, dependencies = {}) {
|
||||
@@ -197,8 +274,8 @@ export async function run(options, dependencies = {}) {
|
||||
const fetchImpl = dependencies.fetchImpl || globalThis.fetch;
|
||||
const results = [];
|
||||
for (const testCase of dataset.cases) {
|
||||
const retrieved = await requestSearch(options, testCase, fetchImpl);
|
||||
results.push(evaluateCase(retrieved, testCase, options.k));
|
||||
const result = await requestSearch(options, testCase, fetchImpl);
|
||||
results.push(evaluateCase(result.retrieved, testCase, options.k, result.noEvidence));
|
||||
}
|
||||
const summary = summarize(results);
|
||||
const failures = thresholdFailures(summary, options);
|
||||
|
||||
Reference in New Issue
Block a user