feat(aihr): harden finance RAG retrieval and evidence gates

This commit is contained in:
key
2026-08-03 22:34:18 +08:00
parent 99e8c509d5
commit 721839c847
22 changed files with 1306 additions and 153 deletions
+106 -29
View File
@@ -11,6 +11,29 @@ function hitSource(hit) {
return typeof hit === 'object' && hit !== null ? String(hit.sourceName || hit.title || '') : '';
}
export function normalizeSourceName(value) {
return String(value || '')
.replace(/^第\s*\d+\s*段[::]\s*/u, '')
.replace(/\.(?:pdf|docx?|pptx?|xlsx?|txt|md)$/iu, '')
.replace(/\s+/gu, '')
.toLowerCase();
}
function sourceMatches(actual, expected) {
const left = normalizeSourceName(actual);
const right = normalizeSourceName(expected);
return Boolean(left && right && (left.includes(right) || right.includes(left)));
}
function requiredSourceMatches(hit, source, authorities, kinds) {
if (!sourceMatches(hitSource(hit), source)) return false;
if (typeof hit !== 'object' || hit === null) return authorities.length === 0 && kinds.length === 0;
const authority = String(hit.sourceAuthority || '').toUpperCase();
const kind = String(hit.sourceKind || '').toUpperCase();
return (authorities.length === 0 || authorities.includes(authority))
&& (kinds.length === 0 || kinds.includes(kind));
}
export function reciprocalRank(retrieved, relevant) {
const relevantSet = new Set(relevant.map(String));
const rank = retrieved.findIndex((hit) => relevantSet.has(hitId(hit)));
@@ -27,38 +50,68 @@ export function recallAtK(retrieved, relevant) {
export function ndcgAtK(retrieved, relevant) {
const relevantSet = new Set(relevant.map(String));
if (relevantSet.size === 0) return 0;
const dcg = retrieved.reduce((sum, hit, index) =>
sum + (relevantSet.has(hitId(hit)) ? 1 / Math.log2(index + 2) : 0), 0);
const credited = new Set();
const dcg = retrieved.reduce((sum, hit, index) => {
const id = hitId(hit);
if (!relevantSet.has(id) || credited.has(id)) return sum;
credited.add(id);
return sum + 1 / Math.log2(index + 2);
}, 0);
const idealLength = Math.min(relevantSet.size, retrieved.length);
const ideal = Array.from({ length: idealLength }, (_, index) => 1 / Math.log2(index + 2))
.reduce((sum, value) => sum + value, 0);
return ideal === 0 ? 0 : dcg / ideal;
}
export function evaluateCase(retrieved, testCase, k) {
export function evaluateCase(retrieved, testCase, k, actualNoEvidence = retrieved.length === 0) {
const ranked = retrieved.slice(0, k);
const relevant = testCase.relevantFragmentIds || [];
const forbiddenIds = new Set((testCase.forbiddenFragmentIds || []).map(String));
const forbiddenSources = new Set((testCase.forbiddenSourceNames || []).map((value) => value.toLowerCase()));
const requiredSources = new Set((testCase.requiredSourceNames || []).map((value) => value.toLowerCase()));
const returnedSources = new Set(ranked.map(hitSource).filter(Boolean).map((value) => value.toLowerCase()));
const forbiddenLeak = ranked.some((hit) => forbiddenIds.has(hitId(hit))
|| forbiddenSources.has(hitSource(hit).toLowerCase()));
const missingRequiredSource = requiredSources.size > 0
&& ![...requiredSources].some((source) => returnedSources.has(source));
const forbiddenSources = testCase.forbiddenSourceNames || [];
const requiredSources = testCase.requiredSourceNames || [];
const requiredAuthorities = (testCase.requiredSourceAuthorities || []).map((value) => String(value).toUpperCase());
const requiredKinds = (testCase.requiredSourceKinds || []).map((value) => String(value).toUpperCase());
const visibleEvidence = ranked.filter((hit) => typeof hit !== 'object' || hit === null
|| hit.selectedForEvidence === undefined || hit.selectedForEvidence === true);
const forbiddenLeak = visibleEvidence.some((hit) => forbiddenIds.has(hitId(hit))
|| forbiddenSources.some((source) => sourceMatches(hitSource(hit), source)));
const missingRequiredSource = requiredSources.length > 0
&& !requiredSources.some((source) => ranked.some((hit) =>
requiredSourceMatches(hit, source, requiredAuthorities, requiredKinds)));
const sourceRanking = requiredSources.length > 0 && relevant.length === 0
? ranked.map((hit, index) => requiredSources.some((source) =>
requiredSourceMatches(hit, source, requiredAuthorities, requiredKinds))
? normalizeSourceName(hitSource(hit)) : `__nonmatching_source_${index}`) : null;
const normalizedRelevantSources = requiredSources.map(normalizeSourceName);
const metricHits = sourceRanking || ranked;
const metricRelevant = sourceRanking ? normalizedRelevantSources : relevant;
const expectedNoEvidence = Boolean(testCase.expectedNoEvidence);
const retrievalEvaluated = metricRelevant.length > 0;
return {
id: testCase.id,
caseType: testCase.caseType || (expectedNoEvidence ? 'no-answer' : 'answerable'),
answerable: !expectedNoEvidence,
recallAtK: expectedNoEvidence ? null : recallAtK(ranked, relevant),
reciprocalRank: expectedNoEvidence ? null : reciprocalRank(ranked, relevant),
ndcgAtK: expectedNoEvidence ? null : ndcgAtK(ranked, relevant),
answerable: retrievalEvaluated,
retrievalEvaluated,
expectedNoEvidence,
recallAtK: retrievalEvaluated ? recallAtK(metricHits, metricRelevant) : null,
recallAt5: retrievalEvaluated ? recallAtK(metricHits.slice(0, 5), metricRelevant) : null,
recallAt10: retrievalEvaluated ? recallAtK(metricHits.slice(0, 10), metricRelevant) : null,
recallAt20: retrievalEvaluated ? recallAtK(metricHits.slice(0, 20), metricRelevant) : null,
reciprocalRank: retrievalEvaluated ? reciprocalRank(metricHits, metricRelevant) : null,
ndcgAtK: retrievalEvaluated ? ndcgAtK(metricHits, metricRelevant) : null,
forbiddenLeak,
wrongSource: forbiddenLeak || missingRequiredSource,
returned: ranked.length,
noEvidence: ranked.length === 0,
noAnswerFalsePositive: expectedNoEvidence && ranked.length > 0
noEvidence: actualNoEvidence,
noAnswerFalsePositive: expectedNoEvidence && !actualNoEvidence,
topCandidates: ranked.slice(0, 5).map((hit, index) => ({
rank: index + 1,
fragmentId: typeof hit === 'object' && hit !== null ? hit.fragmentId : hit,
sourceName: hitSource(hit),
sourceAuthority: typeof hit === 'object' && hit !== null ? hit.sourceAuthority : null,
sourceKind: typeof hit === 'object' && hit !== null ? hit.sourceKind : null,
selectedForEvidence: typeof hit === 'object' && hit !== null ? hit.selectedForEvidence : undefined
}))
};
}
@@ -71,8 +124,8 @@ function average(rows, key) {
}
export function summarize(results) {
const answerable = results.filter((row) => row.answerable);
const noAnswer = results.filter((row) => !row.answerable);
const answerable = results.filter((row) => row.retrievalEvaluated ?? row.answerable);
const noAnswer = results.filter((row) => row.expectedNoEvidence ?? !row.answerable);
const byCaseType = Object.fromEntries([...new Set(results.map((row) => row.caseType))].sort().map((caseType) => {
const rows = results.filter((row) => row.caseType === caseType);
return [caseType, { cases: rows.length, forbiddenLeakageRate: rate(rows, (row) => row.forbiddenLeak) }];
@@ -82,6 +135,9 @@ export function summarize(results) {
answerableCases: answerable.length,
noAnswerCases: noAnswer.length,
recallAtK: average(answerable, 'recallAtK'),
recallAt5: average(answerable, 'recallAt5'),
recallAt10: average(answerable, 'recallAt10'),
recallAt20: average(answerable, 'recallAt20'),
mrr: average(answerable, 'reciprocalRank'),
ndcgAtK: average(answerable, 'ndcgAtK'),
forbiddenLeakageRate: rate(results, (row) => row.forbiddenLeak),
@@ -155,8 +211,10 @@ export function validateDataset(dataset) {
for (const item of dataset.cases) {
if (!item.id || ids.has(item.id) || !item.query) throw new Error(`invalid or duplicate case: ${item.id || '<missing>'}`);
ids.add(item.id);
if (!item.expectedNoEvidence && (!Array.isArray(item.relevantFragmentIds) || item.relevantFragmentIds.length === 0)) {
throw new Error(`answerable case must define relevantFragmentIds: ${item.id}`);
if (!item.expectedNoEvidence
&& (!Array.isArray(item.relevantFragmentIds) || item.relevantFragmentIds.length === 0)
&& (!Array.isArray(item.requiredSourceNames) || item.requiredSourceNames.length === 0)) {
throw new Error(`answerable case must define relevantFragmentIds or requiredSourceNames: ${item.id}`);
}
if (item.expectedNoEvidence && (item.relevantFragmentIds || []).length > 0) {
throw new Error(`no-answer case cannot define relevantFragmentIds: ${item.id}`);
@@ -168,23 +226,42 @@ async function requestSearch(options, testCase, fetchImpl) {
const headers = { 'content-type': 'application/json' };
if (options.token) headers.Authorization = options.token.startsWith('Bearer ') ? options.token : `Bearer ${options.token}`;
if (options.clientid) headers.clientid = options.clientid;
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, '')}/api/knowledge/search`, {
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, '')}/api/knowledge/query`, {
method: 'POST', headers,
body: JSON.stringify({
queryText: testCase.query,
category: testCase.category || undefined,
position: testCase.position || undefined,
source: 'knowledge_search',
limit: options.k
source: testCase.source || 'mobile_uni_agent',
// Response display remains capped at 10; retrievalCandidates is the decoupled pre-generation pool.
limit: Math.min(options.k, 10)
})
});
const body = await response.json();
if (!response.ok || body.code !== 200) {
throw new Error(`case ${testCase.id} search failed with HTTP ${response.status} code ${body.code}`);
throw new Error(`case ${testCase.id} search failed with HTTP ${response.status} code ${body.code}: ${body.msg || ''}`);
}
const snippets = body.data?.snippets || [];
return snippets.filter((snippet) => snippet.fragmentId !== null && snippet.fragmentId !== undefined)
.map((snippet) => ({ fragmentId: snippet.fragmentId, sourceName: snippet.title || '' }));
const candidates = body.data?.retrievalCandidates;
const fallback = body.data?.citations || body.data?.legacy?.snippets || [];
const rows = Array.isArray(candidates) && candidates.length > 0
? [...candidates].sort((left, right) => {
const leftRank = Number.isInteger(left.candidateRank) ? left.candidateRank : Number.MAX_SAFE_INTEGER;
const rightRank = Number.isInteger(right.candidateRank) ? right.candidateRank : Number.MAX_SAFE_INTEGER;
return leftRank - rightRank;
})
: fallback;
return {
noEvidence: Boolean(body.data?.noEvidence),
retrieved: rows.filter((item) => item.fragmentId !== null && item.fragmentId !== undefined)
.map((item) => ({
fragmentId: item.fragmentId,
sourceName: item.title || '',
sourceAuthority: item.sourceAuthority || null,
sourceKind: item.sourceKind || null,
candidateRank: item.candidateRank,
selectedForEvidence: item.selectedForEvidence
}))
};
}
export async function run(options, dependencies = {}) {
@@ -197,8 +274,8 @@ export async function run(options, dependencies = {}) {
const fetchImpl = dependencies.fetchImpl || globalThis.fetch;
const results = [];
for (const testCase of dataset.cases) {
const retrieved = await requestSearch(options, testCase, fetchImpl);
results.push(evaluateCase(retrieved, testCase, options.k));
const result = await requestSearch(options, testCase, fetchImpl);
results.push(evaluateCase(result.retrieved, testCase, options.k, result.noEvidence));
}
const summary = summarize(results);
const failures = thresholdFailures(summary, options);
+44 -2
View File
@@ -1,13 +1,15 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
evaluateCase, ndcgAtK, parseArgs, recallAtK, reciprocalRank, summarize, thresholdFailures, validateDataset
evaluateCase, ndcgAtK, normalizeSourceName, parseArgs, recallAtK, reciprocalRank, summarize,
thresholdFailures, validateDataset
} from './evaluate-knowledge-quality.mjs';
test('ranking metrics use fragment ids and preserve top-k order', () => {
assert.equal(recallAtK(['a', 'b', 'c'], ['b', 'd']), 0.5);
assert.equal(reciprocalRank(['a', 'b'], ['b']), 0.5);
assert.ok(ndcgAtK(['b', 'a'], ['b', 'c']) > 0.61);
assert.equal(ndcgAtK(['b', 'b', 'b'], ['b']), 1);
});
test('forbidden fragments and sources are surfaced as leakage', () => {
@@ -25,7 +27,7 @@ test('forbidden fragments and sources are surfaced as leakage', () => {
test('no-answer cases measure false positives separately from answerable recall', () => {
const answerable = evaluateCase([{ fragmentId: 1, sourceName: 'SOP' }], {
id: 'answerable', relevantFragmentIds: [1], requiredSourceNames: ['SOP']
}, 5);
}, 5, false);
const noAnswer = evaluateCase([{ fragmentId: 9, sourceName: 'Unrelated' }], {
id: 'no-answer', expectedNoEvidence: true, relevantFragmentIds: [], caseType: 'no-answer'
}, 5);
@@ -35,6 +37,45 @@ test('no-answer cases measure false positives separately from answerable recall'
assert.equal(summary.noAnswerPrecision, 0);
});
test('candidate recall is independent from the final no-evidence gate and display prefixes', () => {
const result = evaluateCase([
{
fragmentId: 122126,
sourceName: '第 3 段:银城物业费用报销发票管理操作手册.pdf',
sourceAuthority: 'COMPANY_POLICY',
sourceKind: 'OPERATING_MANUAL',
selectedForEvidence: false
}
], {
id: 'finance',
requiredSourceNames: ['银城物业费用报销发票管理操作手册'],
requiredSourceAuthorities: ['COMPANY_POLICY'],
requiredSourceKinds: ['OPERATING_MANUAL'],
expectedNoEvidence: false
}, 20, true);
assert.equal(result.recallAt5, 1);
assert.equal(result.noEvidence, true);
assert.equal(normalizeSourceName('第 12 段:制度.PDF'), '制度');
});
test('same-name ungoverned copies do not satisfy formal source recall', () => {
const result = evaluateCase([
{
fragmentId: 102169,
sourceName: '银城物业费用报销发票管理操作手册.pdf',
sourceAuthority: 'UNKNOWN',
sourceKind: 'REFERENCE_MATERIAL'
}
], {
id: 'finance-authority',
requiredSourceNames: ['银城物业费用报销发票管理操作手册'],
requiredSourceAuthorities: ['COMPANY_POLICY'],
requiredSourceKinds: ['OPERATING_MANUAL']
}, 20, true);
assert.equal(result.recallAt20, 0);
assert.equal(result.wrongSource, true);
});
test('threshold failures can fail a release gate', () => {
const failures = thresholdFailures({
recallAtK: 0.7, mrr: 0.5, ndcgAtK: 0.6, forbiddenLeakageRate: 0.1,
@@ -49,6 +90,7 @@ test('threshold failures can fail a release gate', () => {
test('dataset contract distinguishes answerable and no-answer cases', () => {
assert.doesNotThrow(() => validateDataset({ schemaVersion: 1, cases: [
{ id: 'a', query: 'q', relevantFragmentIds: [1] },
{ id: 'source', query: 'q-source', requiredSourceNames: ['正式手册'] },
{ id: 'b', query: 'q2', relevantFragmentIds: [], expectedNoEvidence: true }
] }));
assert.throws(() => validateDataset({ schemaVersion: 1, cases: [
+3 -1
View File
@@ -275,6 +275,7 @@ FROM (
UNION ALL SELECT 'aihr_review_batch'
UNION ALL SELECT 'aihr_review_batch_item'
UNION ALL SELECT 'aihr_knowledge_generation'
UNION ALL SELECT 'aihr_query_candidate_trace'
UNION ALL SELECT 'aihr_generation_version'
UNION ALL SELECT 'aihr_version_diff'
UNION ALL SELECT 'aihr_version_rollback'
@@ -948,7 +949,8 @@ REMOTE
[[ "$media_reprocess_contract" = "source_attach_id|tenant_id,source_attach_id,id" ]] \
|| fail "remote media-reprocess schema is invalid: got ${media_reprocess_contract:-missing}"
echo "remote_schema=82/82 $remote_ssh:$remote_db"
echo "remote_schema=83/83 $remote_ssh:$remote_db"
echo "remote_rag_candidate_trace=1/1 $remote_ssh:$remote_db"
echo "remote_media_reprocess_contract=1/1 $remote_ssh:$remote_db"
echo "remote_work_report_idempotency=3/3 $remote_ssh:$remote_db"
echo "remote_learning_task_question_bank=6/6 $remote_ssh:$remote_db"
+1
View File
@@ -36,6 +36,7 @@ docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260812_pipeline_run_sampling_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260813_golden_calibration_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260814_knowledge_privacy_derivative_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260815_rag_candidate_trace_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260815_knowledge_rollout_compat_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260816_knowledge_version_pointers_mysql8.sql"
docker exec -i wygj-mysql mysql -uroot -proot --default-character-set=utf8mb4 ry-vue < "$ROOT_DIR/backend/script/sql/update/aihr_20260817_knowledge_review_assistance_mysql8.sql"
@@ -655,4 +655,13 @@ test "$agent_run_unique" = '0|run_id'
agent_user_index="$(mysql "$DB" -N -B -e "SELECT GROUP_CONCAT(column_name ORDER BY seq_in_index) FROM information_schema.statistics WHERE table_schema='$DB' AND table_name='aihr_agent_run' AND index_name='idx_aihr_agent_run_user';")"
test "$agent_user_index" = 'tenant_id,user_id,create_time'
rag_candidate_trace_migration="$ROOT_DIR/backend/script/sql/update/aihr_20260815_rag_candidate_trace_mysql8.sql"
test -f "$rag_candidate_trace_migration"
mysql "$DB" < "$rag_candidate_trace_migration"
mysql "$DB" < "$rag_candidate_trace_migration"
rag_candidate_trace_columns="$(mysql "$DB" -N -B -e "SELECT GROUP_CONCAT(column_name ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='$DB' AND table_name='aihr_query_candidate_trace';")"
test "$rag_candidate_trace_columns" = 'id,tenant_id,request_id,fragment_id,candidate_rank,channel,channel_rank,raw_score,fusion_score,rerank_score,rerank_model,rerank_applied,source_authority,source_kind,query_variant,selected_for_evidence,decision_reason,create_time'
rag_candidate_trace_unique="$(mysql "$DB" -N -B -e "SELECT CONCAT(MIN(non_unique),'|',GROUP_CONCAT(column_name ORDER BY seq_in_index)) FROM information_schema.statistics WHERE table_schema='$DB' AND table_name='aihr_query_candidate_trace' AND index_name='uk_aihr_query_candidate_channel';")"
test "$rag_candidate_trace_unique" = '0|tenant_id,request_id,fragment_id,channel,query_variant'
echo "PASS: AIHR legacy schema migrations are idempotent, including Agent run audit, practice growth curriculum snapshots, knowledge-space categories, practice evidence, knowledge feedback, candidate interview review, position/SOP qualification contracts, assistant capture, work-report idempotency, direct feedback, broadcast publish/withdraw audit with targeting, delivery controls, topic evidence and file access audit, v1 snapshots, and trusted broadcast question context"
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env node
import { run } from './evaluate-knowledge-quality.mjs';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const baseUrl = process.env.AIHR_BASE_URL || 'https://wygj-api.localhost';
const clientid = '428a8310cd442757ae699df5d894f051';
const dataset = new URL('../tests/fixtures/data_quality/retrieval-finance-regression.json', import.meta.url);
function localUniqueActivePhone() {
const sql = `
select min(s.person_phone)
from aihr_org_snapshot s
where s.tenant_id = '000000'
and lower(coalesce(s.employment_status, 'active')) in ('active', '在职')
and s.person_phone regexp '^[0-9]{11}$'
and (select count(distinct p.ext_party_id) from aihr_org_snapshot p
where p.tenant_id = s.tenant_id and p.person_phone = s.person_phone
and lower(coalesce(p.employment_status, 'active')) in ('active', '在职')) = 1
and (select count(distinct e.person_phone) from aihr_org_snapshot e
where e.tenant_id = s.tenant_id and e.ext_party_id = s.ext_party_id
and lower(coalesce(e.employment_status, 'active')) in ('active', '在职')) = 1`;
return execFileSync('docker', [
'exec', 'wygj-mysql', 'mysql', '-uroot', '-proot', '--default-character-set=utf8mb4',
'ry-vue', '-Nse', sql
], { encoding: 'utf8' }).trim();
}
const phone = process.env.AIHR_VERIFY_PHONE || localUniqueActivePhone();
if (!phone) throw new Error('no bidirectionally unique active local employee is available');
execFileSync('docker', [
'exec', 'wygj-redis', 'redis-cli', '-a', 'ruoyi123', 'del',
`{global:rate_limit:/resource/sms/code:${phone}}:value`,
`global:rate_limit:/resource/sms/code:${phone}`,
`{global:rate_limit:/resource/sms/code:${phone}}:permits`
], { stdio: 'ignore' });
if (new URL(baseUrl).hostname.endsWith('.localhost')) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
}
async function api(endpoint, init = {}) {
const response = await fetch(`${baseUrl.replace(/\/$/, '')}${endpoint}`, {
...init,
headers: { clientid, ...(init.headers || {}) },
signal: AbortSignal.timeout(30_000)
});
const payload = await response.json();
if (!response.ok || Number(payload.code) !== 200) {
throw new Error(`${endpoint} failed: HTTP ${response.status}, code ${payload.code}, ${payload.msg || ''}`);
}
return payload.data;
}
await api(`/resource/sms/code?phonenumber=${encodeURIComponent(phone)}`);
const login = await api('/auth/mobile/sms-login', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
phonenumber: phone,
smsCode: process.env.AIHR_VERIFY_SMS_CODE || '123456',
tenantId: '000000'
})
});
if (!login?.access_token) throw new Error('mobile login returned no access token');
const result = await run({
dataset: fileURLToPath(dataset),
baseUrl,
token: login.access_token,
clientid,
k: 20
});
console.log(JSON.stringify(result, null, 2));
if (result.thresholdPassed === false) process.exitCode = 2;