feat(aihr): harden finance RAG retrieval and evidence gates

This commit is contained in:
key
2026-08-03 22:34:18 +08:00
parent 99e8c509d5
commit 721839c847
22 changed files with 1306 additions and 153 deletions
@@ -57,22 +57,46 @@ public final class AihrKnowledgeQueryDto {
String updatedAt,
String mediaType,
String detailRef,
LocatorSummary locatorSummary
LocatorSummary locatorSummary,
String sourceAuthority,
String sourceKind
) {
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
Long fragmentId, String domain, String status, String occurredAt, String updatedAt,
String mediaType, String detailRef, LocatorSummary locatorSummary) {
this(spaceCode, sourceType, docId, title, snippet, fragmentId, domain, status, occurredAt,
updatedAt, mediaType, detailRef, locatorSummary, null, null);
}
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
Long fragmentId) {
this(spaceCode, sourceType, docId, title, snippet, fragmentId,
"DOCUMENT".equals(sourceType) ? "ENTERPRISE" : sourceType, null, null, null,
null, null, null);
null, null, null, null, null);
}
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
Long fragmentId, String domain, String status, String occurredAt, String updatedAt) {
this(spaceCode, sourceType, docId, title, snippet, fragmentId, domain, status, occurredAt,
updatedAt, null, null, null);
updatedAt, null, null, null, null, null);
}
}
public record QueryEntity(Double amount, String currency, String comparator, String amountBasis) {}
public record QueryPlan(String originalQuery, String normalizedQuery, List<String> alternateQueries,
String intent, String domain, QueryEntity entities,
List<String> requiredAuthorities, boolean needsClarification,
List<String> evidenceNeeds, List<String> clarificationReasons) {}
public record ChannelContribution(String channel, int channelRank, Double rawScore,
Double fusionScore, String queryVariant) {}
public record RetrievalCandidate(Long fragmentId, String title, String sourceAuthority, String sourceKind,
Integer candidateRank, Double fusionScore, Double rerankScore,
String rerankModel, boolean rerankApplied, boolean selectedForEvidence,
String decisionReason, List<ChannelContribution> contributions) {}
public record LocatorSummary(Integer pageNumber, Integer slideNumber, Integer paragraphStart,
Integer paragraphEnd, String sheetName, Integer rowStart, Integer rowEnd,
Long startMs, Long endMs, Long frameMs) {}
@@ -127,8 +151,20 @@ public final class AihrKnowledgeQueryDto {
String rewrittenQuery,
List<Resource> resources,
MemoryCandidateResponse memoryCandidate,
BroadcastContext broadcastContext
BroadcastContext broadcastContext,
QueryPlan queryPlan,
List<RetrievalCandidate> retrievalCandidates
) {
public QueryResponse(String requestId, String queryText, String answer, List<Citation> citations,
List<String> usedSpaceCodes, boolean noEvidence, String promptVersion,
SearchResponse legacy, Object data, String conversationId, Long contextVersion,
String intent, String rewrittenQuery, List<Resource> resources,
MemoryCandidateResponse memoryCandidate, BroadcastContext broadcastContext) {
this(requestId, queryText, answer, citations, usedSpaceCodes, noEvidence, promptVersion,
legacy, data, conversationId, contextVersion, intent, rewrittenQuery, resources,
memoryCandidate, broadcastContext, null, List.of());
}
public QueryResponse(String requestId, String queryText, String answer, List<Citation> citations,
List<String> usedSpaceCodes, boolean noEvidence, String promptVersion,
SearchResponse legacy, Object data, String conversationId, Long contextVersion,
@@ -157,7 +193,7 @@ public final class AihrKnowledgeQueryDto {
public QueryResponse withMemoryCandidate(MemoryCandidateResponse candidate) {
return new QueryResponse(requestId, queryText, answer, citations, usedSpaceCodes, noEvidence,
promptVersion, legacy, data, conversationId, contextVersion, intent, rewrittenQuery,
resources, candidate, broadcastContext);
resources, candidate, broadcastContext, queryPlan, retrievalCandidates);
}
}
}
@@ -7,6 +7,8 @@ import lombok.extern.slf4j.Slf4j;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeAppDto.AuthenticatedApp;
import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal;
import org.dromara.aihr.domain.AihrSopDto.SnippetResponse;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.ChannelContribution;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.RetrievalCandidate;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
@@ -105,6 +107,37 @@ public class AihrKnowledgeQueryAuditService {
recordEvidence(requestId, tenantId, snippets, fragmentIds);
}
public void recordCandidates(String requestId, String tenantId, List<RetrievalCandidate> candidates) {
if (candidates == null || candidates.isEmpty()) return;
for (RetrievalCandidate candidate : candidates) {
if (candidate == null || candidate.fragmentId() == null || candidate.fragmentId() <= 0) continue;
List<ChannelContribution> contributions = candidate.contributions() == null
? List.of() : candidate.contributions();
for (ChannelContribution contribution : contributions) {
try {
jdbcTemplate.update("""
insert into aihr_query_candidate_trace
(tenant_id, request_id, fragment_id, candidate_rank, channel, channel_rank,
raw_score, fusion_score, rerank_score, rerank_model, rerank_applied,
source_authority, source_kind, query_variant, selected_for_evidence,
decision_reason, create_time)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now())
""", tenantId, requestId, candidate.fragmentId(), candidate.candidateRank(),
normalizedChannel(contribution.channel()), contribution.channelRank(),
contribution.rawScore(), candidate.fusionScore(), candidate.rerankScore(),
bounded(candidate.rerankModel(), 100), candidate.rerankApplied(),
bounded(candidate.sourceAuthority(), 30), bounded(candidate.sourceKind(), 30),
bounded(contribution.queryVariant(), 30), candidate.selectedForEvidence(),
bounded(candidate.decisionReason(), 64));
} catch (DataAccessException ex) {
log.warn("knowledge query candidate trace write failed for request {} fragment {}",
requestId, candidate.fragmentId());
break;
}
}
}
}
private static String normalizedChannel(String value) {
if (value == null || value.isBlank()) {
return "UNKNOWN";
@@ -113,6 +146,12 @@ public class AihrKnowledgeQueryAuditService {
return normalized.length() > 20 ? normalized.substring(0, 20) : normalized;
}
private static String bounded(String value, int max) {
if (value == null || value.isBlank()) return null;
String normalized = value.trim();
return normalized.length() <= max ? normalized : normalized.substring(0, max);
}
private String json(List<String> values) {
try {
return objectMapper.writeValueAsString(values == null ? List.of() : values);
@@ -22,6 +22,7 @@ import org.dromara.aihr.memory.AihrMemoryDto.ServiceMemoryRecall;
import org.dromara.aihr.memory.AihrMemoryService;
import org.dromara.aihr.service.AihrModelSeedService;
import org.dromara.aihr.service.AihrSopSeedService;
import org.dromara.aihr.service.AihrSopSeedService.SearchExecution;
import org.dromara.common.core.constant.HttpStatus;
import org.dromara.common.core.enums.UserType;
import org.dromara.common.core.exception.ServiceException;
@@ -368,11 +369,17 @@ public class AihrKnowledgeQueryService {
principal == null ? Set.of() : principal.roles());
SearchRequest searchRequest = new SearchRequest(
request.queryText(), request.category(), request.position(), retrievalSource, request.limit());
SearchResponse legacy = retrievalQuery.equals(request.queryText())
? sopService.searchAuthorized(searchRequest,
principal == null ? "" : principal.extPartyId(), spaceIds)
: sopService.searchAuthorized(searchRequest,
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
SearchExecution execution = sopService.searchAuthorizedDetailed(searchRequest,
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
if (execution == null) {
SearchResponse fallback = retrievalQuery.equals(request.queryText())
? sopService.searchAuthorized(searchRequest,
principal == null ? "" : principal.extPartyId(), spaceIds)
: sopService.searchAuthorized(searchRequest,
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
execution = new SearchExecution(fallback, null, List.of());
}
SearchResponse legacy = execution.response();
List<Citation> citations = citations(
app.tenantId(), spaceIds, legacy.snippets(), formalPolicyOnly, request.queryText());
boolean noEvidence = citations.isEmpty();
@@ -386,8 +393,10 @@ public class AihrKnowledgeQueryService {
citations.isEmpty() ? List.of() : List.of("DOCUMENT"), status, latency, legacy.promptVersion());
auditService.recordEvidence(requestId, app.tenantId(), legacy.snippets(), citations.stream()
.map(Citation::fragmentId).filter(Objects::nonNull).toList());
auditService.recordCandidates(requestId, app.tenantId(), execution.candidates());
return new QueryResponse(requestId, displayQuery, answer, citations, scopeCodes,
noEvidence, legacy.promptVersion(), legacy, null);
noEvidence, legacy.promptVersion(), legacy, null, null, null, "QA", request.queryText(),
List.of(), null, null, execution.queryPlan(), execution.candidates());
} catch (RuntimeException ex) {
auditService.record(requestId, principal, app, displayQuery, scopeCodes, List.of(),
ex instanceof ServiceException serviceEx && serviceEx.getCode() != null && serviceEx.getCode() < 500
@@ -419,7 +428,8 @@ public class AihrKnowledgeQueryService {
documentResponse.usedSpaceCodes(), false, BROADCAST_PROMPT_VERSION, documentResponse.legacy(), null,
documentResponse.conversationId(), documentResponse.contextVersion(), "QA",
documentResponse.rewrittenQuery(), documentResponse.resources(), documentResponse.memoryCandidate(),
new BroadcastContext(broadcast.messageId(), broadcast.title(), broadcast.publishedAt())
new BroadcastContext(broadcast.messageId(), broadcast.title(), broadcast.publishedAt()),
documentResponse.queryPlan(), documentResponse.retrievalCandidates()
);
}
@@ -536,7 +546,8 @@ public class AihrKnowledgeQueryService {
return new QueryResponse(
response.requestId(), question, answer, citations, response.usedSpaceCodes(), noEvidence,
response.promptVersion(), response.legacy(), response.data(), context.conversationId(), nextVersion,
context.intent(), context.rewrittenQuery(), resources, response.memoryCandidate(), response.broadcastContext()
context.intent(), context.rewrittenQuery(), resources, response.memoryCandidate(), response.broadcastContext(),
response.queryPlan(), response.retrievalCandidates()
);
}
@@ -564,7 +575,8 @@ public class AihrKnowledgeQueryService {
String ref = citationDetailService.issue(citation, tenantId);
return new Citation(citation.spaceCode(), citation.sourceType(), citation.docId(), citation.title(),
citation.snippet(), citation.fragmentId(), citation.domain(), citation.status(), citation.occurredAt(),
citation.updatedAt(), location.sourceKind(), ref, location.summary());
citation.updatedAt(), location.sourceKind(), ref, location.summary(),
citation.sourceAuthority(), citation.sourceKind());
}).toList();
}
@@ -611,7 +623,7 @@ public class AihrKnowledgeQueryService {
response.requestId(), response.queryText(), answer, List.copyOf(citations), List.copyOf(usedScopes),
false, response.promptVersion(), response.legacy(), response.data(), response.conversationId(),
response.contextVersion(), response.intent(), response.rewrittenQuery(), response.resources(),
response.memoryCandidate()
response.memoryCandidate(), response.broadcastContext(), response.queryPlan(), response.retrievalCandidates()
);
}
@@ -720,26 +732,7 @@ public class AihrKnowledgeQueryService {
args.add(tenantId);
args.addAll(allowedSpaceIds);
args.addAll(fragmentIds);
String formalSourceClause = formalPolicyOnly ? """
and exists (
select 1
from aihr_knowledge_attach governed_attach
join aihr_knowledge_source_governance governance
on governance.tenant_id = governed_attach.tenant_id
and governance.attachment_id = governed_attach.id
where governed_attach.tenant_id = f.tenant_id
and governed_attach.knowledge_id = f.knowledge_id
and governed_attach.doc_id = f.doc_id
and governed_attach.status = 2
and governance.source_doc_id = f.doc_id
and governance.authority_type = 'FORMAL_POLICY'
and governance.lifecycle_status = 'APPROVED'
and governance.effective_date <= current_date()
and (governance.expires_date is null or governance.expires_date >= current_date())
and char_length(trim(governance.source_version)) > 0
and governance.content_sha256 regexp '^[0-9a-f]{64}$'
)
""" : "";
String formalSourceClause = "";
List<Citation> rows = jdbcTemplate.query("""
select k.code as space_code, f.doc_id,
coalesce((
@@ -756,39 +749,39 @@ public class AihrKnowledgeQueryService {
and governed_title_chunk.published_fragment_id = f.id
limit 1
), a.name, k.name) as title,
f.content, f.id as fragment_id
from aihr_knowledge_fragment f
join aihr_knowledge_info k on k.id = f.knowledge_id and k.tenant_id = f.tenant_id
left join aihr_knowledge_attach a
on a.tenant_id = f.tenant_id and a.knowledge_id = f.knowledge_id and a.doc_id = f.doc_id
where f.tenant_id = ?
and f.knowledge_id in (%s)
and f.id in (%s)
and exists (
select 1
from aihr_chunk_revision governed_chunk
join aihr_data_asset governed_asset
on governed_asset.tenant_id = governed_chunk.tenant_id
and governed_asset.id = governed_chunk.asset_id
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
join aihr_dataset_membership governed_dataset
on governed_dataset.tenant_id = governed_chunk.tenant_id
and governed_dataset.version_id = governed_chunk.version_id
and governed_dataset.dataset_code = 'production'
and governed_dataset.status = 'ACTIVE'
where governed_chunk.tenant_id = f.tenant_id
and governed_chunk.published_fragment_id = f.id
and governed_asset.lifecycle_status = 'PUBLISHED'
and governed_asset.trust_level = 'HUMAN_VERIFIED'
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
)
%s
f.content, f.id as fragment_id, governed_asset.source_authority,
governed_asset.source_type, governed_asset.source_name
from aihr_knowledge_fragment f
join aihr_knowledge_info k on k.id = f.knowledge_id and k.tenant_id = f.tenant_id
left join aihr_knowledge_attach a
on a.tenant_id = f.tenant_id and a.knowledge_id = f.knowledge_id and a.doc_id = f.doc_id
join aihr_chunk_revision governed_chunk
on governed_chunk.tenant_id = f.tenant_id and governed_chunk.published_fragment_id = f.id
join aihr_data_asset governed_asset
on governed_asset.tenant_id = governed_chunk.tenant_id
and governed_asset.id = governed_chunk.asset_id
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
join aihr_dataset_membership governed_dataset
on governed_dataset.tenant_id = governed_chunk.tenant_id
and governed_dataset.version_id = governed_chunk.version_id
and governed_dataset.dataset_code = 'production'
and governed_dataset.status = 'ACTIVE'
where f.tenant_id = ?
and f.knowledge_id in (%s)
and f.id in (%s)
and governed_asset.lifecycle_status = 'PUBLISHED'
and governed_asset.trust_level = 'HUMAN_VERIFIED'
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
%s
""".formatted(placeholders(allowedSpaceIds.size()), placeholders(fragmentIds.size()),
formalSourceClause),
(rs, rowNum) -> new Citation(
rs.getString("space_code"), "DOCUMENT", rs.getString("doc_id"), rs.getString("title"),
rs.getString("content"), rs.getLong("fragment_id")
rs.getString("content"), rs.getLong("fragment_id"), "ENTERPRISE", null, null, null,
null, null, null, rs.getString("source_authority"),
AihrRagQueryPlanner.sourceKind(rs.getString("source_authority"), rs.getString("source_type"),
rs.getString("source_name"))
), args.toArray());
Map<Long, Citation> byId = new LinkedHashMap<>();
rows.forEach(row -> byId.put(row.fragmentId(), row));
@@ -796,13 +789,15 @@ public class AihrKnowledgeQueryService {
.map(byId::get)
.filter(java.util.Objects::nonNull)
.filter(row -> !formalPolicyOnly
|| AihrFormalPolicyClassifier.isRelevantFormalEvidence(queryText, row.snippet()))
|| (Set.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP").contains(row.sourceKind())
&& AihrFormalPolicyClassifier.isRelevantFormalEvidence(queryText, row.snippet())))
.map(row -> new Citation(
row.spaceCode(), row.sourceType(), row.docId(), row.title(),
formalPolicyOnly
? AihrFormalPolicyClassifier.relevantEvidenceSnippet(queryText, row.snippet(), 500)
: truncate(row.snippet(), 500),
row.fragmentId()
row.fragmentId(), row.domain(), row.status(), row.occurredAt(), row.updatedAt(),
row.mediaType(), row.detailRef(), row.locatorSummary(), row.sourceAuthority(), row.sourceKind()
))
.toList();
}
@@ -0,0 +1,236 @@
package org.dromara.aihr.knowledge.service;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryEntity;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryPlan;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** Deterministic, authorization-free planning for bounded enterprise knowledge retrieval. */
public final class AihrRagQueryPlanner {
public static final String FINANCE_REIMBURSEMENT = "FINANCE_REIMBURSEMENT";
public static final String FINANCE_POLICY = "FINANCE_POLICY";
public static final String GENERAL_KNOWLEDGE = "GENERAL_KNOWLEDGE";
private static final List<String> FORMAL_AUTHORITIES =
List.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP");
private static final Pattern ARABIC_AMOUNT = Pattern.compile(
"(?<![0-9A-Za-z])(\\d+(?:\\.\\d+)?)\\s*(?:元|块钱|块|人民币)");
private static final Pattern CHINESE_HUNDREDS = Pattern.compile("([一二三四五六七八九])百(?:\\s*(?:元|块钱|块))?");
private static final Pattern HAN_RUN = Pattern.compile("[\\p{IsHan}]{2,16}");
private static final Pattern LATIN_TERM = Pattern.compile("[A-Za-z][A-Za-z0-9_-]{1,31}");
private static final List<String> BUSINESS_TERMS = List.of(
"费用报销", "报销流程", "发票登记", "财务复审", "零星采购", "小额采购", "个人借款",
"采购审批", "报销单", "借款", "报销", "发票", "采购", "自采", "财务", "报备", "审批",
"材料", "流程", "额度", "月累计", "单笔"
);
private static final List<String> STOP_PHRASES = List.of(
"我有个", "告诉我", "请告诉", "请问", "帮我", "怎么走", "怎么", "如何", "一下", "这个",
"那个", "有没有", "能不能", "可以吗", "是什么", "的", "以下", "以内", "以上", "不超过"
);
private static final Set<String> STOP_TERMS = Set.of(
"我有", "有个", "个报", "告诉", "诉我", "怎么", "么走", "以下", "以内", "以上", "块钱",
"这个", "那个", "一下", "请问", "能不", "不能", "可以", "以吗"
);
private AihrRagQueryPlanner() {}
public static QueryPlan plan(String rawQuery) {
String original = clean(rawQuery);
QueryEntity entities = entities(original);
String domain = domain(original);
boolean finance = isFormalFinanceDomain(domain);
String normalized = normalize(original, domain, entities);
List<String> alternates = alternates(original, domain);
List<String> evidenceNeeds = evidenceNeeds(original, domain, entities);
List<String> reasons = new ArrayList<>();
if (FINANCE_REIMBURSEMENT.equals(domain) && entities.amount() != null
&& "UNKNOWN".equals(entities.amountBasis())) {
reasons.add("金额口径不明确:请确认是单笔金额还是月累计额度");
}
return new QueryPlan(
original,
normalized,
alternates,
"KNOWLEDGE_QA",
domain,
entities,
finance ? FORMAL_AUTHORITIES : List.of(),
!reasons.isEmpty(),
evidenceNeeds,
List.copyOf(reasons)
);
}
public static List<String> sparseTerms(String queryText) {
String original = clean(queryText).toLowerCase(Locale.ROOT);
if (original.isBlank()) return List.of();
LinkedHashSet<String> terms = new LinkedHashSet<>();
for (String businessTerm : BUSINESS_TERMS) {
if (original.contains(businessTerm)) terms.add(businessTerm);
}
Matcher amountMatcher = ARABIC_AMOUNT.matcher(original);
StringBuffer withoutAmounts = new StringBuffer();
while (amountMatcher.find()) {
terms.add(amountMatcher.group(1) + "元");
amountMatcher.appendReplacement(withoutAmounts, " ");
}
amountMatcher.appendTail(withoutAmounts);
String text = normalizeChineseAmounts(withoutAmounts.toString())
.replaceAll("[^\\p{IsHan}A-Za-z0-9_-]+", " ");
for (String stop : STOP_PHRASES) text = text.replace(stop, " ");
Matcher hanMatcher = HAN_RUN.matcher(text);
while (hanMatcher.find() && terms.size() < 24) {
String run = hanMatcher.group();
if (!STOP_TERMS.contains(run)) terms.add(run);
for (int i = 0; i + 2 <= run.length() && terms.size() < 24; i++) {
String term = run.substring(i, i + 2);
if (!STOP_TERMS.contains(term)) terms.add(term);
}
}
Matcher latinMatcher = LATIN_TERM.matcher(text);
while (latinMatcher.find() && terms.size() < 24) terms.add(latinMatcher.group().toLowerCase(Locale.ROOT));
return terms.stream()
.filter(term -> !term.matches("\\d+"))
.filter(term -> !STOP_TERMS.contains(term))
.limit(24)
.toList();
}
public static boolean isFormalFinanceDomain(String domain) {
return FINANCE_REIMBURSEMENT.equals(domain) || FINANCE_POLICY.equals(domain);
}
public static String sourceKind(String sourceAuthority, String sourceType, String sourceName) {
String authority = clean(sourceAuthority).toUpperCase(Locale.ROOT);
String type = clean(sourceType).toUpperCase(Locale.ROOT);
String name = clean(sourceName).toLowerCase(Locale.ROOT);
if (authority.contains("EXPERIENCE") || authority.contains("EMPLOYEE_SUBMISSION")
|| type.contains("INTERVIEW")) return "EXPERIENCE";
if (authority.contains("CASE") || type.contains("CASE")) return "CASE";
if (name.contains("操作手册") || authority.contains("OPERATING_MANUAL")) return "OPERATING_MANUAL";
if (name.contains("sop") || authority.contains("APPROVED_SOP")) return "APPROVED_SOP";
if (authority.contains("POLICY") || authority.contains("COMPANY_POLICY")
|| authority.contains("FORMAL")) return "FORMAL_POLICY";
return "REFERENCE_MATERIAL";
}
public static boolean authorityAllowed(QueryPlan plan, String sourceKind) {
return plan == null || plan.requiredAuthorities().isEmpty()
|| plan.requiredAuthorities().contains(sourceKind);
}
public static boolean domainMatches(QueryPlan plan, String title, String content) {
if (plan == null || !isFormalFinanceDomain(plan.domain())) return true;
String source = clean(title) + " " + clean(content);
if (FINANCE_REIMBURSEMENT.equals(plan.domain())) {
return containsAny(source, "费用报销", "发票登记", "报销单", "财务复审", "个人借款")
|| (containsAny(title, "报销", "发票", "财务", "借款")
&& containsAny(source, "流程", "提交", "登记", "审批", "复审"));
}
return containsAny(title, "财务", "报销", "借款", "发票", "采购")
|| containsAny(source, "财务管理", "费用报销", "个人借款", "采购审批");
}
private static String domain(String query) {
if (containsAny(query, "报销", "走账") && containsAny(query, "采购", "自采", "发票", "费用", "垫付")) {
return FINANCE_REIMBURSEMENT;
}
if (containsAny(query, "报销", "借款", "发票", "财务", "采购", "自采", "报备", "审批")) {
return FINANCE_POLICY;
}
return GENERAL_KNOWLEDGE;
}
private static QueryEntity entities(String query) {
Double amount = null;
Matcher arabic = ARABIC_AMOUNT.matcher(query);
if (arabic.find()) amount = Double.valueOf(arabic.group(1));
if (amount == null) {
Matcher chinese = CHINESE_HUNDREDS.matcher(query);
if (chinese.find()) amount = chineseDigit(chinese.group(1)) * 100.0;
}
String comparator = containsAny(query, "以下", "以内", "不超过", "最多") ? "LTE"
: containsAny(query, "超过", "大于") ? "GT"
: containsAny(query, "以上", "不少于", "至少") ? "GTE" : "UNSPECIFIED";
String basis = containsAny(query, "单笔", "每笔", "一笔") ? "PER_TRANSACTION"
: containsAny(query, "月累计", "每月", "当月", "一个月", "月度") ? "MONTHLY_TOTAL" : "UNKNOWN";
return new QueryEntity(amount, amount == null ? null : "CNY", comparator, basis);
}
private static String normalize(String original, String domain, QueryEntity entities) {
if (!isFormalFinanceDomain(domain)) return original;
LinkedHashSet<String> parts = new LinkedHashSet<>();
if (containsAny(original, "采购", "自采", "零星", "小额")) parts.add("零星采购");
parts.add(FINANCE_REIMBURSEMENT.equals(domain) ? "费用报销流程" : "财务制度");
if (containsAny(original, "发票")) parts.add("发票登记");
if (entities.amount() != null) {
String amount = entities.amount() % 1 == 0
? String.valueOf(entities.amount().longValue()) : String.valueOf(entities.amount());
parts.add(amount + "元" + comparatorText(entities.comparator()));
}
if (!"UNKNOWN".equals(entities.amountBasis())) parts.add(entities.amountBasis());
return String.join(" ", parts);
}
private static List<String> alternates(String original, String domain) {
if (!isFormalFinanceDomain(domain)) return List.of();
LinkedHashSet<String> values = new LinkedHashSet<>();
values.add("费用报销 发票登记 报销单 财务复审");
if (containsAny(original, "采购", "自采", "零星", "小额")) values.add("零星采购 自采 费用报销");
if (containsAny(original, "借款")) values.add("个人借款 费用报销冲抵");
return values.stream().limit(2).toList();
}
private static List<String> evidenceNeeds(String original, String domain, QueryEntity entities) {
if (!isFormalFinanceDomain(domain)) return List.of("DIRECT_ANSWER");
LinkedHashSet<String> needs = new LinkedHashSet<>();
if (entities.amount() != null) {
needs.add("AMOUNT_THRESHOLD");
needs.add("AMOUNT_BASIS");
needs.add("APPLICABILITY");
}
if (containsAny(original, "怎么", "如何", "流程", "走账", "报销")) needs.add("ENTRY_STEPS");
needs.add("REQUIRED_MATERIALS");
needs.add("APPROVAL_REVIEW");
needs.add("EXCEPTIONS");
return List.copyOf(needs);
}
private static String normalizeChineseAmounts(String value) {
Matcher matcher = CHINESE_HUNDREDS.matcher(value);
StringBuffer output = new StringBuffer();
while (matcher.find()) matcher.appendReplacement(output, chineseDigit(matcher.group(1)) + "00元");
matcher.appendTail(output);
return output.toString();
}
private static int chineseDigit(String value) {
return "一二三四五六七八九".indexOf(value) + 1;
}
private static String comparatorText(String comparator) {
return switch (Optional.ofNullable(comparator).orElse("")) {
case "LTE" -> "以下";
case "GT" -> "以上";
case "GTE" -> "及以上";
default -> "";
};
}
private static boolean containsAny(String value, String... terms) {
String source = clean(value).toLowerCase(Locale.ROOT);
for (String term : terms) if (source.contains(term.toLowerCase(Locale.ROOT))) return true;
return false;
}
private static String clean(String value) {
return Optional.ofNullable(value).orElse("").trim();
}
}
@@ -46,6 +46,10 @@ import org.dromara.aihr.knowledge.parse.ParsedDocument.LocatedSegment;
import org.dromara.aihr.knowledge.parse.AihrExtractionQuality;
import org.dromara.aihr.knowledge.parse.TikaKnowledgeDocumentParser;
import org.dromara.aihr.knowledge.service.AihrFormalPolicyClassifier;
import org.dromara.aihr.knowledge.service.AihrRagQueryPlanner;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.ChannelContribution;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryPlan;
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.RetrievalCandidate;
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycle.UsageType;
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycle.ReasonCode;
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycleService;
@@ -126,6 +130,10 @@ public class AihrSopSeedService {
private static final String LOCAL_EMBEDDING_BASE_URL = "local";
private static final int LOCAL_EMBEDDING_DIMENSION = 1536;
private static final int MAX_EVIDENCE_SNIPPETS = 3;
private static final int RETRIEVAL_CANDIDATE_K = 30;
private static final int RETRIEVAL_RERANK_K = 50;
private static final int RETRIEVAL_EVIDENCE_K = 3;
private static final int RETRIEVAL_DISPLAY_K = 3;
private static final KnowledgeDocumentParser KNOWLEDGE_DOCUMENT_PARSER = new TikaKnowledgeDocumentParser();
private static final String NO_CLEAR_SOP_EVIDENCE = "未在已发布 SOP 中找到明确依据";
private static final String LEGACY_NO_CLEAR_SOP_EVIDENCE = "未在已入库 SOP 中找到明确依据";
@@ -227,11 +235,16 @@ public class AihrSopSeedService {
public SearchResponse searchAuthorized(SearchRequest request, String requesterExtPartyId, Set<Long> allowedKnowledgeIds) {
String queryText = request == null ? "" : Optional.ofNullable(request.queryText()).orElse("").trim();
return searchAuthorized(request, requesterExtPartyId, allowedKnowledgeIds, queryText);
return searchAuthorizedDetailed(request, requesterExtPartyId, allowedKnowledgeIds, queryText).response();
}
public SearchResponse searchAuthorized(SearchRequest request, String requesterExtPartyId,
Set<Long> allowedKnowledgeIds, String retrievalQueryText) {
return searchAuthorizedDetailed(request, requesterExtPartyId, allowedKnowledgeIds, retrievalQueryText).response();
}
public SearchExecution searchAuthorizedDetailed(SearchRequest request, String requesterExtPartyId,
Set<Long> allowedKnowledgeIds, String retrievalQueryText) {
if (allowedKnowledgeIds == null || allowedKnowledgeIds.isEmpty()) {
throw new ServiceException("当前请求没有可访问的知识空间", 403);
}
@@ -240,18 +253,20 @@ public class AihrSopSeedService {
String position = firstNonBlank(request == null ? null : request.position(), "生活顾问");
String source = normalizeSearchSource(request == null ? null : request.source());
String requester = isBlank(requesterExtPartyId) ? "" : requesterExtPartyId.trim();
QueryPlan plan = AihrRagQueryPlanner.plan(queryText);
if (isBlank(queryText)) {
return withReviewId(noEvidenceResponse("", "sop"), source, requester);
return new SearchExecution(withReviewId(noEvidenceResponse("", "sop"), source, requester), plan, List.of());
}
boolean formalPolicyOnly = AihrFormalPolicyClassifier.FORMAL_POLICY_SOURCE.equals(source);
SearchResponse response = dbSearch(
SearchExecution execution = dbSearchDetailed(
"", queryText, retrievalQuery, request == null ? null : request.limit(), allowedKnowledgeIds,
formalPolicyOnly);
formalPolicyOnly, plan);
SearchResponse response = execution.response();
if (response == null || isNoEvidenceAnswer(response.answer())) {
recordKnowledgeGap(queryText, "sop", position, source);
response = noEvidenceResponse(queryText, "sop");
if (response == null) response = noEvidenceResponse(queryText, "sop");
}
return withReviewId(response, source, requester);
return new SearchExecution(withReviewId(response, source, requester), plan, execution.candidates());
}
/**
@@ -3418,6 +3433,10 @@ public class AihrSopSeedService {
+ String.join(",", java.util.Collections.nCopies(allowedKnowledgeIds.size(), "?")) + ")";
}
private static String sqlPlaceholders(int count) {
return String.join(",", java.util.Collections.nCopies(count, "?"));
}
private static void addKnowledgeScopeArgs(List<Object> args, Set<Long> allowedKnowledgeIds) {
if (allowedKnowledgeIds != null) {
args.addAll(allowedKnowledgeIds);
@@ -3431,21 +3450,24 @@ public class AihrSopSeedService {
return """
and exists (
select 1
from aihr_knowledge_attach governed_attach
join aihr_knowledge_source_governance governance
on governance.tenant_id = governed_attach.tenant_id
and governance.attachment_id = governed_attach.id
where governed_attach.tenant_id = %1$s.tenant_id
and governed_attach.knowledge_id = %1$s.knowledge_id
and governed_attach.doc_id = %1$s.doc_id
and governed_attach.status = 2
and governance.source_doc_id = %1$s.doc_id
and governance.authority_type = 'FORMAL_POLICY'
and governance.lifecycle_status = 'APPROVED'
and governance.effective_date <= current_date()
and (governance.expires_date is null or governance.expires_date >= current_date())
and char_length(trim(governance.source_version)) > 0
and governance.content_sha256 regexp '^[0-9a-f]{64}$'
from aihr_chunk_revision governed_chunk
join aihr_data_asset governed_asset
on governed_asset.tenant_id = governed_chunk.tenant_id
and governed_asset.id = governed_chunk.asset_id
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
join aihr_dataset_membership governed_dataset
on governed_dataset.tenant_id = governed_chunk.tenant_id
and governed_dataset.version_id = governed_chunk.version_id
and governed_dataset.dataset_code = 'production'
and governed_dataset.status = 'ACTIVE'
where governed_chunk.tenant_id = %1$s.tenant_id
and governed_chunk.published_fragment_id = %1$s.id
and governed_asset.lifecycle_status = 'PUBLISHED'
and governed_asset.trust_level = 'HUMAN_VERIFIED'
and upper(governed_asset.source_authority) in
('COMPANY_POLICY', 'FORMAL_POLICY', 'OPERATING_MANUAL', 'APPROVED_SOP')
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
)
""".formatted(fragmentAlias);
}
@@ -3473,6 +3495,305 @@ public class AihrSopSeedService {
}
}
private SearchExecution dbSearchDetailed(String category, String queryText, String retrievalQueryText,
Integer displayLimit, Set<Long> allowedKnowledgeIds,
boolean formalPolicyOnly, QueryPlan plan) {
try {
boolean requiresFormal = formalPolicyOnly || !plan.requiredAuthorities().isEmpty();
List<CandidateAccumulator> candidates = retrieveCandidates(
category, queryText, retrievalQueryText, allowedKnowledgeIds, plan, requiresFormal);
if (candidates.isEmpty()) return new SearchExecution(null, plan, List.of());
enrichCandidateGovernance(candidates);
Set<Long> rejectedFeedback = downFeedbackFragmentIds(queryText);
for (CandidateAccumulator candidate : candidates) {
if (candidate.hit.fragmentId() != null && rejectedFeedback.contains(candidate.hit.fragmentId())) {
candidate.reject("REJECTED_DOWNSTREAM_FEEDBACK");
} else if (requiresFormal && !formalSourceKind(candidate.sourceKind)) {
candidate.reject("REJECTED_SOURCE_AUTHORITY");
} else if (!AihrRagQueryPlanner.domainMatches(plan, candidate.hit.title(), candidate.hit.content())) {
candidate.reject("REJECTED_DOMAIN_MISMATCH");
}
}
List<CandidateAccumulator> eligible = candidates.stream()
.filter(candidate -> candidate.decisionReason == null)
.sorted(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed())
.toList();
List<CandidateAccumulator> rerankPool = eligible.stream().limit(RETRIEVAL_RERANK_K).toList();
eligible.stream().skip(RETRIEVAL_RERANK_K)
.forEach(candidate -> candidate.reject("OUTSIDE_RERANK_K"));
List<CandidateAccumulator> ranked = rerankCandidates(queryText, rerankPool);
for (int index = 0; index < ranked.size(); index++) ranked.get(index).candidateRank = index + 1;
List<CandidateAccumulator> evidence = ranked.stream().limit(RETRIEVAL_EVIDENCE_K).toList();
if (plan.needsClarification()) {
evidence.forEach(candidate -> candidate.reject("NEEDS_CLARIFICATION_AMOUNT_BASIS"));
String message = "已找到正式的费用报销资料,但当前问题中的金额口径不明确:请确认“"
+ amountText(plan) + "”是单笔金额还是月累计额度。当前资料不能据此推导零星采购必然可以报销。";
return new SearchExecution(clarificationResponse(queryText, category, message), plan,
candidateTraces(candidates));
}
if (requiresFormal && evidence.isEmpty()) {
return new SearchExecution(noEvidenceResponse(queryText, category,
List.of("没有找到当前有效且经过人工审核的正式制度、操作手册或已批准 SOP。")),
plan, candidateTraces(candidates));
}
if (requiresAmountEvidence(plan) && !supportsAmountRule(plan, evidence)) {
evidence.forEach(candidate -> candidate.reject("INSUFFICIENT_AMOUNT_EVIDENCE"));
String message = "已找到正式的费用报销操作资料,但当前正式资料没有证明“" + amountText(plan)
+ "”对应零星采购的适用额度和口径。请由财务负责人确认单笔/月累计规则后再办理。";
return new SearchExecution(clarificationResponse(queryText, category, message), plan,
candidateTraces(candidates));
}
if (!hasRequiredFinanceEvidence(plan, evidence)) {
evidence.forEach(candidate -> candidate.reject("INSUFFICIENT_REQUIRED_EVIDENCE"));
return new SearchExecution(noEvidenceResponse(queryText, category,
List.of("正式资料尚未同时覆盖报销入口、必要材料和审批/复审要求。")),
plan, candidateTraces(candidates));
}
evidence.forEach(candidate -> {
candidate.selectedForEvidence = true;
candidate.decisionReason = "SELECTED_EVIDENCE";
});
ranked.stream().skip(evidence.size()).forEach(candidate -> {
if (candidate.decisionReason == null) candidate.decisionReason = "OUTSIDE_EVIDENCE_K";
});
SearchResponse response = responseFromHits(category, queryText,
evidence.stream().map(candidate -> candidate.hit).toList(), List.of(), displayLimitValue(displayLimit));
return new SearchExecution(response, plan, candidateTraces(candidates));
} catch (DataAccessException ex) {
return new SearchExecution(null, plan, List.of());
}
}
private List<CandidateAccumulator> retrieveCandidates(String category, String queryText,
String retrievalQueryText,
Set<Long> allowedKnowledgeIds, QueryPlan plan,
boolean includePublishedCandidates) {
List<QueryVariant> variants = queryVariants(queryText, retrievalQueryText, plan);
Map<String, CandidateAccumulator> candidates = new LinkedHashMap<>();
for (QueryVariant variant : variants) {
addCandidateHits(candidates, likeHits(category, variant.text(), RETRIEVAL_CANDIDATE_K,
allowedKnowledgeIds, false, includePublishedCandidates), "KEYWORD", variant.code(), true);
addCandidateHits(candidates, fulltextHits(category, variant.text(), RETRIEVAL_CANDIDATE_K,
allowedKnowledgeIds, false, includePublishedCandidates), "FULLTEXT", variant.code(), true);
}
Map<String, List<KnowledgeHit>> vectorHits = vectorHits(
category, variants, RETRIEVAL_CANDIDATE_K, allowedKnowledgeIds, false, includePublishedCandidates);
for (QueryVariant variant : variants) {
addCandidateHits(candidates, vectorHits.getOrDefault(variant.code(), List.of()),
"VECTOR", variant.code(), false);
}
applyFamilyFusion(candidates.values(), true);
applyFamilyFusion(candidates.values(), false);
return candidates.values().stream()
.peek(CandidateAccumulator::finishFusion)
.sorted(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed())
.toList();
}
private static List<QueryVariant> queryVariants(String queryText, String retrievalQueryText, QueryPlan plan) {
LinkedHashMap<String, String> variants = new LinkedHashMap<>();
addQueryVariant(variants, "ORIGINAL", queryText);
if (!Objects.equals(cleanQuery(queryText), cleanQuery(retrievalQueryText))) {
addQueryVariant(variants, "GLOSSARY", retrievalQueryText);
}
addQueryVariant(variants, "NORMALIZED", plan.normalizedQuery());
for (int index = 0; index < plan.alternateQueries().size(); index++) {
addQueryVariant(variants, "ALTERNATE_" + (index + 1), plan.alternateQueries().get(index));
}
return variants.entrySet().stream().map(entry -> new QueryVariant(entry.getKey(), entry.getValue())).toList();
}
private static void addQueryVariant(Map<String, String> variants, String code, String value) {
String query = cleanQuery(value);
if (query.isBlank() || variants.values().stream().anyMatch(existing -> cleanQuery(existing).equals(query))) return;
variants.put(code, value.trim());
}
private static String cleanQuery(String value) {
return Optional.ofNullable(value).orElse("").replaceAll("\\s+", " ").trim().toLowerCase(Locale.ROOT);
}
private static void addCandidateHits(Map<String, CandidateAccumulator> candidates, List<KnowledgeHit> hits,
String channel, String queryVariant, boolean sparse) {
for (int index = 0; index < hits.size(); index++) {
KnowledgeHit hit = hits.get(index);
String key = hit.fragmentId() == null ? hit.docId() + "#" + hit.idx() : String.valueOf(hit.fragmentId());
CandidateAccumulator candidate = candidates.computeIfAbsent(key, ignored -> new CandidateAccumulator(hit));
candidate.add(channel, index + 1, hit.score(), queryVariant, sparse);
}
}
private static void applyFamilyFusion(java.util.Collection<CandidateAccumulator> candidates, boolean sparse) {
List<CandidateAccumulator> ranked = candidates.stream()
.filter(candidate -> sparse ? candidate.sparseSignal > 0 : candidate.vectorSignal > 0)
.sorted(Comparator.comparingDouble((CandidateAccumulator candidate) ->
sparse ? candidate.sparseSignal : candidate.vectorSignal)
.reversed())
.toList();
for (int index = 0; index < ranked.size(); index++) {
double contribution = reciprocalRank(index + 1);
if (sparse) ranked.get(index).sparseFusion = contribution;
else ranked.get(index).vectorFusion = contribution;
}
}
private static double reciprocalRank(int rank) {
return 1.0 / (60 + Math.max(1, rank));
}
private void enrichCandidateGovernance(List<CandidateAccumulator> candidates) {
List<Long> fragmentIds = candidates.stream().map(candidate -> candidate.hit.fragmentId())
.filter(Objects::nonNull).distinct().toList();
if (fragmentIds.isEmpty()) return;
List<Object> args = new ArrayList<>();
args.add(tenantId());
args.addAll(fragmentIds);
jdbcTemplate.query("""
select f.id fragment_id, asset.source_authority, asset.source_type, asset.source_name
from aihr_knowledge_fragment f
join aihr_chunk_revision chunk
on chunk.tenant_id = f.tenant_id and chunk.published_fragment_id = f.id
join aihr_data_asset asset
on asset.tenant_id = chunk.tenant_id and asset.id = chunk.asset_id
and coalesce(asset.published_version_id, asset.current_version_id) = chunk.version_id
where f.tenant_id = ? and f.id in (%s)
""".formatted(sqlPlaceholders(fragmentIds.size())), rs -> {
Long fragmentId = rs.getLong("fragment_id");
candidates.stream().filter(candidate -> Objects.equals(candidate.hit.fragmentId(), fragmentId))
.findFirst().ifPresent(candidate -> {
candidate.sourceAuthority = rsString(rs, "source_authority");
candidate.sourceKind = AihrRagQueryPlanner.sourceKind(candidate.sourceAuthority,
rsString(rs, "source_type"), rsString(rs, "source_name"));
});
}, args.toArray());
}
private static String rsString(java.sql.ResultSet rs, String column) {
try {
return rs.getString(column);
} catch (java.sql.SQLException ex) {
throw new IllegalStateException(ex);
}
}
private List<CandidateAccumulator> rerankCandidates(String queryText, List<CandidateAccumulator> candidates) {
if (candidates.size() < 2) return candidates;
Optional<ChatRuntime> runtime = rerankRuntime();
if (runtime.isEmpty()) return candidates;
try {
ObjectNode body = objectMapper.createObjectNode();
body.put("model", runtime.get().modelName());
body.put("query", AihrSensitiveText.forModel(queryText));
body.put("top_n", candidates.size());
body.put("return_documents", false);
ArrayNode documents = body.putArray("documents");
candidates.forEach(candidate -> documents.add(
AihrSensitiveText.forModel(truncate(candidate.hit.content(), 2000))));
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(normalizeBaseUrl(runtime.get().baseUrl()) + "/rerank"))
.timeout(Duration.ofSeconds(30))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer " + runtime.get().apiKey())
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
.build();
HttpResponse<String> response = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15))
.build().send(request, HttpResponse.BodyHandlers.ofString());
if (!ok(response.statusCode())) throw new IllegalStateException("rerank HTTP " + response.statusCode());
JsonNode results = objectMapper.readTree(response.body()).path("results");
if (!results.isArray() || results.isEmpty()) return candidates;
List<CandidateAccumulator> reranked = new ArrayList<>();
for (JsonNode item : results) {
int index = item.path("index").asInt(-1);
if (index < 0 || index >= candidates.size()) continue;
CandidateAccumulator candidate = candidates.get(index);
candidate.rerankApplied = true;
candidate.rerankModel = runtime.get().modelName();
if (item.path("relevance_score").isNumber()) {
candidate.rerankScore = item.path("relevance_score").asDouble();
}
if (!reranked.contains(candidate)) reranked.add(candidate);
}
for (CandidateAccumulator candidate : candidates) if (!reranked.contains(candidate)) reranked.add(candidate);
return reranked;
} catch (Exception ex) {
log.warn("candidate rerank failed, keep family RRF order(处理错误已隐藏)");
return candidates;
}
}
private static boolean formalSourceKind(String sourceKind) {
return Set.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP").contains(sourceKind);
}
private static boolean requiresAmountEvidence(QueryPlan plan) {
return plan != null && AihrRagQueryPlanner.isFormalFinanceDomain(plan.domain())
&& plan.entities() != null && plan.entities().amount() != null;
}
private static boolean supportsAmountRule(QueryPlan plan, List<CandidateAccumulator> evidence) {
if (!requiresAmountEvidence(plan)) return true;
String amount = amountValue(plan);
String content = evidence.stream().map(candidate -> candidate.hit.content()).filter(Objects::nonNull)
.reduce("", (left, right) -> left + " " + right);
boolean amountFound = content.contains(amount + "元") || content.contains(amount + " 元")
|| content.contains(chineseAmount(plan.entities().amount()));
boolean basisFound = "PER_TRANSACTION".equals(plan.entities().amountBasis())
? containsAny(content, "单笔", "每笔", "一笔")
: "MONTHLY_TOTAL".equals(plan.entities().amountBasis())
? containsAny(content, "月累计", "每月", "当月", "月度额度") : false;
return amountFound && basisFound;
}
private static boolean hasRequiredFinanceEvidence(QueryPlan plan, List<CandidateAccumulator> evidence) {
if (plan == null || !AihrRagQueryPlanner.isFormalFinanceDomain(plan.domain())) return !evidence.isEmpty();
String content = evidence.stream().map(candidate -> candidate.hit.content()).filter(Objects::nonNull)
.reduce("", (left, right) -> left + " " + right);
return containsAny(content, "费用报销", "报销单", "发票登记")
&& containsAny(content, "发票", "票据", "附件")
&& containsAny(content, "提交", "审批", "复审", "流程结束");
}
private static String amountValue(QueryPlan plan) {
double value = plan.entities().amount();
return value % 1 == 0 ? String.valueOf((long) value) : String.valueOf(value);
}
private static String amountText(QueryPlan plan) {
if (plan == null || plan.entities() == null || plan.entities().amount() == null) return "该金额";
String comparator = switch (Optional.ofNullable(plan.entities().comparator()).orElse("")) {
case "LTE" -> "以下";
case "GT" -> "以上";
case "GTE" -> "及以上";
default -> "";
};
return amountValue(plan) + "元" + comparator;
}
private static String chineseAmount(Double amount) {
if (amount == null || amount % 100 != 0 || amount < 100 || amount > 900) return "";
return "一二三四五六七八九".substring((int) (amount / 100) - 1, (int) (amount / 100)) + "百";
}
private static int displayLimitValue(Integer limit) {
if (limit == null) return RETRIEVAL_DISPLAY_K;
return Math.max(1, Math.min(limit, RETRIEVAL_DISPLAY_K));
}
private static SearchResponse clarificationResponse(String queryText, String category, String message) {
return new SearchResponse(queryText, firstNonBlank(category, "sop"), message, "正式证据不足,未生成流程",
List.of(), List.of(), List.of(), records(), List.of(), List.of(message), null, null);
}
private static List<RetrievalCandidate> candidateTraces(List<CandidateAccumulator> candidates) {
return candidates.stream()
.sorted(Comparator.comparing((CandidateAccumulator candidate) -> candidate.candidateRank,
Comparator.nullsLast(Integer::compareTo))
.thenComparing(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed()))
.map(CandidateAccumulator::toTrace)
.toList();
}
private static String normalizeFeedbackRequestId(String value) {
if (isBlank(value)) {
return null;
@@ -3571,6 +3892,14 @@ public class AihrSopSeedService {
return "and " + AihrKnowledgeRolloutService.servingFragmentExistsSql(fragmentAlias);
}
private static String candidateLifecycleScopeSql(String fragmentAlias, boolean includePublishedCandidates) {
if (!includePublishedCandidates) {
return servingLifecycleScopeSql(fragmentAlias);
}
return "and (" + AihrKnowledgeRolloutService.servingFragmentExistsSql(fragmentAlias)
+ " or " + publishedLifecycleExistsSql(fragmentAlias) + ")";
}
private static String publishedLifecycleExistsSql(String fragmentAlias) {
return """
exists (
@@ -3664,6 +3993,40 @@ public class AihrSopSeedService {
return List.of();
}
private Map<String, List<KnowledgeHit>> vectorHits(String category, List<QueryVariant> variants, int limit,
Set<Long> allowedKnowledgeIds,
boolean formalPolicyOnly) {
return vectorHits(category, variants, limit, allowedKnowledgeIds, formalPolicyOnly, false);
}
private Map<String, List<KnowledgeHit>> vectorHits(String category, List<QueryVariant> variants, int limit,
Set<Long> allowedKnowledgeIds,
boolean formalPolicyOnly,
boolean includePublishedCandidates) {
if (variants == null || variants.isEmpty()) return Map.of();
List<String> queries = variants.stream().map(QueryVariant::text).toList();
for (EmbeddingRuntime runtime : embeddingRuntimes()) {
try {
List<String> embeddings = callEmbeddings(runtime, queries);
if (embeddings.size() != variants.size()) continue;
Map<String, List<KnowledgeHit>> results = new LinkedHashMap<>();
for (int index = 0; index < variants.size(); index++) {
List<KnowledgeHit> hits = new ArrayList<>();
for (VectorMatch match : queryQdrant(
category, embeddings.get(index), limit, allowedKnowledgeIds)) {
hydrateQdrantHit(match, allowedKnowledgeIds, formalPolicyOnly, includePublishedCandidates)
.ifPresent(hits::add);
}
results.put(variants.get(index).code(), List.copyOf(hits));
}
return results;
} catch (Exception ignored) {
// Vector recall is additive; try the next configured runtime before sparse-only fallback.
}
}
return Map.of();
}
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit) {
return fulltextHits(category, queryText, limit, null);
}
@@ -3674,6 +4037,12 @@ public class AihrSopSeedService {
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit,
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly) {
return fulltextHits(category, queryText, limit, allowedKnowledgeIds, formalPolicyOnly, false);
}
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit,
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly,
boolean includePublishedCandidates) {
String sql = """
select f.id as fragment_id,
coalesce(a.name, i.name) as title,
@@ -3695,7 +4064,8 @@ public class AihrSopSeedService {
order by score desc, f.idx asc
limit ?
""".formatted(knowledgeScopeSql("f", allowedKnowledgeIds),
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"));
formalSourceScopeSql("f", formalPolicyOnly),
candidateLifecycleScopeSql("f", includePublishedCandidates));
List<Object> args = new ArrayList<>();
args.add(queryText);
args.add(tenantId());
@@ -3727,6 +4097,12 @@ public class AihrSopSeedService {
private List<KnowledgeHit> likeHits(String category, String queryText, int limit,
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly) {
return likeHits(category, queryText, limit, allowedKnowledgeIds, formalPolicyOnly, false);
}
private List<KnowledgeHit> likeHits(String category, String queryText, int limit,
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly,
boolean includePublishedCandidates) {
List<String> terms = searchTerms(queryText);
if (terms.isEmpty()) {
return List.of();
@@ -3758,7 +4134,8 @@ public class AihrSopSeedService {
order by score desc, f.idx asc
limit ?
""".formatted(String.join(" + ", scoreParts), knowledgeScopeSql("f", allowedKnowledgeIds),
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"),
formalSourceScopeSql("f", formalPolicyOnly),
candidateLifecycleScopeSql("f", includePublishedCandidates),
String.join(" or ", whereParts));
List<Object> args = new ArrayList<>();
terms.stream().map(AihrSopSeedService::likePattern).forEach(args::add);
@@ -3791,6 +4168,12 @@ public class AihrSopSeedService {
private Optional<KnowledgeHit> hydrateQdrantHit(VectorMatch match, Set<Long> allowedKnowledgeIds,
boolean formalPolicyOnly) {
return hydrateQdrantHit(match, allowedKnowledgeIds, formalPolicyOnly, false);
}
private Optional<KnowledgeHit> hydrateQdrantHit(VectorMatch match, Set<Long> allowedKnowledgeIds,
boolean formalPolicyOnly,
boolean includePublishedCandidates) {
String sql = """
select f.id as fragment_id,
coalesce(a.name, i.name) as title,
@@ -3811,7 +4194,8 @@ public class AihrSopSeedService {
and f.idx = ?
limit 1
""".formatted(knowledgeScopeSql("f", allowedKnowledgeIds),
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"));
formalSourceScopeSql("f", formalPolicyOnly),
candidateLifecycleScopeSql("f", includePublishedCandidates));
List<Object> args = new ArrayList<>();
args.add(tenantId());
addKnowledgeScopeArgs(args, allowedKnowledgeIds);
@@ -3862,15 +4246,21 @@ public class AihrSopSeedService {
}
private SearchResponse responseFromHits(String category, String queryText, List<KnowledgeHit> hits, List<String> extraCautions) {
return responseFromHits(category, queryText, hits, extraCautions, RETRIEVAL_DISPLAY_K);
}
private SearchResponse responseFromHits(String category, String queryText, List<KnowledgeHit> hits,
List<String> extraCautions, int displayLimit) {
List<KnowledgeHit> evidenceHits = evidenceHits(queryText, hits);
KnowledgeHit first = evidenceHits.get(0);
String responseCategory = isBlank(category) ? first.category() : category;
List<SnippetResponse> snippets = evidenceHits.stream()
List<KnowledgeHit> displayHits = displayHits(evidenceHits, displayLimit);
List<SnippetResponse> snippets = displayHits.stream()
.map(hit -> new SnippetResponse("第 " + hit.idx() + " 段:" + hit.title(),
displaySnippetText(hit.content()), hit.fragmentId(), hit.score(), hit.retrievalChannel()))
.toList();
Map<String, DocResponse> docs = new LinkedHashMap<>();
for (KnowledgeHit hit : evidenceHits) {
for (KnowledgeHit hit : displayHits) {
docs.putIfAbsent(hit.title(), new DocResponse(hit.title(), hit.description(), "命中 " + scoreText(hit.score()), "pill-success"));
}
SearchDigest digest = searchDigest(queryText, evidenceHits);
@@ -3914,6 +4304,11 @@ public class AihrSopSeedService {
return hits.stream().limit(MAX_EVIDENCE_SNIPPETS).toList();
}
static List<KnowledgeHit> displayHits(List<KnowledgeHit> evidenceHits, Integer limit) {
if (evidenceHits == null || evidenceHits.isEmpty()) return List.of();
return evidenceHits.stream().limit(displayLimitValue(limit)).toList();
}
private static boolean queryMentionsTitle(String queryText, String title) {
String query = evidenceText(queryText);
String docTitle = evidenceText(title);
@@ -4201,26 +4596,7 @@ public class AihrSopSeedService {
}
private static List<String> searchTerms(String queryText) {
String cleaned = Optional.ofNullable(queryText).orElse("")
.toLowerCase()
.replaceAll("[^\\p{IsHan}\\p{Alnum}]+", " ")
.trim();
if (cleaned.isEmpty()) {
return List.of();
}
LinkedHashSet<String> terms = new LinkedHashSet<>();
for (String token : cleaned.split("\\s+")) {
if (token.length() >= 2 && token.length() <= 16) {
terms.add(token);
}
for (int i = 0; i + 2 <= token.length() && terms.size() < 12; i++) {
terms.add(token.substring(i, i + 2));
}
if (terms.size() >= 12) {
break;
}
}
return terms.stream().limit(12).toList();
return AihrRagQueryPlanner.sparseTerms(queryText);
}
private static String likePattern(String term) {
@@ -5474,6 +5850,63 @@ public class AihrSopSeedService {
static record FeedbackFilterResult(List<KnowledgeHit> hits, List<String> cautions) {
}
public record SearchExecution(SearchResponse response, QueryPlan queryPlan,
List<RetrievalCandidate> candidates) {
public SearchExecution {
candidates = candidates == null ? List.of() : List.copyOf(candidates);
}
}
private record QueryVariant(String code, String text) {}
private static final class CandidateAccumulator {
private final KnowledgeHit hit;
private final List<ChannelContribution> contributions = new ArrayList<>();
private double sparseSignal;
private double vectorSignal;
private double sparseFusion;
private double vectorFusion;
private double fusionScore;
private String sourceAuthority = "UNKNOWN";
private String sourceKind = "REFERENCE_MATERIAL";
private Integer candidateRank;
private Double rerankScore;
private String rerankModel;
private boolean rerankApplied;
private boolean selectedForEvidence;
private String decisionReason;
private CandidateAccumulator(KnowledgeHit hit) {
this.hit = hit;
}
private void add(String channel, int channelRank, double rawScore, String queryVariant, boolean sparse) {
double signal = reciprocalRank(channelRank);
contributions.add(new ChannelContribution(channel, channelRank, rawScore, signal, queryVariant));
if (sparse) sparseSignal = Math.max(sparseSignal, signal);
else vectorSignal = Math.max(vectorSignal, signal);
}
private void finishFusion() {
fusionScore = sparseFusion + vectorFusion;
}
private double fusionScore() {
return fusionScore;
}
private void reject(String reason) {
selectedForEvidence = false;
if (decisionReason == null || "OUTSIDE_EVIDENCE_K".equals(decisionReason)) decisionReason = reason;
}
private RetrievalCandidate toTrace() {
return new RetrievalCandidate(hit.fragmentId(), hit.title(), sourceAuthority, sourceKind,
candidateRank, fusionScore, rerankScore, rerankModel, rerankApplied, selectedForEvidence,
firstNonBlank(decisionReason, "CANDIDATE"), List.copyOf(contributions));
}
}
private record KnowledgeConfig(long knowledgeId, int blockSize, int overlap) {
}
@@ -0,0 +1,61 @@
package org.dromara.aihr.knowledge.service;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@Tag("dev")
class AihrRagQueryPlannerTest {
@Test
void removesNumericNgramNoiseButPreservesTypedAmountAndBusinessIntent() {
var query = "我有个报销500块钱,告诉我怎么走流程。";
var plan = AihrRagQueryPlanner.plan(query);
var terms = AihrRagQueryPlanner.sparseTerms(query);
assertThat(plan.originalQuery()).isEqualTo(query);
assertThat(plan.domain()).isEqualTo(AihrRagQueryPlanner.FINANCE_POLICY);
assertThat(plan.entities().amount()).isEqualTo(500D);
assertThat(plan.entities().currency()).isEqualTo("CNY");
assertThat(terms).contains("500元", "报销", "流程");
assertThat(terms).doesNotContain("50", "00", "销5", "0块", "我有", "告诉");
}
@Test
void plansAmbiguousPurchaseThresholdAsFormalFinanceClarification() {
List<String> queries = List.of(
"500元以下的零星采购怎么报销?",
"五百块的小额自采怎么走账?"
);
for (String query : queries) {
var plan = AihrRagQueryPlanner.plan(query);
assertThat(plan.domain()).isEqualTo(AihrRagQueryPlanner.FINANCE_REIMBURSEMENT);
assertThat(plan.entities().amount()).isEqualTo(500D);
assertThat(plan.entities().amountBasis()).isEqualTo("UNKNOWN");
assertThat(plan.needsClarification()).isTrue();
assertThat(plan.requiredAuthorities())
.containsExactly("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP");
assertThat(plan.evidenceNeeds())
.contains("AMOUNT_THRESHOLD", "AMOUNT_BASIS", "ENTRY_STEPS", "REQUIRED_MATERIALS",
"APPROVAL_REVIEW", "EXCEPTIONS");
}
}
@Test
void preservesExplicitAmountBasisAndRejectsExperienceAsFormalAuthority() {
var plan = AihrRagQueryPlanner.plan("月累计500元以内的零星采购怎么报销?");
assertThat(plan.entities().amountBasis()).isEqualTo("MONTHLY_TOTAL");
assertThat(plan.needsClarification()).isFalse();
assertThat(AihrRagQueryPlanner.sourceKind("EMPLOYEE_SUBMISSION", "INTERVIEW", "员工访谈"))
.isEqualTo("EXPERIENCE");
assertThat(AihrRagQueryPlanner.authorityAllowed(plan, "EXPERIENCE")).isFalse();
assertThat(AihrRagQueryPlanner.sourceKind("COMPANY_POLICY", "PDF", "费用报销操作手册.pdf"))
.isEqualTo("OPERATING_MANUAL");
}
}
@@ -152,6 +152,20 @@ public class AihrSopSeedServiceTest {
assertEquals(3L, evidence.get(2).fragmentId());
}
@Test
@Tag("dev")
public void displayLimitDoesNotReduceEvidencePool() {
List<AihrSopSeedService.KnowledgeHit> evidence = AihrSopSeedService.evidenceHits(
"费用报销的发票怎么登记?",
List.of(hit(1L, "费用报销操作手册"), hit(2L, "费用报销操作手册"), hit(3L, "费用报销操作手册")));
List<AihrSopSeedService.KnowledgeHit> displayed = AihrSopSeedService.displayHits(evidence, 1);
assertEquals(3, evidence.size());
assertEquals(1, displayed.size());
assertEquals(1L, displayed.get(0).fragmentId());
}
@Test
@Tag("dev")
public void allDownVotedHitsAreSuppressed() {
@@ -667,12 +681,14 @@ public class AihrSopSeedServiceTest {
assertTrue(code.contains("searchAuthorized(SearchRequest request, String requesterExtPartyId, Set<Long> allowedKnowledgeIds)"));
assertTrue(code.contains(".knowledge_id in ("));
assertTrue(code.contains("qdrantMatchAny(\"knowledge_id\", knowledgeIds)"));
assertTrue(code.contains("aihr_knowledge_source_governance"));
assertTrue(code.contains("governance.source_doc_id"));
assertTrue(code.contains("governance.authority_type = 'FORMAL_POLICY'"));
assertTrue(code.contains("governance.lifecycle_status = 'APPROVED'"));
assertTrue(code.contains("governance.effective_date <= current_date()"));
assertTrue(code.contains("governance.content_sha256 regexp '^[0-9a-f]{64}$'"));
assertTrue(code.contains("governed_dataset.dataset_code = 'production'"));
assertTrue(code.contains("governed_dataset.status = 'ACTIVE'"));
assertTrue(code.contains("governed_asset.lifecycle_status = 'PUBLISHED'"));
assertTrue(code.contains("governed_asset.trust_level = 'HUMAN_VERIFIED'"));
assertTrue(code.contains("upper(governed_asset.source_authority) in"));
assertTrue(code.contains("'COMPANY_POLICY', 'FORMAL_POLICY', 'OPERATING_MANUAL', 'APPROVED_SOP'"));
assertTrue(code.contains("governed_asset.effective_from is null"));
assertTrue(code.contains("governed_asset.effective_to is null"));
AihrSopSeedService service = new AihrSopSeedService(
new ObjectMapper(), mock(JdbcTemplate.class), null, "", null, null, mock(AihrModelSeedService.class));
@@ -680,6 +696,27 @@ public class AihrSopSeedServiceTest {
() -> service.searchAuthorized(new AihrSopDto.SearchRequest("问题", "sop", null, null, 5), "", Set.of()));
}
@Test
@Tag("dev")
public void formalCandidateRecallIncludesPublishedGovernedFragmentsWithoutChangingDefaultRollout() throws Exception {
Method candidateScope = AihrSopSeedService.class.getDeclaredMethod(
"candidateLifecycleScopeSql", String.class, boolean.class);
Method servingScope = AihrSopSeedService.class.getDeclaredMethod("servingLifecycleScopeSql", String.class);
Method publishedScope = AihrSopSeedService.class.getDeclaredMethod("publishedLifecycleExistsSql", String.class);
candidateScope.setAccessible(true);
servingScope.setAccessible(true);
publishedScope.setAccessible(true);
String serving = (String) servingScope.invoke(null, "f");
String published = (String) publishedScope.invoke(null, "f");
String defaultCandidates = (String) candidateScope.invoke(null, "f", false);
String formalCandidates = (String) candidateScope.invoke(null, "f", true);
assertEquals(serving, defaultCandidates);
assertTrue(formalCandidates.contains(serving.substring("and ".length())));
assertTrue(formalCandidates.contains(published));
}
@Test
@Tag("dev")
public void legacySearchControllerDelegatesToAuthorizedQueryService() throws Exception {
@@ -0,0 +1,29 @@
-- Generation-safe RAG candidate observability. Additive and idempotent on MySQL 8.
CREATE TABLE IF NOT EXISTS `aihr_query_candidate_trace` (
`id` bigint NOT NULL AUTO_INCREMENT,
`tenant_id` varchar(20) NOT NULL,
`request_id` varchar(64) NOT NULL,
`fragment_id` bigint NOT NULL,
`candidate_rank` int DEFAULT NULL COMMENT 'Rank after rerank, null when rejected before rerank',
`channel` varchar(20) NOT NULL COMMENT 'KEYWORD/FULLTEXT/VECTOR',
`channel_rank` int NOT NULL,
`raw_score` decimal(18,10) DEFAULT NULL,
`fusion_score` decimal(18,10) DEFAULT NULL,
`rerank_score` decimal(18,10) DEFAULT NULL,
`rerank_model` varchar(100) DEFAULT NULL,
`rerank_applied` tinyint(1) NOT NULL DEFAULT 0,
`source_authority` varchar(30) DEFAULT NULL,
`source_kind` varchar(30) DEFAULT NULL,
`query_variant` varchar(30) NOT NULL COMMENT 'ORIGINAL/NORMALIZED/GLOSSARY/ALTERNATE_n',
`selected_for_evidence` tinyint(1) NOT NULL DEFAULT 0,
`decision_reason` varchar(64) NOT NULL,
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_aihr_query_candidate_channel`
(`tenant_id`, `request_id`, `fragment_id`, `channel`, `query_variant`),
KEY `idx_aihr_query_candidate_request` (`tenant_id`, `request_id`, `candidate_rank`),
KEY `idx_aihr_query_candidate_reason` (`tenant_id`, `decision_reason`, `create_time`),
KEY `idx_aihr_query_candidate_fragment` (`tenant_id`, `fragment_id`, `create_time`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
COMMENT='Pre-generation retrieval candidates, channel contributions, rerank and gate decisions';