feat(aihr): harden finance RAG retrieval and evidence gates
This commit is contained in:
+41
-5
@@ -57,22 +57,46 @@ public final class AihrKnowledgeQueryDto {
|
||||
String updatedAt,
|
||||
String mediaType,
|
||||
String detailRef,
|
||||
LocatorSummary locatorSummary
|
||||
LocatorSummary locatorSummary,
|
||||
String sourceAuthority,
|
||||
String sourceKind
|
||||
) {
|
||||
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
|
||||
Long fragmentId, String domain, String status, String occurredAt, String updatedAt,
|
||||
String mediaType, String detailRef, LocatorSummary locatorSummary) {
|
||||
this(spaceCode, sourceType, docId, title, snippet, fragmentId, domain, status, occurredAt,
|
||||
updatedAt, mediaType, detailRef, locatorSummary, null, null);
|
||||
}
|
||||
|
||||
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
|
||||
Long fragmentId) {
|
||||
this(spaceCode, sourceType, docId, title, snippet, fragmentId,
|
||||
"DOCUMENT".equals(sourceType) ? "ENTERPRISE" : sourceType, null, null, null,
|
||||
null, null, null);
|
||||
null, null, null, null, null);
|
||||
}
|
||||
|
||||
public Citation(String spaceCode, String sourceType, String docId, String title, String snippet,
|
||||
Long fragmentId, String domain, String status, String occurredAt, String updatedAt) {
|
||||
this(spaceCode, sourceType, docId, title, snippet, fragmentId, domain, status, occurredAt,
|
||||
updatedAt, null, null, null);
|
||||
updatedAt, null, null, null, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
public record QueryEntity(Double amount, String currency, String comparator, String amountBasis) {}
|
||||
|
||||
public record QueryPlan(String originalQuery, String normalizedQuery, List<String> alternateQueries,
|
||||
String intent, String domain, QueryEntity entities,
|
||||
List<String> requiredAuthorities, boolean needsClarification,
|
||||
List<String> evidenceNeeds, List<String> clarificationReasons) {}
|
||||
|
||||
public record ChannelContribution(String channel, int channelRank, Double rawScore,
|
||||
Double fusionScore, String queryVariant) {}
|
||||
|
||||
public record RetrievalCandidate(Long fragmentId, String title, String sourceAuthority, String sourceKind,
|
||||
Integer candidateRank, Double fusionScore, Double rerankScore,
|
||||
String rerankModel, boolean rerankApplied, boolean selectedForEvidence,
|
||||
String decisionReason, List<ChannelContribution> contributions) {}
|
||||
|
||||
public record LocatorSummary(Integer pageNumber, Integer slideNumber, Integer paragraphStart,
|
||||
Integer paragraphEnd, String sheetName, Integer rowStart, Integer rowEnd,
|
||||
Long startMs, Long endMs, Long frameMs) {}
|
||||
@@ -127,8 +151,20 @@ public final class AihrKnowledgeQueryDto {
|
||||
String rewrittenQuery,
|
||||
List<Resource> resources,
|
||||
MemoryCandidateResponse memoryCandidate,
|
||||
BroadcastContext broadcastContext
|
||||
BroadcastContext broadcastContext,
|
||||
QueryPlan queryPlan,
|
||||
List<RetrievalCandidate> retrievalCandidates
|
||||
) {
|
||||
public QueryResponse(String requestId, String queryText, String answer, List<Citation> citations,
|
||||
List<String> usedSpaceCodes, boolean noEvidence, String promptVersion,
|
||||
SearchResponse legacy, Object data, String conversationId, Long contextVersion,
|
||||
String intent, String rewrittenQuery, List<Resource> resources,
|
||||
MemoryCandidateResponse memoryCandidate, BroadcastContext broadcastContext) {
|
||||
this(requestId, queryText, answer, citations, usedSpaceCodes, noEvidence, promptVersion,
|
||||
legacy, data, conversationId, contextVersion, intent, rewrittenQuery, resources,
|
||||
memoryCandidate, broadcastContext, null, List.of());
|
||||
}
|
||||
|
||||
public QueryResponse(String requestId, String queryText, String answer, List<Citation> citations,
|
||||
List<String> usedSpaceCodes, boolean noEvidence, String promptVersion,
|
||||
SearchResponse legacy, Object data, String conversationId, Long contextVersion,
|
||||
@@ -157,7 +193,7 @@ public final class AihrKnowledgeQueryDto {
|
||||
public QueryResponse withMemoryCandidate(MemoryCandidateResponse candidate) {
|
||||
return new QueryResponse(requestId, queryText, answer, citations, usedSpaceCodes, noEvidence,
|
||||
promptVersion, legacy, data, conversationId, contextVersion, intent, rewrittenQuery,
|
||||
resources, candidate, broadcastContext);
|
||||
resources, candidate, broadcastContext, queryPlan, retrievalCandidates);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+39
@@ -7,6 +7,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeAppDto.AuthenticatedApp;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SnippetResponse;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.ChannelContribution;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.RetrievalCandidate;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -105,6 +107,37 @@ public class AihrKnowledgeQueryAuditService {
|
||||
recordEvidence(requestId, tenantId, snippets, fragmentIds);
|
||||
}
|
||||
|
||||
public void recordCandidates(String requestId, String tenantId, List<RetrievalCandidate> candidates) {
|
||||
if (candidates == null || candidates.isEmpty()) return;
|
||||
for (RetrievalCandidate candidate : candidates) {
|
||||
if (candidate == null || candidate.fragmentId() == null || candidate.fragmentId() <= 0) continue;
|
||||
List<ChannelContribution> contributions = candidate.contributions() == null
|
||||
? List.of() : candidate.contributions();
|
||||
for (ChannelContribution contribution : contributions) {
|
||||
try {
|
||||
jdbcTemplate.update("""
|
||||
insert into aihr_query_candidate_trace
|
||||
(tenant_id, request_id, fragment_id, candidate_rank, channel, channel_rank,
|
||||
raw_score, fusion_score, rerank_score, rerank_model, rerank_applied,
|
||||
source_authority, source_kind, query_variant, selected_for_evidence,
|
||||
decision_reason, create_time)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now())
|
||||
""", tenantId, requestId, candidate.fragmentId(), candidate.candidateRank(),
|
||||
normalizedChannel(contribution.channel()), contribution.channelRank(),
|
||||
contribution.rawScore(), candidate.fusionScore(), candidate.rerankScore(),
|
||||
bounded(candidate.rerankModel(), 100), candidate.rerankApplied(),
|
||||
bounded(candidate.sourceAuthority(), 30), bounded(candidate.sourceKind(), 30),
|
||||
bounded(contribution.queryVariant(), 30), candidate.selectedForEvidence(),
|
||||
bounded(candidate.decisionReason(), 64));
|
||||
} catch (DataAccessException ex) {
|
||||
log.warn("knowledge query candidate trace write failed for request {} fragment {}",
|
||||
requestId, candidate.fragmentId());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String normalizedChannel(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return "UNKNOWN";
|
||||
@@ -113,6 +146,12 @@ public class AihrKnowledgeQueryAuditService {
|
||||
return normalized.length() > 20 ? normalized.substring(0, 20) : normalized;
|
||||
}
|
||||
|
||||
private static String bounded(String value, int max) {
|
||||
if (value == null || value.isBlank()) return null;
|
||||
String normalized = value.trim();
|
||||
return normalized.length() <= max ? normalized : normalized.substring(0, max);
|
||||
}
|
||||
|
||||
private String json(List<String> values) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(values == null ? List.of() : values);
|
||||
|
||||
+56
-61
@@ -22,6 +22,7 @@ import org.dromara.aihr.memory.AihrMemoryDto.ServiceMemoryRecall;
|
||||
import org.dromara.aihr.memory.AihrMemoryService;
|
||||
import org.dromara.aihr.service.AihrModelSeedService;
|
||||
import org.dromara.aihr.service.AihrSopSeedService;
|
||||
import org.dromara.aihr.service.AihrSopSeedService.SearchExecution;
|
||||
import org.dromara.common.core.constant.HttpStatus;
|
||||
import org.dromara.common.core.enums.UserType;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
@@ -368,11 +369,17 @@ public class AihrKnowledgeQueryService {
|
||||
principal == null ? Set.of() : principal.roles());
|
||||
SearchRequest searchRequest = new SearchRequest(
|
||||
request.queryText(), request.category(), request.position(), retrievalSource, request.limit());
|
||||
SearchResponse legacy = retrievalQuery.equals(request.queryText())
|
||||
? sopService.searchAuthorized(searchRequest,
|
||||
principal == null ? "" : principal.extPartyId(), spaceIds)
|
||||
: sopService.searchAuthorized(searchRequest,
|
||||
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
|
||||
SearchExecution execution = sopService.searchAuthorizedDetailed(searchRequest,
|
||||
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
|
||||
if (execution == null) {
|
||||
SearchResponse fallback = retrievalQuery.equals(request.queryText())
|
||||
? sopService.searchAuthorized(searchRequest,
|
||||
principal == null ? "" : principal.extPartyId(), spaceIds)
|
||||
: sopService.searchAuthorized(searchRequest,
|
||||
principal == null ? "" : principal.extPartyId(), spaceIds, retrievalQuery);
|
||||
execution = new SearchExecution(fallback, null, List.of());
|
||||
}
|
||||
SearchResponse legacy = execution.response();
|
||||
List<Citation> citations = citations(
|
||||
app.tenantId(), spaceIds, legacy.snippets(), formalPolicyOnly, request.queryText());
|
||||
boolean noEvidence = citations.isEmpty();
|
||||
@@ -386,8 +393,10 @@ public class AihrKnowledgeQueryService {
|
||||
citations.isEmpty() ? List.of() : List.of("DOCUMENT"), status, latency, legacy.promptVersion());
|
||||
auditService.recordEvidence(requestId, app.tenantId(), legacy.snippets(), citations.stream()
|
||||
.map(Citation::fragmentId).filter(Objects::nonNull).toList());
|
||||
auditService.recordCandidates(requestId, app.tenantId(), execution.candidates());
|
||||
return new QueryResponse(requestId, displayQuery, answer, citations, scopeCodes,
|
||||
noEvidence, legacy.promptVersion(), legacy, null);
|
||||
noEvidence, legacy.promptVersion(), legacy, null, null, null, "QA", request.queryText(),
|
||||
List.of(), null, null, execution.queryPlan(), execution.candidates());
|
||||
} catch (RuntimeException ex) {
|
||||
auditService.record(requestId, principal, app, displayQuery, scopeCodes, List.of(),
|
||||
ex instanceof ServiceException serviceEx && serviceEx.getCode() != null && serviceEx.getCode() < 500
|
||||
@@ -419,7 +428,8 @@ public class AihrKnowledgeQueryService {
|
||||
documentResponse.usedSpaceCodes(), false, BROADCAST_PROMPT_VERSION, documentResponse.legacy(), null,
|
||||
documentResponse.conversationId(), documentResponse.contextVersion(), "QA",
|
||||
documentResponse.rewrittenQuery(), documentResponse.resources(), documentResponse.memoryCandidate(),
|
||||
new BroadcastContext(broadcast.messageId(), broadcast.title(), broadcast.publishedAt())
|
||||
new BroadcastContext(broadcast.messageId(), broadcast.title(), broadcast.publishedAt()),
|
||||
documentResponse.queryPlan(), documentResponse.retrievalCandidates()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -536,7 +546,8 @@ public class AihrKnowledgeQueryService {
|
||||
return new QueryResponse(
|
||||
response.requestId(), question, answer, citations, response.usedSpaceCodes(), noEvidence,
|
||||
response.promptVersion(), response.legacy(), response.data(), context.conversationId(), nextVersion,
|
||||
context.intent(), context.rewrittenQuery(), resources, response.memoryCandidate(), response.broadcastContext()
|
||||
context.intent(), context.rewrittenQuery(), resources, response.memoryCandidate(), response.broadcastContext(),
|
||||
response.queryPlan(), response.retrievalCandidates()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -564,7 +575,8 @@ public class AihrKnowledgeQueryService {
|
||||
String ref = citationDetailService.issue(citation, tenantId);
|
||||
return new Citation(citation.spaceCode(), citation.sourceType(), citation.docId(), citation.title(),
|
||||
citation.snippet(), citation.fragmentId(), citation.domain(), citation.status(), citation.occurredAt(),
|
||||
citation.updatedAt(), location.sourceKind(), ref, location.summary());
|
||||
citation.updatedAt(), location.sourceKind(), ref, location.summary(),
|
||||
citation.sourceAuthority(), citation.sourceKind());
|
||||
}).toList();
|
||||
}
|
||||
|
||||
@@ -611,7 +623,7 @@ public class AihrKnowledgeQueryService {
|
||||
response.requestId(), response.queryText(), answer, List.copyOf(citations), List.copyOf(usedScopes),
|
||||
false, response.promptVersion(), response.legacy(), response.data(), response.conversationId(),
|
||||
response.contextVersion(), response.intent(), response.rewrittenQuery(), response.resources(),
|
||||
response.memoryCandidate()
|
||||
response.memoryCandidate(), response.broadcastContext(), response.queryPlan(), response.retrievalCandidates()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -720,26 +732,7 @@ public class AihrKnowledgeQueryService {
|
||||
args.add(tenantId);
|
||||
args.addAll(allowedSpaceIds);
|
||||
args.addAll(fragmentIds);
|
||||
String formalSourceClause = formalPolicyOnly ? """
|
||||
and exists (
|
||||
select 1
|
||||
from aihr_knowledge_attach governed_attach
|
||||
join aihr_knowledge_source_governance governance
|
||||
on governance.tenant_id = governed_attach.tenant_id
|
||||
and governance.attachment_id = governed_attach.id
|
||||
where governed_attach.tenant_id = f.tenant_id
|
||||
and governed_attach.knowledge_id = f.knowledge_id
|
||||
and governed_attach.doc_id = f.doc_id
|
||||
and governed_attach.status = 2
|
||||
and governance.source_doc_id = f.doc_id
|
||||
and governance.authority_type = 'FORMAL_POLICY'
|
||||
and governance.lifecycle_status = 'APPROVED'
|
||||
and governance.effective_date <= current_date()
|
||||
and (governance.expires_date is null or governance.expires_date >= current_date())
|
||||
and char_length(trim(governance.source_version)) > 0
|
||||
and governance.content_sha256 regexp '^[0-9a-f]{64}$'
|
||||
)
|
||||
""" : "";
|
||||
String formalSourceClause = "";
|
||||
List<Citation> rows = jdbcTemplate.query("""
|
||||
select k.code as space_code, f.doc_id,
|
||||
coalesce((
|
||||
@@ -756,39 +749,39 @@ public class AihrKnowledgeQueryService {
|
||||
and governed_title_chunk.published_fragment_id = f.id
|
||||
limit 1
|
||||
), a.name, k.name) as title,
|
||||
f.content, f.id as fragment_id
|
||||
from aihr_knowledge_fragment f
|
||||
join aihr_knowledge_info k on k.id = f.knowledge_id and k.tenant_id = f.tenant_id
|
||||
left join aihr_knowledge_attach a
|
||||
on a.tenant_id = f.tenant_id and a.knowledge_id = f.knowledge_id and a.doc_id = f.doc_id
|
||||
where f.tenant_id = ?
|
||||
and f.knowledge_id in (%s)
|
||||
and f.id in (%s)
|
||||
and exists (
|
||||
select 1
|
||||
from aihr_chunk_revision governed_chunk
|
||||
join aihr_data_asset governed_asset
|
||||
on governed_asset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_asset.id = governed_chunk.asset_id
|
||||
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
|
||||
join aihr_dataset_membership governed_dataset
|
||||
on governed_dataset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_dataset.version_id = governed_chunk.version_id
|
||||
and governed_dataset.dataset_code = 'production'
|
||||
and governed_dataset.status = 'ACTIVE'
|
||||
where governed_chunk.tenant_id = f.tenant_id
|
||||
and governed_chunk.published_fragment_id = f.id
|
||||
and governed_asset.lifecycle_status = 'PUBLISHED'
|
||||
and governed_asset.trust_level = 'HUMAN_VERIFIED'
|
||||
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
|
||||
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
|
||||
)
|
||||
%s
|
||||
f.content, f.id as fragment_id, governed_asset.source_authority,
|
||||
governed_asset.source_type, governed_asset.source_name
|
||||
from aihr_knowledge_fragment f
|
||||
join aihr_knowledge_info k on k.id = f.knowledge_id and k.tenant_id = f.tenant_id
|
||||
left join aihr_knowledge_attach a
|
||||
on a.tenant_id = f.tenant_id and a.knowledge_id = f.knowledge_id and a.doc_id = f.doc_id
|
||||
join aihr_chunk_revision governed_chunk
|
||||
on governed_chunk.tenant_id = f.tenant_id and governed_chunk.published_fragment_id = f.id
|
||||
join aihr_data_asset governed_asset
|
||||
on governed_asset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_asset.id = governed_chunk.asset_id
|
||||
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
|
||||
join aihr_dataset_membership governed_dataset
|
||||
on governed_dataset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_dataset.version_id = governed_chunk.version_id
|
||||
and governed_dataset.dataset_code = 'production'
|
||||
and governed_dataset.status = 'ACTIVE'
|
||||
where f.tenant_id = ?
|
||||
and f.knowledge_id in (%s)
|
||||
and f.id in (%s)
|
||||
and governed_asset.lifecycle_status = 'PUBLISHED'
|
||||
and governed_asset.trust_level = 'HUMAN_VERIFIED'
|
||||
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
|
||||
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
|
||||
%s
|
||||
""".formatted(placeholders(allowedSpaceIds.size()), placeholders(fragmentIds.size()),
|
||||
formalSourceClause),
|
||||
(rs, rowNum) -> new Citation(
|
||||
rs.getString("space_code"), "DOCUMENT", rs.getString("doc_id"), rs.getString("title"),
|
||||
rs.getString("content"), rs.getLong("fragment_id")
|
||||
rs.getString("content"), rs.getLong("fragment_id"), "ENTERPRISE", null, null, null,
|
||||
null, null, null, rs.getString("source_authority"),
|
||||
AihrRagQueryPlanner.sourceKind(rs.getString("source_authority"), rs.getString("source_type"),
|
||||
rs.getString("source_name"))
|
||||
), args.toArray());
|
||||
Map<Long, Citation> byId = new LinkedHashMap<>();
|
||||
rows.forEach(row -> byId.put(row.fragmentId(), row));
|
||||
@@ -796,13 +789,15 @@ public class AihrKnowledgeQueryService {
|
||||
.map(byId::get)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.filter(row -> !formalPolicyOnly
|
||||
|| AihrFormalPolicyClassifier.isRelevantFormalEvidence(queryText, row.snippet()))
|
||||
|| (Set.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP").contains(row.sourceKind())
|
||||
&& AihrFormalPolicyClassifier.isRelevantFormalEvidence(queryText, row.snippet())))
|
||||
.map(row -> new Citation(
|
||||
row.spaceCode(), row.sourceType(), row.docId(), row.title(),
|
||||
formalPolicyOnly
|
||||
? AihrFormalPolicyClassifier.relevantEvidenceSnippet(queryText, row.snippet(), 500)
|
||||
: truncate(row.snippet(), 500),
|
||||
row.fragmentId()
|
||||
row.fragmentId(), row.domain(), row.status(), row.occurredAt(), row.updatedAt(),
|
||||
row.mediaType(), row.detailRef(), row.locatorSummary(), row.sourceAuthority(), row.sourceKind()
|
||||
))
|
||||
.toList();
|
||||
}
|
||||
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
package org.dromara.aihr.knowledge.service;
|
||||
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryEntity;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryPlan;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Deterministic, authorization-free planning for bounded enterprise knowledge retrieval. */
|
||||
public final class AihrRagQueryPlanner {
|
||||
|
||||
public static final String FINANCE_REIMBURSEMENT = "FINANCE_REIMBURSEMENT";
|
||||
public static final String FINANCE_POLICY = "FINANCE_POLICY";
|
||||
public static final String GENERAL_KNOWLEDGE = "GENERAL_KNOWLEDGE";
|
||||
private static final List<String> FORMAL_AUTHORITIES =
|
||||
List.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP");
|
||||
private static final Pattern ARABIC_AMOUNT = Pattern.compile(
|
||||
"(?<![0-9A-Za-z])(\\d+(?:\\.\\d+)?)\\s*(?:元|块钱|块|人民币)");
|
||||
private static final Pattern CHINESE_HUNDREDS = Pattern.compile("([一二三四五六七八九])百(?:\\s*(?:元|块钱|块))?");
|
||||
private static final Pattern HAN_RUN = Pattern.compile("[\\p{IsHan}]{2,16}");
|
||||
private static final Pattern LATIN_TERM = Pattern.compile("[A-Za-z][A-Za-z0-9_-]{1,31}");
|
||||
private static final List<String> BUSINESS_TERMS = List.of(
|
||||
"费用报销", "报销流程", "发票登记", "财务复审", "零星采购", "小额采购", "个人借款",
|
||||
"采购审批", "报销单", "借款", "报销", "发票", "采购", "自采", "财务", "报备", "审批",
|
||||
"材料", "流程", "额度", "月累计", "单笔"
|
||||
);
|
||||
private static final List<String> STOP_PHRASES = List.of(
|
||||
"我有个", "告诉我", "请告诉", "请问", "帮我", "怎么走", "怎么", "如何", "一下", "这个",
|
||||
"那个", "有没有", "能不能", "可以吗", "是什么", "的", "以下", "以内", "以上", "不超过"
|
||||
);
|
||||
private static final Set<String> STOP_TERMS = Set.of(
|
||||
"我有", "有个", "个报", "告诉", "诉我", "怎么", "么走", "以下", "以内", "以上", "块钱",
|
||||
"这个", "那个", "一下", "请问", "能不", "不能", "可以", "以吗"
|
||||
);
|
||||
|
||||
private AihrRagQueryPlanner() {}
|
||||
|
||||
public static QueryPlan plan(String rawQuery) {
|
||||
String original = clean(rawQuery);
|
||||
QueryEntity entities = entities(original);
|
||||
String domain = domain(original);
|
||||
boolean finance = isFormalFinanceDomain(domain);
|
||||
String normalized = normalize(original, domain, entities);
|
||||
List<String> alternates = alternates(original, domain);
|
||||
List<String> evidenceNeeds = evidenceNeeds(original, domain, entities);
|
||||
List<String> reasons = new ArrayList<>();
|
||||
if (FINANCE_REIMBURSEMENT.equals(domain) && entities.amount() != null
|
||||
&& "UNKNOWN".equals(entities.amountBasis())) {
|
||||
reasons.add("金额口径不明确:请确认是单笔金额还是月累计额度");
|
||||
}
|
||||
return new QueryPlan(
|
||||
original,
|
||||
normalized,
|
||||
alternates,
|
||||
"KNOWLEDGE_QA",
|
||||
domain,
|
||||
entities,
|
||||
finance ? FORMAL_AUTHORITIES : List.of(),
|
||||
!reasons.isEmpty(),
|
||||
evidenceNeeds,
|
||||
List.copyOf(reasons)
|
||||
);
|
||||
}
|
||||
|
||||
public static List<String> sparseTerms(String queryText) {
|
||||
String original = clean(queryText).toLowerCase(Locale.ROOT);
|
||||
if (original.isBlank()) return List.of();
|
||||
LinkedHashSet<String> terms = new LinkedHashSet<>();
|
||||
for (String businessTerm : BUSINESS_TERMS) {
|
||||
if (original.contains(businessTerm)) terms.add(businessTerm);
|
||||
}
|
||||
Matcher amountMatcher = ARABIC_AMOUNT.matcher(original);
|
||||
StringBuffer withoutAmounts = new StringBuffer();
|
||||
while (amountMatcher.find()) {
|
||||
terms.add(amountMatcher.group(1) + "元");
|
||||
amountMatcher.appendReplacement(withoutAmounts, " ");
|
||||
}
|
||||
amountMatcher.appendTail(withoutAmounts);
|
||||
String text = normalizeChineseAmounts(withoutAmounts.toString())
|
||||
.replaceAll("[^\\p{IsHan}A-Za-z0-9_-]+", " ");
|
||||
for (String stop : STOP_PHRASES) text = text.replace(stop, " ");
|
||||
Matcher hanMatcher = HAN_RUN.matcher(text);
|
||||
while (hanMatcher.find() && terms.size() < 24) {
|
||||
String run = hanMatcher.group();
|
||||
if (!STOP_TERMS.contains(run)) terms.add(run);
|
||||
for (int i = 0; i + 2 <= run.length() && terms.size() < 24; i++) {
|
||||
String term = run.substring(i, i + 2);
|
||||
if (!STOP_TERMS.contains(term)) terms.add(term);
|
||||
}
|
||||
}
|
||||
Matcher latinMatcher = LATIN_TERM.matcher(text);
|
||||
while (latinMatcher.find() && terms.size() < 24) terms.add(latinMatcher.group().toLowerCase(Locale.ROOT));
|
||||
return terms.stream()
|
||||
.filter(term -> !term.matches("\\d+"))
|
||||
.filter(term -> !STOP_TERMS.contains(term))
|
||||
.limit(24)
|
||||
.toList();
|
||||
}
|
||||
|
||||
public static boolean isFormalFinanceDomain(String domain) {
|
||||
return FINANCE_REIMBURSEMENT.equals(domain) || FINANCE_POLICY.equals(domain);
|
||||
}
|
||||
|
||||
public static String sourceKind(String sourceAuthority, String sourceType, String sourceName) {
|
||||
String authority = clean(sourceAuthority).toUpperCase(Locale.ROOT);
|
||||
String type = clean(sourceType).toUpperCase(Locale.ROOT);
|
||||
String name = clean(sourceName).toLowerCase(Locale.ROOT);
|
||||
if (authority.contains("EXPERIENCE") || authority.contains("EMPLOYEE_SUBMISSION")
|
||||
|| type.contains("INTERVIEW")) return "EXPERIENCE";
|
||||
if (authority.contains("CASE") || type.contains("CASE")) return "CASE";
|
||||
if (name.contains("操作手册") || authority.contains("OPERATING_MANUAL")) return "OPERATING_MANUAL";
|
||||
if (name.contains("sop") || authority.contains("APPROVED_SOP")) return "APPROVED_SOP";
|
||||
if (authority.contains("POLICY") || authority.contains("COMPANY_POLICY")
|
||||
|| authority.contains("FORMAL")) return "FORMAL_POLICY";
|
||||
return "REFERENCE_MATERIAL";
|
||||
}
|
||||
|
||||
public static boolean authorityAllowed(QueryPlan plan, String sourceKind) {
|
||||
return plan == null || plan.requiredAuthorities().isEmpty()
|
||||
|| plan.requiredAuthorities().contains(sourceKind);
|
||||
}
|
||||
|
||||
public static boolean domainMatches(QueryPlan plan, String title, String content) {
|
||||
if (plan == null || !isFormalFinanceDomain(plan.domain())) return true;
|
||||
String source = clean(title) + " " + clean(content);
|
||||
if (FINANCE_REIMBURSEMENT.equals(plan.domain())) {
|
||||
return containsAny(source, "费用报销", "发票登记", "报销单", "财务复审", "个人借款")
|
||||
|| (containsAny(title, "报销", "发票", "财务", "借款")
|
||||
&& containsAny(source, "流程", "提交", "登记", "审批", "复审"));
|
||||
}
|
||||
return containsAny(title, "财务", "报销", "借款", "发票", "采购")
|
||||
|| containsAny(source, "财务管理", "费用报销", "个人借款", "采购审批");
|
||||
}
|
||||
|
||||
private static String domain(String query) {
|
||||
if (containsAny(query, "报销", "走账") && containsAny(query, "采购", "自采", "发票", "费用", "垫付")) {
|
||||
return FINANCE_REIMBURSEMENT;
|
||||
}
|
||||
if (containsAny(query, "报销", "借款", "发票", "财务", "采购", "自采", "报备", "审批")) {
|
||||
return FINANCE_POLICY;
|
||||
}
|
||||
return GENERAL_KNOWLEDGE;
|
||||
}
|
||||
|
||||
private static QueryEntity entities(String query) {
|
||||
Double amount = null;
|
||||
Matcher arabic = ARABIC_AMOUNT.matcher(query);
|
||||
if (arabic.find()) amount = Double.valueOf(arabic.group(1));
|
||||
if (amount == null) {
|
||||
Matcher chinese = CHINESE_HUNDREDS.matcher(query);
|
||||
if (chinese.find()) amount = chineseDigit(chinese.group(1)) * 100.0;
|
||||
}
|
||||
String comparator = containsAny(query, "以下", "以内", "不超过", "最多") ? "LTE"
|
||||
: containsAny(query, "超过", "大于") ? "GT"
|
||||
: containsAny(query, "以上", "不少于", "至少") ? "GTE" : "UNSPECIFIED";
|
||||
String basis = containsAny(query, "单笔", "每笔", "一笔") ? "PER_TRANSACTION"
|
||||
: containsAny(query, "月累计", "每月", "当月", "一个月", "月度") ? "MONTHLY_TOTAL" : "UNKNOWN";
|
||||
return new QueryEntity(amount, amount == null ? null : "CNY", comparator, basis);
|
||||
}
|
||||
|
||||
private static String normalize(String original, String domain, QueryEntity entities) {
|
||||
if (!isFormalFinanceDomain(domain)) return original;
|
||||
LinkedHashSet<String> parts = new LinkedHashSet<>();
|
||||
if (containsAny(original, "采购", "自采", "零星", "小额")) parts.add("零星采购");
|
||||
parts.add(FINANCE_REIMBURSEMENT.equals(domain) ? "费用报销流程" : "财务制度");
|
||||
if (containsAny(original, "发票")) parts.add("发票登记");
|
||||
if (entities.amount() != null) {
|
||||
String amount = entities.amount() % 1 == 0
|
||||
? String.valueOf(entities.amount().longValue()) : String.valueOf(entities.amount());
|
||||
parts.add(amount + "元" + comparatorText(entities.comparator()));
|
||||
}
|
||||
if (!"UNKNOWN".equals(entities.amountBasis())) parts.add(entities.amountBasis());
|
||||
return String.join(" ", parts);
|
||||
}
|
||||
|
||||
private static List<String> alternates(String original, String domain) {
|
||||
if (!isFormalFinanceDomain(domain)) return List.of();
|
||||
LinkedHashSet<String> values = new LinkedHashSet<>();
|
||||
values.add("费用报销 发票登记 报销单 财务复审");
|
||||
if (containsAny(original, "采购", "自采", "零星", "小额")) values.add("零星采购 自采 费用报销");
|
||||
if (containsAny(original, "借款")) values.add("个人借款 费用报销冲抵");
|
||||
return values.stream().limit(2).toList();
|
||||
}
|
||||
|
||||
private static List<String> evidenceNeeds(String original, String domain, QueryEntity entities) {
|
||||
if (!isFormalFinanceDomain(domain)) return List.of("DIRECT_ANSWER");
|
||||
LinkedHashSet<String> needs = new LinkedHashSet<>();
|
||||
if (entities.amount() != null) {
|
||||
needs.add("AMOUNT_THRESHOLD");
|
||||
needs.add("AMOUNT_BASIS");
|
||||
needs.add("APPLICABILITY");
|
||||
}
|
||||
if (containsAny(original, "怎么", "如何", "流程", "走账", "报销")) needs.add("ENTRY_STEPS");
|
||||
needs.add("REQUIRED_MATERIALS");
|
||||
needs.add("APPROVAL_REVIEW");
|
||||
needs.add("EXCEPTIONS");
|
||||
return List.copyOf(needs);
|
||||
}
|
||||
|
||||
private static String normalizeChineseAmounts(String value) {
|
||||
Matcher matcher = CHINESE_HUNDREDS.matcher(value);
|
||||
StringBuffer output = new StringBuffer();
|
||||
while (matcher.find()) matcher.appendReplacement(output, chineseDigit(matcher.group(1)) + "00元");
|
||||
matcher.appendTail(output);
|
||||
return output.toString();
|
||||
}
|
||||
|
||||
private static int chineseDigit(String value) {
|
||||
return "一二三四五六七八九".indexOf(value) + 1;
|
||||
}
|
||||
|
||||
private static String comparatorText(String comparator) {
|
||||
return switch (Optional.ofNullable(comparator).orElse("")) {
|
||||
case "LTE" -> "以下";
|
||||
case "GT" -> "以上";
|
||||
case "GTE" -> "及以上";
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
|
||||
private static boolean containsAny(String value, String... terms) {
|
||||
String source = clean(value).toLowerCase(Locale.ROOT);
|
||||
for (String term : terms) if (source.contains(term.toLowerCase(Locale.ROOT))) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String clean(String value) {
|
||||
return Optional.ofNullable(value).orElse("").trim();
|
||||
}
|
||||
}
|
||||
+479
-46
@@ -46,6 +46,10 @@ import org.dromara.aihr.knowledge.parse.ParsedDocument.LocatedSegment;
|
||||
import org.dromara.aihr.knowledge.parse.AihrExtractionQuality;
|
||||
import org.dromara.aihr.knowledge.parse.TikaKnowledgeDocumentParser;
|
||||
import org.dromara.aihr.knowledge.service.AihrFormalPolicyClassifier;
|
||||
import org.dromara.aihr.knowledge.service.AihrRagQueryPlanner;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.ChannelContribution;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryPlan;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.RetrievalCandidate;
|
||||
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycle.UsageType;
|
||||
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycle.ReasonCode;
|
||||
import org.dromara.aihr.knowledge.quality.AihrKnowledgeLifecycleService;
|
||||
@@ -126,6 +130,10 @@ public class AihrSopSeedService {
|
||||
private static final String LOCAL_EMBEDDING_BASE_URL = "local";
|
||||
private static final int LOCAL_EMBEDDING_DIMENSION = 1536;
|
||||
private static final int MAX_EVIDENCE_SNIPPETS = 3;
|
||||
private static final int RETRIEVAL_CANDIDATE_K = 30;
|
||||
private static final int RETRIEVAL_RERANK_K = 50;
|
||||
private static final int RETRIEVAL_EVIDENCE_K = 3;
|
||||
private static final int RETRIEVAL_DISPLAY_K = 3;
|
||||
private static final KnowledgeDocumentParser KNOWLEDGE_DOCUMENT_PARSER = new TikaKnowledgeDocumentParser();
|
||||
private static final String NO_CLEAR_SOP_EVIDENCE = "未在已发布 SOP 中找到明确依据";
|
||||
private static final String LEGACY_NO_CLEAR_SOP_EVIDENCE = "未在已入库 SOP 中找到明确依据";
|
||||
@@ -227,11 +235,16 @@ public class AihrSopSeedService {
|
||||
|
||||
public SearchResponse searchAuthorized(SearchRequest request, String requesterExtPartyId, Set<Long> allowedKnowledgeIds) {
|
||||
String queryText = request == null ? "" : Optional.ofNullable(request.queryText()).orElse("").trim();
|
||||
return searchAuthorized(request, requesterExtPartyId, allowedKnowledgeIds, queryText);
|
||||
return searchAuthorizedDetailed(request, requesterExtPartyId, allowedKnowledgeIds, queryText).response();
|
||||
}
|
||||
|
||||
public SearchResponse searchAuthorized(SearchRequest request, String requesterExtPartyId,
|
||||
Set<Long> allowedKnowledgeIds, String retrievalQueryText) {
|
||||
return searchAuthorizedDetailed(request, requesterExtPartyId, allowedKnowledgeIds, retrievalQueryText).response();
|
||||
}
|
||||
|
||||
public SearchExecution searchAuthorizedDetailed(SearchRequest request, String requesterExtPartyId,
|
||||
Set<Long> allowedKnowledgeIds, String retrievalQueryText) {
|
||||
if (allowedKnowledgeIds == null || allowedKnowledgeIds.isEmpty()) {
|
||||
throw new ServiceException("当前请求没有可访问的知识空间", 403);
|
||||
}
|
||||
@@ -240,18 +253,20 @@ public class AihrSopSeedService {
|
||||
String position = firstNonBlank(request == null ? null : request.position(), "生活顾问");
|
||||
String source = normalizeSearchSource(request == null ? null : request.source());
|
||||
String requester = isBlank(requesterExtPartyId) ? "" : requesterExtPartyId.trim();
|
||||
QueryPlan plan = AihrRagQueryPlanner.plan(queryText);
|
||||
if (isBlank(queryText)) {
|
||||
return withReviewId(noEvidenceResponse("", "sop"), source, requester);
|
||||
return new SearchExecution(withReviewId(noEvidenceResponse("", "sop"), source, requester), plan, List.of());
|
||||
}
|
||||
boolean formalPolicyOnly = AihrFormalPolicyClassifier.FORMAL_POLICY_SOURCE.equals(source);
|
||||
SearchResponse response = dbSearch(
|
||||
SearchExecution execution = dbSearchDetailed(
|
||||
"", queryText, retrievalQuery, request == null ? null : request.limit(), allowedKnowledgeIds,
|
||||
formalPolicyOnly);
|
||||
formalPolicyOnly, plan);
|
||||
SearchResponse response = execution.response();
|
||||
if (response == null || isNoEvidenceAnswer(response.answer())) {
|
||||
recordKnowledgeGap(queryText, "sop", position, source);
|
||||
response = noEvidenceResponse(queryText, "sop");
|
||||
if (response == null) response = noEvidenceResponse(queryText, "sop");
|
||||
}
|
||||
return withReviewId(response, source, requester);
|
||||
return new SearchExecution(withReviewId(response, source, requester), plan, execution.candidates());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3418,6 +3433,10 @@ public class AihrSopSeedService {
|
||||
+ String.join(",", java.util.Collections.nCopies(allowedKnowledgeIds.size(), "?")) + ")";
|
||||
}
|
||||
|
||||
private static String sqlPlaceholders(int count) {
|
||||
return String.join(",", java.util.Collections.nCopies(count, "?"));
|
||||
}
|
||||
|
||||
private static void addKnowledgeScopeArgs(List<Object> args, Set<Long> allowedKnowledgeIds) {
|
||||
if (allowedKnowledgeIds != null) {
|
||||
args.addAll(allowedKnowledgeIds);
|
||||
@@ -3431,21 +3450,24 @@ public class AihrSopSeedService {
|
||||
return """
|
||||
and exists (
|
||||
select 1
|
||||
from aihr_knowledge_attach governed_attach
|
||||
join aihr_knowledge_source_governance governance
|
||||
on governance.tenant_id = governed_attach.tenant_id
|
||||
and governance.attachment_id = governed_attach.id
|
||||
where governed_attach.tenant_id = %1$s.tenant_id
|
||||
and governed_attach.knowledge_id = %1$s.knowledge_id
|
||||
and governed_attach.doc_id = %1$s.doc_id
|
||||
and governed_attach.status = 2
|
||||
and governance.source_doc_id = %1$s.doc_id
|
||||
and governance.authority_type = 'FORMAL_POLICY'
|
||||
and governance.lifecycle_status = 'APPROVED'
|
||||
and governance.effective_date <= current_date()
|
||||
and (governance.expires_date is null or governance.expires_date >= current_date())
|
||||
and char_length(trim(governance.source_version)) > 0
|
||||
and governance.content_sha256 regexp '^[0-9a-f]{64}$'
|
||||
from aihr_chunk_revision governed_chunk
|
||||
join aihr_data_asset governed_asset
|
||||
on governed_asset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_asset.id = governed_chunk.asset_id
|
||||
and coalesce(governed_asset.published_version_id, governed_asset.current_version_id) = governed_chunk.version_id
|
||||
join aihr_dataset_membership governed_dataset
|
||||
on governed_dataset.tenant_id = governed_chunk.tenant_id
|
||||
and governed_dataset.version_id = governed_chunk.version_id
|
||||
and governed_dataset.dataset_code = 'production'
|
||||
and governed_dataset.status = 'ACTIVE'
|
||||
where governed_chunk.tenant_id = %1$s.tenant_id
|
||||
and governed_chunk.published_fragment_id = %1$s.id
|
||||
and governed_asset.lifecycle_status = 'PUBLISHED'
|
||||
and governed_asset.trust_level = 'HUMAN_VERIFIED'
|
||||
and upper(governed_asset.source_authority) in
|
||||
('COMPANY_POLICY', 'FORMAL_POLICY', 'OPERATING_MANUAL', 'APPROVED_SOP')
|
||||
and (governed_asset.effective_from is null or governed_asset.effective_from <= current_date())
|
||||
and (governed_asset.effective_to is null or governed_asset.effective_to >= current_date())
|
||||
)
|
||||
""".formatted(fragmentAlias);
|
||||
}
|
||||
@@ -3473,6 +3495,305 @@ public class AihrSopSeedService {
|
||||
}
|
||||
}
|
||||
|
||||
private SearchExecution dbSearchDetailed(String category, String queryText, String retrievalQueryText,
|
||||
Integer displayLimit, Set<Long> allowedKnowledgeIds,
|
||||
boolean formalPolicyOnly, QueryPlan plan) {
|
||||
try {
|
||||
boolean requiresFormal = formalPolicyOnly || !plan.requiredAuthorities().isEmpty();
|
||||
List<CandidateAccumulator> candidates = retrieveCandidates(
|
||||
category, queryText, retrievalQueryText, allowedKnowledgeIds, plan, requiresFormal);
|
||||
if (candidates.isEmpty()) return new SearchExecution(null, plan, List.of());
|
||||
enrichCandidateGovernance(candidates);
|
||||
Set<Long> rejectedFeedback = downFeedbackFragmentIds(queryText);
|
||||
for (CandidateAccumulator candidate : candidates) {
|
||||
if (candidate.hit.fragmentId() != null && rejectedFeedback.contains(candidate.hit.fragmentId())) {
|
||||
candidate.reject("REJECTED_DOWNSTREAM_FEEDBACK");
|
||||
} else if (requiresFormal && !formalSourceKind(candidate.sourceKind)) {
|
||||
candidate.reject("REJECTED_SOURCE_AUTHORITY");
|
||||
} else if (!AihrRagQueryPlanner.domainMatches(plan, candidate.hit.title(), candidate.hit.content())) {
|
||||
candidate.reject("REJECTED_DOMAIN_MISMATCH");
|
||||
}
|
||||
}
|
||||
List<CandidateAccumulator> eligible = candidates.stream()
|
||||
.filter(candidate -> candidate.decisionReason == null)
|
||||
.sorted(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed())
|
||||
.toList();
|
||||
List<CandidateAccumulator> rerankPool = eligible.stream().limit(RETRIEVAL_RERANK_K).toList();
|
||||
eligible.stream().skip(RETRIEVAL_RERANK_K)
|
||||
.forEach(candidate -> candidate.reject("OUTSIDE_RERANK_K"));
|
||||
List<CandidateAccumulator> ranked = rerankCandidates(queryText, rerankPool);
|
||||
for (int index = 0; index < ranked.size(); index++) ranked.get(index).candidateRank = index + 1;
|
||||
|
||||
List<CandidateAccumulator> evidence = ranked.stream().limit(RETRIEVAL_EVIDENCE_K).toList();
|
||||
if (plan.needsClarification()) {
|
||||
evidence.forEach(candidate -> candidate.reject("NEEDS_CLARIFICATION_AMOUNT_BASIS"));
|
||||
String message = "已找到正式的费用报销资料,但当前问题中的金额口径不明确:请确认“"
|
||||
+ amountText(plan) + "”是单笔金额还是月累计额度。当前资料不能据此推导零星采购必然可以报销。";
|
||||
return new SearchExecution(clarificationResponse(queryText, category, message), plan,
|
||||
candidateTraces(candidates));
|
||||
}
|
||||
if (requiresFormal && evidence.isEmpty()) {
|
||||
return new SearchExecution(noEvidenceResponse(queryText, category,
|
||||
List.of("没有找到当前有效且经过人工审核的正式制度、操作手册或已批准 SOP。")),
|
||||
plan, candidateTraces(candidates));
|
||||
}
|
||||
if (requiresAmountEvidence(plan) && !supportsAmountRule(plan, evidence)) {
|
||||
evidence.forEach(candidate -> candidate.reject("INSUFFICIENT_AMOUNT_EVIDENCE"));
|
||||
String message = "已找到正式的费用报销操作资料,但当前正式资料没有证明“" + amountText(plan)
|
||||
+ "”对应零星采购的适用额度和口径。请由财务负责人确认单笔/月累计规则后再办理。";
|
||||
return new SearchExecution(clarificationResponse(queryText, category, message), plan,
|
||||
candidateTraces(candidates));
|
||||
}
|
||||
if (!hasRequiredFinanceEvidence(plan, evidence)) {
|
||||
evidence.forEach(candidate -> candidate.reject("INSUFFICIENT_REQUIRED_EVIDENCE"));
|
||||
return new SearchExecution(noEvidenceResponse(queryText, category,
|
||||
List.of("正式资料尚未同时覆盖报销入口、必要材料和审批/复审要求。")),
|
||||
plan, candidateTraces(candidates));
|
||||
}
|
||||
evidence.forEach(candidate -> {
|
||||
candidate.selectedForEvidence = true;
|
||||
candidate.decisionReason = "SELECTED_EVIDENCE";
|
||||
});
|
||||
ranked.stream().skip(evidence.size()).forEach(candidate -> {
|
||||
if (candidate.decisionReason == null) candidate.decisionReason = "OUTSIDE_EVIDENCE_K";
|
||||
});
|
||||
SearchResponse response = responseFromHits(category, queryText,
|
||||
evidence.stream().map(candidate -> candidate.hit).toList(), List.of(), displayLimitValue(displayLimit));
|
||||
return new SearchExecution(response, plan, candidateTraces(candidates));
|
||||
} catch (DataAccessException ex) {
|
||||
return new SearchExecution(null, plan, List.of());
|
||||
}
|
||||
}
|
||||
|
||||
private List<CandidateAccumulator> retrieveCandidates(String category, String queryText,
|
||||
String retrievalQueryText,
|
||||
Set<Long> allowedKnowledgeIds, QueryPlan plan,
|
||||
boolean includePublishedCandidates) {
|
||||
List<QueryVariant> variants = queryVariants(queryText, retrievalQueryText, plan);
|
||||
Map<String, CandidateAccumulator> candidates = new LinkedHashMap<>();
|
||||
for (QueryVariant variant : variants) {
|
||||
addCandidateHits(candidates, likeHits(category, variant.text(), RETRIEVAL_CANDIDATE_K,
|
||||
allowedKnowledgeIds, false, includePublishedCandidates), "KEYWORD", variant.code(), true);
|
||||
addCandidateHits(candidates, fulltextHits(category, variant.text(), RETRIEVAL_CANDIDATE_K,
|
||||
allowedKnowledgeIds, false, includePublishedCandidates), "FULLTEXT", variant.code(), true);
|
||||
}
|
||||
Map<String, List<KnowledgeHit>> vectorHits = vectorHits(
|
||||
category, variants, RETRIEVAL_CANDIDATE_K, allowedKnowledgeIds, false, includePublishedCandidates);
|
||||
for (QueryVariant variant : variants) {
|
||||
addCandidateHits(candidates, vectorHits.getOrDefault(variant.code(), List.of()),
|
||||
"VECTOR", variant.code(), false);
|
||||
}
|
||||
applyFamilyFusion(candidates.values(), true);
|
||||
applyFamilyFusion(candidates.values(), false);
|
||||
return candidates.values().stream()
|
||||
.peek(CandidateAccumulator::finishFusion)
|
||||
.sorted(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed())
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static List<QueryVariant> queryVariants(String queryText, String retrievalQueryText, QueryPlan plan) {
|
||||
LinkedHashMap<String, String> variants = new LinkedHashMap<>();
|
||||
addQueryVariant(variants, "ORIGINAL", queryText);
|
||||
if (!Objects.equals(cleanQuery(queryText), cleanQuery(retrievalQueryText))) {
|
||||
addQueryVariant(variants, "GLOSSARY", retrievalQueryText);
|
||||
}
|
||||
addQueryVariant(variants, "NORMALIZED", plan.normalizedQuery());
|
||||
for (int index = 0; index < plan.alternateQueries().size(); index++) {
|
||||
addQueryVariant(variants, "ALTERNATE_" + (index + 1), plan.alternateQueries().get(index));
|
||||
}
|
||||
return variants.entrySet().stream().map(entry -> new QueryVariant(entry.getKey(), entry.getValue())).toList();
|
||||
}
|
||||
|
||||
private static void addQueryVariant(Map<String, String> variants, String code, String value) {
|
||||
String query = cleanQuery(value);
|
||||
if (query.isBlank() || variants.values().stream().anyMatch(existing -> cleanQuery(existing).equals(query))) return;
|
||||
variants.put(code, value.trim());
|
||||
}
|
||||
|
||||
private static String cleanQuery(String value) {
|
||||
return Optional.ofNullable(value).orElse("").replaceAll("\\s+", " ").trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static void addCandidateHits(Map<String, CandidateAccumulator> candidates, List<KnowledgeHit> hits,
|
||||
String channel, String queryVariant, boolean sparse) {
|
||||
for (int index = 0; index < hits.size(); index++) {
|
||||
KnowledgeHit hit = hits.get(index);
|
||||
String key = hit.fragmentId() == null ? hit.docId() + "#" + hit.idx() : String.valueOf(hit.fragmentId());
|
||||
CandidateAccumulator candidate = candidates.computeIfAbsent(key, ignored -> new CandidateAccumulator(hit));
|
||||
candidate.add(channel, index + 1, hit.score(), queryVariant, sparse);
|
||||
}
|
||||
}
|
||||
|
||||
private static void applyFamilyFusion(java.util.Collection<CandidateAccumulator> candidates, boolean sparse) {
|
||||
List<CandidateAccumulator> ranked = candidates.stream()
|
||||
.filter(candidate -> sparse ? candidate.sparseSignal > 0 : candidate.vectorSignal > 0)
|
||||
.sorted(Comparator.comparingDouble((CandidateAccumulator candidate) ->
|
||||
sparse ? candidate.sparseSignal : candidate.vectorSignal)
|
||||
.reversed())
|
||||
.toList();
|
||||
for (int index = 0; index < ranked.size(); index++) {
|
||||
double contribution = reciprocalRank(index + 1);
|
||||
if (sparse) ranked.get(index).sparseFusion = contribution;
|
||||
else ranked.get(index).vectorFusion = contribution;
|
||||
}
|
||||
}
|
||||
|
||||
private static double reciprocalRank(int rank) {
|
||||
return 1.0 / (60 + Math.max(1, rank));
|
||||
}
|
||||
|
||||
private void enrichCandidateGovernance(List<CandidateAccumulator> candidates) {
|
||||
List<Long> fragmentIds = candidates.stream().map(candidate -> candidate.hit.fragmentId())
|
||||
.filter(Objects::nonNull).distinct().toList();
|
||||
if (fragmentIds.isEmpty()) return;
|
||||
List<Object> args = new ArrayList<>();
|
||||
args.add(tenantId());
|
||||
args.addAll(fragmentIds);
|
||||
jdbcTemplate.query("""
|
||||
select f.id fragment_id, asset.source_authority, asset.source_type, asset.source_name
|
||||
from aihr_knowledge_fragment f
|
||||
join aihr_chunk_revision chunk
|
||||
on chunk.tenant_id = f.tenant_id and chunk.published_fragment_id = f.id
|
||||
join aihr_data_asset asset
|
||||
on asset.tenant_id = chunk.tenant_id and asset.id = chunk.asset_id
|
||||
and coalesce(asset.published_version_id, asset.current_version_id) = chunk.version_id
|
||||
where f.tenant_id = ? and f.id in (%s)
|
||||
""".formatted(sqlPlaceholders(fragmentIds.size())), rs -> {
|
||||
Long fragmentId = rs.getLong("fragment_id");
|
||||
candidates.stream().filter(candidate -> Objects.equals(candidate.hit.fragmentId(), fragmentId))
|
||||
.findFirst().ifPresent(candidate -> {
|
||||
candidate.sourceAuthority = rsString(rs, "source_authority");
|
||||
candidate.sourceKind = AihrRagQueryPlanner.sourceKind(candidate.sourceAuthority,
|
||||
rsString(rs, "source_type"), rsString(rs, "source_name"));
|
||||
});
|
||||
}, args.toArray());
|
||||
}
|
||||
|
||||
private static String rsString(java.sql.ResultSet rs, String column) {
|
||||
try {
|
||||
return rs.getString(column);
|
||||
} catch (java.sql.SQLException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private List<CandidateAccumulator> rerankCandidates(String queryText, List<CandidateAccumulator> candidates) {
|
||||
if (candidates.size() < 2) return candidates;
|
||||
Optional<ChatRuntime> runtime = rerankRuntime();
|
||||
if (runtime.isEmpty()) return candidates;
|
||||
try {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", runtime.get().modelName());
|
||||
body.put("query", AihrSensitiveText.forModel(queryText));
|
||||
body.put("top_n", candidates.size());
|
||||
body.put("return_documents", false);
|
||||
ArrayNode documents = body.putArray("documents");
|
||||
candidates.forEach(candidate -> documents.add(
|
||||
AihrSensitiveText.forModel(truncate(candidate.hit.content(), 2000))));
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(normalizeBaseUrl(runtime.get().baseUrl()) + "/rerank"))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Authorization", "Bearer " + runtime.get().apiKey())
|
||||
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
|
||||
.build();
|
||||
HttpResponse<String> response = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15))
|
||||
.build().send(request, HttpResponse.BodyHandlers.ofString());
|
||||
if (!ok(response.statusCode())) throw new IllegalStateException("rerank HTTP " + response.statusCode());
|
||||
JsonNode results = objectMapper.readTree(response.body()).path("results");
|
||||
if (!results.isArray() || results.isEmpty()) return candidates;
|
||||
List<CandidateAccumulator> reranked = new ArrayList<>();
|
||||
for (JsonNode item : results) {
|
||||
int index = item.path("index").asInt(-1);
|
||||
if (index < 0 || index >= candidates.size()) continue;
|
||||
CandidateAccumulator candidate = candidates.get(index);
|
||||
candidate.rerankApplied = true;
|
||||
candidate.rerankModel = runtime.get().modelName();
|
||||
if (item.path("relevance_score").isNumber()) {
|
||||
candidate.rerankScore = item.path("relevance_score").asDouble();
|
||||
}
|
||||
if (!reranked.contains(candidate)) reranked.add(candidate);
|
||||
}
|
||||
for (CandidateAccumulator candidate : candidates) if (!reranked.contains(candidate)) reranked.add(candidate);
|
||||
return reranked;
|
||||
} catch (Exception ex) {
|
||||
log.warn("candidate rerank failed, keep family RRF order(处理错误已隐藏)");
|
||||
return candidates;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean formalSourceKind(String sourceKind) {
|
||||
return Set.of("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP").contains(sourceKind);
|
||||
}
|
||||
|
||||
private static boolean requiresAmountEvidence(QueryPlan plan) {
|
||||
return plan != null && AihrRagQueryPlanner.isFormalFinanceDomain(plan.domain())
|
||||
&& plan.entities() != null && plan.entities().amount() != null;
|
||||
}
|
||||
|
||||
private static boolean supportsAmountRule(QueryPlan plan, List<CandidateAccumulator> evidence) {
|
||||
if (!requiresAmountEvidence(plan)) return true;
|
||||
String amount = amountValue(plan);
|
||||
String content = evidence.stream().map(candidate -> candidate.hit.content()).filter(Objects::nonNull)
|
||||
.reduce("", (left, right) -> left + " " + right);
|
||||
boolean amountFound = content.contains(amount + "元") || content.contains(amount + " 元")
|
||||
|| content.contains(chineseAmount(plan.entities().amount()));
|
||||
boolean basisFound = "PER_TRANSACTION".equals(plan.entities().amountBasis())
|
||||
? containsAny(content, "单笔", "每笔", "一笔")
|
||||
: "MONTHLY_TOTAL".equals(plan.entities().amountBasis())
|
||||
? containsAny(content, "月累计", "每月", "当月", "月度额度") : false;
|
||||
return amountFound && basisFound;
|
||||
}
|
||||
|
||||
private static boolean hasRequiredFinanceEvidence(QueryPlan plan, List<CandidateAccumulator> evidence) {
|
||||
if (plan == null || !AihrRagQueryPlanner.isFormalFinanceDomain(plan.domain())) return !evidence.isEmpty();
|
||||
String content = evidence.stream().map(candidate -> candidate.hit.content()).filter(Objects::nonNull)
|
||||
.reduce("", (left, right) -> left + " " + right);
|
||||
return containsAny(content, "费用报销", "报销单", "发票登记")
|
||||
&& containsAny(content, "发票", "票据", "附件")
|
||||
&& containsAny(content, "提交", "审批", "复审", "流程结束");
|
||||
}
|
||||
|
||||
private static String amountValue(QueryPlan plan) {
|
||||
double value = plan.entities().amount();
|
||||
return value % 1 == 0 ? String.valueOf((long) value) : String.valueOf(value);
|
||||
}
|
||||
|
||||
private static String amountText(QueryPlan plan) {
|
||||
if (plan == null || plan.entities() == null || plan.entities().amount() == null) return "该金额";
|
||||
String comparator = switch (Optional.ofNullable(plan.entities().comparator()).orElse("")) {
|
||||
case "LTE" -> "以下";
|
||||
case "GT" -> "以上";
|
||||
case "GTE" -> "及以上";
|
||||
default -> "";
|
||||
};
|
||||
return amountValue(plan) + "元" + comparator;
|
||||
}
|
||||
|
||||
private static String chineseAmount(Double amount) {
|
||||
if (amount == null || amount % 100 != 0 || amount < 100 || amount > 900) return "";
|
||||
return "一二三四五六七八九".substring((int) (amount / 100) - 1, (int) (amount / 100)) + "百";
|
||||
}
|
||||
|
||||
private static int displayLimitValue(Integer limit) {
|
||||
if (limit == null) return RETRIEVAL_DISPLAY_K;
|
||||
return Math.max(1, Math.min(limit, RETRIEVAL_DISPLAY_K));
|
||||
}
|
||||
|
||||
private static SearchResponse clarificationResponse(String queryText, String category, String message) {
|
||||
return new SearchResponse(queryText, firstNonBlank(category, "sop"), message, "正式证据不足,未生成流程",
|
||||
List.of(), List.of(), List.of(), records(), List.of(), List.of(message), null, null);
|
||||
}
|
||||
|
||||
private static List<RetrievalCandidate> candidateTraces(List<CandidateAccumulator> candidates) {
|
||||
return candidates.stream()
|
||||
.sorted(Comparator.comparing((CandidateAccumulator candidate) -> candidate.candidateRank,
|
||||
Comparator.nullsLast(Integer::compareTo))
|
||||
.thenComparing(Comparator.comparingDouble(CandidateAccumulator::fusionScore).reversed()))
|
||||
.map(CandidateAccumulator::toTrace)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static String normalizeFeedbackRequestId(String value) {
|
||||
if (isBlank(value)) {
|
||||
return null;
|
||||
@@ -3571,6 +3892,14 @@ public class AihrSopSeedService {
|
||||
return "and " + AihrKnowledgeRolloutService.servingFragmentExistsSql(fragmentAlias);
|
||||
}
|
||||
|
||||
private static String candidateLifecycleScopeSql(String fragmentAlias, boolean includePublishedCandidates) {
|
||||
if (!includePublishedCandidates) {
|
||||
return servingLifecycleScopeSql(fragmentAlias);
|
||||
}
|
||||
return "and (" + AihrKnowledgeRolloutService.servingFragmentExistsSql(fragmentAlias)
|
||||
+ " or " + publishedLifecycleExistsSql(fragmentAlias) + ")";
|
||||
}
|
||||
|
||||
private static String publishedLifecycleExistsSql(String fragmentAlias) {
|
||||
return """
|
||||
exists (
|
||||
@@ -3664,6 +3993,40 @@ public class AihrSopSeedService {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private Map<String, List<KnowledgeHit>> vectorHits(String category, List<QueryVariant> variants, int limit,
|
||||
Set<Long> allowedKnowledgeIds,
|
||||
boolean formalPolicyOnly) {
|
||||
return vectorHits(category, variants, limit, allowedKnowledgeIds, formalPolicyOnly, false);
|
||||
}
|
||||
|
||||
private Map<String, List<KnowledgeHit>> vectorHits(String category, List<QueryVariant> variants, int limit,
|
||||
Set<Long> allowedKnowledgeIds,
|
||||
boolean formalPolicyOnly,
|
||||
boolean includePublishedCandidates) {
|
||||
if (variants == null || variants.isEmpty()) return Map.of();
|
||||
List<String> queries = variants.stream().map(QueryVariant::text).toList();
|
||||
for (EmbeddingRuntime runtime : embeddingRuntimes()) {
|
||||
try {
|
||||
List<String> embeddings = callEmbeddings(runtime, queries);
|
||||
if (embeddings.size() != variants.size()) continue;
|
||||
Map<String, List<KnowledgeHit>> results = new LinkedHashMap<>();
|
||||
for (int index = 0; index < variants.size(); index++) {
|
||||
List<KnowledgeHit> hits = new ArrayList<>();
|
||||
for (VectorMatch match : queryQdrant(
|
||||
category, embeddings.get(index), limit, allowedKnowledgeIds)) {
|
||||
hydrateQdrantHit(match, allowedKnowledgeIds, formalPolicyOnly, includePublishedCandidates)
|
||||
.ifPresent(hits::add);
|
||||
}
|
||||
results.put(variants.get(index).code(), List.copyOf(hits));
|
||||
}
|
||||
return results;
|
||||
} catch (Exception ignored) {
|
||||
// Vector recall is additive; try the next configured runtime before sparse-only fallback.
|
||||
}
|
||||
}
|
||||
return Map.of();
|
||||
}
|
||||
|
||||
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit) {
|
||||
return fulltextHits(category, queryText, limit, null);
|
||||
}
|
||||
@@ -3674,6 +4037,12 @@ public class AihrSopSeedService {
|
||||
|
||||
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit,
|
||||
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly) {
|
||||
return fulltextHits(category, queryText, limit, allowedKnowledgeIds, formalPolicyOnly, false);
|
||||
}
|
||||
|
||||
private List<KnowledgeHit> fulltextHits(String category, String queryText, int limit,
|
||||
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly,
|
||||
boolean includePublishedCandidates) {
|
||||
String sql = """
|
||||
select f.id as fragment_id,
|
||||
coalesce(a.name, i.name) as title,
|
||||
@@ -3695,7 +4064,8 @@ public class AihrSopSeedService {
|
||||
order by score desc, f.idx asc
|
||||
limit ?
|
||||
""".formatted(knowledgeScopeSql("f", allowedKnowledgeIds),
|
||||
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"));
|
||||
formalSourceScopeSql("f", formalPolicyOnly),
|
||||
candidateLifecycleScopeSql("f", includePublishedCandidates));
|
||||
List<Object> args = new ArrayList<>();
|
||||
args.add(queryText);
|
||||
args.add(tenantId());
|
||||
@@ -3727,6 +4097,12 @@ public class AihrSopSeedService {
|
||||
|
||||
private List<KnowledgeHit> likeHits(String category, String queryText, int limit,
|
||||
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly) {
|
||||
return likeHits(category, queryText, limit, allowedKnowledgeIds, formalPolicyOnly, false);
|
||||
}
|
||||
|
||||
private List<KnowledgeHit> likeHits(String category, String queryText, int limit,
|
||||
Set<Long> allowedKnowledgeIds, boolean formalPolicyOnly,
|
||||
boolean includePublishedCandidates) {
|
||||
List<String> terms = searchTerms(queryText);
|
||||
if (terms.isEmpty()) {
|
||||
return List.of();
|
||||
@@ -3758,7 +4134,8 @@ public class AihrSopSeedService {
|
||||
order by score desc, f.idx asc
|
||||
limit ?
|
||||
""".formatted(String.join(" + ", scoreParts), knowledgeScopeSql("f", allowedKnowledgeIds),
|
||||
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"),
|
||||
formalSourceScopeSql("f", formalPolicyOnly),
|
||||
candidateLifecycleScopeSql("f", includePublishedCandidates),
|
||||
String.join(" or ", whereParts));
|
||||
List<Object> args = new ArrayList<>();
|
||||
terms.stream().map(AihrSopSeedService::likePattern).forEach(args::add);
|
||||
@@ -3791,6 +4168,12 @@ public class AihrSopSeedService {
|
||||
|
||||
private Optional<KnowledgeHit> hydrateQdrantHit(VectorMatch match, Set<Long> allowedKnowledgeIds,
|
||||
boolean formalPolicyOnly) {
|
||||
return hydrateQdrantHit(match, allowedKnowledgeIds, formalPolicyOnly, false);
|
||||
}
|
||||
|
||||
private Optional<KnowledgeHit> hydrateQdrantHit(VectorMatch match, Set<Long> allowedKnowledgeIds,
|
||||
boolean formalPolicyOnly,
|
||||
boolean includePublishedCandidates) {
|
||||
String sql = """
|
||||
select f.id as fragment_id,
|
||||
coalesce(a.name, i.name) as title,
|
||||
@@ -3811,7 +4194,8 @@ public class AihrSopSeedService {
|
||||
and f.idx = ?
|
||||
limit 1
|
||||
""".formatted(knowledgeScopeSql("f", allowedKnowledgeIds),
|
||||
formalSourceScopeSql("f", formalPolicyOnly), servingLifecycleScopeSql("f"));
|
||||
formalSourceScopeSql("f", formalPolicyOnly),
|
||||
candidateLifecycleScopeSql("f", includePublishedCandidates));
|
||||
List<Object> args = new ArrayList<>();
|
||||
args.add(tenantId());
|
||||
addKnowledgeScopeArgs(args, allowedKnowledgeIds);
|
||||
@@ -3862,15 +4246,21 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
private SearchResponse responseFromHits(String category, String queryText, List<KnowledgeHit> hits, List<String> extraCautions) {
|
||||
return responseFromHits(category, queryText, hits, extraCautions, RETRIEVAL_DISPLAY_K);
|
||||
}
|
||||
|
||||
private SearchResponse responseFromHits(String category, String queryText, List<KnowledgeHit> hits,
|
||||
List<String> extraCautions, int displayLimit) {
|
||||
List<KnowledgeHit> evidenceHits = evidenceHits(queryText, hits);
|
||||
KnowledgeHit first = evidenceHits.get(0);
|
||||
String responseCategory = isBlank(category) ? first.category() : category;
|
||||
List<SnippetResponse> snippets = evidenceHits.stream()
|
||||
List<KnowledgeHit> displayHits = displayHits(evidenceHits, displayLimit);
|
||||
List<SnippetResponse> snippets = displayHits.stream()
|
||||
.map(hit -> new SnippetResponse("第 " + hit.idx() + " 段:" + hit.title(),
|
||||
displaySnippetText(hit.content()), hit.fragmentId(), hit.score(), hit.retrievalChannel()))
|
||||
.toList();
|
||||
Map<String, DocResponse> docs = new LinkedHashMap<>();
|
||||
for (KnowledgeHit hit : evidenceHits) {
|
||||
for (KnowledgeHit hit : displayHits) {
|
||||
docs.putIfAbsent(hit.title(), new DocResponse(hit.title(), hit.description(), "命中 " + scoreText(hit.score()), "pill-success"));
|
||||
}
|
||||
SearchDigest digest = searchDigest(queryText, evidenceHits);
|
||||
@@ -3914,6 +4304,11 @@ public class AihrSopSeedService {
|
||||
return hits.stream().limit(MAX_EVIDENCE_SNIPPETS).toList();
|
||||
}
|
||||
|
||||
static List<KnowledgeHit> displayHits(List<KnowledgeHit> evidenceHits, Integer limit) {
|
||||
if (evidenceHits == null || evidenceHits.isEmpty()) return List.of();
|
||||
return evidenceHits.stream().limit(displayLimitValue(limit)).toList();
|
||||
}
|
||||
|
||||
private static boolean queryMentionsTitle(String queryText, String title) {
|
||||
String query = evidenceText(queryText);
|
||||
String docTitle = evidenceText(title);
|
||||
@@ -4201,26 +4596,7 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
private static List<String> searchTerms(String queryText) {
|
||||
String cleaned = Optional.ofNullable(queryText).orElse("")
|
||||
.toLowerCase()
|
||||
.replaceAll("[^\\p{IsHan}\\p{Alnum}]+", " ")
|
||||
.trim();
|
||||
if (cleaned.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
LinkedHashSet<String> terms = new LinkedHashSet<>();
|
||||
for (String token : cleaned.split("\\s+")) {
|
||||
if (token.length() >= 2 && token.length() <= 16) {
|
||||
terms.add(token);
|
||||
}
|
||||
for (int i = 0; i + 2 <= token.length() && terms.size() < 12; i++) {
|
||||
terms.add(token.substring(i, i + 2));
|
||||
}
|
||||
if (terms.size() >= 12) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return terms.stream().limit(12).toList();
|
||||
return AihrRagQueryPlanner.sparseTerms(queryText);
|
||||
}
|
||||
|
||||
private static String likePattern(String term) {
|
||||
@@ -5474,6 +5850,63 @@ public class AihrSopSeedService {
|
||||
static record FeedbackFilterResult(List<KnowledgeHit> hits, List<String> cautions) {
|
||||
}
|
||||
|
||||
public record SearchExecution(SearchResponse response, QueryPlan queryPlan,
|
||||
List<RetrievalCandidate> candidates) {
|
||||
public SearchExecution {
|
||||
candidates = candidates == null ? List.of() : List.copyOf(candidates);
|
||||
}
|
||||
}
|
||||
|
||||
private record QueryVariant(String code, String text) {}
|
||||
|
||||
private static final class CandidateAccumulator {
|
||||
private final KnowledgeHit hit;
|
||||
private final List<ChannelContribution> contributions = new ArrayList<>();
|
||||
private double sparseSignal;
|
||||
private double vectorSignal;
|
||||
private double sparseFusion;
|
||||
private double vectorFusion;
|
||||
private double fusionScore;
|
||||
private String sourceAuthority = "UNKNOWN";
|
||||
private String sourceKind = "REFERENCE_MATERIAL";
|
||||
private Integer candidateRank;
|
||||
private Double rerankScore;
|
||||
private String rerankModel;
|
||||
private boolean rerankApplied;
|
||||
private boolean selectedForEvidence;
|
||||
private String decisionReason;
|
||||
|
||||
private CandidateAccumulator(KnowledgeHit hit) {
|
||||
this.hit = hit;
|
||||
}
|
||||
|
||||
private void add(String channel, int channelRank, double rawScore, String queryVariant, boolean sparse) {
|
||||
double signal = reciprocalRank(channelRank);
|
||||
contributions.add(new ChannelContribution(channel, channelRank, rawScore, signal, queryVariant));
|
||||
if (sparse) sparseSignal = Math.max(sparseSignal, signal);
|
||||
else vectorSignal = Math.max(vectorSignal, signal);
|
||||
}
|
||||
|
||||
private void finishFusion() {
|
||||
fusionScore = sparseFusion + vectorFusion;
|
||||
}
|
||||
|
||||
private double fusionScore() {
|
||||
return fusionScore;
|
||||
}
|
||||
|
||||
private void reject(String reason) {
|
||||
selectedForEvidence = false;
|
||||
if (decisionReason == null || "OUTSIDE_EVIDENCE_K".equals(decisionReason)) decisionReason = reason;
|
||||
}
|
||||
|
||||
private RetrievalCandidate toTrace() {
|
||||
return new RetrievalCandidate(hit.fragmentId(), hit.title(), sourceAuthority, sourceKind,
|
||||
candidateRank, fusionScore, rerankScore, rerankModel, rerankApplied, selectedForEvidence,
|
||||
firstNonBlank(decisionReason, "CANDIDATE"), List.copyOf(contributions));
|
||||
}
|
||||
}
|
||||
|
||||
private record KnowledgeConfig(long knowledgeId, int blockSize, int overlap) {
|
||||
}
|
||||
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package org.dromara.aihr.knowledge.service;
|
||||
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Tag("dev")
|
||||
class AihrRagQueryPlannerTest {
|
||||
|
||||
@Test
|
||||
void removesNumericNgramNoiseButPreservesTypedAmountAndBusinessIntent() {
|
||||
var query = "我有个报销500块钱,告诉我怎么走流程。";
|
||||
|
||||
var plan = AihrRagQueryPlanner.plan(query);
|
||||
var terms = AihrRagQueryPlanner.sparseTerms(query);
|
||||
|
||||
assertThat(plan.originalQuery()).isEqualTo(query);
|
||||
assertThat(plan.domain()).isEqualTo(AihrRagQueryPlanner.FINANCE_POLICY);
|
||||
assertThat(plan.entities().amount()).isEqualTo(500D);
|
||||
assertThat(plan.entities().currency()).isEqualTo("CNY");
|
||||
assertThat(terms).contains("500元", "报销", "流程");
|
||||
assertThat(terms).doesNotContain("50", "00", "销5", "0块", "我有", "告诉");
|
||||
}
|
||||
|
||||
@Test
|
||||
void plansAmbiguousPurchaseThresholdAsFormalFinanceClarification() {
|
||||
List<String> queries = List.of(
|
||||
"500元以下的零星采购怎么报销?",
|
||||
"五百块的小额自采怎么走账?"
|
||||
);
|
||||
|
||||
for (String query : queries) {
|
||||
var plan = AihrRagQueryPlanner.plan(query);
|
||||
assertThat(plan.domain()).isEqualTo(AihrRagQueryPlanner.FINANCE_REIMBURSEMENT);
|
||||
assertThat(plan.entities().amount()).isEqualTo(500D);
|
||||
assertThat(plan.entities().amountBasis()).isEqualTo("UNKNOWN");
|
||||
assertThat(plan.needsClarification()).isTrue();
|
||||
assertThat(plan.requiredAuthorities())
|
||||
.containsExactly("FORMAL_POLICY", "OPERATING_MANUAL", "APPROVED_SOP");
|
||||
assertThat(plan.evidenceNeeds())
|
||||
.contains("AMOUNT_THRESHOLD", "AMOUNT_BASIS", "ENTRY_STEPS", "REQUIRED_MATERIALS",
|
||||
"APPROVAL_REVIEW", "EXCEPTIONS");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesExplicitAmountBasisAndRejectsExperienceAsFormalAuthority() {
|
||||
var plan = AihrRagQueryPlanner.plan("月累计500元以内的零星采购怎么报销?");
|
||||
|
||||
assertThat(plan.entities().amountBasis()).isEqualTo("MONTHLY_TOTAL");
|
||||
assertThat(plan.needsClarification()).isFalse();
|
||||
assertThat(AihrRagQueryPlanner.sourceKind("EMPLOYEE_SUBMISSION", "INTERVIEW", "员工访谈"))
|
||||
.isEqualTo("EXPERIENCE");
|
||||
assertThat(AihrRagQueryPlanner.authorityAllowed(plan, "EXPERIENCE")).isFalse();
|
||||
assertThat(AihrRagQueryPlanner.sourceKind("COMPANY_POLICY", "PDF", "费用报销操作手册.pdf"))
|
||||
.isEqualTo("OPERATING_MANUAL");
|
||||
}
|
||||
}
|
||||
+43
-6
@@ -152,6 +152,20 @@ public class AihrSopSeedServiceTest {
|
||||
assertEquals(3L, evidence.get(2).fragmentId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void displayLimitDoesNotReduceEvidencePool() {
|
||||
List<AihrSopSeedService.KnowledgeHit> evidence = AihrSopSeedService.evidenceHits(
|
||||
"费用报销的发票怎么登记?",
|
||||
List.of(hit(1L, "费用报销操作手册"), hit(2L, "费用报销操作手册"), hit(3L, "费用报销操作手册")));
|
||||
|
||||
List<AihrSopSeedService.KnowledgeHit> displayed = AihrSopSeedService.displayHits(evidence, 1);
|
||||
|
||||
assertEquals(3, evidence.size());
|
||||
assertEquals(1, displayed.size());
|
||||
assertEquals(1L, displayed.get(0).fragmentId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void allDownVotedHitsAreSuppressed() {
|
||||
@@ -667,12 +681,14 @@ public class AihrSopSeedServiceTest {
|
||||
assertTrue(code.contains("searchAuthorized(SearchRequest request, String requesterExtPartyId, Set<Long> allowedKnowledgeIds)"));
|
||||
assertTrue(code.contains(".knowledge_id in ("));
|
||||
assertTrue(code.contains("qdrantMatchAny(\"knowledge_id\", knowledgeIds)"));
|
||||
assertTrue(code.contains("aihr_knowledge_source_governance"));
|
||||
assertTrue(code.contains("governance.source_doc_id"));
|
||||
assertTrue(code.contains("governance.authority_type = 'FORMAL_POLICY'"));
|
||||
assertTrue(code.contains("governance.lifecycle_status = 'APPROVED'"));
|
||||
assertTrue(code.contains("governance.effective_date <= current_date()"));
|
||||
assertTrue(code.contains("governance.content_sha256 regexp '^[0-9a-f]{64}$'"));
|
||||
assertTrue(code.contains("governed_dataset.dataset_code = 'production'"));
|
||||
assertTrue(code.contains("governed_dataset.status = 'ACTIVE'"));
|
||||
assertTrue(code.contains("governed_asset.lifecycle_status = 'PUBLISHED'"));
|
||||
assertTrue(code.contains("governed_asset.trust_level = 'HUMAN_VERIFIED'"));
|
||||
assertTrue(code.contains("upper(governed_asset.source_authority) in"));
|
||||
assertTrue(code.contains("'COMPANY_POLICY', 'FORMAL_POLICY', 'OPERATING_MANUAL', 'APPROVED_SOP'"));
|
||||
assertTrue(code.contains("governed_asset.effective_from is null"));
|
||||
assertTrue(code.contains("governed_asset.effective_to is null"));
|
||||
|
||||
AihrSopSeedService service = new AihrSopSeedService(
|
||||
new ObjectMapper(), mock(JdbcTemplate.class), null, "", null, null, mock(AihrModelSeedService.class));
|
||||
@@ -680,6 +696,27 @@ public class AihrSopSeedServiceTest {
|
||||
() -> service.searchAuthorized(new AihrSopDto.SearchRequest("问题", "sop", null, null, 5), "", Set.of()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void formalCandidateRecallIncludesPublishedGovernedFragmentsWithoutChangingDefaultRollout() throws Exception {
|
||||
Method candidateScope = AihrSopSeedService.class.getDeclaredMethod(
|
||||
"candidateLifecycleScopeSql", String.class, boolean.class);
|
||||
Method servingScope = AihrSopSeedService.class.getDeclaredMethod("servingLifecycleScopeSql", String.class);
|
||||
Method publishedScope = AihrSopSeedService.class.getDeclaredMethod("publishedLifecycleExistsSql", String.class);
|
||||
candidateScope.setAccessible(true);
|
||||
servingScope.setAccessible(true);
|
||||
publishedScope.setAccessible(true);
|
||||
|
||||
String serving = (String) servingScope.invoke(null, "f");
|
||||
String published = (String) publishedScope.invoke(null, "f");
|
||||
String defaultCandidates = (String) candidateScope.invoke(null, "f", false);
|
||||
String formalCandidates = (String) candidateScope.invoke(null, "f", true);
|
||||
|
||||
assertEquals(serving, defaultCandidates);
|
||||
assertTrue(formalCandidates.contains(serving.substring("and ".length())));
|
||||
assertTrue(formalCandidates.contains(published));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void legacySearchControllerDelegatesToAuthorizedQueryService() throws Exception {
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Generation-safe RAG candidate observability. Additive and idempotent on MySQL 8.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `aihr_query_candidate_trace` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT,
|
||||
`tenant_id` varchar(20) NOT NULL,
|
||||
`request_id` varchar(64) NOT NULL,
|
||||
`fragment_id` bigint NOT NULL,
|
||||
`candidate_rank` int DEFAULT NULL COMMENT 'Rank after rerank, null when rejected before rerank',
|
||||
`channel` varchar(20) NOT NULL COMMENT 'KEYWORD/FULLTEXT/VECTOR',
|
||||
`channel_rank` int NOT NULL,
|
||||
`raw_score` decimal(18,10) DEFAULT NULL,
|
||||
`fusion_score` decimal(18,10) DEFAULT NULL,
|
||||
`rerank_score` decimal(18,10) DEFAULT NULL,
|
||||
`rerank_model` varchar(100) DEFAULT NULL,
|
||||
`rerank_applied` tinyint(1) NOT NULL DEFAULT 0,
|
||||
`source_authority` varchar(30) DEFAULT NULL,
|
||||
`source_kind` varchar(30) DEFAULT NULL,
|
||||
`query_variant` varchar(30) NOT NULL COMMENT 'ORIGINAL/NORMALIZED/GLOSSARY/ALTERNATE_n',
|
||||
`selected_for_evidence` tinyint(1) NOT NULL DEFAULT 0,
|
||||
`decision_reason` varchar(64) NOT NULL,
|
||||
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uk_aihr_query_candidate_channel`
|
||||
(`tenant_id`, `request_id`, `fragment_id`, `channel`, `query_variant`),
|
||||
KEY `idx_aihr_query_candidate_request` (`tenant_id`, `request_id`, `candidate_rank`),
|
||||
KEY `idx_aihr_query_candidate_reason` (`tenant_id`, `decision_reason`, `create_time`),
|
||||
KEY `idx_aihr_query_candidate_fragment` (`tenant_id`, `fragment_id`, `create_time`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
|
||||
COMMENT='Pre-generation retrieval candidates, channel contributions, rerank and gate decisions';
|
||||
Reference in New Issue
Block a user