From 99e8c509d520fe5777fcb3518e80b01f58cdad64 Mon Sep 17 00:00:00 2001 From: key Date: Mon, 3 Aug 2026 16:00:45 +0800 Subject: [PATCH] fix: harden knowledge lifecycle rollout --- .../AihrKnowledgeIndexOutboxService.java | 146 +++++++---- .../AihrKnowledgeRetentionService.java | 248 +++++++++++++----- .../AihrKnowledgeShadowRolloutService.java | 105 ++++++-- ...AihrKnowledgeVersionGovernanceService.java | 7 +- .../aihr/service/AihrSopSeedService.java | 147 ++++++++++- .../AihrDataQualityLifecycleSchemaTest.java | 3 +- .../AihrKnowledgeGovernanceSafetyTest.java | 33 ++- .../AihrKnowledgeIndexOutboxServiceTest.java | 15 ++ ...AihrKnowledgeShadowRolloutServiceTest.java | 86 ++++++ .../aihr/service/AihrSopSeedServiceTest.java | 4 + ..._20260821_knowledge_shadow_gate_mysql8.sql | 21 ++ docs/API_INTEGRATION.md | 4 +- docs/BRD_IMPLEMENTATION_AUDIT.md | 7 +- docs/BRD_PRODUCTION_MIGRATION_RUNBOOK.md | 4 +- docs/DIGITAL_ASSET_PROCESSING_GOVERNANCE.md | 2 + frontend/src/api/aihr/processing.ts | 4 + frontend/src/views/knowledge/processing.vue | 27 +- scripts/release-preflight.sh | 2 + 18 files changed, 709 insertions(+), 156 deletions(-) create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutServiceTest.java diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxService.java index b8382636..8efe77d1 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxService.java @@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; import java.util.List; @@ -19,6 +20,7 @@ public class AihrKnowledgeIndexOutboxService { private final JdbcTemplate jdbcTemplate; private final AihrSopSeedService sopSeedService; + private final TransactionTemplate transactionTemplate; @Value("${aihr.qdrant.governed-collection:${AIHR_QDRANT_GOVERNED_COLLECTION:aihr_knowledge_governed_v1}}") private String productionCollection = AihrKnowledgeRolloutService.DEFAULT_GOVERNED_COLLECTION; @@ -102,15 +104,18 @@ public class AihrKnowledgeIndexOutboxService { return; } try { - Integer vectors = TenantHelper.dynamic(row.tenantId(), () -> { + AihrSopSeedService.VectorizationResult vectorization = TenantHelper.dynamic(row.tenantId(), () -> { if ("DELETE".equals(row.operation())) { sopSeedService.deleteVectorDocument(row.knowledgeId(), row.docId(), row.collection(), row.generation()); return null; } - return sopSeedService.vectorizePublishedDocument(row.knowledgeId(), row.docId(), + return sopSeedService.vectorizePublishedDocumentWithManifest(row.knowledgeId(), row.docId(), row.versionId(), row.collection(), row.generation()); }); + if ("UPSERT".equals(row.operation()) && vectorization != null && vectorization.count() > 0) { + recordGenerationVectorization(row, vectorization); + } jdbcTemplate.update(""" update aihr_index_outbox set status = 'SUCCEEDED', last_error = null, update_time = now() @@ -121,51 +126,9 @@ public class AihrKnowledgeIndexOutboxService { set index_status = ?, update_time = now() where tenant_id = ? and id = ? and coalesce(published_version_id, current_version_id) = ? - """, "DELETE".equals(row.operation()) || vectors == null || vectors == 0 ? "NOT_REQUIRED" : "READY", + """, "DELETE".equals(row.operation()) || vectorization == null || vectorization.count() == 0 + ? "NOT_REQUIRED" : "READY", row.tenantId(), row.assetId(), row.versionId()); - if ("UPSERT".equals(row.operation()) && vectors != null && vectors > 0) { - jdbcTemplate.update(""" - insert into aihr_knowledge_generation - (tenant_id, knowledge_id, generation, collection_name, status, - version_count, point_count, create_time, ready_time, activated_time) - values (?, ?, ?, ?, ?, 0, 0, now(), ?, ?) - on duplicate key update collection_name = values(collection_name), - status = case when status = 'FAILED' then 'BUILDING' else status end, - ready_time = coalesce(ready_time, values(ready_time)), - activated_time = coalesce(activated_time, values(activated_time)), last_error = null - """, row.tenantId(), row.knowledgeId(), row.generation(), row.collection(), - row.generation() == row.activeGeneration() ? "ACTIVE" : "BUILDING", - row.generation() == row.activeGeneration() ? java.time.LocalDateTime.now() : null, - row.generation() == row.activeGeneration() ? java.time.LocalDateTime.now() : null); - jdbcTemplate.update(""" - insert into aihr_generation_version - (tenant_id, knowledge_id, generation, asset_id, version_id, content_sha256, - point_count, status, create_time) - select a.tenant_id, a.knowledge_id, ?, a.id, ?, v.content_sha256, ?, 'INCLUDED', now() - from aihr_data_asset a - join aihr_data_version v on v.tenant_id = a.tenant_id and v.id = ? - where a.tenant_id = ? and a.id = ? - on duplicate key update version_id = values(version_id), - content_sha256 = values(content_sha256), point_count = values(point_count), status = 'INCLUDED' - """, row.generation(), row.versionId(), vectors, row.versionId(), - row.tenantId(), row.assetId()); - jdbcTemplate.update(""" - update aihr_knowledge_generation generation_row - set version_count = (select count(*) from aihr_generation_version member - where member.tenant_id = generation_row.tenant_id - and member.knowledge_id = generation_row.knowledge_id - and member.generation = generation_row.generation - and member.status = 'INCLUDED'), - point_count = (select coalesce(sum(member.point_count), 0) - from aihr_generation_version member - where member.tenant_id = generation_row.tenant_id - and member.knowledge_id = generation_row.knowledge_id - and member.generation = generation_row.generation - and member.status = 'INCLUDED') - where generation_row.tenant_id = ? and generation_row.knowledge_id = ? - and generation_row.generation = ? - """, row.tenantId(), row.knowledgeId(), row.generation()); - } } catch (RuntimeException ex) { String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage(); int attempted = row.attemptCount() + 1; @@ -200,6 +163,84 @@ public class AihrKnowledgeIndexOutboxService { } } + private void recordGenerationVectorization(OutboxRow row, + AihrSopSeedService.VectorizationResult vectorization) { + String mismatch = transactionTemplate.execute(status -> { + List rows = jdbcTemplate.query(""" + select embedding_model, embedding_dimension, status, last_error + from aihr_knowledge_generation + where tenant_id = ? and knowledge_id = ? and generation = ? for update + """, (rs, rowNum) -> new GenerationManifest(rs.getString("embedding_model"), + rs.getObject("embedding_dimension", Integer.class), rs.getString("status"), + rs.getString("last_error")), row.tenantId(), row.knowledgeId(), row.generation()); + GenerationManifest existing = rows.isEmpty() ? null : rows.get(0); + if (existing != null && existing.lastError() != null + && existing.lastError().startsWith("EMBEDDING_MANIFEST_MISMATCH")) { + return existing.lastError(); + } + if (existing != null && !manifestCompatible(existing.embeddingModel(), existing.embeddingDimension(), + vectorization.embeddingModel(), vectorization.embeddingDimension())) { + String reason = "EMBEDDING_MANIFEST_MISMATCH expected=" + existing.embeddingModel() + "/" + + existing.embeddingDimension() + " actual=" + vectorization.embeddingModel() + "/" + + vectorization.embeddingDimension(); + jdbcTemplate.update(""" + update aihr_knowledge_generation set status = 'FAILED', last_error = ?, ready_time = null + where tenant_id = ? and knowledge_id = ? and generation = ? + """, reason, row.tenantId(), row.knowledgeId(), row.generation()); + return reason; + } + jdbcTemplate.update(""" + insert into aihr_knowledge_generation + (tenant_id, knowledge_id, generation, collection_name, status, + version_count, point_count, embedding_model, embedding_dimension, + create_time, ready_time, activated_time) + values (?, ?, ?, ?, ?, 0, 0, ?, ?, now(), ?, ?) + on duplicate key update collection_name = values(collection_name), + embedding_model = coalesce(embedding_model, values(embedding_model)), + embedding_dimension = coalesce(embedding_dimension, values(embedding_dimension)), + status = case when status = 'FAILED' then 'BUILDING' else status end, + ready_time = coalesce(ready_time, values(ready_time)), + activated_time = coalesce(activated_time, values(activated_time)), last_error = null + """, row.tenantId(), row.knowledgeId(), row.generation(), row.collection(), + row.generation() == row.activeGeneration() ? "ACTIVE" : "BUILDING", + vectorization.embeddingModel(), vectorization.embeddingDimension(), + row.generation() == row.activeGeneration() ? java.time.LocalDateTime.now() : null, + row.generation() == row.activeGeneration() ? java.time.LocalDateTime.now() : null); + jdbcTemplate.update(""" + insert into aihr_generation_version + (tenant_id, knowledge_id, generation, asset_id, version_id, content_sha256, + point_count, status, create_time) + select a.tenant_id, a.knowledge_id, ?, a.id, ?, v.content_sha256, ?, 'INCLUDED', now() + from aihr_data_asset a + join aihr_data_version v on v.tenant_id = a.tenant_id and v.id = ? + where a.tenant_id = ? and a.id = ? + on duplicate key update version_id = values(version_id), + content_sha256 = values(content_sha256), point_count = values(point_count), status = 'INCLUDED' + """, row.generation(), row.versionId(), vectorization.count(), row.versionId(), + row.tenantId(), row.assetId()); + jdbcTemplate.update(""" + update aihr_knowledge_generation generation_row + set version_count = (select count(*) from aihr_generation_version member + where member.tenant_id = generation_row.tenant_id + and member.knowledge_id = generation_row.knowledge_id + and member.generation = generation_row.generation + and member.status = 'INCLUDED'), + point_count = (select coalesce(sum(member.point_count), 0) + from aihr_generation_version member + where member.tenant_id = generation_row.tenant_id + and member.knowledge_id = generation_row.knowledge_id + and member.generation = generation_row.generation + and member.status = 'INCLUDED') + where generation_row.tenant_id = ? and generation_row.knowledge_id = ? + and generation_row.generation = ? + """, row.tenantId(), row.knowledgeId(), row.generation()); + return null; + }); + if (mismatch != null) { + throw new IllegalStateException(mismatch); + } + } + public OutboxHealth health(String tenantId) { OutboxCounts counts = jdbcTemplate.queryForObject(""" select sum(case when status = 'PENDING' then 1 else 0 end) pending_count, @@ -240,6 +281,15 @@ public class AihrKnowledgeIndexOutboxService { || ("DELETE".equals(operation) && !"DEPRECATED".equals(lifecycleStatus)); } + static boolean manifestCompatible(String expectedModel, Integer expectedDimension, + String actualModel, int actualDimension) { + if (actualModel == null || actualModel.isBlank() || actualDimension <= 0) { + return false; + } + return (expectedModel == null || expectedModel.isBlank() || expectedModel.equals(actualModel)) + && (expectedDimension == null || expectedDimension <= 0 || expectedDimension == actualDimension); + } + public record OutboxHealth(int pending, int processing, int failed, int deadLetter, long oldestPendingSeconds, int staleAssets, boolean vectorMatched, Integer mysqlFragments, Integer ungovernedFragments, Long qdrantPoints, @@ -255,4 +305,8 @@ public class AihrKnowledgeIndexOutboxService { String docId, String lifecycleStatus, long currentVersionId, long activeGeneration) { } + + private record GenerationManifest(String embeddingModel, Integer embeddingDimension, + String status, String lastError) { + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java index a94b93ce..ecd1659a 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java @@ -15,10 +15,12 @@ import org.springframework.jdbc.support.GeneratedKeyHolder; import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; import java.sql.PreparedStatement; import java.sql.Statement; import java.time.LocalDateTime; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -33,9 +35,9 @@ public class AihrKnowledgeRetentionService { private final JdbcTemplate jdbcTemplate; private final ObjectMapper objectMapper; - private final AihrKnowledgeLifecycleService lifecycleService; private final AihrSopSeedService sopSeedService; private final ISysOssService ossService; + private final TransactionTemplate transactionTemplate; public ImpactAnalysis impact(String tenantId, long assetId) { AssetRetention asset = requireAsset(tenantId, assetId, false); @@ -62,17 +64,28 @@ public class AihrKnowledgeRetentionService { int generations = count(""" select count(*) from aihr_generation_version where tenant_id = ? and asset_id = ? """, tenantId, assetId); + int activeGenerationMemberships = count(""" + select count(*) + from aihr_generation_version member + where member.tenant_id = ? and member.asset_id = ? and member.status = 'INCLUDED' + and member.generation = coalesce(( + select scope.active_generation from aihr_knowledge_rollout_scope scope + where scope.tenant_id = member.tenant_id and scope.knowledge_id = member.knowledge_id + limit 1), 1) + """, tenantId, assetId); boolean canRequestPurge = canRequestPurge(asset.lifecycleStatus(), asset.retentionClass(), - asset.legalHold(), learningTasks, broadcasts); + asset.legalHold(), learningTasks, broadcasts, activeGenerationMemberships); List blockers = new java.util.ArrayList<>(); if (asset.legalHold()) blockers.add("LEGAL_HOLD"); if ("PERMANENT".equals(asset.retentionClass())) blockers.add("PERMANENT_RETENTION"); if ("PUBLISHED".equals(asset.lifecycleStatus())) blockers.add("WITHDRAW_FIRST"); if (learningTasks > 0) blockers.add("LEARNING_TASK_REFERENCE"); if (broadcasts > 0) blockers.add("BROADCAST_REFERENCE"); + if (activeGenerationMemberships > 0) blockers.add("ACTIVE_GENERATION_REFERENCE"); return new ImpactAnalysis(assetId, asset.lifecycleStatus(), asset.retentionClass(), asset.legalHold(), asset.attachmentId(), asset.rawOssId(), fragments, publishedFragments, queryEvidence, - learningTasks, broadcasts, generations, canRequestPurge, List.copyOf(blockers)); + learningTasks, broadcasts, generations, activeGenerationMemberships, + canRequestPurge, List.copyOf(blockers)); } @Transactional @@ -145,13 +158,52 @@ public class AihrKnowledgeRetentionService { return requireRequest(tenantId, requestId, false); } - @Transactional public DeletionRequest execute(String tenantId, long requestId, long executorId) { if (executorId <= 0) throw new ServiceException("A signed-in executor is required", HttpStatus.BAD_REQUEST); + ExecutionPlan plan = transactionTemplate.execute(status -> prepareExecution(tenantId, requestId, executorId)); + if (plan == null) { + throw new ServiceException("Failed to prepare physical purge", HttpStatus.CONFLICT); + } + if (plan.completed()) { + return plan.request(); + } + try { + TenantHelper.dynamic(tenantId, () -> { + sopSeedService.deleteLegacyVectorDocument(plan.asset().knowledgeId(), plan.asset().docId()); + for (VectorDeletionTarget target : plan.vectorTargets()) { + sopSeedService.deleteVectorDocument(plan.asset().knowledgeId(), plan.asset().docId(), + target.collection(), target.generation()); + } + return null; + }); + if (plan.asset().rawOssId() != null && strictCount( + "select count(*) from aihr_knowledge_attach where tenant_id = ? and oss_id = ?", + tenantId, plan.asset().rawOssId()) <= 1 && strictCount( + "select count(*) from sys_oss where tenant_id = ? and oss_id = ?", + tenantId, plan.asset().rawOssId()) > 0) { + ossService.deleteWithValidByIds(List.of(plan.asset().rawOssId()), true); + } + transactionTemplate.executeWithoutResult(status -> completePurge(tenantId, requestId)); + } catch (RuntimeException ex) { + transactionTemplate.executeWithoutResult(status -> markPurgeFailed(tenantId, requestId, ex)); + throw new ServiceException("Physical purge failed; the request remains retryable", 503) + .setDetailMessage(ex.getMessage()); + } + return requireRequest(tenantId, requestId, false); + } + + private ExecutionPlan prepareExecution(String tenantId, long requestId, long executorId) { DeletionRequest request = requireRequest(tenantId, requestId, true); - if ("EXECUTED".equals(request.status())) return request; - if (!Set.of("APPROVED", "FAILED").contains(request.status())) { - throw new ServiceException("Purge request is not approved", HttpStatus.CONFLICT); + if ("EXECUTED".equals(request.status())) { + return new ExecutionPlan(request, null, List.of(), true); + } + boolean staleExecution = "EXECUTING".equals(request.status()) && strictCount(""" + select count(*) from aihr_deletion_request + where tenant_id = ? and id = ? and status = 'EXECUTING' + and update_time < date_sub(now(), interval 10 minute) + """, tenantId, requestId) > 0; + if (!Set.of("APPROVED", "FAILED").contains(request.status()) && !staleExecution) { + throw new ServiceException("Purge request is not approved or is already executing", HttpStatus.CONFLICT); } if (request.requesterId() == executorId) { throw new ServiceException("Requester cannot execute the irreversible purge", HttpStatus.CONFLICT); @@ -163,36 +215,69 @@ public class AihrKnowledgeRetentionService { if (asset.legalHold() || "PUBLISHED".equals(asset.lifecycleStatus())) { throw new ServiceException("Legal hold or active publication blocks purge", HttpStatus.CONFLICT); } - jdbcTemplate.update(""" - update aihr_deletion_request set status = 'EXECUTING', executed_by = ?, last_error = null, update_time = now() - where tenant_id = ? and id = ? and status in ('APPROVED','FAILED') - """, executorId, tenantId, requestId); - try { - AihrKnowledgeRolloutService.ModeGeneration target = rolloutTarget(tenantId, asset.knowledgeId()); - TenantHelper.dynamic(tenantId, () -> { - sopSeedService.deleteVectorDocument(asset.knowledgeId(), asset.docId(), - target.governedCollection(), target.activeGeneration()); - return null; - }); - if (asset.rawOssId() != null && count( - "select count(*) from aihr_knowledge_attach where tenant_id = ? and oss_id = ?", - tenantId, asset.rawOssId()) <= 1) { - ossService.deleteWithValidByIds(List.of(asset.rawOssId()), true); - } - purgeDatabase(tenantId, asset); - jdbcTemplate.update(""" - update aihr_deletion_request - set status = 'EXECUTED', executed_time = now(), last_error = null, update_time = now() - where tenant_id = ? and id = ? - """, tenantId, requestId); - } catch (RuntimeException ex) { - jdbcTemplate.update(""" - update aihr_deletion_request set status = 'FAILED', last_error = left(?, 1000), update_time = now() - where tenant_id = ? and id = ? - """, ex.getMessage(), tenantId, requestId); - return requireRequest(tenantId, requestId, false); + if (strictCount(""" + select count(*) + from aihr_generation_version member + where member.tenant_id = ? and member.asset_id = ? and member.status = 'INCLUDED' + and member.generation = coalesce(( + select scope.active_generation from aihr_knowledge_rollout_scope scope + where scope.tenant_id = member.tenant_id and scope.knowledge_id = member.knowledge_id + limit 1), 1) + """, tenantId, asset.assetId()) > 0) { + throw new ServiceException( + "Active vector generation still contains this asset; activate a replacement generation first", + HttpStatus.CONFLICT); } - return requireRequest(tenantId, requestId, false); + jdbcTemplate.update(""" + update aihr_deletion_request + set status = 'EXECUTING', executed_by = ?, last_error = null, update_time = now() + where tenant_id = ? and id = ? + """, executorId, tenantId, requestId); + LinkedHashSet targets = new LinkedHashSet<>(jdbcTemplate.query(""" + select distinct vector_target.collection_name, vector_target.generation + from ( + select g.collection_name, member.generation + from aihr_generation_version member + join aihr_knowledge_generation g on g.tenant_id = member.tenant_id + and g.knowledge_id = member.knowledge_id and g.generation = member.generation + where member.tenant_id = ? and member.asset_id = ? + union all + select target_collection, index_generation + from aihr_index_outbox + where tenant_id = ? and asset_id = ? + ) vector_target + order by vector_target.generation + """, (rs, rowNum) -> new VectorDeletionTarget(rs.getString("collection_name"), + rs.getLong("generation")), tenantId, asset.assetId(), tenantId, asset.assetId())); + AihrKnowledgeRolloutService.ModeGeneration active = rolloutTarget(tenantId, asset.knowledgeId()); + targets.add(new VectorDeletionTarget(active.governedCollection(), active.activeGeneration())); + return new ExecutionPlan(request, asset, List.copyOf(targets), false); + } + + private void completePurge(String tenantId, long requestId) { + DeletionRequest request = requireRequest(tenantId, requestId, true); + if ("EXECUTED".equals(request.status())) { + return; + } + if (!"EXECUTING".equals(request.status())) { + throw new ServiceException("Purge request execution state changed", HttpStatus.CONFLICT); + } + AssetRetention asset = requireAsset(tenantId, request.assetId(), true); + purgeDatabase(tenantId, asset); + jdbcTemplate.update(""" + update aihr_deletion_request + set status = 'EXECUTED', executed_time = now(), last_error = null, update_time = now() + where tenant_id = ? and id = ? and status = 'EXECUTING' + """, tenantId, requestId); + } + + private void markPurgeFailed(String tenantId, long requestId, RuntimeException error) { + String message = error.getMessage() == null ? error.getClass().getSimpleName() : error.getMessage(); + jdbcTemplate.update(""" + update aihr_deletion_request + set status = 'FAILED', last_error = left(?, 1000), update_time = now() + where tenant_id = ? and id = ? and status = 'EXECUTING' + """, message, tenantId, requestId); } public List requests(String tenantId, String status, int limit) { @@ -269,40 +354,74 @@ public class AihrKnowledgeRetentionService { } private void purgeDatabase(String tenantId, AssetRetention asset) { - safeUpdate("delete from aihr_query_evidence where tenant_id = ? and asset_id = ?", + List affectedGenerations = jdbcTemplate.query(""" + select distinct knowledge_id, generation + from aihr_generation_version + where tenant_id = ? and asset_id = ? + order by knowledge_id, generation + """, (rs, rowNum) -> new GenerationRef(rs.getLong("knowledge_id"), rs.getLong("generation")), tenantId, asset.assetId()); - safeUpdate("delete from aihr_knowledge_fragment_locator where tenant_id = ? and doc_id = ?", + jdbcTemplate.update("delete from aihr_query_evidence where tenant_id = ? and asset_id = ?", + tenantId, asset.assetId()); + jdbcTemplate.update("delete from aihr_knowledge_fragment_locator where tenant_id = ? and doc_id = ?", tenantId, asset.docId()); - safeUpdate("delete from aihr_knowledge_fragment where tenant_id = ? and knowledge_id = ? and doc_id = ?", + jdbcTemplate.update("delete from aihr_knowledge_fragment where tenant_id = ? and knowledge_id = ? and doc_id = ?", tenantId, asset.knowledgeId(), asset.docId()); - safeUpdate(""" + jdbcTemplate.update(""" delete from aihr_claim_conflict where tenant_id = ? and ( left_claim_id in (select id from aihr_knowledge_claim where tenant_id = ? and asset_id = ?) or right_claim_id in (select id from aihr_knowledge_claim where tenant_id = ? and asset_id = ?)) """, tenantId, tenantId, asset.assetId(), tenantId, asset.assetId()); - safeUpdate(""" + jdbcTemplate.update(""" delete from aihr_duplicate_relation where tenant_id = ? and (left_asset_id = ? or right_asset_id = ?) """, tenantId, asset.assetId(), asset.assetId()); - safeUpdate("delete from aihr_review_batch_item where tenant_id = ? and asset_id = ?", + jdbcTemplate.update("delete from aihr_review_batch_item where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); - safeUpdate(""" + jdbcTemplate.update(""" delete from aihr_dataset_membership where tenant_id = ? and version_id in (select id from aihr_data_version where tenant_id = ? and asset_id = ?) """, tenantId, tenantId, asset.assetId()); - safeUpdate("delete from aihr_quality_assessment where tenant_id = ? and asset_id = ?", + jdbcTemplate.update("delete from aihr_quality_assessment where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); for (String table : List.of("aihr_generation_version", "aihr_index_outbox", "aihr_quality_issue", "aihr_review_assistance", "aihr_version_diff", "aihr_version_rollback")) { - safeUpdate("delete from " + table + " where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); + jdbcTemplate.update("delete from " + table + " where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); } - safeUpdate("delete from aihr_knowledge_claim where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); - safeUpdate("delete from aihr_review_decision where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); - safeUpdate("delete from aihr_chunk_revision where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); - safeUpdate("delete from aihr_data_version where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); - safeUpdate("delete from aihr_data_asset where tenant_id = ? and id = ?", tenantId, asset.assetId()); + for (GenerationRef generation : affectedGenerations) { + jdbcTemplate.update(""" + update aihr_knowledge_generation generation_row + set version_count = (select count(*) from aihr_generation_version member + where member.tenant_id = generation_row.tenant_id + and member.knowledge_id = generation_row.knowledge_id + and member.generation = generation_row.generation + and member.status = 'INCLUDED'), + point_count = (select coalesce(sum(member.point_count), 0) + from aihr_generation_version member + where member.tenant_id = generation_row.tenant_id + and member.knowledge_id = generation_row.knowledge_id + and member.generation = generation_row.generation + and member.status = 'INCLUDED'), + status = 'FAILED', manifest_sha256 = null, + last_error = concat('INVALIDATED_BY_ASSET_PURGE:', ?) + where generation_row.tenant_id = ? and generation_row.knowledge_id = ? + and generation_row.generation = ? and generation_row.status <> 'ACTIVE' + """, asset.assetId(), tenantId, generation.knowledgeId(), generation.generation()); + jdbcTemplate.update(""" + update aihr_knowledge_rollout_scope + set candidate_generation = null, + reason = concat('Candidate invalidated by physical purge of asset ', ?), + update_time = now() + where tenant_id = ? and knowledge_id = ? and candidate_generation = ? + """, asset.assetId(), tenantId, generation.knowledgeId(), generation.generation()); + } + jdbcTemplate.update("delete from aihr_knowledge_claim where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); + jdbcTemplate.update("delete from aihr_review_decision where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); + jdbcTemplate.update("delete from aihr_chunk_revision where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); + jdbcTemplate.update("delete from aihr_data_version where tenant_id = ? and asset_id = ?", tenantId, asset.assetId()); + jdbcTemplate.update("delete from aihr_data_asset where tenant_id = ? and id = ?", tenantId, asset.assetId()); if (asset.attachmentId() != null) { - safeUpdate("delete from aihr_knowledge_attach where tenant_id = ? and id = ?", + jdbcTemplate.update("delete from aihr_knowledge_attach where tenant_id = ? and id = ?", tenantId, asset.attachmentId()); } } @@ -371,12 +490,9 @@ public class AihrKnowledgeRetentionService { } } - private void safeUpdate(String sql, Object... args) { - try { - jdbcTemplate.update(sql, args); - } catch (DataAccessException ignored) { - // Optional consumers can be absent in an incremental deployment; reconciliation records residue. - } + private int strictCount(String sql, Object... args) { + Integer value = jdbcTemplate.queryForObject(sql, Integer.class, args); + return value == null ? 0 : value; } private String json(Object value) { @@ -398,9 +514,10 @@ public class AihrKnowledgeRetentionService { } static boolean canRequestPurge(String lifecycleStatus, String retentionClass, boolean legalHold, - int learningTasks, int broadcasts) { + int learningTasks, int broadcasts, int activeGenerationMemberships) { return !legalHold && !"PERMANENT".equals(normalize(retentionClass)) - && !"PUBLISHED".equals(normalize(lifecycleStatus)) && learningTasks == 0 && broadcasts == 0; + && !"PUBLISHED".equals(normalize(lifecycleStatus)) && learningTasks == 0 && broadcasts == 0 + && activeGenerationMemberships == 0; } private static String normalize(String value) { @@ -420,7 +537,8 @@ public class AihrKnowledgeRetentionService { public record ImpactAnalysis(long assetId, String lifecycleStatus, String retentionClass, boolean legalHold, Long attachmentId, Long rawOssId, int chunks, int publishedFragments, int queryEvidence, int learningTasks, - int broadcasts, int generations, boolean canRequestPurge, + int broadcasts, int generations, int activeGenerationMemberships, + boolean canRequestPurge, List blockers) { } @@ -447,4 +565,14 @@ public class AihrKnowledgeRetentionService { Long rawOssId, String lifecycleStatus, Long publishedVersionId, String retentionClass, boolean legalHold) { } + + private record VectorDeletionTarget(String collection, long generation) { + } + + private record GenerationRef(long knowledgeId, long generation) { + } + + private record ExecutionPlan(DeletionRequest request, AssetRetention asset, + List vectorTargets, boolean completed) { + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutService.java index d054680a..4ee1268c 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutService.java @@ -3,8 +3,10 @@ package org.dromara.aihr.knowledge.quality; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; +import org.dromara.aihr.service.AihrSopSeedService; import org.dromara.common.core.constant.HttpStatus; import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.tenant.helper.TenantHelper; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.support.GeneratedKeyHolder; import org.springframework.jdbc.support.KeyHolder; @@ -29,6 +31,7 @@ public class AihrKnowledgeShadowRolloutService { private final JdbcTemplate jdbcTemplate; private final ObjectMapper objectMapper; + private final AihrSopSeedService sopSeedService; @Transactional public GenerationBuild startBuild(String tenantId, long knowledgeId, long reviewerId, BuildCommand command) { @@ -73,7 +76,8 @@ public class AihrKnowledgeShadowRolloutService { set candidate_generation = ?, updated_by = ?, reason = ?, update_time = now() where tenant_id = ? and knowledge_id = ? """, generation, reviewerId, command.reason().trim(), tenantId, knowledgeId); - return new GenerationBuild(knowledgeId, generation, collection, "BUILDING", jobs, 0, 0); + return new GenerationBuild(knowledgeId, generation, collection, "BUILDING", jobs, 0, 0, + null, null, null); } @Transactional @@ -104,16 +108,28 @@ public class AihrKnowledgeShadowRolloutService { where o.tenant_id = ? and a.knowledge_id = ? and o.index_generation = ? and o.status in ('PENDING','PROCESSING') """, tenantId, knowledgeId, generation); - String status = failed > 0 ? "FAILED" : (pending == 0 && included == expected ? "READY" : "BUILDING"); + GenerationManifest manifest = generationManifest(tenantId, knowledgeId, generation); + boolean manifestReady = manifest != null && manifest.embeddingModel() != null + && !manifest.embeddingModel().isBlank() && manifest.embeddingDimension() != null + && manifest.embeddingDimension() > 0; + String failureReason = failed > 0 ? "INDEX_OUTBOX_FAILURE" + : (pending == 0 && expected == 0 ? "EMPTY_GENERATION" + : (pending == 0 && included == expected && !manifestReady ? "EMBEDDING_MANIFEST_MISSING" : null)); + String status = failureReason != null ? "FAILED" + : (pending == 0 && included == expected ? "READY" : "BUILDING"); + String manifestSha256 = "READY".equals(status) + ? generationManifestSha256(tenantId, knowledgeId, generation, manifest) : null; jdbcTemplate.update(""" update aihr_knowledge_generation set status = ?, ready_time = case when ? = 'READY' then coalesce(ready_time, now()) else ready_time end, - last_error = case when ? = 'FAILED' then 'INDEX_OUTBOX_FAILURE' else null end + manifest_sha256 = case when ? = 'READY' then ? else manifest_sha256 end, + last_error = ? where tenant_id = ? and knowledge_id = ? and generation = ? - """, status, status, status, tenantId, knowledgeId, generation); + """, status, status, status, manifestSha256, failureReason, tenantId, knowledgeId, generation); GenerationBuild result = buildStatus(tenantId, knowledgeId, generation); return new GenerationBuild(result.knowledgeId(), result.generation(), result.collection(), status, - expected, included, failed + pending); + expected, included, failed + pending, result.embeddingModel(), result.embeddingDimension(), + result.manifestSha256()); } @Transactional @@ -125,26 +141,24 @@ public class AihrKnowledgeShadowRolloutService { if (!"SHADOW".equals(scope.mode()) || scope.candidateGeneration() == null) { throw new ServiceException("Shadow comparison requires a candidate generation", HttpStatus.CONFLICT); } + GenerationManifest manifest = generationManifest(tenantId, knowledgeId, scope.candidateGeneration()); + if (manifest == null || !"READY".equals(manifest.status())) { + throw new ServiceException("Shadow comparison requires a READY candidate generation", + HttpStatus.CONFLICT); + } + if (manifest.embeddingModel() == null || manifest.embeddingModel().isBlank() + || manifest.embeddingDimension() == null || manifest.embeddingDimension() <= 0) { + throw new ServiceException("Candidate generation embedding manifest is incomplete", + HttpStatus.CONFLICT); + } String query = command.queryText().trim().substring(0, Math.min(1000, command.queryText().trim().length())); long started = System.nanoTime(); - List legacy = jdbcTemplate.queryForList(""" - select f.id from aihr_knowledge_fragment f - join aihr_knowledge_attach a on a.tenant_id = f.tenant_id - and a.knowledge_id = f.knowledge_id and a.doc_id = f.doc_id and a.status = 2 - where f.tenant_id = ? and f.knowledge_id = ? and f.content like concat('%', ?, '%') - order by f.id limit 5 - """, Long.class, tenantId, knowledgeId, query); - List governed = jdbcTemplate.queryForList(""" - select c.published_fragment_id - from aihr_chunk_revision c - join aihr_data_asset a on a.tenant_id = c.tenant_id and a.id = c.asset_id - and a.published_version_id = c.version_id - join aihr_dataset_membership d on d.tenant_id = c.tenant_id and d.version_id = c.version_id - and d.dataset_code = 'production' and d.status = 'ACTIVE' - where c.tenant_id = ? and a.knowledge_id = ? and a.lifecycle_status = 'PUBLISHED' - and c.published_fragment_id is not null and c.content like concat('%', ?, '%') - order by c.chunk_index, c.id limit 5 - """, Long.class, tenantId, knowledgeId, query); + List legacy = TenantHelper.dynamic(tenantId, + () -> sopSeedService.shadowServingFragmentIds(knowledgeId, query, 5)); + List governed = TenantHelper.dynamic(tenantId, + () -> sopSeedService.shadowCandidateFragmentIds(knowledgeId, query, 5, + manifest.collection(), scope.candidateGeneration(), manifest.embeddingModel(), + manifest.embeddingDimension())); long latencyMs = Math.max(0, (System.nanoTime() - started) / 1_000_000L); boolean top1 = !legacy.isEmpty() && !governed.isEmpty() && legacy.get(0).equals(governed.get(0)); Set intersection = new LinkedHashSet<>(legacy); @@ -305,10 +319,13 @@ public class AihrKnowledgeShadowRolloutService { update aihr_knowledge_generation set status = 'RETIRED', retire_after = now() where tenant_id = ? and knowledge_id = ? and generation = ? """, tenantId, knowledgeId, window.toGeneration()); - jdbcTemplate.update(""" + int restored = jdbcTemplate.update(""" update aihr_knowledge_generation set status = 'ACTIVE', retire_after = null - where tenant_id = ? and knowledge_id = ? and generation = ? + where tenant_id = ? and knowledge_id = ? and generation = ? and status = 'RETIRED' """, tenantId, knowledgeId, window.fromGeneration()); + if (restored != 1) { + throw new ServiceException("Rollback generation is no longer restorable", HttpStatus.CONFLICT); + } jdbcTemplate.update(""" update aihr_knowledge_rollout_scope set mode = 'SHADOW', active_generation = ?, candidate_generation = ?, @@ -329,15 +346,42 @@ public class AihrKnowledgeShadowRolloutService { private GenerationBuild buildStatus(String tenantId, long knowledgeId, long generation) { List rows = jdbcTemplate.query(""" - select generation, collection_name, status, version_count, point_count + select generation, collection_name, status, version_count, point_count, + embedding_model, embedding_dimension, manifest_sha256 from aihr_knowledge_generation where tenant_id = ? and knowledge_id = ? and generation = ? """, (rs, rowNum) -> new GenerationBuild(knowledgeId, rs.getLong("generation"), rs.getString("collection_name"), rs.getString("status"), rs.getInt("version_count"), - rs.getInt("point_count"), 0), tenantId, knowledgeId, generation); + rs.getInt("point_count"), 0, rs.getString("embedding_model"), + rs.getObject("embedding_dimension", Integer.class), rs.getString("manifest_sha256")), + tenantId, knowledgeId, generation); if (rows.isEmpty()) throw new ServiceException("Generation does not exist", HttpStatus.NOT_FOUND); return rows.get(0); } + private GenerationManifest generationManifest(String tenantId, long knowledgeId, long generation) { + List rows = jdbcTemplate.query(""" + select collection_name, status, embedding_model, embedding_dimension + from aihr_knowledge_generation + where tenant_id = ? and knowledge_id = ? and generation = ? limit 1 + """, (rs, rowNum) -> new GenerationManifest(rs.getString("collection_name"), + rs.getString("status"), rs.getString("embedding_model"), + rs.getObject("embedding_dimension", Integer.class)), tenantId, knowledgeId, generation); + return rows.isEmpty() ? null : rows.get(0); + } + + private String generationManifestSha256(String tenantId, long knowledgeId, long generation, + GenerationManifest manifest) { + List members = jdbcTemplate.queryForList(""" + select concat(asset_id, ':', version_id, ':', content_sha256, ':', point_count) + from aihr_generation_version + where tenant_id = ? and knowledge_id = ? and generation = ? and status = 'INCLUDED' + order by asset_id + """, String.class, tenantId, knowledgeId, generation); + return AihrKnowledgeLifecycleService.sha256(String.join("\n", List.of( + manifest.collection(), String.valueOf(generation), manifest.embeddingModel(), + String.valueOf(manifest.embeddingDimension()), String.join("\n", members)))); + } + private RolloutTarget lockScope(String tenantId, long knowledgeId) { return requireScope(tenantId, knowledgeId, true); } @@ -392,7 +436,8 @@ public class AihrKnowledgeShadowRolloutService { } public record GenerationBuild(long knowledgeId, long generation, String collection, String status, - int expectedVersions, int includedVersions, int outstandingJobs) { + int expectedVersions, int includedVersions, int outstandingJobs, + String embeddingModel, Integer embeddingDimension, String manifestSha256) { } public record ShadowComparison(long knowledgeId, long generation, String querySha256, @@ -422,6 +467,10 @@ public class AihrKnowledgeShadowRolloutService { Long candidateGeneration) { } + private record GenerationManifest(String collection, String status, String embeddingModel, + Integer embeddingDimension) { + } + private record GateMetrics(int sampleCount, double top1Rate, double overlapAvg, double emptyRate) { } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeVersionGovernanceService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeVersionGovernanceService.java index 873ee9b7..3f28ffaa 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeVersionGovernanceService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeVersionGovernanceService.java @@ -195,12 +195,14 @@ public class AihrKnowledgeVersionGovernanceService { public List generations(String tenantId, long knowledgeId) { return jdbcTemplate.query(""" select generation, collection_name, base_generation, status, version_count, point_count, - manifest_sha256, create_time, ready_time, activated_time, retire_after, last_error + embedding_model, embedding_dimension, manifest_sha256, + create_time, ready_time, activated_time, retire_after, last_error from aihr_knowledge_generation where tenant_id = ? and knowledge_id = ? order by generation desc """, (rs, rowNum) -> new GenerationSummary(rs.getLong("generation"), rs.getString("collection_name"), rs.getObject("base_generation", Long.class), rs.getString("status"), rs.getInt("version_count"), rs.getInt("point_count"), + rs.getString("embedding_model"), rs.getObject("embedding_dimension", Integer.class), rs.getString("manifest_sha256"), rs.getObject("create_time", LocalDateTime.class), rs.getObject("ready_time", LocalDateTime.class), rs.getObject("activated_time", LocalDateTime.class), rs.getObject("retire_after", LocalDateTime.class), rs.getString("last_error")), tenantId, knowledgeId); @@ -284,7 +286,8 @@ public class AihrKnowledgeVersionGovernanceService { } public record GenerationSummary(long generation, String collection, Long baseGeneration, String status, - int versionCount, int pointCount, String manifestSha256, + int versionCount, int pointCount, String embeddingModel, + Integer embeddingDimension, String manifestSha256, LocalDateTime createTime, LocalDateTime readyTime, LocalDateTime activatedTime, LocalDateTime retireAfter, String lastError) { } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSopSeedService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSopSeedService.java index b24b5218..ee938f30 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSopSeedService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSopSeedService.java @@ -1,5 +1,6 @@ package org.dromara.aihr.service; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -2652,12 +2653,20 @@ public class AihrSopSeedService { public int vectorizePublishedDocument(long knowledgeId, String docId) { AihrKnowledgeRolloutService.ModeGeneration rollout = rolloutModeGeneration(knowledgeId); - return vectorizePublishedDocument(knowledgeId, docId, null, - firstNonBlank(rollout.governedCollection(), governedQdrantCollection()), rollout.activeGeneration()); + return vectorizePublishedDocumentWithManifest(knowledgeId, docId, null, + firstNonBlank(rollout.governedCollection(), governedQdrantCollection()), rollout.activeGeneration()).count(); } public int vectorizePublishedDocument(long knowledgeId, String docId, Long expectedVersionId, String targetCollection, long targetGeneration) { + return vectorizePublishedDocumentWithManifest(knowledgeId, docId, expectedVersionId, + targetCollection, targetGeneration).count(); + } + + public VectorizationResult vectorizePublishedDocumentWithManifest(long knowledgeId, String docId, + Long expectedVersionId, + String targetCollection, + long targetGeneration) { List rows = jdbcTemplate.query(""" select i.name, f.content from aihr_knowledge_fragment f @@ -2677,8 +2686,9 @@ public class AihrSopSeedService { List fragments = rows.stream().map(DocumentVectorRow::content).toList(); EmbeddingData data = generateEmbeddings(fragments); if (data.embeddings().isEmpty()) { - return 0; + throw new ServiceException("Published document embedding returned no vectors", 503); } + int dimension = requireEmbeddingDimension(data.embeddings()); for (int index = 0; index < data.embeddings().size(); index++) { jdbcTemplate.update(""" update aihr_knowledge_fragment @@ -2694,7 +2704,7 @@ public class AihrSopSeedService { } catch (Exception ex) { throw new ServiceException("Vector index upsert failed").setDetailMessage(ex.getMessage()); } - return data.embeddings().size(); + return new VectorizationResult(data.embeddings().size(), data.modelName(), dimension); } public void deleteVectorDocument(long knowledgeId, String docId) { @@ -4565,6 +4575,132 @@ public class AihrSopSeedService { } } + public void deleteLegacyVectorDocument(long knowledgeId, String docId) { + deleteVectorDocument(knowledgeId, docId, legacyQdrantCollection(), null); + } + + public List shadowServingFragmentIds(long knowledgeId, String queryText, int limit) { + int boundedLimit = Math.max(1, Math.min(limit, 20)); + return dbHits("", queryText, boundedLimit, Set.of(knowledgeId)).stream() + .map(KnowledgeHit::fragmentId) + .filter(Objects::nonNull) + .toList(); + } + + public List shadowCandidateFragmentIds(long knowledgeId, String queryText, int limit, + String collection, long generation, + String embeddingModel, int embeddingDimension) { + if (knowledgeId <= 0 || isBlank(queryText) || generation <= 0 || isBlank(collection) + || isBlank(embeddingModel) || embeddingDimension <= 0) { + throw new ServiceException("Candidate vector query manifest is incomplete", 409); + } + EmbeddingRuntime runtime = embeddingRuntimes().stream() + .filter(candidate -> embeddingModel.equals(candidate.modelName()) + && embeddingDimension == candidate.dimension()) + .findFirst() + .orElseThrow(() -> new ServiceException( + "Candidate embedding model is no longer available", 409)); + try { + List embeddings = callEmbeddings(runtime, List.of(queryText)); + if (embeddings.size() != 1 || requireEmbeddingDimension(embeddings) != embeddingDimension) { + throw new IllegalStateException("candidate query embedding dimension mismatch"); + } + int boundedLimit = Math.max(1, Math.min(limit, 20)); + ObjectNode body = objectMapper.createObjectNode(); + body.set("query", objectMapper.readTree(embeddings.get(0))); + body.set("filter", qdrantProductionFilterForSpaces(Set.of(knowledgeId), null, null, generation)); + body.put("limit", boundedLimit); + body.put("with_payload", true); + body.put("with_vector", false); + HttpResponse response = qdrantRequest( + "POST", "/collections/" + collection + "/points/query", body); + if (!ok(response.statusCode())) { + throw new IllegalStateException("candidate qdrant query HTTP " + response.statusCode()); + } + JsonNode points = objectMapper.readTree(response.body()).path("result").path("points"); + if (!points.isArray()) { + throw new IllegalStateException("candidate qdrant response missing points"); + } + List orderedFragmentIds = new ArrayList<>(); + for (JsonNode point : points) { + JsonNode payload = point.path("payload"); + long fragmentId = payload.path("fragment_id").asLong(0L); + if (fragmentId <= 0) { + throw new IllegalStateException("candidate qdrant point is missing fragment lineage"); + } + orderedFragmentIds.add(fragmentId); + } + return validateCandidateFragmentLineage(knowledgeId, generation, orderedFragmentIds); + } catch (ServiceException ex) { + throw ex; + } catch (Exception ex) { + throw new ServiceException("Candidate vector query failed", 503).setDetailMessage(ex.getMessage()); + } + } + + private List validateCandidateFragmentLineage(long knowledgeId, long generation, + List orderedFragmentIds) { + if (orderedFragmentIds.isEmpty()) { + return List.of(); + } + String placeholders = String.join(",", java.util.Collections.nCopies(orderedFragmentIds.size(), "?")); + List args = new ArrayList<>(); + args.add(tenantId()); + args.add(knowledgeId); + args.addAll(orderedFragmentIds); + Set authorized = new LinkedHashSet<>(jdbcTemplate.queryForList(""" + select c.published_fragment_id + from aihr_chunk_revision c + join aihr_data_asset a on a.tenant_id = c.tenant_id and a.id = c.asset_id + and a.published_version_id = c.version_id + join aihr_dataset_membership d on d.tenant_id = c.tenant_id and d.version_id = c.version_id + and d.dataset_code = 'production' and d.status = 'ACTIVE' + join aihr_generation_version g on g.tenant_id = c.tenant_id + and g.knowledge_id = a.knowledge_id and g.generation = ? + and g.asset_id = a.id and g.version_id = c.version_id and g.status = 'INCLUDED' + where c.tenant_id = ? and a.knowledge_id = ? + and a.lifecycle_status = 'PUBLISHED' and a.trust_level = 'HUMAN_VERIFIED' + and (a.effective_from is null or a.effective_from <= current_date()) + and (a.effective_to is null or a.effective_to >= current_date()) + and c.published_fragment_id in (%s) + """.formatted(placeholders), Long.class, + prependGenerationArguments(args, generation).toArray())); + if (authorized.size() != new LinkedHashSet<>(orderedFragmentIds).size()) { + throw new ServiceException("Candidate vector lineage no longer matches the published generation", 409); + } + return orderedFragmentIds.stream().filter(authorized::contains).toList(); + } + + private static List prependGenerationArguments(List arguments, long generation) { + List ordered = new ArrayList<>(); + ordered.add(generation); + ordered.addAll(arguments); + return ordered; + } + + private int requireEmbeddingDimension(List embeddings) { + Integer dimension = null; + for (String embedding : embeddings) { + try { + JsonNode vector = objectMapper.readTree(embedding); + if (!vector.isArray() || vector.isEmpty()) { + throw new IllegalStateException("embedding vector is empty or malformed"); + } + if (dimension == null) { + dimension = vector.size(); + } else if (dimension != vector.size()) { + throw new IllegalStateException("embedding vectors have inconsistent dimensions"); + } + } catch (JsonProcessingException ex) { + throw new IllegalStateException("embedding vector is not valid JSON", ex); + } + } + if (dimension == null || dimension <= 0) { + throw new IllegalStateException("embedding vector dimension is unavailable"); + } + return dimension; + } + private List legacyVectorScopes(Set allowedKnowledgeIds) { if (allowedKnowledgeIds != null && !allowedKnowledgeIds.isEmpty()) { return allowedKnowledgeIds.stream().map(id -> new RolloutVectorScope(id, "LEGACY", @@ -5357,6 +5493,9 @@ public class AihrSopSeedService { private record EmbeddingData(String modelName, List embeddings) { } + public record VectorizationResult(int count, String embeddingModel, int embeddingDimension) { + } + private record VectorDbStats(int fragments, int ungovernedFragments, int embeddedFragments, String embeddingModel, Integer embeddingDimension) { } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrDataQualityLifecycleSchemaTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrDataQualityLifecycleSchemaTest.java index 266bfc69..ccb22d5d 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrDataQualityLifecycleSchemaTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrDataQualityLifecycleSchemaTest.java @@ -281,7 +281,8 @@ class AihrDataQualityLifecycleSchemaTest { assertThat(sql).contains("`aihr_shadow_comparison`", "top1_agreement", "overlap_at_5", "`aihr_activation_gate`", "threshold_json", "evidence_json", - "`aihr_generation_activation`", "rollback_deadline", "ACTIVATE", "ROLLBACK"); + "`aihr_generation_activation`", "rollback_deadline", "ACTIVATE", "ROLLBACK", + "embedding_model", "embedding_dimension"); assertThat(sql.toLowerCase()).doesNotContain("update aihr_knowledge_rollout_scope"); } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeGovernanceSafetyTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeGovernanceSafetyTest.java index 7713952e..50a77aff 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeGovernanceSafetyTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeGovernanceSafetyTest.java @@ -4,6 +4,8 @@ import org.dromara.common.core.exception.ServiceException; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.LocalDateTime; import static org.assertj.core.api.Assertions.assertThat; @@ -31,11 +33,12 @@ class AihrKnowledgeGovernanceSafetyTest { @Test void referencedOrActiveAssetsCannotEnterPhysicalPurge() { - assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 0, 0)).isTrue(); - assertThat(AihrKnowledgeRetentionService.canRequestPurge("PUBLISHED", "STANDARD", false, 0, 0)).isFalse(); - assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 1, 0)).isFalse(); - assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 0, 1)).isFalse(); - assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", true, 0, 0)).isFalse(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 0, 0, 0)).isTrue(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("PUBLISHED", "STANDARD", false, 0, 0, 0)).isFalse(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 1, 0, 0)).isFalse(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 0, 1, 0)).isFalse(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", true, 0, 0, 0)).isFalse(); + assertThat(AihrKnowledgeRetentionService.canRequestPurge("DEPRECATED", "STANDARD", false, 0, 0, 1)).isFalse(); } @Test @@ -60,7 +63,25 @@ class AihrKnowledgeGovernanceSafetyTest { existing.runKey(), null, "LOW_RISK", true, 50, null))).isTrue(); assertThat(AihrKnowledgeMigrationOrchestrationService.sameManifest(existing, new AihrKnowledgeMigrationOrchestrationService.CreateRun( - existing.runKey(), null, "ALL", true, 50, null))).isFalse(); + existing.runKey(), null, "ALL", true, 50, null))).isFalse(); + } + + @Test + void physicalPurgeUsesRetryableExternalCleanupAndAtomicDatabaseDeletion() throws Exception { + Path source = Path.of("src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java"); + if (!Files.exists(source)) { + source = Path.of("ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeRetentionService.java"); + } + String code = Files.readString(source); + + assertThat(code).contains("prepareExecution", "completePurge", "markPurgeFailed", + "deleteLegacyVectorDocument", "VectorDeletionTarget", "transactionTemplate.executeWithoutResult", + "ACTIVE_GENERATION_REFERENCE", "INVALIDATED_BY_ASSET_PURGE", "manifest_sha256 = null"); + assertThat(code).doesNotContain("private void safeUpdate"); + + Path rolloutSource = source.getParent().resolve("AihrKnowledgeShadowRolloutService.java"); + String rolloutCode = Files.readString(rolloutSource); + assertThat(rolloutCode).contains("and status = 'RETIRED'", "Rollback generation is no longer restorable"); } private static AihrKnowledgeMigrationOrchestrationService.MigrationRun migrationRun( diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxServiceTest.java index afcfa1cb..190e0848 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxServiceTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeIndexOutboxServiceTest.java @@ -25,4 +25,19 @@ class AihrKnowledgeIndexOutboxServiceTest { assertThat(AihrKnowledgeIndexOutboxService.isObsolete("UPSERT", 10, "PUBLISHED", 10)).isFalse(); assertThat(AihrKnowledgeIndexOutboxService.isObsolete("DELETE", 10, "DEPRECATED", 10)).isFalse(); } + + @Test + @Tag("dev") + void generationManifestRejectsMixedModelsAndDimensions() { + assertThat(AihrKnowledgeIndexOutboxService.manifestCompatible( + null, null, "local-hash-v1", 256)).isTrue(); + assertThat(AihrKnowledgeIndexOutboxService.manifestCompatible( + "local-hash-v1", 256, "local-hash-v1", 256)).isTrue(); + assertThat(AihrKnowledgeIndexOutboxService.manifestCompatible( + "local-hash-v1", 256, "other-model", 256)).isFalse(); + assertThat(AihrKnowledgeIndexOutboxService.manifestCompatible( + "local-hash-v1", 256, "local-hash-v1", 1024)).isFalse(); + assertThat(AihrKnowledgeIndexOutboxService.manifestCompatible( + "local-hash-v1", 256, "", 0)).isFalse(); + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutServiceTest.java new file mode 100644 index 00000000..08021f5e --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/quality/AihrKnowledgeShadowRolloutServiceTest.java @@ -0,0 +1,86 @@ +package org.dromara.aihr.knowledge.quality; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.dromara.aihr.service.AihrSopSeedService; +import org.dromara.common.core.exception.ServiceException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@Tag("dev") +class AihrKnowledgeShadowRolloutServiceTest { + + @Test + void candidateGenerationMustBeReadyBeforeAComparison() throws Exception { + JdbcTemplate jdbc = rolloutJdbc("BUILDING"); + AihrSopSeedService sop = mock(AihrSopSeedService.class); + AihrKnowledgeShadowRolloutService service = new AihrKnowledgeShadowRolloutService( + jdbc, new ObjectMapper(), sop); + + ServiceException error = assertThrows(ServiceException.class, + () -> service.compare("000000", 10L, + new AihrKnowledgeShadowRolloutService.ComparisonCommand("收费标准", "MANUAL_SAMPLE"))); + + assertEquals("Shadow comparison requires a READY candidate generation", error.getMessage()); + verifyNoInteractions(sop); + } + + @Test + void strictCandidateVectorFailureCannotCreateActivationEvidence() throws Exception { + JdbcTemplate jdbc = rolloutJdbc("READY"); + AihrSopSeedService sop = mock(AihrSopSeedService.class); + when(sop.shadowServingFragmentIds(10L, "收费标准", 5)).thenReturn(List.of(101L)); + when(sop.shadowCandidateFragmentIds(10L, "收费标准", 5, + "aihr_knowledge_governed_v1", 2L, "local-hash-v1", 256)) + .thenThrow(new ServiceException("Candidate vector query failed", 503)); + AihrKnowledgeShadowRolloutService service = new AihrKnowledgeShadowRolloutService( + jdbc, new ObjectMapper(), sop); + + assertThrows(ServiceException.class, + () -> service.compare("000000", 10L, + new AihrKnowledgeShadowRolloutService.ComparisonCommand("收费标准", "MANUAL_SAMPLE"))); + + verify(jdbc, never()).update(contains("insert into aihr_shadow_comparison"), any(Object[].class)); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static JdbcTemplate rolloutJdbc(String generationStatus) throws Exception { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + ResultSet scope = mock(ResultSet.class); + when(scope.getString("mode")).thenReturn("SHADOW"); + when(scope.getString("governed_collection")).thenReturn("aihr_knowledge_governed_v1"); + when(scope.getLong("active_generation")).thenReturn(1L); + when(scope.getObject("candidate_generation", Long.class)).thenReturn(2L); + when(jdbc.query(contains("aihr_knowledge_rollout_scope"), any(RowMapper.class), any(Object[].class))) + .thenAnswer(invocation -> { + RowMapper mapper = invocation.getArgument(1); + return List.of(mapper.mapRow(scope, 0)); + }); + + ResultSet generation = mock(ResultSet.class); + when(generation.getString("collection_name")).thenReturn("aihr_knowledge_governed_v1"); + when(generation.getString("status")).thenReturn(generationStatus); + when(generation.getString("embedding_model")).thenReturn("local-hash-v1"); + when(generation.getObject("embedding_dimension", Integer.class)).thenReturn(256); + when(jdbc.query(contains("from aihr_knowledge_generation"), any(RowMapper.class), any(Object[].class))) + .thenAnswer(invocation -> { + RowMapper mapper = invocation.getArgument(1); + return List.of(mapper.mapRow(generation, 0)); + }); + return jdbc; + } +} diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrSopSeedServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrSopSeedServiceTest.java index c3d8674a..384880a3 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrSopSeedServiceTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrSopSeedServiceTest.java @@ -648,6 +648,10 @@ public class AihrSopSeedServiceTest { assertTrue(code.contains("payload.put(\"trust_level\", \"HUMAN_VERIFIED\")")); assertTrue(code.contains("payload.put(\"index_generation\", generation)")); assertTrue(code.contains("qdrantProductionFilterForSpaces(knowledgeIds, null, category, generation)")); + assertTrue(code.contains("shadowCandidateFragmentIds")); + assertTrue(code.contains("qdrantProductionFilterForSpaces(Set.of(knowledgeId), null, null, generation)")); + assertTrue(code.contains("candidate qdrant point is missing fragment lineage")); + assertTrue(code.contains("Candidate vector query failed")); assertTrue(code.contains("legacyQdrantCollection()")); assertTrue(code.contains("governedQdrantCollection()")); } diff --git a/backend/script/sql/update/aihr_20260821_knowledge_shadow_gate_mysql8.sql b/backend/script/sql/update/aihr_20260821_knowledge_shadow_gate_mysql8.sql index 14a46ed5..fd4d163f 100644 --- a/backend/script/sql/update/aihr_20260821_knowledge_shadow_gate_mysql8.sql +++ b/backend/script/sql/update/aihr_20260821_knowledge_shadow_gate_mysql8.sql @@ -1,5 +1,25 @@ -- Shadow retrieval evidence and explicit generation activation gates. +SET @aihr_column_exists := ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'aihr_knowledge_generation' + AND column_name = 'embedding_model' +); +SET @aihr_shadow_ddl := IF(@aihr_column_exists = 0, + 'ALTER TABLE `aihr_knowledge_generation` ADD COLUMN `embedding_model` varchar(100) DEFAULT NULL AFTER `point_count`', + 'SELECT 1'); +PREPARE aihr_shadow_stmt FROM @aihr_shadow_ddl; EXECUTE aihr_shadow_stmt; DEALLOCATE PREPARE aihr_shadow_stmt; + +SET @aihr_column_exists := ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'aihr_knowledge_generation' + AND column_name = 'embedding_dimension' +); +SET @aihr_shadow_ddl := IF(@aihr_column_exists = 0, + 'ALTER TABLE `aihr_knowledge_generation` ADD COLUMN `embedding_dimension` int DEFAULT NULL AFTER `embedding_model`', + 'SELECT 1'); +PREPARE aihr_shadow_stmt FROM @aihr_shadow_ddl; EXECUTE aihr_shadow_stmt; DEALLOCATE PREPARE aihr_shadow_stmt; + CREATE TABLE IF NOT EXISTS `aihr_shadow_comparison` ( `id` bigint NOT NULL AUTO_INCREMENT, `tenant_id` varchar(20) NOT NULL, @@ -78,3 +98,4 @@ SET @aihr_shadow_ddl := CONCAT('ALTER TABLE `aihr_generation_activation` CONVERT PREPARE aihr_shadow_stmt FROM @aihr_shadow_ddl; EXECUTE aihr_shadow_stmt; DEALLOCATE PREPARE aihr_shadow_stmt; SET @aihr_shadow_collation := NULL; SET @aihr_shadow_ddl := NULL; +SET @aihr_column_exists := NULL; diff --git a/docs/API_INTEGRATION.md b/docs/API_INTEGRATION.md index 0b1b0d74..7ca6db35 100644 --- a/docs/API_INTEGRATION.md +++ b/docs/API_INTEGRATION.md @@ -330,11 +330,11 @@ POST /api/knowledge/quality/rollout/spaces/{knowledgeId}/generation-rollback 审核包只提供风险、重复、冲突、质量问题和建议动作，不代替人工决定。低风险批量发布必须由人工显式发起、携带幂等 `batchKey`，逐项重新校验候选版本；`MEDIUM/HIGH/CRITICAL` 不进入批量发布。并发审核通过 `expectedVersionId` 拒绝已经换过候选版的陈旧操作。 -撤回是可恢复的在线下线：立即停用生产成员并排队删除当前 generation 的向量，但保留版本和片段供审计/回滚。物理清理只能针对已撤回资产，先生成影响快照，再进入 `PENDING -> APPROVED -> EXECUTING -> EXECUTED/FAILED`；申请人与批准/执行人必须不同，`legal_hold`、`PERMANENT` 或未到 `execute_after` 一律拒绝。保留期为 `TEMPORARY=7`、`STANDARD=90`、`REGULATED=365` 天。独占 OSS、MySQL 版本/片段/证据和对应向量按审计请求清理，`aihr_deletion_request` 永久保留为墓碑；`reconciliation-runs` 用于发现 MySQL、OSS 元数据、向量计数和 chunk 向量血缘漂移。 +撤回是可恢复的在线下线：立即停用生产成员并排队删除当前 generation 的向量，但保留版本和片段供审计/回滚。物理清理只能针对已撤回资产，先生成影响快照，再进入 `PENDING -> APPROVED -> EXECUTING -> EXECUTED/FAILED`；申请人与批准/执行人必须不同，`legal_hold`、`PERMANENT`、未到 `execute_after` 或活动 generation 仍包含该资产时一律拒绝。活动代次引用存在时，必须先构建并人工启用一个不含该资产的新代次。保留期为 `TEMPORARY=7`、`STANDARD=90`、`REGULATED=365` 天。执行过程先准备固定向量目标清单，再以可重试方式清理 legacy、全部治理代次和独占 OSS，最后在单个数据库事务内删除版本、片段和证据；任一外部或数据库步骤失败都记录为 `FAILED` 并允许重试。清理会重算受影响非活动代次的成员数和点数、清空旧 manifest、标记代次失效并解除失效候选指针，不能把已被清理破坏的旧代次用于回滚。`aihr_deletion_request` 永久保留为墓碑；`reconciliation-runs` 用于发现 MySQL、OSS 元数据、向量计数和 chunk 向量血缘漂移。 历史补链使用迁移清单而不是无状态循环调用。每个 `migration-run` 冻结租户、可选知识空间、风险范围、dry-run、批大小和 manifest hash；`next` 只推进已持久化游标并记录批次证据。`LOW_RISK` 只扫描租户可验证 OSS 原件，`QUARANTINE_FIRST` 只扫描缺失原件且需要隔离的资料，`ALL` 扫描全部候选。失败附件进入显式 dead letter，最多按退避策略重试，不能因为单项失败回滚或隐藏仍由 LEGACY 服务的线上资料。正式任务必须提交 `verifiedDryRunId`，服务端只接受已完成且知识空间、风险范围、批大小完全一致的演练；管理端创建正式任务后仍需人工逐批推进，不会自动修改资料。 -空间发布固定为 `LEGACY -> SHADOW -> ENFORCED`。SHADOW 仍服务旧链路，候选 generation 在独立 collection/generation 中全量构建；管理端可记录真实问题的双链路对照。启用门禁要求候选 generation 为 `READY`、至少 20 个样本、Top1 一致率不低于 90%、Top5 平均重合率不低于 80%、治理结果空集率不高于 1%，且无覆盖缺口、开放关键问题或索引死信。旧 transition 接口不能直接进入 ENFORCED；只有人工使用最新 `PASSED` gate 调用 `activate` 才能切换。原 generation 保留 7 天回退窗口，回退后恢复 SHADOW。 +空间发布固定为 `LEGACY -> SHADOW -> ENFORCED`。SHADOW 仍服务旧链路，候选 generation 在独立 collection/generation 中全量构建；每个候选清单记录实际 embedding 模型、维度、成员数、点数和确定性 SHA-256，混用模型/维度或清单不完整会使 generation 失败，不能进入 `READY`。管理端可记录真实问题的双链路对照：旧链路结果来自当前线上召回，候选结果必须由 Qdrant 按租户、知识空间、生产数据集、已发布生命周期、人工可信级别和精确 generation 过滤；向量服务或模型失败时不写伪对照证据。启用门禁要求候选 generation 为 `READY`、至少 20 个样本、Top1 一致率不低于 90%、Top5 平均重合率不低于 80%、治理结果空集率不高于 1%，且无覆盖缺口、开放关键问题或索引死信。旧 transition 接口不能直接进入 ENFORCED；只有人工使用最新 `PASSED` gate 调用 `activate` 才能切换。原 generation 保留 7 天回退窗口；只有仍为 `RETIRED` 且未被物理清理失效的原代次可回退，回退后恢复 SHADOW。 `publish` 必须带人工审核意见、用途、来源权威级别和来源版本。请求中的 `acceptedReasonCodes` 只能确认当前版本仍开放的软提示，服务端会再次统计开放问题，遗漏任何一项都拒绝发布；`HARD` 问题不能在发布动作中接受，必须修订原资料并生成新版本，或通过后续独立的显式纠错流程解决。规范性知识仅接受 `FORMAL_POLICY / COMPANY_POLICY / REGULATION / APPROVED_SOP` 且版本非空；同名已发布资料内容变化会产生 `VERSION_CONFLICT`，规范性知识必须再传 `supersedesAssetId`，服务端验证同租户、同来源名和旧资产仍已发布后，在同一事务内撤回旧版本并发布新版本。 diff --git a/docs/BRD_IMPLEMENTATION_AUDIT.md b/docs/BRD_IMPLEMENTATION_AUDIT.md index f32ed2d0..a4f73fb2 100644 --- a/docs/BRD_IMPLEMENTATION_AUDIT.md +++ b/docs/BRD_IMPLEMENTATION_AUDIT.md @@ -29,8 +29,8 @@ | 层级 | 当前状态 | 证据与边界 | |---|---|---| | 已实现 | 是 | 已有阶段一主功能；当前工作树另实现大喇叭投放控制、反馈转人工、正式学习任务/随机组卷、案例经验治理、原生异常恢复、上传容量门禁，以及数字资产兼容放量、版本、审核、向量代次、撤回/回滚、保留清理、历史迁移和影子启用全链路；接口事实见 [API_INTEGRATION.md](API_INTEGRATION.md) | -| 本地自动化/API 验证 | 是 | 当前工作树管理端生产构建通过；后端完整测试 `851/851`，失败/错误/跳过均为 `0`；发布预检范围契约通过，生命周期迁移已在现有 MySQL 连续执行两次且资产数保持 `104`，未创建清理或迁移任务 | -| 本地浏览器/视觉验证 | 部分 | `390×844` 已实际点击员工“今日/练/问”、大喇叭登录门禁和直通车登录门禁，无横向溢出或重叠；2026-08-03 已在当前工作树管理端登录并实际打开空间放量、迁移/清理、版本审核辅助和影子启用对话框，桌面布局无明显重叠或截断，未提交任何状态变更。窄屏管理端本轮未复核 | +| 本地自动化/API 验证 | 是 | 当前工作树管理端定向 ESLint 与生产构建通过；后端完整测试 `855/855`，失败/错误/跳过均为 `0`；发布预检范围契约通过，生命周期迁移已在现有 MySQL 连续执行两次且资产数保持 `104`，未创建清理或迁移任务 | +| 本地浏览器/视觉验证 | 部分 | `390×844` 已实际点击员工“今日/练/问”、大喇叭登录门禁和直通车登录门禁，无横向溢出或重叠；2026-08-03 已在当前工作树管理端登录并实际打开空间放量、迁移/清理、版本审核辅助和影子启用对话框，最新构建重启后再次复核桌面布局和对话框交互，无明显重叠或截断，未提交任何状态变更。窄屏管理端本轮未复核 | | 已部署 | 部分 | 2026-07-25 至 2026-07-31 的既有生产基线已部署；当前工作树的 `aihr_20260815_*` 至 `aihr_20260821_*` 生命周期迁移及对应后端、管理端均未部署，线上资料未发生本批变更 | | 生产基础验证 | 是 | 服务 `active`；根站、租户接口和移动首页 API 正常；媒体重试列/索引契约通过，4 个历史零片段任务均恢复为已完成且有片段/向量。当前未执行包含移动 H5 的完整包匹配 | | 生产完整业务验收 | 部分 | 公司消息已完成认证态全员文件消息和直通车闭环，单台 Android 标准基座已完成生产 ASR；训练写入、主管复盘、定向人群正反例、正式处理人绑定、其他真机媒体与严格试点窗口仍未完成 | @@ -50,7 +50,8 @@ - 执行顺序已经重新编排，不是把自动化辅助作为七阶段之外的平行项目：阶段 0 兼容层必须先落地，随后七阶段能力按依赖进入同一发布计划；系统先生成清单、风险、差异、影响和建议，人工只负责业务判断、异人批准和最终启用。 - 既有资料默认继续由 `LEGACY` 服务。历史补链必须先创建固定范围的 dry-run，只有无失败完成且知识空间、风险范围、批大小完全匹配的演练，才能生成正式任务；正式任务仍需人工逐批推进，单项失败进入 dead letter，不中断旧读链路。 - 新上传或修订只推进候选版本，生效版本继续提供检索；发布、撤回、回滚和 generation 切换均保留审计证据。日常删除语义为可恢复撤回，物理清理另受保留期、法务冻结、引用影响和异人控制约束。 -- 生产启用固定为单空间 `LEGACY -> SHADOW -> ENFORCED`。SHADOW 期间构建隔离候选 generation 并记录真实问题对照；覆盖、索引、质量和样本门禁通过后，才允许人工确认启用，并保留七天 generation 回退窗口。 +- 生产启用固定为单空间 `LEGACY -> SHADOW -> ENFORCED`。SHADOW 期间构建隔离候选 generation，记录实际 embedding 模型、维度和确定性 manifest，并以当前线上召回和严格过滤候选 Qdrant 记录真实问题对照；检索失败不生成伪证据。覆盖、索引、质量和样本门禁通过后，才允许人工确认启用，并保留七天 generation 回退窗口。 +- 物理清理新增活动 generation 硬门禁：仍被当前代次包含的撤回资料必须先构建并启用排除该资料的新代次。清理按固定目标重试外部删除，再以单事务删除数据库链路；受影响非活动代次自动重算成员/点数、清空 manifest 并标记失效，已失效代次不能恢复为活动代次。 - 本地浏览器仅执行只读查看和打开对话框。验收前后数据库均为资产 `104`、非活动删除状态 `0`、清理申请 `0`、迁移任务 `0`、影子样本 `0`、generation 启用 `0`；已有一个本地 SHADOW 空间保持不变。以上不能外推为生产部署或线上迁移完成。 ### 2026-07-29 资料处理媒体恢复发布边界 diff --git a/docs/BRD_PRODUCTION_MIGRATION_RUNBOOK.md b/docs/BRD_PRODUCTION_MIGRATION_RUNBOOK.md index 30c79f30..bbbb8504 100644 --- a/docs/BRD_PRODUCTION_MIGRATION_RUNBOOK.md +++ b/docs/BRD_PRODUCTION_MIGRATION_RUNBOOK.md @@ -180,9 +180,9 @@ mysql --default-character-set=utf8mb4 "$DB_NAME" < backend/script/sql/aihr_perso 随后按 `20260808 -> 20260809 -> 20260810 -> 20260811 -> 20260812 -> 20260813 -> 20260814` 执行质量监控、人工术语修订、AI 案例来源、规则演进、处理批次、黄金集校准和隐私派生迁移。`20260811` 只增加规则草稿、状态审计、影子对照和抽样复核；`20260812` 只增加 `pipeline_run` 与自动生成待人工复核样本的调度，不启用规则执法，不改变任何资产生命周期。`20260814` 只增加独立脱敏正文、脱敏文件名、派生 hash、隐私状态、规则版本和复扫摘要字段，历史版本默认保持 `NOT_PROCESSED`，不得通过 SQL 伪造脱敏完成或自动获批。完整迁移后仍须最后执行排序规则兼容迁移。 -生命周期兼容发布继续按 `20260815 -> 20260816 -> 20260817 -> 20260818 -> 20260819 -> 20260820 -> 20260821` 执行：先建立按空间的 `LEGACY/SHADOW/ENFORCED` 兼容范围，再增加候选/生效双指针、审核辅助、不可变 generation 与回滚证据、保留和异人删除、历史迁移清单、影子指标与启用门禁。七个脚本均为增量结构迁移；不得在 SQL 阶段自动发布、撤回、删除或切换现有空间。迁移完成后所有空间仍应为 LEGACY 或保持迁移前显式模式，历史资料继续可见。生产发布必须先只读统计资产、版本、生产成员、fragment、OSS 和 Qdrant 数量并备份；先以单空间 dry-run 和 SHADOW 验证，达到门禁后由人工启用。正式迁移任务必须通过 `verified_dry_run_id` 绑定同知识空间、同风险范围、同批大小且无失败的已完成演练；创建正式任务不自动推进批次。任何补链失败都停留在迁移 dead letter，不得切断旧读路径。 +生命周期兼容发布继续按 `20260815 -> 20260816 -> 20260817 -> 20260818 -> 20260819 -> 20260820 -> 20260821` 执行。`20260815` 是七阶段共同依赖的阶段 0，不是平行的第八项：先建立按空间的 `LEGACY/SHADOW/ENFORCED` 兼容范围和只读基线，再增加候选/生效双指针、审核辅助、不可变 generation 与回滚证据、保留和异人删除、历史迁移清单、影子指标与启用门禁。七个脚本均为增量结构迁移；不得在 SQL 阶段自动发布、撤回、删除或切换现有空间。迁移完成后所有空间仍应为 LEGACY 或保持迁移前显式模式，历史资料继续可见。生产发布必须先只读统计资产、版本、生产成员、fragment、OSS 和 Qdrant 数量并备份；先以单空间 dry-run 和 SHADOW 验证，达到门禁后由人工启用。正式迁移任务必须通过 `verified_dry_run_id` 绑定同知识空间、同风险范围、同批大小且无失败的已完成演练；创建正式任务不自动推进批次。任何补链失败都停留在迁移 dead letter，不得切断旧读路径。系统负责生成范围清单、差异、风险、影响、候选 generation 和门禁证据，人工只保留业务事实裁决、异人审批、抽样复核和最终切换。 -生产物理删除不是部署步骤，也不得与 schema/JAR 发布同窗口批量执行。日常纠错只做 `withdraw`；`purge` 必须有业务/法务确认的保留分类、影响快照、异人批准和已到期 `execute_after`。发布后先运行 reconciliation 并处理不一致，再考虑清理。ENFORCED 启用后旧 generation 保留七天；窗口内指标恶化可从管理端回退上一代，超过窗口必须重新构建候选 generation 和重新通过门禁。 +生产物理删除不是部署步骤，也不得与 schema/JAR 发布同窗口批量执行。日常纠错只做 `withdraw`；`purge` 必须有业务/法务确认的保留分类、影响快照、异人批准和已到期 `execute_after`。活动 generation 仍包含该资产时，先构建并启用排除该资产的新 generation，系统不会允许直接清理；清理成功后，受影响的非活动代次会被自动重算并标记失效，不能再作为回滚目标。发布后先运行 reconciliation 并处理不一致，再考虑清理。ENFORCED 启用后旧 generation 保留七天；窗口内指标恶化且旧代次未失效时可从管理端回退，超过窗口或旧代次已失效时必须重新构建候选 generation 和重新通过门禁。 8 月 1 日受控内部试点的正式来源注册是独立的业务数据变更，不随通用 schema 自动执行。先逐一核对附件 `id + doc_id`、原文件 SHA-256、发文号/版本、生效日期和适用范围，再由已授权负责人执行 `backend/script/sql/ops/aihr_20260730_aug1_formal_source_registry_mysql8.sql`。执行后必须只读确认恰好 10 条 `FORMAL_POLICY + APPROVED` 记录；任何缺失都保持无正式依据，不得把访谈、经验萃取、AI 生成内容或草稿补进白名单。 diff --git a/docs/DIGITAL_ASSET_PROCESSING_GOVERNANCE.md b/docs/DIGITAL_ASSET_PROCESSING_GOVERNANCE.md index 03a02de3..66ce9118 100644 --- a/docs/DIGITAL_ASSET_PROCESSING_GOVERNANCE.md +++ b/docs/DIGITAL_ASSET_PROCESSING_GOVERNANCE.md @@ -599,6 +599,8 @@ legacy preview（只读盘点） - 撤回后 MySQL 立即不可见，Qdrant 删除最终一致率 100%。 - 黄金问题达到该批预设 Recall/MRR/nDCG 和无答案阈值。 - 回滚只切换生产可见性/索引 generation，不删除原件、版本和审核证据。 +- 候选 generation 的 embedding 模型、维度、成员数、点数和 manifest hash 完整且一致；影子样本来自旧链路与严格过滤候选 Qdrant 的真实召回，检索失败不得生成比较证据。 +- 物理清理前活动 generation 不得仍含目标资产；清理后受影响非活动代次必须重算计数、清空 manifest 并失效，失效代次不得回滚。 ## 16. 黄金集、测试与标定 diff --git a/frontend/src/api/aihr/processing.ts b/frontend/src/api/aihr/processing.ts index e951a3e3..6a3ae99a 100644 --- a/frontend/src/api/aihr/processing.ts +++ b/frontend/src/api/aihr/processing.ts @@ -251,6 +251,7 @@ export type DeletionImpact = { learningTasks: number; broadcasts: number; generations: number; + activeGenerationMemberships: number; canRequestPurge: boolean; blockers: string[]; }; @@ -296,6 +297,9 @@ export type GenerationBuild = { expectedVersions: number; includedVersions: number; outstandingJobs: number; + embeddingModel?: string; + embeddingDimension?: number; + manifestSha256?: string; }; export type ActivationGate = { diff --git a/frontend/src/views/knowledge/processing.vue b/frontend/src/views/knowledge/processing.vue index c57b7bae..e26eeb47 100644 --- a/frontend/src/views/knowledge/processing.vue +++ b/frontend/src/views/knowledge/processing.vue @@ -936,6 +936,12 @@ :type="deletionImpact.canRequestPurge ? 'warning' : 'info'" :closable="false" /> + 构建候选代次 刷新构建状态 + @@ -1285,6 +1303,7 @@ import { type DeletionImpact, type PurgeRequest, type MigrationRun, + type GenerationBuild, type ActivationGate, type KnowledgeGlossaryTerm, type KnowledgeRolloutMode, @@ -1337,6 +1356,7 @@ const rolloutTransitioningId = ref(null); const rolloutScopes = ref([]); const rolloutDialogVisible = ref(false); const selectedRolloutScope = ref(null); +const candidateBuild = ref(null); const rolloutGate = ref(null); const rolloutQuery = ref(''); const rolloutAssisting = ref(false); @@ -1870,9 +1890,11 @@ async function executePurge(request: PurgeRequest) { function openRolloutAssistant(scope: KnowledgeRolloutScope) { selectedRolloutScope.value = scope; + candidateBuild.value = null; rolloutGate.value = null; rolloutQuery.value = ''; rolloutDialogVisible.value = true; + if (scope.candidateGeneration) void refreshCandidateGeneration(); } async function buildCandidateGeneration() { @@ -1880,7 +1902,7 @@ async function buildCandidateGeneration() { if (!scope) return; rolloutAssisting.value = true; try { - await startGenerationBuild(scope.knowledgeId, '管理端启动候选代次全量构建'); + candidateBuild.value = (await startGenerationBuild(scope.knowledgeId, '管理端启动候选代次全量构建')).data; await loadRolloutScopes(); selectedRolloutScope.value = rolloutScopes.value.find((item) => item.knowledgeId === scope.knowledgeId) || scope; ElMessage.success('候选代次已排队构建'); @@ -1897,6 +1919,7 @@ async function refreshCandidateGeneration() { rolloutAssisting.value = true; try { const result = (await refreshGenerationBuild(scope.knowledgeId)).data; + candidateBuild.value = result; ElMessage.info(`候选 G${result.generation}：${result.status}，已纳入 ${result.includedVersions}/${result.expectedVersions}`); } catch { ElMessage.error('候选代次状态刷新失败'); @@ -1914,7 +1937,7 @@ async function runShadowComparison() { rolloutQuery.value = ''; ElMessage.success('影子对照样本已记录，不影响当前在线结果'); } catch { - ElMessage.error('影子对照失败，请确认候选代次已创建'); + ElMessage.error('影子对照失败，请确认候选代次已就绪且向量服务可用'); } finally { rolloutAssisting.value = false; } diff --git a/scripts/release-preflight.sh b/scripts/release-preflight.sh index d3d98664..a1ed49d6 100755 --- a/scripts/release-preflight.sh +++ b/scripts/release-preflight.sh @@ -875,6 +875,8 @@ FROM ( UNION ALL SELECT 'aihr_chunk_revision', 'index_generation' UNION ALL SELECT 'aihr_chunk_revision', 'vector_point_id' UNION ALL SELECT 'aihr_index_outbox', 'index_generation' + UNION ALL SELECT 'aihr_knowledge_generation', 'embedding_model' + UNION ALL SELECT 'aihr_knowledge_generation', 'embedding_dimension' UNION ALL SELECT 'aihr_migration_run', 'verified_dry_run_id' ) required LEFT JOIN information_schema.columns actual