From 775a326ae26930cccfc1e5b9d8e43f5392bf952b Mon Sep 17 00:00:00 2001 From: let5sne Date: Fri, 24 Jul 2026 20:27:03 +0800 Subject: [PATCH] fix(agent): route media by intent --- .../aihr/agent/AihrAgentController.java | 21 +++++ .../aihr/agent/AihrAgentOrchestrator.java | 40 +++++++++- .../service/AihrKnowledgeQueryService.java | 47 +++++++++-- .../AihrAgentControllerContractTest.java | 10 +++ .../aihr/agent/AihrAgentMediaTest.java | 79 +++++++++++++++++++ .../AihrKnowledgeQueryServiceTest.java | 76 ++++++++++++++++++ 6 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentMediaTest.java diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentController.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentController.java index bb6725b5..06c83801 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentController.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentController.java @@ -5,10 +5,14 @@ import lombok.RequiredArgsConstructor; import org.dromara.aihr.agent.AihrAgentDto.AgentRequest; import org.dromara.aihr.agent.AihrAgentDto.AgentResponse; import org.dromara.common.core.domain.R; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; @RestController @RequiredArgsConstructor @@ -22,4 +26,21 @@ public class AihrAgentController { public R messages(@RequestBody AgentRequest request) { return R.ok(orchestrator.handle(request)); } + + @SaCheckLogin + @PostMapping(value = "/messages/media", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public R media(@RequestPart("file") MultipartFile file, + @RequestParam(value = "question", required = false) String question, + @RequestParam(value = "conversationId", required = false) String conversationId, + @RequestParam(value = "contextVersion", required = false) Long contextVersion, + @RequestParam(value = "projectCode", required = false) String projectCode, + @RequestParam(value = "broadcastMessageId", required = false) Long broadcastMessageId, + @RequestParam(value = "externalConsent", defaultValue = "false") + boolean externalConsent) { + String prompt = question == null || question.isBlank() ? "请分析附件内容" : question.trim(); + return R.ok(orchestrator.handleMedia( + new AgentRequest(prompt, conversationId, contextVersion, projectCode, broadcastMessageId, externalConsent), + file + )); + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentOrchestrator.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentOrchestrator.java index 067b0218..db41464a 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentOrchestrator.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/agent/AihrAgentOrchestrator.java @@ -13,8 +13,10 @@ import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryResponse; import org.dromara.aihr.knowledge.service.AihrKnowledgeDataToolService.CurrentTaskSummary; import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver; import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService; +import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode; import org.dromara.common.core.exception.ServiceException; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; import java.time.OffsetDateTime; import java.util.List; @@ -30,9 +32,7 @@ public class AihrAgentOrchestrator { private final AihrKnowledgeQueryService queryService; public AgentResponse handle(AgentRequest request) { - if (request == null || request.question() == null || request.question().isBlank()) { - throw new ServiceException("问题不能为空", 400); - } + requireQuestion(request); AgentPlan plan = planner.plan(request.question(), false); policy.authorize(principalResolver.current(), plan, request.externalConsent()); return switch (plan.intent()) { @@ -47,6 +47,23 @@ public class AihrAgentOrchestrator { }; } + public AgentResponse handleMedia(AgentRequest request, MultipartFile file) { + requireQuestion(request); + if (file == null || file.isEmpty()) { + throw new ServiceException("附件不能为空", 400); + } + AgentPlan plan = planner.plan(request.question(), true); + policy.authorize(principalResolver.current(), plan, request.externalConsent()); + MediaMode mode = needsKnowledge(request.question()) + ? MediaMode.MEDIA_WITH_KNOWLEDGE + : MediaMode.MEDIA_ONLY; + QueryResponse response = queryService.queryInternalMedia(new QueryRequest( + request.question(), List.of(), "sop", null, "mobile_uni_agent", 5, null, + request.conversationId(), request.contextVersion(), request.projectCode(), request.broadcastMessageId() + ), file, mode); + return fromResponse(plan, response, "MEDIA"); + } + private AgentResponse fromQuery(AgentPlan plan, AgentRequest request) { String toolCode = switch (plan.tool()) { case MY_CURRENT_TASKS, MY_PRACTICE_SUMMARY, TEAM_PRACTICE_SUMMARY -> plan.tool().name(); @@ -56,8 +73,11 @@ public class AihrAgentOrchestrator { plan.rewrittenRequest(), List.of(), "sop", null, "mobile_uni_agent", 5, toolCode, request.conversationId(), request.contextVersion(), request.projectCode(), request.broadcastMessageId() )); + return fromResponse(plan, response, toolCode == null ? "ENTERPRISE_KNOWLEDGE" : "LIVE_DATA"); + } + + private AgentResponse fromResponse(AgentPlan plan, QueryResponse response, String sourceType) { AgentStatus status = status(response); - String sourceType = toolCode == null ? "ENTERPRISE_KNOWLEDGE" : "LIVE_DATA"; String sourceLabel = response.citations().isEmpty() ? sourceType : response.citations().get(0).title(); return new AgentResponse( @@ -67,6 +87,18 @@ public class AihrAgentOrchestrator { ); } + private static void requireQuestion(AgentRequest request) { + if (request == null || request.question() == null || request.question().isBlank()) { + throw new ServiceException("问题不能为空", 400); + } + } + + private static boolean needsKnowledge(String question) { + String value = question == null ? "" : question; + return List.of("怎么办", "怎么处理", "如何处理", "sop", "SOP", "制度", "流程", "规定", + "故障", "漏水", "电梯", "现场").stream().anyMatch(value::contains); + } + private static AgentStatus status(QueryResponse response) { if (response.data() instanceof CurrentTaskSummary summary && "UNAVAILABLE".equals(summary.state())) { return AgentStatus.UNAVAILABLE; diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/service/AihrKnowledgeQueryService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/service/AihrKnowledgeQueryService.java index f244266b..6e942f39 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/service/AihrKnowledgeQueryService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/knowledge/service/AihrKnowledgeQueryService.java @@ -40,6 +40,11 @@ import java.util.UUID; @Service public class AihrKnowledgeQueryService { + public enum MediaMode { + MEDIA_ONLY, + MEDIA_WITH_KNOWLEDGE + } + private final AihrKnowledgePrincipalResolver principalResolver; private final AihrKnowledgeAppService appService; private final AihrKnowledgeAccessService accessService; @@ -177,6 +182,10 @@ public class AihrKnowledgeQueryService { } public QueryResponse queryInternalMedia(QueryRequest rawRequest, MultipartFile file) { + return queryInternalMedia(rawRequest, file, MediaMode.MEDIA_WITH_KNOWLEDGE); + } + + public QueryResponse queryInternalMedia(QueryRequest rawRequest, MultipartFile file, MediaMode mode) { QueryRequest request = validate(rawRequest); if (request.broadcastMessageId() != null) { throw new ServiceException("公司消息追问暂不支持现场附件", HttpStatus.BAD_REQUEST); @@ -196,11 +205,26 @@ public class AihrKnowledgeQueryService { } String combinedQuery = truncate(context.rewrittenQuery(), 400) + "\n附件现场信息：" + truncate(extractedContext, 520); - QueryResponse response = queryDocuments(principal, app, spaceIds, new QueryRequest( - combinedQuery, request.spaceCodes(), request.category(), request.position(), - request.source(), request.limit(), null, request.conversationId(), request.contextVersion(), - request.projectCode() - ), request.queryText()); + QueryResponse response; + if (mode == MediaMode.MEDIA_ONLY) { + String requestId = UUID.randomUUID().toString().replace("-", ""); + Citation citation = new Citation(null, "MEDIA", "media:" + requestId, + "本次附件识别", truncate(extractedContext, 520), null); + auditService.record(requestId, principal, app, request.queryText(), List.of(), List.of("MEDIA"), + "SUCCESS", 0L, "media-only-v1"); + response = new QueryResponse( + requestId, request.queryText(), "附件识别结果：" + truncate(extractedContext, 520), + List.of(citation), List.of(), false, "media-only-v1", null, + Map.of("observation", truncate(extractedContext, 2000)) + ); + } else { + response = queryDocuments(principal, app, spaceIds, new QueryRequest( + combinedQuery, request.spaceCodes(), request.category(), request.position(), + request.source(), request.limit(), null, request.conversationId(), request.contextVersion(), + request.projectCode() + ), request.queryText()); + response = withMediaContextWhenNoEvidence(response, extractedContext); + } QueryResponse completed = complete(principal, app, spaceIds, request.queryText(), context, response); MemoryCandidateResponse candidate = memoryService.detectForQuery( principal, combinedQuery, context.conversationId(), completed.requestId(), @@ -209,6 +233,19 @@ public class AihrKnowledgeQueryService { return completed.withMemoryCandidate(candidate); } + private static QueryResponse withMediaContextWhenNoEvidence(QueryResponse response, String extractedContext) { + if (!response.noEvidence()) { + return response; + } + String answer = "附件识别结果：" + truncate(extractedContext, 240) + "\n\n" + response.answer(); + return new QueryResponse( + response.requestId(), response.queryText(), answer, response.citations(), response.usedSpaceCodes(), + true, response.promptVersion(), response.legacy(), response.data(), response.conversationId(), + response.contextVersion(), response.intent(), response.rewrittenQuery(), response.resources(), + response.memoryCandidate(), response.broadcastContext() + ); + } + public QueryResponse queryExternal(AuthenticatedApp app, QueryRequest rawRequest) { QueryRequest request = validate(rawRequest); if (hasText(request.toolCode()) || hasText(request.conversationId()) || request.contextVersion() != null diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentControllerContractTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentControllerContractTest.java index 0f6b3c2e..dced0307 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentControllerContractTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentControllerContractTest.java @@ -38,4 +38,14 @@ class AihrAgentControllerContractTest { assertTrue(fields.stream().noneMatch(name -> name.contains("userid") || name.contains("employeeid") || name.contains("role") || name.contains("tool"))); } + + @Test + void mediaEndpointAlsoRequiresLogin() throws Exception { + var method = AihrAgentController.class.getMethod("media", + org.springframework.web.multipart.MultipartFile.class, + String.class, String.class, Long.class, String.class, Long.class, boolean.class); + + assertNotNull(method.getAnnotation(SaCheckLogin.class)); + assertEquals("/messages/media", method.getAnnotation(PostMapping.class).value()[0]); + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentMediaTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentMediaTest.java new file mode 100644 index 00000000..ea9b4abe --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/agent/AihrAgentMediaTest.java @@ -0,0 +1,79 @@ +package org.dromara.aihr.agent; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.dromara.aihr.agent.AihrAgentDto.AgentRequest; +import org.dromara.aihr.agent.AihrAgentDto.AgentStatus; +import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal; +import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryRequest; +import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryResponse; +import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver; +import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; +import java.util.Set; + +import static org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode.MEDIA_ONLY; +import static org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode.MEDIA_WITH_KNOWLEDGE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Tag("dev") +class AihrAgentMediaTest { + + @Test + void genericImageQuestionUsesMediaOnly() { + var query = new CapturingMediaService(); + var response = orchestrator(query).handleMedia(request("图中是什么宠物"), + new MockMultipartFile("file", "pet.jpg", "image/jpeg", new byte[]{1})); + + assertEquals(MEDIA_ONLY, query.mode); + assertEquals(AgentStatus.COMPLETED, response.status()); + } + + @Test + void workplaceImageQuestionCombinesMediaWithKnowledge() { + var query = new CapturingMediaService(); + orchestrator(query).handleMedia(request("图中电梯门打不开怎么办"), + new MockMultipartFile("file", "lift.jpg", "image/jpeg", new byte[]{1})); + + assertEquals(MEDIA_WITH_KNOWLEDGE, query.mode); + } + + private static AihrAgentOrchestrator orchestrator(CapturingMediaService query) { + return new AihrAgentOrchestrator( + new AihrAgentPlanner(null, new ObjectMapper()), + new AihrAgentPolicy(), + new AihrKnowledgePrincipalResolver(null) { + @Override + public AihrKnowledgePrincipal current() { + return new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1", + Set.of("employee"), Set.of("P1"), "app"); + } + }, + query + ); + } + + private static AgentRequest request(String question) { + return new AgentRequest(question, "conversation_1", 0L, "P1", null, false); + } + + private static final class CapturingMediaService extends AihrKnowledgeQueryService { + private MediaMode mode; + + private CapturingMediaService() { + super(null, null, null, null, null, null, null, null, null); + } + + @Override + public QueryResponse queryInternalMedia(QueryRequest request, MultipartFile file, MediaMode mode) { + this.mode = mode; + return new QueryResponse("request-1", request.queryText(), "附件识别结果：图中是一只金毛犬", + List.of(), List.of(), false, "media-only-v1", null, null, + "conversation_1", 1L, "QA", request.queryText(), List.of()); + } + } +} diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/AihrKnowledgeQueryServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/AihrKnowledgeQueryServiceTest.java index 8687674b..3bb97682 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/AihrKnowledgeQueryServiceTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/knowledge/AihrKnowledgeQueryServiceTest.java @@ -201,6 +201,82 @@ class AihrKnowledgeQueryServiceTest { assertEquals(true, requestCaptor.getValue().queryText().contains("电梯门无法打开")); } + @Test + void genericMediaQuestionSkipsRagAndAppendsTheObservedAnswer() { + var resolver = mock(AihrKnowledgePrincipalResolver.class); + var appService = mock(AihrKnowledgeAppService.class); + var access = mock(AihrKnowledgeAccessService.class); + var sop = mock(AihrSopSeedService.class); + var conversations = mock(AihrKnowledgeConversationService.class); + var principal = new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1", + Set.of("employee"), Set.of("P1"), "app"); + var app = new AuthenticatedApp(1L, "000000", "yc_mobile", "员工端", "SESSION", 60, null); + var request = new QueryRequest("图中是什么宠物", List.of(), "sop", null, "mobile_uni_agent", 5, null, + "conversation_123", 0L, "P1"); + var context = new ConversationContext("conversation_123", true, true, 0L, List.of(), + "图中是什么宠物", "QA", null, "P1", null); + var media = new MockMultipartFile("file", "pet.jpg", "image/jpeg", new byte[]{1}); + when(resolver.current()).thenReturn(principal); + when(appService.requireSessionApp("000000", "app")).thenReturn(app); + when(access.resolveInternalSpaceIds(principal, app, List.of(), "READ")).thenReturn(Set.of(9L)); + when(conversations.prepare(principal, app, request)).thenReturn(context); + when(conversations.append(any(), any(), any(), anyString(), anyString(), any())).thenReturn(1L); + when(sop.extractQuestionMedia(media, "图中是什么宠物")).thenReturn("图中是一只金毛犬"); + var service = new AihrKnowledgeQueryService(resolver, appService, access, sop, + mock(AihrKnowledgeQueryAuditService.class), mock(JdbcTemplate.class), + mock(AihrKnowledgeDataToolService.class), conversations, mock(AihrMemoryService.class)); + + var result = service.queryInternalMedia(request, media, AihrKnowledgeQueryService.MediaMode.MEDIA_ONLY); + + assertFalse(result.noEvidence()); + assertTrue(result.answer().contains("图中是一只金毛犬")); + verify(sop, never()).searchAuthorized(any(), any(), any()); + var appendedAnswer = ArgumentCaptor.forClass(String.class); + verify(conversations).append(any(), any(), any(), anyString(), appendedAnswer.capture(), any()); + assertTrue(appendedAnswer.getValue().contains("图中是一只金毛犬")); + } + + @Test + void workMediaWithoutSopEvidenceKeepsObservationBeforeConversationAppend() { + var resolver = mock(AihrKnowledgePrincipalResolver.class); + var appService = mock(AihrKnowledgeAppService.class); + var access = mock(AihrKnowledgeAccessService.class); + var sop = mock(AihrSopSeedService.class); + var jdbc = mock(JdbcTemplate.class); + var conversations = mock(AihrKnowledgeConversationService.class); + var principal = new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1", + Set.of("employee"), Set.of("P1"), "app"); + var app = new AuthenticatedApp(1L, "000000", "yc_mobile", "员工端", "SESSION", 60, null); + var request = new QueryRequest("电梯门打不开怎么办", List.of(), "sop", null, "mobile_uni_agent", 5, null, + "conversation_123", 0L, "P1"); + var context = new ConversationContext("conversation_123", true, true, 0L, List.of(), + "电梯门打不开怎么办", "QA", null, "P1", null); + var media = new MockMultipartFile("file", "lift.jpg", "image/jpeg", new byte[]{1}); + when(resolver.current()).thenReturn(principal); + when(appService.requireSessionApp("000000", "app")).thenReturn(app); + when(access.resolveInternalSpaceIds(principal, app, List.of(), "READ")).thenReturn(Set.of(9L)); + when(conversations.prepare(principal, app, request)).thenReturn(context); + when(conversations.append(any(), any(), any(), anyString(), anyString(), any())).thenReturn(1L); + when(jdbc.queryForList(any(String.class), any(Class.class), any(Object[].class))).thenReturn(List.of("sop")); + when(sop.extractQuestionMedia(media, "电梯门打不开怎么办")).thenReturn("画面显示电梯门无法打开"); + when(sop.searchAuthorized(any(), any(), any())).thenReturn( + new SearchResponse("现场问题", "sop", "未找到明确制度依据。", "", List.of(), List.of(), + List.of(), List.of(), List.of(), List.of(), "test-v1", null)); + var service = new AihrKnowledgeQueryService(resolver, appService, access, sop, + mock(AihrKnowledgeQueryAuditService.class), jdbc, mock(AihrKnowledgeDataToolService.class), + conversations, mock(AihrMemoryService.class)); + + var result = service.queryInternalMedia( + request, media, AihrKnowledgeQueryService.MediaMode.MEDIA_WITH_KNOWLEDGE); + + assertTrue(result.noEvidence()); + assertTrue(result.answer().contains("附件识别结果：画面显示电梯门无法打开")); + assertTrue(result.answer().contains("未找到明确制度依据")); + var appendedAnswer = ArgumentCaptor.forClass(String.class); + verify(conversations).append(any(), any(), any(), anyString(), appendedAnswer.capture(), any()); + assertTrue(appendedAnswer.getValue().contains("附件识别结果：画面显示电梯门无法打开")); + } + @Test void fileFollowupUsesConversationRewriteAndReturnsOnlyAuthorizedAttachment() throws Exception { var resolver = mock(AihrKnowledgePrincipalResolver.class);