fix(agent): route media by intent

2026-07-24 20:27:03 +08:00
parent ffe830f64a
commit 775a326ae2
6 changed files with 264 additions and 9 deletions
@@ -5,10 +5,14 @@ import lombok.RequiredArgsConstructor;
 import org.dromara.aihr.agent.AihrAgentDto.AgentRequest;
 import org.dromara.aihr.agent.AihrAgentDto.AgentResponse;
 import org.dromara.common.core.domain.R;
+import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;

@RestController
@RequiredArgsConstructor
@@ -22,4 +26,21 @@ public class AihrAgentController {
    public R<AgentResponse> messages(@RequestBody AgentRequest request) {
        return R.ok(orchestrator.handle(request));
    }
+
+    @SaCheckLogin
+    @PostMapping(value = "/messages/media", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    public R<AgentResponse> media(@RequestPart("file") MultipartFile file,
+                                  @RequestParam(value = "question", required = false) String question,
+                                  @RequestParam(value = "conversationId", required = false) String conversationId,
+                                  @RequestParam(value = "contextVersion", required = false) Long contextVersion,
+                                  @RequestParam(value = "projectCode", required = false) String projectCode,
+                                  @RequestParam(value = "broadcastMessageId", required = false) Long broadcastMessageId,
+                                  @RequestParam(value = "externalConsent", defaultValue = "false")
+                                  boolean externalConsent) {
+        String prompt = question == null || question.isBlank() ? "请分析附件内容" : question.trim();
+        return R.ok(orchestrator.handleMedia(
+            new AgentRequest(prompt, conversationId, contextVersion, projectCode, broadcastMessageId, externalConsent),
+            file
+        ));
+    }
 }
@@ -13,8 +13,10 @@ import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryResponse;
 import org.dromara.aihr.knowledge.service.AihrKnowledgeDataToolService.CurrentTaskSummary;
 import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver;
 import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService;
+import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode;
 import org.dromara.common.core.exception.ServiceException;
 import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;

 import java.time.OffsetDateTime;
 import java.util.List;
@@ -30,9 +32,7 @@ public class AihrAgentOrchestrator {
    private final AihrKnowledgeQueryService queryService;

    public AgentResponse handle(AgentRequest request) {
-        if (request == null || request.question() == null || request.question().isBlank()) {
-            throw new ServiceException("问题不能为空", 400);
-        }
+        requireQuestion(request);
        AgentPlan plan = planner.plan(request.question(), false);
        policy.authorize(principalResolver.current(), plan, request.externalConsent());
        return switch (plan.intent()) {
@@ -47,6 +47,23 @@ public class AihrAgentOrchestrator {
        };
    }

+    public AgentResponse handleMedia(AgentRequest request, MultipartFile file) {
+        requireQuestion(request);
+        if (file == null || file.isEmpty()) {
+            throw new ServiceException("附件不能为空", 400);
+        }
+        AgentPlan plan = planner.plan(request.question(), true);
+        policy.authorize(principalResolver.current(), plan, request.externalConsent());
+        MediaMode mode = needsKnowledge(request.question())
+            ? MediaMode.MEDIA_WITH_KNOWLEDGE
+            : MediaMode.MEDIA_ONLY;
+        QueryResponse response = queryService.queryInternalMedia(new QueryRequest(
+            request.question(), List.of(), "sop", null, "mobile_uni_agent", 5, null,
+            request.conversationId(), request.contextVersion(), request.projectCode(), request.broadcastMessageId()
+        ), file, mode);
+        return fromResponse(plan, response, "MEDIA");
+    }
+
    private AgentResponse fromQuery(AgentPlan plan, AgentRequest request) {
        String toolCode = switch (plan.tool()) {
            case MY_CURRENT_TASKS, MY_PRACTICE_SUMMARY, TEAM_PRACTICE_SUMMARY -> plan.tool().name();
@@ -56,8 +73,11 @@ public class AihrAgentOrchestrator {
            plan.rewrittenRequest(), List.of(), "sop", null, "mobile_uni_agent", 5, toolCode,
            request.conversationId(), request.contextVersion(), request.projectCode(), request.broadcastMessageId()
        ));
+        return fromResponse(plan, response, toolCode == null ? "ENTERPRISE_KNOWLEDGE" : "LIVE_DATA");
+    }
+
+    private AgentResponse fromResponse(AgentPlan plan, QueryResponse response, String sourceType) {
        AgentStatus status = status(response);
-        String sourceType = toolCode == null ? "ENTERPRISE_KNOWLEDGE" : "LIVE_DATA";
        String sourceLabel = response.citations().isEmpty() ? sourceType
            : response.citations().get(0).title();
        return new AgentResponse(
@@ -67,6 +87,18 @@ public class AihrAgentOrchestrator {
        );
    }

+    private static void requireQuestion(AgentRequest request) {
+        if (request == null || request.question() == null || request.question().isBlank()) {
+            throw new ServiceException("问题不能为空", 400);
+        }
+    }
+
+    private static boolean needsKnowledge(String question) {
+        String value = question == null ? "" : question;
+        return List.of("怎么办", "怎么处理", "如何处理", "sop", "SOP", "制度", "流程", "规定",
+            "故障", "漏水", "电梯", "现场").stream().anyMatch(value::contains);
+    }
+
    private static AgentStatus status(QueryResponse response) {
        if (response.data() instanceof CurrentTaskSummary summary && "UNAVAILABLE".equals(summary.state())) {
            return AgentStatus.UNAVAILABLE;
@@ -40,6 +40,11 @@ import java.util.UUID;
@Service
 public class AihrKnowledgeQueryService {

+    public enum MediaMode {
+        MEDIA_ONLY,
+        MEDIA_WITH_KNOWLEDGE
+    }
+
    private final AihrKnowledgePrincipalResolver principalResolver;
    private final AihrKnowledgeAppService appService;
    private final AihrKnowledgeAccessService accessService;
@@ -177,6 +182,10 @@ public class AihrKnowledgeQueryService {
    }

    public QueryResponse queryInternalMedia(QueryRequest rawRequest, MultipartFile file) {
+        return queryInternalMedia(rawRequest, file, MediaMode.MEDIA_WITH_KNOWLEDGE);
+    }
+
+    public QueryResponse queryInternalMedia(QueryRequest rawRequest, MultipartFile file, MediaMode mode) {
        QueryRequest request = validate(rawRequest);
        if (request.broadcastMessageId() != null) {
            throw new ServiceException("公司消息追问暂不支持现场附件", HttpStatus.BAD_REQUEST);
@@ -196,11 +205,26 @@ public class AihrKnowledgeQueryService {
        }
        String combinedQuery = truncate(context.rewrittenQuery(), 400)
            + "\n附件现场信息：" + truncate(extractedContext, 520);
-        QueryResponse response = queryDocuments(principal, app, spaceIds, new QueryRequest(
-            combinedQuery, request.spaceCodes(), request.category(), request.position(),
-            request.source(), request.limit(), null, request.conversationId(), request.contextVersion(),
-            request.projectCode()
-        ), request.queryText());
+        QueryResponse response;
+        if (mode == MediaMode.MEDIA_ONLY) {
+            String requestId = UUID.randomUUID().toString().replace("-", "");
+            Citation citation = new Citation(null, "MEDIA", "media:" + requestId,
+                "本次附件识别", truncate(extractedContext, 520), null);
+            auditService.record(requestId, principal, app, request.queryText(), List.of(), List.of("MEDIA"),
+                "SUCCESS", 0L, "media-only-v1");
+            response = new QueryResponse(
+                requestId, request.queryText(), "附件识别结果：" + truncate(extractedContext, 520),
+                List.of(citation), List.of(), false, "media-only-v1", null,
+                Map.of("observation", truncate(extractedContext, 2000))
+            );
+        } else {
+            response = queryDocuments(principal, app, spaceIds, new QueryRequest(
+                combinedQuery, request.spaceCodes(), request.category(), request.position(),
+                request.source(), request.limit(), null, request.conversationId(), request.contextVersion(),
+                request.projectCode()
+            ), request.queryText());
+            response = withMediaContextWhenNoEvidence(response, extractedContext);
+        }
        QueryResponse completed = complete(principal, app, spaceIds, request.queryText(), context, response);
        MemoryCandidateResponse candidate = memoryService.detectForQuery(
            principal, combinedQuery, context.conversationId(), completed.requestId(),
@@ -209,6 +233,19 @@ public class AihrKnowledgeQueryService {
        return completed.withMemoryCandidate(candidate);
    }

+    private static QueryResponse withMediaContextWhenNoEvidence(QueryResponse response, String extractedContext) {
+        if (!response.noEvidence()) {
+            return response;
+        }
+        String answer = "附件识别结果：" + truncate(extractedContext, 240) + "\n\n" + response.answer();
+        return new QueryResponse(
+            response.requestId(), response.queryText(), answer, response.citations(), response.usedSpaceCodes(),
+            true, response.promptVersion(), response.legacy(), response.data(), response.conversationId(),
+            response.contextVersion(), response.intent(), response.rewrittenQuery(), response.resources(),
+            response.memoryCandidate(), response.broadcastContext()
+        );
+    }
+
    public QueryResponse queryExternal(AuthenticatedApp app, QueryRequest rawRequest) {
        QueryRequest request = validate(rawRequest);
        if (hasText(request.toolCode()) || hasText(request.conversationId()) || request.contextVersion() != null
@@ -38,4 +38,14 @@ class AihrAgentControllerContractTest {
        assertTrue(fields.stream().noneMatch(name ->
            name.contains("userid") || name.contains("employeeid") || name.contains("role") || name.contains("tool")));
    }
+
+    @Test
+    void mediaEndpointAlsoRequiresLogin() throws Exception {
+        var method = AihrAgentController.class.getMethod("media",
+            org.springframework.web.multipart.MultipartFile.class,
+            String.class, String.class, Long.class, String.class, Long.class, boolean.class);
+
+        assertNotNull(method.getAnnotation(SaCheckLogin.class));
+        assertEquals("/messages/media", method.getAnnotation(PostMapping.class).value()[0]);
+    }
 }
@@ -0,0 +1,79 @@
+package org.dromara.aihr.agent;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.dromara.aihr.agent.AihrAgentDto.AgentRequest;
+import org.dromara.aihr.agent.AihrAgentDto.AgentStatus;
+import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal;
+import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryRequest;
+import org.dromara.aihr.knowledge.domain.AihrKnowledgeQueryDto.QueryResponse;
+import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver;
+import org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode.MEDIA_ONLY;
+import static org.dromara.aihr.knowledge.service.AihrKnowledgeQueryService.MediaMode.MEDIA_WITH_KNOWLEDGE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("dev")
+class AihrAgentMediaTest {
+
+    @Test
+    void genericImageQuestionUsesMediaOnly() {
+        var query = new CapturingMediaService();
+        var response = orchestrator(query).handleMedia(request("图中是什么宠物"),
+            new MockMultipartFile("file", "pet.jpg", "image/jpeg", new byte[]{1}));
+
+        assertEquals(MEDIA_ONLY, query.mode);
+        assertEquals(AgentStatus.COMPLETED, response.status());
+    }
+
+    @Test
+    void workplaceImageQuestionCombinesMediaWithKnowledge() {
+        var query = new CapturingMediaService();
+        orchestrator(query).handleMedia(request("图中电梯门打不开怎么办"),
+            new MockMultipartFile("file", "lift.jpg", "image/jpeg", new byte[]{1}));
+
+        assertEquals(MEDIA_WITH_KNOWLEDGE, query.mode);
+    }
+
+    private static AihrAgentOrchestrator orchestrator(CapturingMediaService query) {
+        return new AihrAgentOrchestrator(
+            new AihrAgentPlanner(null, new ObjectMapper()),
+            new AihrAgentPolicy(),
+            new AihrKnowledgePrincipalResolver(null) {
+                @Override
+                public AihrKnowledgePrincipal current() {
+                    return new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1",
+                        Set.of("employee"), Set.of("P1"), "app");
+                }
+            },
+            query
+        );
+    }
+
+    private static AgentRequest request(String question) {
+        return new AgentRequest(question, "conversation_1", 0L, "P1", null, false);
+    }
+
+    private static final class CapturingMediaService extends AihrKnowledgeQueryService {
+        private MediaMode mode;
+
+        private CapturingMediaService() {
+            super(null, null, null, null, null, null, null, null, null);
+        }
+
+        @Override
+        public QueryResponse queryInternalMedia(QueryRequest request, MultipartFile file, MediaMode mode) {
+            this.mode = mode;
+            return new QueryResponse("request-1", request.queryText(), "附件识别结果：图中是一只金毛犬",
+                List.of(), List.of(), false, "media-only-v1", null, null,
+                "conversation_1", 1L, "QA", request.queryText(), List.of());
+        }
+    }
+}
@@ -201,6 +201,82 @@ class AihrKnowledgeQueryServiceTest {
        assertEquals(true, requestCaptor.getValue().queryText().contains("电梯门无法打开"));
    }

+    @Test
+    void genericMediaQuestionSkipsRagAndAppendsTheObservedAnswer() {
+        var resolver = mock(AihrKnowledgePrincipalResolver.class);
+        var appService = mock(AihrKnowledgeAppService.class);
+        var access = mock(AihrKnowledgeAccessService.class);
+        var sop = mock(AihrSopSeedService.class);
+        var conversations = mock(AihrKnowledgeConversationService.class);
+        var principal = new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1",
+            Set.of("employee"), Set.of("P1"), "app");
+        var app = new AuthenticatedApp(1L, "000000", "yc_mobile", "员工端", "SESSION", 60, null);
+        var request = new QueryRequest("图中是什么宠物", List.of(), "sop", null, "mobile_uni_agent", 5, null,
+            "conversation_123", 0L, "P1");
+        var context = new ConversationContext("conversation_123", true, true, 0L, List.of(),
+            "图中是什么宠物", "QA", null, "P1", null);
+        var media = new MockMultipartFile("file", "pet.jpg", "image/jpeg", new byte[]{1});
+        when(resolver.current()).thenReturn(principal);
+        when(appService.requireSessionApp("000000", "app")).thenReturn(app);
+        when(access.resolveInternalSpaceIds(principal, app, List.of(), "READ")).thenReturn(Set.of(9L));
+        when(conversations.prepare(principal, app, request)).thenReturn(context);
+        when(conversations.append(any(), any(), any(), anyString(), anyString(), any())).thenReturn(1L);
+        when(sop.extractQuestionMedia(media, "图中是什么宠物")).thenReturn("图中是一只金毛犬");
+        var service = new AihrKnowledgeQueryService(resolver, appService, access, sop,
+            mock(AihrKnowledgeQueryAuditService.class), mock(JdbcTemplate.class),
+            mock(AihrKnowledgeDataToolService.class), conversations, mock(AihrMemoryService.class));
+
+        var result = service.queryInternalMedia(request, media, AihrKnowledgeQueryService.MediaMode.MEDIA_ONLY);
+
+        assertFalse(result.noEvidence());
+        assertTrue(result.answer().contains("图中是一只金毛犬"));
+        verify(sop, never()).searchAuthorized(any(), any(), any());
+        var appendedAnswer = ArgumentCaptor.forClass(String.class);
+        verify(conversations).append(any(), any(), any(), anyString(), appendedAnswer.capture(), any());
+        assertTrue(appendedAnswer.getValue().contains("图中是一只金毛犬"));
+    }
+
+    @Test
+    void workMediaWithoutSopEvidenceKeepsObservationBeforeConversationAppend() {
+        var resolver = mock(AihrKnowledgePrincipalResolver.class);
+        var appService = mock(AihrKnowledgeAppService.class);
+        var access = mock(AihrKnowledgeAccessService.class);
+        var sop = mock(AihrSopSeedService.class);
+        var jdbc = mock(JdbcTemplate.class);
+        var conversations = mock(AihrKnowledgeConversationService.class);
+        var principal = new AihrKnowledgePrincipal("000000", 1L, "app_user", "employee-1",
+            Set.of("employee"), Set.of("P1"), "app");
+        var app = new AuthenticatedApp(1L, "000000", "yc_mobile", "员工端", "SESSION", 60, null);
+        var request = new QueryRequest("电梯门打不开怎么办", List.of(), "sop", null, "mobile_uni_agent", 5, null,
+            "conversation_123", 0L, "P1");
+        var context = new ConversationContext("conversation_123", true, true, 0L, List.of(),
+            "电梯门打不开怎么办", "QA", null, "P1", null);
+        var media = new MockMultipartFile("file", "lift.jpg", "image/jpeg", new byte[]{1});
+        when(resolver.current()).thenReturn(principal);
+        when(appService.requireSessionApp("000000", "app")).thenReturn(app);
+        when(access.resolveInternalSpaceIds(principal, app, List.of(), "READ")).thenReturn(Set.of(9L));
+        when(conversations.prepare(principal, app, request)).thenReturn(context);
+        when(conversations.append(any(), any(), any(), anyString(), anyString(), any())).thenReturn(1L);
+        when(jdbc.queryForList(any(String.class), any(Class.class), any(Object[].class))).thenReturn(List.of("sop"));
+        when(sop.extractQuestionMedia(media, "电梯门打不开怎么办")).thenReturn("画面显示电梯门无法打开");
+        when(sop.searchAuthorized(any(), any(), any())).thenReturn(
+            new SearchResponse("现场问题", "sop", "未找到明确制度依据。", "", List.of(), List.of(),
+                List.of(), List.of(), List.of(), List.of(), "test-v1", null));
+        var service = new AihrKnowledgeQueryService(resolver, appService, access, sop,
+            mock(AihrKnowledgeQueryAuditService.class), jdbc, mock(AihrKnowledgeDataToolService.class),
+            conversations, mock(AihrMemoryService.class));
+
+        var result = service.queryInternalMedia(
+            request, media, AihrKnowledgeQueryService.MediaMode.MEDIA_WITH_KNOWLEDGE);
+
+        assertTrue(result.noEvidence());
+        assertTrue(result.answer().contains("附件识别结果：画面显示电梯门无法打开"));
+        assertTrue(result.answer().contains("未找到明确制度依据"));
+        var appendedAnswer = ArgumentCaptor.forClass(String.class);
+        verify(conversations).append(any(), any(), any(), anyString(), appendedAnswer.capture(), any());
+        assertTrue(appendedAnswer.getValue().contains("附件识别结果：画面显示电梯门无法打开"));
+    }
+
    @Test
    void fileFollowupUsesConversationRewriteAndReturnsOnlyAuthorizedAttachment() throws Exception {
        var resolver = mock(AihrKnowledgePrincipalResolver.class);