From 57866afc603b3fb2d147a5a3f20ed26800bbaf44 Mon Sep 17 00:00:00 2001 From: let5sne Date: Sun, 19 Jul 2026 22:58:20 +0800 Subject: [PATCH] feat(web-ai): support qwen image search safely --- .../aihr/community/AihrCommunityPolicy.java | 10 ++ .../aihr/webai/AihrWebAiController.java | 11 ++ .../org/dromara/aihr/webai/AihrWebAiDto.java | 1 + .../AihrWebAiImageUploadLimitFilter.java | 148 ++++++++++++++++++ .../dromara/aihr/webai/AihrWebAiService.java | 62 +++++++- .../aihr/webai/AihrWebSearchClient.java | 90 ++++++++++- .../AihrWebSearchProviderAdminController.java | 4 +- .../webai/AihrWebSearchProviderService.java | 30 +++- .../aihr/webai/AihrWebAiSecurityTest.java | 32 ++++ mobile-uni/src/pages/user/web-ai/index.vue | 68 ++++++-- mobile-uni/src/services/web-ai.ts | 58 +++++-- mobile-uni/src/types/api.ts | 1 + 12 files changed, 472 insertions(+), 43 deletions(-) create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiImageUploadLimitFilter.java diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/community/AihrCommunityPolicy.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/community/AihrCommunityPolicy.java index 55d5f052..42089e72 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/community/AihrCommunityPolicy.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/community/AihrCommunityPolicy.java @@ -10,6 +10,7 @@ public final class AihrCommunityPolicy { private static final Set MODERATOR_ROLES = Set.of( "supervisor", "manager", "project_manager", "admin", "superadmin", "hr_operator" ); + private static final Set WEB_SEARCH_PROVIDER_ROLES = Set.of("admin", "superadmin", "hr_operator"); private AihrCommunityPolicy() { } @@ -32,6 +33,15 @@ public final class AihrCommunityPolicy { .anyMatch(MODERATOR_ROLES::contains); } + public static boolean canManageWebSearchProviders(AihrKnowledgePrincipal principal) { + if (principal == null) { + return false; + } + return principal.roles().stream() + .map(role -> role == null ? "" : role.trim().toLowerCase(Locale.ROOT)) + .anyMatch(WEB_SEARCH_PROVIDER_ROLES::contains); + } + public static String moderatedStatus(String decision) { String normalized = decision == null ? "" : decision.trim().toLowerCase(Locale.ROOT); return switch (normalized) { diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiController.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiController.java index 3d077570..5d655b22 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiController.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiController.java @@ -6,11 +6,15 @@ import org.dromara.aihr.webai.AihrWebAiDto.QueryRequest; import org.dromara.aihr.webai.AihrWebAiDto.QueryResponse; import org.dromara.aihr.webai.AihrWebAiDto.CapabilitiesResponse; import org.dromara.common.core.domain.R; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; @RestController @RequiredArgsConstructor @@ -29,4 +33,11 @@ public class AihrWebAiController { public R query(@RequestBody QueryRequest request) { return R.ok(webAiService.query(request)); } + + @PostMapping(value = "/query-image", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public R queryImage(@RequestPart("file") MultipartFile file, + @RequestParam("question") String question, + @RequestParam("consent") Boolean consent) { + return R.ok(webAiService.queryImage(new QueryRequest(question, consent), file)); + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiDto.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiDto.java index 9eb98be9..44277b60 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiDto.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiDto.java @@ -21,6 +21,7 @@ public final class AihrWebAiDto { } public record CapabilitiesResponse(boolean enabled, + boolean supportsImage, String provider, boolean requiresConsent, int maxQuestionLength, diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiImageUploadLimitFilter.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiImageUploadLimitFilter.java new file mode 100644 index 00000000..521746b9 --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiImageUploadLimitFilter.java @@ -0,0 +1,148 @@ +package org.dromara.aihr.webai; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +/** Limits this high-cost multipart route before Spring can spool the upload to disk. */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 20) +public class AihrWebAiImageUploadLimitFilter extends OncePerRequestFilter { + + static final long MAX_FILE_BYTES = 4L * 1024 * 1024; + private static final long MAX_REQUEST_BYTES = MAX_FILE_BYTES + 64L * 1024; + private static final String ROUTE = "/api/aihr/web-ai/query-image"; + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + return !"POST".equalsIgnoreCase(request.getMethod()) || !request.getRequestURI().endsWith(ROUTE); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + if (request.getContentLengthLong() > MAX_REQUEST_BYTES) { + tooLarge(response); + return; + } + try { + filterChain.doFilter(new LimitedRequest(request, MAX_REQUEST_BYTES), response); + } catch (ServletException | RuntimeException error) { + if (causedByLimit(error)) { + tooLarge(response); + return; + } + throw error; + } catch (IOException error) { + if (causedByLimit(error)) { + tooLarge(response); + return; + } + throw error; + } + } + + private static boolean causedByLimit(Throwable error) { + Throwable current = error; + while (current != null) { + if (current instanceof RequestTooLargeException) { + return true; + } + current = current.getCause(); + } + return false; + } + + private static void tooLarge(HttpServletResponse response) throws IOException { + if (response.isCommitted()) { + return; + } + response.resetBuffer(); + response.setStatus(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE); + response.setContentType("application/json;charset=UTF-8"); + response.getWriter().write("{\"code\":413,\"msg\":\"图片不能超过 4MB\"}"); + } + + private static final class LimitedRequest extends HttpServletRequestWrapper { + private final long maxBytes; + + private LimitedRequest(HttpServletRequest request, long maxBytes) { + super(request); + this.maxBytes = maxBytes; + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return new LimitedInputStream(super.getInputStream(), maxBytes); + } + + @Override + public BufferedReader getReader() throws IOException { + String charset = getCharacterEncoding(); + return new BufferedReader(new InputStreamReader(getInputStream(), charset == null ? StandardCharsets.UTF_8 : java.nio.charset.Charset.forName(charset))); + } + } + + private static final class LimitedInputStream extends ServletInputStream { + private final ServletInputStream delegate; + private final long maxBytes; + private long bytesRead; + + private LimitedInputStream(ServletInputStream delegate, long maxBytes) { + this.delegate = delegate; + this.maxBytes = maxBytes; + } + + @Override + public int read() throws IOException { + int value = delegate.read(); + if (value >= 0) count(1); + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int read = delegate.read(bytes, offset, length); + if (read > 0) count(read); + return read; + } + + @Override + public boolean isFinished() { + return delegate.isFinished(); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setReadListener(ReadListener readListener) { + delegate.setReadListener(readListener); + } + + private void count(int amount) throws RequestTooLargeException { + bytesRead += amount; + if (bytesRead > maxBytes) { + throw new RequestTooLargeException(); + } + } + } + + private static final class RequestTooLargeException extends IOException { + } +} diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiService.java index a23d77d2..caa81abe 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebAiService.java @@ -7,8 +7,12 @@ import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver; import org.dromara.aihr.webai.AihrWebAiDto.QueryRequest; import org.dromara.aihr.webai.AihrWebAiDto.QueryResponse; import org.dromara.aihr.webai.AihrWebAiDto.CapabilitiesResponse; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.redis.utils.RedisUtils; +import org.redisson.api.RateType; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; import java.util.List; @@ -25,6 +29,14 @@ public class AihrWebAiService { private volatile boolean auditTableReady; public QueryResponse query(QueryRequest request) { + return query(request, null); + } + + public QueryResponse queryImage(QueryRequest request, MultipartFile image) { + return query(request, image); + } + + private QueryResponse query(QueryRequest request, MultipartFile image) { AihrKnowledgePrincipal principal = principalResolver.current(); String rawQuestion = normalizeQuestion(request == null ? null : request.question()); String question = AihrWebPrivacy.forExternalSearch(rawQuestion); @@ -32,33 +44,50 @@ public class AihrWebAiService { AihrWebSearchProviderService.RuntimeProvider runtime = searchClient.runtime(principal); String provider = runtime == null ? clean(properties.getProvider(), "tavily") : runtime.name(); if (request == null || !Boolean.TRUE.equals(request.consent())) { - audit(principal, question, provider, "CONSENT_REQUIRED", 0); + audit(principal, question, provider, auditStatus(image, "CONSENT_REQUIRED"), 0); return new QueryResponse("web", "CONSENT_REQUIRED", "", List.of(), provider, "请确认同意将脱敏后的问题发送到外部检索服务", redacted, true); } if (runtime == null) { - audit(principal, question, provider, "DISABLED", 0); + audit(principal, question, provider, auditStatus(image, "DISABLED"), 0); return new QueryResponse("web", "DISABLED", "", List.of(), provider, "全网检索默认关闭，请由管理员配置合规的外部检索服务后启用", redacted, true); } + if (image != null && !"qwen".equalsIgnoreCase(runtime.code())) { + audit(principal, question, provider, "IMAGE_UNSUPPORTED", 0); + return new QueryResponse("web", "IMAGE_UNSUPPORTED", "", List.of(), provider, + "当前全网检索提供方不支持图片识图，请切换至千问联网检索", redacted, true); + } + if (image != null) { + enforceImageRateLimit(principal); + } + AihrWebSearchClient.ImageInput imageInput; try { - AihrWebSearchClient.SearchResult result = searchClient.search(runtime, question); + imageInput = image == null ? null : searchClient.imageInput(image); + } catch (RuntimeException error) { + audit(principal, question, provider, "IMAGE_REJECTED", 0); + throw error; + } + try { + AihrWebSearchClient.SearchResult result = imageInput == null + ? searchClient.search(runtime, question) + : searchClient.searchWithImage(runtime, question, imageInput); if (result.sources().isEmpty()) { - audit(principal, question, provider, "NO_SOURCES", 0); + audit(principal, question, provider, auditStatus(image, "NO_SOURCES"), 0); return new QueryResponse("web", "NO_SOURCES", "", List.of(), provider, "没有找到可公开核验的来源，本次不生成答案", redacted, true); } if (result.answer() == null || result.answer().isBlank()) { - audit(principal, question, provider, "NO_ANSWER", result.sources().size()); + audit(principal, question, provider, auditStatus(image, "NO_ANSWER"), result.sources().size()); return new QueryResponse("web", "NO_ANSWER", "", result.sources(), provider, "已找到公开来源，但外部服务未返回可核验答案；请直接查看来源", redacted, true); } - audit(principal, question, provider, "OK", result.sources().size()); + audit(principal, question, provider, auditStatus(image, "OK"), result.sources().size()); return new QueryResponse("web", "OK", result.answer(), result.sources(), provider, "回答来自全网公开资料，不代表企业内部 SOP；请核对下方来源", redacted, true); } catch (RuntimeException error) { log.warn("web AI query failed provider={} (details hidden)", provider); - audit(principal, question, provider, "FAILED", 0); + audit(principal, question, provider, auditStatus(image, "FAILED"), 0); return new QueryResponse("web", "FAILED", "", List.of(), provider, "全网检索暂时不可用，本次未生成答案", redacted, true); } @@ -67,7 +96,7 @@ public class AihrWebAiService { public CapabilitiesResponse capabilities() { AihrKnowledgePrincipal principal = principalResolver.current(); AihrWebSearchProviderService.RuntimeProvider runtime = searchClient.runtime(principal); - return new CapabilitiesResponse(runtime != null, + return new CapabilitiesResponse(runtime != null, runtime != null && "qwen".equalsIgnoreCase(runtime.code()), runtime == null ? clean(properties.getProvider(), "tavily") : runtime.name(), true, 500, runtime == null ? "全网检索未配置，企业内部问师傅不受影响" @@ -134,4 +163,21 @@ public class AihrWebAiService { String provider = clean(value, "unknown"); return provider.length() <= 40 ? provider : provider.substring(0, 40); } + + private static String auditStatus(MultipartFile image, String status) { + return image == null ? status : "IMAGE_" + status; + } + + private static void enforceImageRateLimit(AihrKnowledgePrincipal principal) { + try { + String key = "web-ai:image:" + principal.tenantId() + ":" + principal.userId(); + if (RedisUtils.rateLimiter(key, RateType.OVERALL, 6, 60) < 0) { + throw new ServiceException("图片联网识图请求过于频繁，请稍后重试", 429); + } + } catch (ServiceException error) { + throw error; + } catch (RuntimeException error) { + throw new ServiceException("图片联网识图限流服务暂不可用", 503); + } + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchClient.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchClient.java index 54eaf002..d0d41bf3 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchClient.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchClient.java @@ -8,8 +8,17 @@ import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal; import org.dromara.aihr.webai.AihrWebSearchProviderService.RuntimeProvider; import org.dromara.common.core.exception.ServiceException; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import javax.imageio.ImageIO; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.StringReader; import java.net.Inet4Address; import java.net.Inet6Address; @@ -20,6 +29,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; import java.util.ArrayList; +import java.util.Base64; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -34,6 +44,9 @@ public class AihrWebSearchClient { private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(5); private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(12); private static final Duration QWEN_REQUEST_TIMEOUT = Duration.ofSeconds(40); + private static final long QWEN_IMAGE_MAX_BYTES = 4L * 1024 * 1024; + private static final long QWEN_IMAGE_MAX_PIXELS = 12_000_000L; + private static final int QWEN_IMAGE_MAX_EDGE = 2048; private static final Pattern PUBLIC_URL = Pattern.compile("https?://[^\\s\\]\\)\\\"'<>]+"); private final AihrWebAiProperties properties; @@ -51,7 +64,7 @@ public class AihrWebSearchClient { SearchResult search(RuntimeProvider runtime, String question) { String endpoint = clean(runtime.endpoint()); String apiKey = clean(runtime.apiKey()); - if (!isPublicHttpsUrl(endpoint)) { + if (!AihrWebSearchProviderService.isAllowedProviderEndpoint(runtime.code(), endpoint)) { throw new ServiceException("全网检索地址未配置或不安全"); } if (apiKey.isBlank()) { @@ -67,6 +80,21 @@ public class AihrWebSearchClient { return searchTavily(runtime, question); } + SearchResult searchWithImage(RuntimeProvider runtime, String question, ImageInput image) { + String endpoint = clean(runtime.endpoint()); + String apiKey = clean(runtime.apiKey()); + if (!AihrWebSearchProviderService.isAllowedProviderEndpoint(runtime.code(), endpoint)) { + throw new ServiceException("全网检索地址未配置或不安全"); + } + if (apiKey.isBlank()) { + throw new ServiceException("全网检索密钥未配置"); + } + if (!"qwen".equalsIgnoreCase(clean(runtime.code()))) { + throw new ServiceException("当前全网检索提供方不支持图片识图"); + } + return searchQwen(runtime, question, image); + } + private SearchResult searchTavily(RuntimeProvider runtime, String question) { try { ObjectNode body = objectMapper.createObjectNode(); @@ -101,6 +129,10 @@ public class AihrWebSearchClient { } private SearchResult searchQwen(RuntimeProvider runtime, String question) { + return searchQwen(runtime, question, null); + } + + private SearchResult searchQwen(RuntimeProvider runtime, String question, ImageInput image) { String agentId = clean(runtime.agentId()); String agentVersion = clean(runtime.agentVersion()); if (agentId.isBlank() || agentVersion.isBlank()) { @@ -111,7 +143,15 @@ public class AihrWebSearchClient { ObjectNode input = body.putObject("input"); ObjectNode message = input.putArray("messages").addObject(); message.put("role", "user"); - message.put("content", question); + if (image == null) { + message.put("content", question); + } else { + var content = message.putArray("content"); + content.addObject().put("type", "image_url") + .putObject("image_url") + .put("url", "data:" + image.mimeType() + ";base64," + Base64.getEncoder().encodeToString(image.bytes())); + content.addObject().put("type", "text").put("text", question); + } ObjectNode options = body.putObject("parameters").putObject("agent_options"); options.put("agent_id", agentId); options.put("agent_version", agentVersion); @@ -162,6 +202,49 @@ public class AihrWebSearchClient { clean(properties.getEndpoint()), clean(properties.getApiKey()), "", "", true, true, null); } + static ImageInput imageInput(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new ServiceException("请选择图片"); + } + if (file.getSize() > QWEN_IMAGE_MAX_BYTES) { + throw new ServiceException("图片不能超过 4MB"); + } + try { + BufferedImage source = ImageIO.read(new ByteArrayInputStream(file.getBytes())); + if (source == null || source.getWidth() <= 0 || source.getHeight() <= 0) { + throw new ServiceException("仅支持 JPG、PNG 或 BMP 图片"); + } + if ((long) source.getWidth() * source.getHeight() > QWEN_IMAGE_MAX_PIXELS) { + throw new ServiceException("图片像素不能超过1200万"); + } + BufferedImage normalized = normalizeImage(source); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + if (!ImageIO.write(normalized, "jpg", output) || output.size() > QWEN_IMAGE_MAX_BYTES) { + throw new ServiceException("图片内容过于复杂，请压缩后重试"); + } + return new ImageInput(output.toByteArray(), "image/jpeg"); + } catch (IOException error) { + throw new ServiceException("图片读取失败，请重新选择"); + } + } + + private static BufferedImage normalizeImage(BufferedImage source) { + double scale = Math.min(1d, QWEN_IMAGE_MAX_EDGE / (double) Math.max(source.getWidth(), source.getHeight())); + int width = Math.max(1, (int) Math.round(source.getWidth() * scale)); + int height = Math.max(1, (int) Math.round(source.getHeight() * scale)); + BufferedImage target = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + Graphics2D graphics = target.createGraphics(); + try { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, width, height); + graphics.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC); + graphics.drawImage(source, 0, 0, width, height, null); + return target; + } finally { + graphics.dispose(); + } + } + static SearchResult parseTavilyResponse(ObjectMapper mapper, String body, int limit) throws Exception { JsonNode root = mapper.readTree(body == null ? "{}" : body); List sources = new ArrayList<>(); @@ -374,4 +457,7 @@ public class AihrWebSearchClient { public record SearchResult(String answer, List sources) { } + + record ImageInput(byte[] bytes, String mimeType) { + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderAdminController.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderAdminController.java index aedc9c5b..2be6fa0d 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderAdminController.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderAdminController.java @@ -70,8 +70,8 @@ public class AihrWebSearchProviderAdminController { private AihrKnowledgePrincipal requireModerator() { AihrKnowledgePrincipal principal = principalResolver.current(); - if (!AihrCommunityPolicy.canModerate(principal)) { - throw new ServiceException("仅主管或管理员可管理全网检索提供方", 403); + if (!AihrCommunityPolicy.canManageWebSearchProviders(principal)) { + throw new ServiceException("仅系统管理员可管理全网检索提供方", 403); } return principal; } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderService.java index b62d6e7f..26795d05 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/webai/AihrWebSearchProviderService.java @@ -12,6 +12,7 @@ import org.springframework.transaction.annotation.Transactional; import java.sql.PreparedStatement; import java.sql.Statement; +import java.net.URI; import java.time.LocalDateTime; import java.util.List; import java.util.Locale; @@ -88,7 +89,7 @@ public class AihrWebSearchProviderService { ensureTable(); if (enabled) { RuntimeProvider selected = provider(principal, id); - if (!AihrWebSearchClient.isPublicHttpsUrl(selected.endpoint()) || !hasText(selected.apiKey()) + if (!isAllowedProviderEndpoint(selected.code(), selected.endpoint()) || !hasText(selected.apiKey()) || ("qwen".equals(selected.code()) && (!hasText(selected.agentId()) || !hasText(selected.agentVersion())))) { throw new IllegalArgumentException("请先配置安全的公开接口和访问密钥"); @@ -185,8 +186,8 @@ public class AihrWebSearchProviderService { if (!Set.of("tavily", "qwen").contains(code)) { throw new IllegalArgumentException("当前仅支持 Tavily 或千问联网检索"); } - if (!AihrWebSearchClient.isPublicHttpsUrl(endpoint)) { - throw new IllegalArgumentException("检索地址必须是可公开访问的 HTTPS 地址"); + if (!isAllowedProviderEndpoint(code, endpoint)) { + throw new IllegalArgumentException("检索地址与提供方不匹配或不受信任"); } String apiKey = clean(request.apiKey()); if (!idRequired && apiKey.isBlank()) { @@ -242,6 +243,29 @@ public class AihrWebSearchProviderService { return value != null && !value.isBlank(); } + static boolean isAllowedProviderEndpoint(String providerCode, String endpoint) { + String code = clean(providerCode).toLowerCase(Locale.ROOT); + if (!AihrWebSearchClient.isPublicHttpsUrl(endpoint)) { + return false; + } + try { + URI uri = URI.create(endpoint.trim()); + String host = uri.getHost() == null ? "" : uri.getHost().toLowerCase(Locale.ROOT); + String path = uri.getPath() == null ? "" : uri.getPath(); + return switch (code) { + case "qwen" -> "dashscope.aliyuncs.com".equals(host) + && "/api/v2/apps/web-search-agent/chat/completions".equals(path) + && uri.getQuery() == null; + case "tavily" -> "api.tavily.com".equals(host) + && "/search".equals(path) + && uri.getQuery() == null; + default -> false; + }; + } catch (IllegalArgumentException error) { + return false; + } + } + private static String clean(String value) { return value == null ? "" : value.trim(); } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/webai/AihrWebAiSecurityTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/webai/AihrWebAiSecurityTest.java index 4d724523..fa6809c1 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/webai/AihrWebAiSecurityTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/webai/AihrWebAiSecurityTest.java @@ -13,7 +13,11 @@ import org.dromara.common.core.exception.ServiceException; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.mock.web.MockMultipartFile; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; @@ -214,6 +218,34 @@ class AihrWebAiSecurityTest { assertTrue(safe.contains("已脱敏")); } + @Test + @Tag("dev") + void qwenImageInputReencodesOnlyDecodableImages() throws Exception { + BufferedImage source = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(source, "png", output); + var png = new MockMultipartFile("file", "photo.png", "image/png", output.toByteArray()); + + AihrWebSearchClient.ImageInput image = AihrWebSearchClient.imageInput(png); + assertEquals("image/jpeg", image.mimeType()); + assertTrue(image.bytes().length > 3); + assertEquals((byte) 0xff, image.bytes()[0]); + assertEquals((byte) 0xd8, image.bytes()[1]); + assertThrows(ServiceException.class, () -> AihrWebSearchClient.imageInput( + new MockMultipartFile("file", "not-an-image.jpg", "image/jpeg", + new byte[]{(byte) 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}) + )); + } + + @Test + @Tag("dev") + void providersCannotSendSecretsToArbitraryPublicHosts() { + assertTrue(AihrWebSearchProviderService.isAllowedProviderEndpoint("qwen", + "https://dashscope.aliyuncs.com/api/v2/apps/web-search-agent/chat/completions")); + assertFalse(AihrWebSearchProviderService.isAllowedProviderEndpoint("qwen", "https://example.com/collect")); + assertFalse(AihrWebSearchProviderService.isAllowedProviderEndpoint("tavily", "https://api.tavily.com/other")); + } + @Test @Tag("dev") void databaseProviderSecretUsesAuthenticatedEncryption() { diff --git a/mobile-uni/src/pages/user/web-ai/index.vue b/mobile-uni/src/pages/user/web-ai/index.vue index d99f74ac..ff7b5469 100644 --- a/mobile-uni/src/pages/user/web-ai/index.vue +++ b/mobile-uni/src/pages/user/web-ai/index.vue @@ -42,6 +42,7 @@ 🎤 语音转文字 + 📷 图片识图{{ msg.imageName ? ` · ${msg.imageName}` : '' }} {{ msg.text }} @@ -103,10 +104,16 @@ :transcribing="voiceTranscribing" placeholder="例如：最近物业服务行业有哪些官方新规？" :maxlength="capabilities?.maxQuestionLength || 500" - :panel-items="[{ key: 'audio', icon: 'sound', label: '选音频转文字' }]" + :media-name="selectedImage?.name || ''" + media-kind="image" + :panel-items="[ + { key: 'image', icon: 'image', label: '图片识图', disabled: !capabilities?.supportsImage }, + { key: 'audio', icon: 'sound', label: '选音频转文字' } + ]" @send="handleComposerSend" @voice-blob="handleVoiceBlob" @panel="handlePanelAction" + @clear-media="clearImage" /> @@ -114,11 +121,12 @@