From e0490b28cbf0b757154215b9d9bd8ec75e2faded Mon Sep 17 00:00:00 2001 From: let5sne Date: Fri, 3 Jul 2026 20:55:38 +0800 Subject: [PATCH] feat(aihr): add asr/tts speech gateway and mobile voice practice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 /api/ai/asr(multipart转写)与 /api/ai/tts(mp3 base64 dataURL),走 OpenAI 兼容 audio 接口(硅基流动 SenseVoice/CosyVoice2,复用模型管理 配置,新增 asr/tts 类目);移动端对练页接语音输入(MediaRecorder)与 客户台词自动播报,未配置/失败降级文本。ASR 限 5MB、TTS 限 300 字, multipart 头部字段去 CRLF 防注入。 --- .../aihr/controller/AihrSpeechController.java | 66 +++++++ .../dromara/aihr/domain/AihrSpeechDto.java | 16 ++ .../aihr/service/AihrSpeechService.java | 169 ++++++++++++++++++ frontend/src/views/system/model/index.vue | 2 + mobile/src/App.vue | 82 +++++++++ 5 files changed, 335 insertions(+) create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrSpeechController.java create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrSpeechDto.java create mode 100644 backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrSpeechController.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrSpeechController.java new file mode 100644 index 00000000..de4af3df --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrSpeechController.java @@ -0,0 +1,66 @@ +package org.dromara.aihr.controller; + +import lombok.RequiredArgsConstructor; +import org.dromara.aihr.domain.AihrSpeechDto.AsrResponse; +import org.dromara.aihr.domain.AihrSpeechDto.TtsRequest; +import org.dromara.aihr.domain.AihrSpeechDto.TtsResponse; +import org.dromara.aihr.service.AihrSpeechService; +import org.dromara.common.core.domain.R; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.util.Base64; + +/** + * 对练语音 API：ASR 语音输入 + TTS 客户语音播报。未配置语音模型时返回失败，前端降级为文本。 + */ +@RequiredArgsConstructor +@RestController +@RequestMapping("/api/ai") +public class AihrSpeechController { + + private static final String NOT_CONFIGURED_HINT = "语音模型未配置：请在「模型管理」启用 asr/tts 类目模型（如硅基流动 SenseVoice/CosyVoice2）"; + private static final long MAX_ASR_BYTES = 5L * 1024 * 1024; + private static final int MAX_TTS_CHARS = 300; + + private final AihrSpeechService speechService; + + @PostMapping(value = "/asr", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public R asr(@RequestPart("file") MultipartFile file) throws IOException { + if (!speechService.asrConfigured()) { + return R.fail(NOT_CONFIGURED_HINT); + } + if (file == null || file.isEmpty()) { + return R.fail("请上传音频文件"); + } + if (file.getSize() > MAX_ASR_BYTES) { + return R.fail("音频过大（上限 5MB），请缩短录音时长"); + } + return speechService.transcribe(file.getBytes(), file.getOriginalFilename(), file.getContentType()) + .map(text -> R.ok(new AsrResponse(text, "openai-compatible"))) + .orElseGet(() -> R.fail("语音识别失败，请改用文字输入")); + } + + @PostMapping("/tts") + public R tts(@RequestBody TtsRequest request) { + if (!speechService.ttsConfigured()) { + return R.fail(NOT_CONFIGURED_HINT); + } + String text = request == null || request.text() == null ? "" : request.text().trim(); + if (text.isEmpty()) { + return R.fail("合成文本不能为空"); + } + if (text.length() > MAX_TTS_CHARS) { + text = text.substring(0, MAX_TTS_CHARS); + } + return speechService.synthesize(text, request == null ? null : request.voice()) + .map(audio -> R.ok(new TtsResponse("data:audio/mp3;base64," + Base64.getEncoder().encodeToString(audio), "openai-compatible"))) + .orElseGet(() -> R.fail("语音合成失败，已降级为文本展示")); + } +} diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrSpeechDto.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrSpeechDto.java new file mode 100644 index 00000000..fbf2278d --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrSpeechDto.java @@ -0,0 +1,16 @@ +package org.dromara.aihr.domain; + +public final class AihrSpeechDto { + + private AihrSpeechDto() { + } + + public record AsrResponse(String text, String source) { + } + + public record TtsRequest(String text, String voice) { + } + + public record TtsResponse(String audioUrl, String source) { + } +} diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java new file mode 100644 index 00000000..c2ffc63f --- /dev/null +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java @@ -0,0 +1,169 @@ +package org.dromara.aihr.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.dromara.aihr.service.AihrModelSeedService.SpeechModel; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; + +/** + * ASR/TTS 网关：调用 OpenAI 兼容 audio 接口（硅基流动 SenseVoice/CosyVoice2 等）。 + * 模型未配置或调用失败返回 empty，由前端降级为纯文本对练。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AihrSpeechService { + + private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(15); + private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(60); + private static final String DEFAULT_TTS_VOICE = "anna"; + + private final AihrModelSeedService modelService; + private final ObjectMapper objectMapper; + + public boolean asrConfigured() { + return modelService.speechModel("asr").isPresent(); + } + + public boolean ttsConfigured() { + return modelService.speechModel("tts").isPresent(); + } + + /** + * 语音转写：POST {base}/audio/transcriptions（multipart）。 + */ + public Optional transcribe(byte[] audio, String filename, String contentType) { + Optional model = modelService.speechModel("asr"); + if (model.isEmpty() || audio == null || audio.length == 0) { + return Optional.empty(); + } + SpeechModel runtime = model.get(); + try { + String boundary = "----aihr-" + UUID.randomUUID(); + byte[] body = multipartBody(boundary, runtime.modelName(), audio, filename, contentType); + HttpRequest httpRequest = authorized(runtime, "/audio/transcriptions") + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .build(); + HttpResponse response = client().send(httpRequest, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() < 200 || response.statusCode() >= 300) { + log.warn("asr http {}: {}", response.statusCode(), truncate(response.body())); + return Optional.empty(); + } + String text = objectMapper.readTree(response.body()).path("text").asText(""); + return text.isBlank() ? Optional.empty() : Optional.of(text.trim()); + } catch (Exception e) { + log.warn("asr call failed: {}", e.getMessage()); + return Optional.empty(); + } + } + + /** + * 语音合成：POST {base}/audio/speech，返回 mp3 字节。 + */ + public Optional synthesize(String text, String voice) { + Optional model = modelService.speechModel("tts"); + if (model.isEmpty() || text == null || text.isBlank()) { + return Optional.empty(); + } + SpeechModel runtime = model.get(); + try { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", runtime.modelName()); + body.put("input", text.trim()); + body.put("voice", resolveVoice(runtime.modelName(), voice)); + body.put("response_format", "mp3"); + HttpRequest httpRequest = authorized(runtime, "/audio/speech") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body))) + .build(); + HttpResponse response = client().send(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() < 200 || response.statusCode() >= 300) { + log.warn("tts http {}: {}", response.statusCode(), truncate(new String(response.body(), StandardCharsets.UTF_8))); + return Optional.empty(); + } + return response.body().length == 0 ? Optional.empty() : Optional.of(response.body()); + } catch (Exception e) { + log.warn("tts call failed: {}", e.getMessage()); + return Optional.empty(); + } + } + + /** + * 硅基流动 voice 格式为 "{model}:{voice}"；调用方只传短名（如 anna/粤语音色名）时自动补模型前缀。 + */ + private static String resolveVoice(String modelName, String voice) { + String value = voice == null || voice.isBlank() ? DEFAULT_TTS_VOICE : voice.trim(); + return value.contains(":") ? value : modelName + ":" + value; + } + + private HttpRequest.Builder authorized(SpeechModel runtime, String path) { + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(normalizeBaseUrl(runtime.baseUrl()) + path)) + .timeout(REQUEST_TIMEOUT); + if (runtime.apiKey() != null && !runtime.apiKey().isBlank()) { + builder.header("Authorization", "Bearer " + runtime.apiKey()); + } + return builder; + } + + private static HttpClient client() { + return HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build(); + } + + private static byte[] multipartBody(String boundary, String modelName, byte[] audio, String filename, String contentType) throws IOException { + // 客户端可控字段进 multipart 头之前必须去掉 CRLF/引号，防止外发请求头注入 + String safeName = sanitizeHeaderValue(filename, "audio.webm"); + String safeType = sanitizeHeaderValue(contentType, "application/octet-stream"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeAscii(out, "--" + boundary + "\r\n"); + writeAscii(out, "Content-Disposition: form-data; name=\"model\"\r\n\r\n"); + out.write(modelName.getBytes(StandardCharsets.UTF_8)); + writeAscii(out, "\r\n--" + boundary + "\r\n"); + writeAscii(out, "Content-Disposition: form-data; name=\"file\"; filename=\"" + safeName + "\"\r\n"); + writeAscii(out, "Content-Type: " + safeType + "\r\n\r\n"); + out.write(audio); + writeAscii(out, "\r\n--" + boundary + "--\r\n"); + return out.toByteArray(); + } + + private static String sanitizeHeaderValue(String value, String fallback) { + if (value == null || value.isBlank()) { + return fallback; + } + String sanitized = value.replaceAll("[\\r\\n\"\\\\]", "").trim(); + return sanitized.isBlank() ? fallback : sanitized; + } + + private static void writeAscii(ByteArrayOutputStream out, String value) throws IOException { + out.write(value.getBytes(StandardCharsets.US_ASCII)); + } + + private static String normalizeBaseUrl(String baseUrl) { + String normalized = baseUrl == null ? "" : baseUrl.trim(); + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private static String truncate(String value) { + if (value == null || value.length() <= 240) { + return value; + } + return value.substring(0, 240); + } +} diff --git a/frontend/src/views/system/model/index.vue b/frontend/src/views/system/model/index.vue index c391efd8..2d0c27b9 100644 --- a/frontend/src/views/system/model/index.vue +++ b/frontend/src/views/system/model/index.vue @@ -207,6 +207,8 @@ + + diff --git a/mobile/src/App.vue b/mobile/src/App.vue index d646c9b8..6ee2be09 100644 --- a/mobile/src/App.vue +++ b/mobile/src/App.vue @@ -208,6 +208,10 @@