feat(aihr): add asr/tts speech gateway and mobile voice practice

新增 /api/ai/asr(multipart转写)与 /api/ai/tts(mp3 base64 dataURL),走 OpenAI 兼容 audio 接口(硅基流动 SenseVoice/CosyVoice2,复用模型管理配置,新增 asr/tts 类目);移动端对练页接语音输入(MediaRecorder)与客户台词自动播报,未配置/失败降级文本。ASR 限 5MB、TTS 限 300 字, multipart 头部字段去 CRLF 防注入。
2026-07-03 20:55:38 +08:00
parent 96c70a618f
commit e0490b28cb
5 changed files with 335 additions and 0 deletions
@@ -0,0 +1,66 @@
+package org.dromara.aihr.controller;
+
+import lombok.RequiredArgsConstructor;
+import org.dromara.aihr.domain.AihrSpeechDto.AsrResponse;
+import org.dromara.aihr.domain.AihrSpeechDto.TtsRequest;
+import org.dromara.aihr.domain.AihrSpeechDto.TtsResponse;
+import org.dromara.aihr.service.AihrSpeechService;
+import org.dromara.common.core.domain.R;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.Base64;
+
+/**
+ * 对练语音 API：ASR 语音输入 + TTS 客户语音播报。未配置语音模型时返回失败，前端降级为文本。
+ */
+@RequiredArgsConstructor
+@RestController
+@RequestMapping("/api/ai")
+public class AihrSpeechController {
+
+    private static final String NOT_CONFIGURED_HINT = "语音模型未配置：请在「模型管理」启用 asr/tts 类目模型（如硅基流动 SenseVoice/CosyVoice2）";
+    private static final long MAX_ASR_BYTES = 5L * 1024 * 1024;
+    private static final int MAX_TTS_CHARS = 300;
+
+    private final AihrSpeechService speechService;
+
+    @PostMapping(value = "/asr", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    public R<AsrResponse> asr(@RequestPart("file") MultipartFile file) throws IOException {
+        if (!speechService.asrConfigured()) {
+            return R.fail(NOT_CONFIGURED_HINT);
+        }
+        if (file == null || file.isEmpty()) {
+            return R.fail("请上传音频文件");
+        }
+        if (file.getSize() > MAX_ASR_BYTES) {
+            return R.fail("音频过大（上限 5MB），请缩短录音时长");
+        }
+        return speechService.transcribe(file.getBytes(), file.getOriginalFilename(), file.getContentType())
+            .map(text -> R.ok(new AsrResponse(text, "openai-compatible")))
+            .orElseGet(() -> R.fail("语音识别失败，请改用文字输入"));
+    }
+
+    @PostMapping("/tts")
+    public R<TtsResponse> tts(@RequestBody TtsRequest request) {
+        if (!speechService.ttsConfigured()) {
+            return R.fail(NOT_CONFIGURED_HINT);
+        }
+        String text = request == null || request.text() == null ? "" : request.text().trim();
+        if (text.isEmpty()) {
+            return R.fail("合成文本不能为空");
+        }
+        if (text.length() > MAX_TTS_CHARS) {
+            text = text.substring(0, MAX_TTS_CHARS);
+        }
+        return speechService.synthesize(text, request == null ? null : request.voice())
+            .map(audio -> R.ok(new TtsResponse("data:audio/mp3;base64," + Base64.getEncoder().encodeToString(audio), "openai-compatible")))
+            .orElseGet(() -> R.fail("语音合成失败，已降级为文本展示"));
+    }
+}
@@ -0,0 +1,16 @@
+package org.dromara.aihr.domain;
+
+public final class AihrSpeechDto {
+
+    private AihrSpeechDto() {
+    }
+
+    public record AsrResponse(String text, String source) {
+    }
+
+    public record TtsRequest(String text, String voice) {
+    }
+
+    public record TtsResponse(String audioUrl, String source) {
+    }
+}
@@ -0,0 +1,169 @@
+package org.dromara.aihr.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.dromara.aihr.service.AihrModelSeedService.SpeechModel;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.UUID;
+
+/**
+ * ASR/TTS 网关：调用 OpenAI 兼容 audio 接口（硅基流动 SenseVoice/CosyVoice2 等）。
+ * 模型未配置或调用失败返回 empty，由前端降级为纯文本对练。
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class AihrSpeechService {
+
+    private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(15);
+    private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(60);
+    private static final String DEFAULT_TTS_VOICE = "anna";
+
+    private final AihrModelSeedService modelService;
+    private final ObjectMapper objectMapper;
+
+    public boolean asrConfigured() {
+        return modelService.speechModel("asr").isPresent();
+    }
+
+    public boolean ttsConfigured() {
+        return modelService.speechModel("tts").isPresent();
+    }
+
+    /**
+     * 语音转写：POST {base}/audio/transcriptions（multipart）。
+     */
+    public Optional<String> transcribe(byte[] audio, String filename, String contentType) {
+        Optional<SpeechModel> model = modelService.speechModel("asr");
+        if (model.isEmpty() || audio == null || audio.length == 0) {
+            return Optional.empty();
+        }
+        SpeechModel runtime = model.get();
+        try {
+            String boundary = "----aihr-" + UUID.randomUUID();
+            byte[] body = multipartBody(boundary, runtime.modelName(), audio, filename, contentType);
+            HttpRequest httpRequest = authorized(runtime, "/audio/transcriptions")
+                .header("Content-Type", "multipart/form-data; boundary=" + boundary)
+                .POST(HttpRequest.BodyPublishers.ofByteArray(body))
+                .build();
+            HttpResponse<String> response = client().send(httpRequest, HttpResponse.BodyHandlers.ofString());
+            if (response.statusCode() < 200 || response.statusCode() >= 300) {
+                log.warn("asr http {}: {}", response.statusCode(), truncate(response.body()));
+                return Optional.empty();
+            }
+            String text = objectMapper.readTree(response.body()).path("text").asText("");
+            return text.isBlank() ? Optional.empty() : Optional.of(text.trim());
+        } catch (Exception e) {
+            log.warn("asr call failed: {}", e.getMessage());
+            return Optional.empty();
+        }
+    }
+
+    /**
+     * 语音合成：POST {base}/audio/speech，返回 mp3 字节。
+     */
+    public Optional<byte[]> synthesize(String text, String voice) {
+        Optional<SpeechModel> model = modelService.speechModel("tts");
+        if (model.isEmpty() || text == null || text.isBlank()) {
+            return Optional.empty();
+        }
+        SpeechModel runtime = model.get();
+        try {
+            ObjectNode body = objectMapper.createObjectNode();
+            body.put("model", runtime.modelName());
+            body.put("input", text.trim());
+            body.put("voice", resolveVoice(runtime.modelName(), voice));
+            body.put("response_format", "mp3");
+            HttpRequest httpRequest = authorized(runtime, "/audio/speech")
+                .header("Content-Type", "application/json")
+                .POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
+                .build();
+            HttpResponse<byte[]> response = client().send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
+            if (response.statusCode() < 200 || response.statusCode() >= 300) {
+                log.warn("tts http {}: {}", response.statusCode(), truncate(new String(response.body(), StandardCharsets.UTF_8)));
+                return Optional.empty();
+            }
+            return response.body().length == 0 ? Optional.empty() : Optional.of(response.body());
+        } catch (Exception e) {
+            log.warn("tts call failed: {}", e.getMessage());
+            return Optional.empty();
+        }
+    }
+
+    /**
+     * 硅基流动 voice 格式为 "{model}:{voice}"；调用方只传短名（如 anna/粤语音色名）时自动补模型前缀。
+     */
+    private static String resolveVoice(String modelName, String voice) {
+        String value = voice == null || voice.isBlank() ? DEFAULT_TTS_VOICE : voice.trim();
+        return value.contains(":") ? value : modelName + ":" + value;
+    }
+
+    private HttpRequest.Builder authorized(SpeechModel runtime, String path) {
+        HttpRequest.Builder builder = HttpRequest.newBuilder()
+            .uri(URI.create(normalizeBaseUrl(runtime.baseUrl()) + path))
+            .timeout(REQUEST_TIMEOUT);
+        if (runtime.apiKey() != null && !runtime.apiKey().isBlank()) {
+            builder.header("Authorization", "Bearer " + runtime.apiKey());
+        }
+        return builder;
+    }
+
+    private static HttpClient client() {
+        return HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build();
+    }
+
+    private static byte[] multipartBody(String boundary, String modelName, byte[] audio, String filename, String contentType) throws IOException {
+        // 客户端可控字段进 multipart 头之前必须去掉 CRLF/引号，防止外发请求头注入
+        String safeName = sanitizeHeaderValue(filename, "audio.webm");
+        String safeType = sanitizeHeaderValue(contentType, "application/octet-stream");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        writeAscii(out, "--" + boundary + "\r\n");
+        writeAscii(out, "Content-Disposition: form-data; name=\"model\"\r\n\r\n");
+        out.write(modelName.getBytes(StandardCharsets.UTF_8));
+        writeAscii(out, "\r\n--" + boundary + "\r\n");
+        writeAscii(out, "Content-Disposition: form-data; name=\"file\"; filename=\"" + safeName + "\"\r\n");
+        writeAscii(out, "Content-Type: " + safeType + "\r\n\r\n");
+        out.write(audio);
+        writeAscii(out, "\r\n--" + boundary + "--\r\n");
+        return out.toByteArray();
+    }
+
+    private static String sanitizeHeaderValue(String value, String fallback) {
+        if (value == null || value.isBlank()) {
+            return fallback;
+        }
+        String sanitized = value.replaceAll("[\\r\\n\"\\\\]", "").trim();
+        return sanitized.isBlank() ? fallback : sanitized;
+    }
+
+    private static void writeAscii(ByteArrayOutputStream out, String value) throws IOException {
+        out.write(value.getBytes(StandardCharsets.US_ASCII));
+    }
+
+    private static String normalizeBaseUrl(String baseUrl) {
+        String normalized = baseUrl == null ? "" : baseUrl.trim();
+        while (normalized.endsWith("/")) {
+            normalized = normalized.substring(0, normalized.length() - 1);
+        }
+        return normalized;
+    }
+
+    private static String truncate(String value) {
+        if (value == null || value.length() <= 240) {
+            return value;
+        }
+        return value.substring(0, 240);
+    }
+}