admin/prop-ai-hr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(aihr): add asr/tts speech gateway and mobile voice practice

Browse Source 
新增 /api/ai/asr(multipart转写)与 /api/ai/tts(mp3 base64 dataURL),走
OpenAI 兼容 audio 接口(硅基流动 SenseVoice/CosyVoice2,复用模型管理
配置,新增 asr/tts 类目);移动端对练页接语音输入(MediaRecorder)与
客户台词自动播报,未配置/失败降级文本。ASR 限 5MB、TTS 限 300 字,
multipart 头部字段去 CRLF 防注入。
This commit is contained in:
admin
2026-07-03 20:55:38 +08:00
parent 96c70a618f
commit e0490b28cb
5 changed files with 335 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrSpeechController.java
+66
View File
@@ -0,0 +1,66 @@
package org.dromara.aihr.controller;
import lombok.RequiredArgsConstructor;
import org.dromara.aihr.domain.AihrSpeechDto.AsrResponse;
import org.dromara.aihr.domain.AihrSpeechDto.TtsRequest;
import org.dromara.aihr.domain.AihrSpeechDto.TtsResponse;
import org.dromara.aihr.service.AihrSpeechService;
import org.dromara.common.core.domain.R;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.Base64;
/**
* 对练语音 API：ASR 语音输入 + TTS 客户语音播报。未配置语音模型时返回失败，前端降级为文本。
*/
@RequiredArgsConstructor
@RestController
@RequestMapping("/api/ai")
public class AihrSpeechController {
private static final String NOT_CONFIGURED_HINT = "语音模型未配置：请在「模型管理」启用 asr/tts 类目模型（如硅基流动 SenseVoice/CosyVoice2）";
private static final long MAX_ASR_BYTES = 5L * 1024 * 1024;
private static final int MAX_TTS_CHARS = 300;
private final AihrSpeechService speechService;
@PostMapping(value = "/asr", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public R<AsrResponse> asr(@RequestPart("file") MultipartFile file) throws IOException {
if (!speechService.asrConfigured()) {
return R.fail(NOT_CONFIGURED_HINT);
}
if (file == null || file.isEmpty()) {
return R.fail("请上传音频文件");
}
if (file.getSize() > MAX_ASR_BYTES) {
return R.fail("音频过大（上限 5MB），请缩短录音时长");
}
return speechService.transcribe(file.getBytes(), file.getOriginalFilename(), file.getContentType())
.map(text -> R.ok(new AsrResponse(text, "openai-compatible")))
.orElseGet(() -> R.fail("语音识别失败，请改用文字输入"));
}
@PostMapping("/tts")
public R<TtsResponse> tts(@RequestBody TtsRequest request) {
if (!speechService.ttsConfigured()) {
return R.fail(NOT_CONFIGURED_HINT);
}
String text = request == null || request.text() == null ? "" : request.text().trim();
if (text.isEmpty()) {
return R.fail("合成文本不能为空");
}
if (text.length() > MAX_TTS_CHARS) {
text = text.substring(0, MAX_TTS_CHARS);
}
return speechService.synthesize(text, request == null ? null : request.voice())
.map(audio -> R.ok(new TtsResponse("data:audio/mp3;base64," + Base64.getEncoder().encodeToString(audio), "openai-compatible")))
.orElseGet(() -> R.fail("语音合成失败，已降级为文本展示"));
}
}
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrSpeechDto.java
+16
View File
@@ -0,0 +1,16 @@
package org.dromara.aihr.domain;
public final class AihrSpeechDto {
private AihrSpeechDto() {
}
public record AsrResponse(String text, String source) {
}
public record TtsRequest(String text, String voice) {
}
public record TtsResponse(String audioUrl, String source) {
}
}
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java
+169
View File
@@ -0,0 +1,169 @@
package org.dromara.aihr.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.dromara.aihr.service.AihrModelSeedService.SpeechModel;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Optional;
import java.util.UUID;
/**
* ASR/TTS 网关：调用 OpenAI 兼容 audio 接口（硅基流动 SenseVoice/CosyVoice2 等）。
* 模型未配置或调用失败返回 empty，由前端降级为纯文本对练。
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class AihrSpeechService {
private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(15);
private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(60);
private static final String DEFAULT_TTS_VOICE = "anna";
private final AihrModelSeedService modelService;
private final ObjectMapper objectMapper;
public boolean asrConfigured() {
return modelService.speechModel("asr").isPresent();
}
public boolean ttsConfigured() {
return modelService.speechModel("tts").isPresent();
}
/**
* 语音转写：POST {base}/audio/transcriptions（multipart）。
*/
public Optional<String> transcribe(byte[] audio, String filename, String contentType) {
Optional<SpeechModel> model = modelService.speechModel("asr");
if (model.isEmpty() || audio == null || audio.length == 0) {
return Optional.empty();
}
SpeechModel runtime = model.get();
try {
String boundary = "----aihr-" + UUID.randomUUID();
byte[] body = multipartBody(boundary, runtime.modelName(), audio, filename, contentType);
HttpRequest httpRequest = authorized(runtime, "/audio/transcriptions")
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
.POST(HttpRequest.BodyPublishers.ofByteArray(body))
.build();
HttpResponse<String> response = client().send(httpRequest, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
log.warn("asr http {}: {}", response.statusCode(), truncate(response.body()));
return Optional.empty();
}
String text = objectMapper.readTree(response.body()).path("text").asText("");
return text.isBlank() ? Optional.empty() : Optional.of(text.trim());
} catch (Exception e) {
log.warn("asr call failed: {}", e.getMessage());
return Optional.empty();
}
}
/**
* 语音合成：POST {base}/audio/speech，返回 mp3 字节。
*/
public Optional<byte[]> synthesize(String text, String voice) {
Optional<SpeechModel> model = modelService.speechModel("tts");
if (model.isEmpty() || text == null || text.isBlank()) {
return Optional.empty();
}
SpeechModel runtime = model.get();
try {
ObjectNode body = objectMapper.createObjectNode();
body.put("model", runtime.modelName());
body.put("input", text.trim());
body.put("voice", resolveVoice(runtime.modelName(), voice));
body.put("response_format", "mp3");
HttpRequest httpRequest = authorized(runtime, "/audio/speech")
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
.build();
HttpResponse<byte[]> response = client().send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
log.warn("tts http {}: {}", response.statusCode(), truncate(new String(response.body(), StandardCharsets.UTF_8)));
return Optional.empty();
}
return response.body().length == 0 ? Optional.empty() : Optional.of(response.body());
} catch (Exception e) {
log.warn("tts call failed: {}", e.getMessage());
return Optional.empty();
}
}
/**
* 硅基流动 voice 格式为 "{model}:{voice}"；调用方只传短名（如 anna/粤语音色名）时自动补模型前缀。
*/
private static String resolveVoice(String modelName, String voice) {
String value = voice == null || voice.isBlank() ? DEFAULT_TTS_VOICE : voice.trim();
return value.contains(":") ? value : modelName + ":" + value;
}
private HttpRequest.Builder authorized(SpeechModel runtime, String path) {
HttpRequest.Builder builder = HttpRequest.newBuilder()
.uri(URI.create(normalizeBaseUrl(runtime.baseUrl()) + path))
.timeout(REQUEST_TIMEOUT);
if (runtime.apiKey() != null && !runtime.apiKey().isBlank()) {
builder.header("Authorization", "Bearer " + runtime.apiKey());
}
return builder;
}
private static HttpClient client() {
return HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build();
}
private static byte[] multipartBody(String boundary, String modelName, byte[] audio, String filename, String contentType) throws IOException {
// 客户端可控字段进 multipart 头之前必须去掉 CRLF/引号，防止外发请求头注入
String safeName = sanitizeHeaderValue(filename, "audio.webm");
String safeType = sanitizeHeaderValue(contentType, "application/octet-stream");
ByteArrayOutputStream out = new ByteArrayOutputStream();
writeAscii(out, "--" + boundary + "\r\n");
writeAscii(out, "Content-Disposition: form-data; name=\"model\"\r\n\r\n");
out.write(modelName.getBytes(StandardCharsets.UTF_8));
writeAscii(out, "\r\n--" + boundary + "\r\n");
writeAscii(out, "Content-Disposition: form-data; name=\"file\"; filename=\"" + safeName + "\"\r\n");
writeAscii(out, "Content-Type: " + safeType + "\r\n\r\n");
out.write(audio);
writeAscii(out, "\r\n--" + boundary + "--\r\n");
return out.toByteArray();
}
private static String sanitizeHeaderValue(String value, String fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
String sanitized = value.replaceAll("[\\r\\n\"\\\\]", "").trim();
return sanitized.isBlank() ? fallback : sanitized;
}
private static void writeAscii(ByteArrayOutputStream out, String value) throws IOException {
out.write(value.getBytes(StandardCharsets.US_ASCII));
}
private static String normalizeBaseUrl(String baseUrl) {
String normalized = baseUrl == null ? "" : baseUrl.trim();
while (normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private static String truncate(String value) {
if (value == null || value.length() <= 240) {
return value;
}
return value.substring(0, 240);
}
}
frontend/src/views/system/model/index.vue
+2
View File
@@ -207,6 +207,8 @@
<el-option label="对话 chat" value="chat" />
<el-option label="向量 vector" value="vector" />
<el-option label="重排 rerank" value="rerank" />
<el-option label="语音识别 asr" value="asr" />
<el-option label="语音合成 tts" value="tts" />
</el-select>
</el-form-item>
<el-form-item label="模型名称" required>
mobile/src/App.vue
+82
View File
@@ -208,6 +208,10 @@
</div>
<textarea v-model.trim="practiceDraft" rows="3" placeholder="输入你的回复" />
<div class="practice-actions">
<button type="button" :disabled="asrBusy" @click="toggleRecording">
{{ recording ? '停止录音' : asrBusy ? '识别中' : '语音输入' }}
</button>
<button type="button" @click="toggleVoice">{{ voiceEnabled ? '播报:开' : '播报:关' }}</button>
<button type="button" @click="fillPracticeReply">填入建议回复</button>
<button type="button" :disabled="practiceStatus === 'starting' || practiceStatus === 'submitting'" @click="submitMobilePractice">
{{ practiceStatus === 'submitting' ? '提交中' : '提交本轮' }}
@@ -498,6 +502,12 @@ const selectedReview = ref<PracticeReviewDetail | null>(null);
const reviewMessage = ref('');
const markingReviewed = ref(false);
const competencyProfile = ref<CompetencyProfile | null>(null);
const voiceEnabled = ref(true);
const recording = ref(false);
const asrBusy = ref(false);
let mediaRecorder: MediaRecorder | null = null;
let recordedChunks: Blob[] = [];
let customerAudio: HTMLAudioElement | null = null;
const syncPath = () => window.history.pushState({}, '', rolePaths[roleKey.value]);
const tap = (name: string) => {
@@ -531,10 +541,79 @@ const readApi = async <T,>(response: Response): Promise<T> => {
const apiHeaders = () => ({
'Content-Type': 'application/json',
...authOnlyHeaders()
});
// multipart 上传不能手动设置 Content-Type，浏览器需自动带 boundary
const authOnlyHeaders = () => ({
...(authToken.value ? { Authorization: `Bearer ${authToken.value}` } : {}),
...(clientId.value ? { clientid: clientId.value } : {})
});
const playCustomerVoice = async (text: string) => {
if (!voiceEnabled.value || !text) return;
try {
const data = await readApi<{ audioUrl: string }>(
await fetch('/dev-api/api/ai/tts', {
method: 'POST',
headers: apiHeaders(),
body: JSON.stringify({ text, voice: '' })
})
);
customerAudio?.pause();
customerAudio = new Audio(data.audioUrl);
void customerAudio.play();
} catch {
// TTS 未配置或失败时静默降级为纯文本
}
};
const toggleVoice = () => {
voiceEnabled.value = !voiceEnabled.value;
if (!voiceEnabled.value) customerAudio?.pause();
};
const toggleRecording = async () => {
if (recording.value) {
mediaRecorder?.stop();
return;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
recordedChunks = [];
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.ondataavailable = (event) => {
if (event.data.size > 0) recordedChunks.push(event.data);
};
mediaRecorder.onstop = () => {
stream.getTracks().forEach((track) => track.stop());
recording.value = false;
void transcribeRecording(new Blob(recordedChunks, { type: mediaRecorder?.mimeType || 'audio/webm' }));
};
mediaRecorder.start();
recording.value = true;
practiceMessage.value = '';
} catch {
practiceMessage.value = '无法访问麦克风，请改用文字输入';
}
};
const transcribeRecording = async (blob: Blob) => {
asrBusy.value = true;
try {
const form = new FormData();
form.append('file', blob, 'practice.webm');
const data = await readApi<{ text: string }>(
await fetch('/dev-api/api/ai/asr', { method: 'POST', headers: authOnlyHeaders(), body: form })
);
practiceDraft.value = data.text;
} catch (error) {
practiceMessage.value = error instanceof Error ? error.message : '语音识别失败，请改用文字输入';
} finally {
asrBusy.value = false;
}
};
const handlePrimaryAction = () => {
if (roleKey.value !== 'user') {
tap(workerRole.value.primary.cta);
@@ -550,6 +629,9 @@ const appendPracticeTurn = (role: PracticeRole, text: string) => {
coach: 'AI教练'
};
practiceTurns.value.push({ role, label: labels[role], text });
if (role === 'customer') {
void playCustomerVoice(text);
}
};
const fillPracticeReply = () => {