admin/prop-ai-hr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(speech): add qwen tts voice preferences

Browse Source
This commit is contained in:
admin
2026-07-24 01:54:37 +08:00
parent faa2a77e9a
commit 474d46e728
8 changed files with 523 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrModelSeedService.java
+39 -14
View File
@@ -58,6 +58,12 @@ public class AihrModelSeedService {
@Value("${aihr.ai-runtime.speech-enabled:${AIHR_AI_SPEECH_ENABLED:true}}")
private boolean speechEnabled;
@Value("${aihr.realtime-practice.endpoint:${AIHR_QWEN_REALTIME_ENDPOINT:}}")
private String qwenRealtimeEndpoint;
@Value("${aihr.realtime-practice.api-key:${AIHR_QWEN_REALTIME_API_KEY:}}")
private String qwenRealtimeApiKey;
public List<ProviderResponse> providers() {
List<ProviderResponse> rows = dbProviders();
return rows.isEmpty() ? seedProviders() : mergeProviders(rows);
@@ -267,7 +273,7 @@ public class AihrModelSeedService {
rs.getString("resolved_api_key")
), tenantId(), category);
return rows.stream()
.filter(model -> configured(model.baseUrl(), model.modelName(), model.apiKey()))
.filter(model -> configured(model.baseUrl(), model.modelName(), model.apiKey()) || qwenTtsConfigured(model))
.findFirst();
} catch (DataAccessException e) {
log.debug("aihr speech model db fallback（处理错误已隐藏）");
@@ -282,6 +288,16 @@ public class AihrModelSeedService {
public record SpeechModel(String providerCode, String modelName, String baseUrl, String apiKey) {
}
private boolean qwenTtsConfigured(SpeechModel model) {
return isDashScopeQwenTts(model) && !isBlank(qwenRealtimeEndpoint) && !isBlank(qwenRealtimeApiKey);
}
private static boolean isDashScopeQwenTts(SpeechModel model) {
return model != null
&& ("dashscope".equalsIgnoreCase(model.providerCode()) || "qianwen".equalsIgnoreCase(model.providerCode()))
&& "qwen-audio-3.0-tts-flash".equals(model.modelName());
}
/**
* 供其他模块(如三角色对练)复用的 chat 调用：模型未配置或调用失败返回 empty，由调用方决定兜底。
*/
@@ -354,19 +370,28 @@ public class AihrModelSeedService {
on p.tenant_id = c.tenant_id and p.provider_code = c.provider_code
where c.tenant_id = ?
order by case c.category when 'chat' then 0 when 'vector' then 1 when 'rerank' then 2 else 9 end, c.id asc
""", (rs, rowNum) -> new ConfigResponse(
rs.getLong("id"),
rs.getString("category"),
rs.getString("model_name"),
rs.getString("provider_code"),
normalizeModelDescribe(rs.getString("model_describe")),
rs.getObject("model_dimension", Integer.class),
rs.getString("model_show"),
rs.getString("api_host"),
rs.getInt("enabled") == 1,
!isBlank(rs.getString("resolved_api_key")),
configured(rs.getString("resolved_api_host"), rs.getString("model_name"), rs.getString("resolved_api_key"))
), tenantId());
""", (rs, rowNum) -> {
SpeechModel runtime = new SpeechModel(
rs.getString("provider_code"),
rs.getString("model_name"),
rs.getString("resolved_api_host"),
rs.getString("resolved_api_key")
);
boolean qwenTts = qwenTtsConfigured(runtime);
return new ConfigResponse(
rs.getLong("id"),
rs.getString("category"),
rs.getString("model_name"),
rs.getString("provider_code"),
normalizeModelDescribe(rs.getString("model_describe")),
rs.getObject("model_dimension", Integer.class),
rs.getString("model_show"),
rs.getString("api_host"),
rs.getInt("enabled") == 1,
qwenTts || !isBlank(runtime.apiKey()),
qwenTts || configured(runtime.baseUrl(), runtime.modelName(), runtime.apiKey())
);
}, tenantId());
} catch (DataAccessException e) {
log.debug("aihr model config db fallback（处理错误已隐藏）");
return List.of();
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrSpeechService.java
+165 -6
View File
@@ -6,6 +6,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.dromara.aihr.domain.AihrSpeechDto.VoiceProfile;
import org.dromara.aihr.service.AihrModelSeedService.SpeechModel;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
@@ -33,7 +34,10 @@ public class AihrSpeechService {
private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(15);
private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(5);
private static final Duration DASHSCOPE_TTS_TIMEOUT = Duration.ofSeconds(20);
private static final String DEFAULT_TTS_VOICE = "anna";
private static final String QWEN_TTS_MODEL = "qwen-audio-3.0-tts-flash";
private static final String QWEN_TTS_DEFAULT_VOICE = "longanhuan_v3.6";
private static final Pattern SAFE_VOICE = Pattern.compile("[A-Za-z0-9_./:-]{1,200}");
private static final Map<String, String> ROLE_VOICES = Map.of(
"mentor", "speech:shifu-warm-v1:cm3hz4wfz02jy106j6z6muix7:ysvbyzypjgmnceokxedr",
@@ -47,15 +51,48 @@ public class AihrSpeechService {
"intense", "请用情绪强烈、急切、有压力且明显不满的语气说",
"calm", "请用平静、自然的语气说"
);
private static final Map<String, String> DIALECT_PROMPTS = Map.of(
"mandarin", "请使用自然普通话表达",
"cantonese", "请使用自然粤语口语表达",
"sichuanese", "请使用自然四川话口语表达"
private static final Map<String, String> DIALECT_PROMPTS = Map.ofEntries(
Map.entry("mandarin", "请使用自然普通话表达"),
Map.entry("cantonese", "请使用自然粤语口语表达"),
Map.entry("chongqing", "请使用自然重庆话口语表达"),
Map.entry("northeastern", "请使用自然东北话口语表达"),
Map.entry("gansu", "请使用自然甘肃话口语表达"),
Map.entry("guizhou", "请使用自然贵州话口语表达"),
Map.entry("zhejiang", "请使用自然浙江话口语表达"),
Map.entry("hebei", "请使用自然河北话口语表达"),
Map.entry("henan", "请使用自然河南话口语表达"),
Map.entry("hubei", "请使用自然湖北话口语表达"),
Map.entry("hunan", "请使用自然湖南话口语表达"),
Map.entry("jiangxi", "请使用自然江西话口语表达"),
Map.entry("ningbo", "请使用自然宁波话口语表达"),
Map.entry("ningxia", "请使用自然宁夏话口语表达"),
Map.entry("qingdao", "请使用自然青岛话口语表达"),
Map.entry("shaanxi", "请使用自然陕西话口语表达"),
Map.entry("shanxi", "请使用自然山西话口语表达"),
Map.entry("shandong", "请使用自然山东话口语表达"),
Map.entry("shanghai", "请使用自然上海话口语表达"),
Map.entry("sichuanese", "请使用自然四川话口语表达"),
Map.entry("yunnan", "请使用自然云南话口语表达")
);
private static final Map<String, String> QWEN_TTS_ROLE_VOICES = Map.of(
"mentor", QWEN_TTS_DEFAULT_VOICE,
"customer", QWEN_TTS_DEFAULT_VOICE,
"interviewer", QWEN_TTS_DEFAULT_VOICE,
"neutral", QWEN_TTS_DEFAULT_VOICE
);
private static final java.util.Set<String> QWEN_TTS_VOICES = java.util.Set.of(
"longanhuan_v3.6", "longjielidou_v3.6", "loongeva_v3.6", "loongjohn"
);
private final AihrModelSeedService modelService;
private final ObjectMapper objectMapper;
@Value("${aihr.realtime-practice.endpoint:${AIHR_QWEN_REALTIME_ENDPOINT:}}")
private String qwenRealtimeEndpoint;
@Value("${aihr.realtime-practice.api-key:${AIHR_QWEN_REALTIME_API_KEY:}}")
private String qwenRealtimeApiKey;
public boolean asrConfigured() {
return modelService.speechModel("asr").isPresent();
}
@@ -71,7 +108,7 @@ public class AihrSpeechService {
if (rawDialect.isEmpty() || DIALECT_PROMPTS.containsKey(rawDialect.toLowerCase(Locale.ROOT))) {
return Optional.empty();
}
return Optional.of("dialect 仅支持 mandarin、cantonese、sichuanese");
return Optional.of("dialect 不受支持");
}
/**
@@ -117,6 +154,9 @@ public class AihrSpeechService {
}
SpeechModel runtime = model.get();
try {
if (isDashScopeQwenTts(runtime)) {
return synthesizeDashScopeQwen(runtime, text, voice, voiceProfile);
}
ObjectNode body = buildRequestBody(objectMapper, runtime, text, voice, voiceProfile);
HttpRequest httpRequest = authorized(runtime, "/audio/speech")
.header("Content-Type", "application/json")
@@ -134,6 +174,38 @@ public class AihrSpeechService {
}
}
private Optional<byte[]> synthesizeDashScopeQwen(SpeechModel runtime, String text, String voice, VoiceProfile voiceProfile) {
if (isBlank(qwenRealtimeEndpoint) || isBlank(qwenRealtimeApiKey)) {
return Optional.empty();
}
try {
ObjectNode body = buildDashScopeRequestBody(objectMapper, runtime, text, voiceProfile == null
? new VoiceProfile("neutral", voice, null, null, null)
: withLegacyVoice(voiceProfile, voice));
HttpRequest request = HttpRequest.newBuilder()
.uri(dashScopeSynthesisUri(qwenRealtimeEndpoint))
.timeout(DASHSCOPE_TTS_TIMEOUT)
.header("Content-Type", "application/json")
.header("Authorization", "Bearer " + qwenRealtimeApiKey.trim())
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
.build();
HttpResponse<String> response = client().send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
log.warn("dashscope tts http {}（外部响应体已隐藏）", response.statusCode());
return Optional.empty();
}
Optional<String> audioUrl = dashScopeAudioUrl(objectMapper, response.body());
if (audioUrl.isEmpty()) {
log.warn("dashscope tts response missing audio url");
return Optional.empty();
}
return downloadDashScopeAudio(audioUrl.get());
} catch (Exception e) {
log.warn("dashscope tts call failed（处理错误已隐藏）");
return Optional.empty();
}
}
static ObjectNode buildRequestBody(ObjectMapper objectMapper, SpeechModel runtime, String text,
String legacyVoice, VoiceProfile voiceProfile) {
String role = normalizedRole(voiceProfile == null ? null : voiceProfile.role());
@@ -142,6 +214,9 @@ public class AihrSpeechService {
if (requestedVoice == null) {
requestedVoice = safeVoice(legacyVoice);
}
if (requestedVoice != null && QWEN_TTS_VOICES.contains(requestedVoice)) {
requestedVoice = null;
}
if (requestedVoice == null && expressiveCosyVoice) {
requestedVoice = ROLE_VOICES.get(role);
}
@@ -165,6 +240,80 @@ public class AihrSpeechService {
return body;
}
static ObjectNode buildDashScopeRequestBody(ObjectMapper objectMapper, SpeechModel runtime, String text,
VoiceProfile voiceProfile) {
String role = normalizedRole(voiceProfile == null ? null : voiceProfile.role());
String emotion = normalizedEmotion(voiceProfile == null ? null : voiceProfile.emotion(), role);
String dialect = normalizedDialect(voiceProfile == null ? null : voiceProfile.dialect());
String requestedVoice = safeVoice(voiceProfile == null ? null : voiceProfile.voice());
String voice = requestedVoice != null && QWEN_TTS_VOICES.contains(requestedVoice)
? requestedVoice
: QWEN_TTS_ROLE_VOICES.getOrDefault(role, QWEN_TTS_DEFAULT_VOICE);
ObjectNode input = objectMapper.createObjectNode();
input.put("text", AihrSensitiveText.forModel(text.trim()));
input.put("voice", voice);
input.put("format", "mp3");
input.put("sample_rate", 24000);
input.put("rate", resolveSpeed(voiceProfile == null ? null : voiceProfile.speed(), emotion));
String instruction = expressivePrompt(emotion, dialect);
if (!instruction.isBlank()) {
input.put("instruction", instruction);
}
ObjectNode body = objectMapper.createObjectNode();
body.put("model", runtime == null || isBlank(runtime.modelName()) ? QWEN_TTS_MODEL : runtime.modelName());
body.set("input", input);
return body;
}
static URI dashScopeSynthesisUri(String endpoint) {
URI configured = URI.create(endpoint == null ? "" : endpoint.trim());
String scheme = configured.getScheme();
String authority = configured.getRawAuthority();
if (("https".equalsIgnoreCase(scheme) || "http".equalsIgnoreCase(scheme)) && authority != null && !authority.isBlank()) {
return URI.create(scheme + "://" + authority + "/api/v1/services/audio/tts/SpeechSynthesizer");
}
throw new IllegalArgumentException("invalid DashScope endpoint");
}
static Optional<String> dashScopeAudioUrl(ObjectMapper objectMapper, String responseBody) {
try {
String value = objectMapper.readTree(responseBody == null ? "" : responseBody)
.path("output").path("audio").path("url").asText("").trim();
URI uri = value.isBlank() ? null : URI.create(value);
if (uri == null || !("http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme()))
|| uri.getHost() == null || !uri.getHost().endsWith(".oss-cn-beijing.aliyuncs.com")) {
return Optional.empty();
}
return Optional.of(uri.toString());
} catch (Exception e) {
return Optional.empty();
}
}
private Optional<byte[]> downloadDashScopeAudio(String url) {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(DASHSCOPE_TTS_TIMEOUT)
.GET()
.build();
HttpResponse<byte[]> response = client().send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() < 200 || response.statusCode() >= 300 || response.body().length == 0) {
return Optional.empty();
}
return Optional.of(response.body());
} catch (Exception e) {
return Optional.empty();
}
}
private static VoiceProfile withLegacyVoice(VoiceProfile profile, String legacyVoice) {
if (safeVoice(profile.voice()) != null || safeVoice(legacyVoice) == null) {
return profile;
}
return new VoiceProfile(profile.role(), legacyVoice, profile.speed(), profile.emotion(), profile.dialect());
}
/**
* 硅基流动 voice 格式为 "{model}:{voice}"；调用方只传短名（如 anna/粤语音色名）时自动补模型前缀。
*/
@@ -196,7 +345,7 @@ public class AihrSpeechService {
if (value.isBlank() || DIALECT_PROMPTS.containsKey(value)) {
return value;
}
throw new IllegalArgumentException("dialect 仅支持 mandarin、cantonese、sichuanese");
throw new IllegalArgumentException("dialect 不受支持");
}
private static String expressivePrompt(String emotion, String dialect) {
@@ -236,6 +385,12 @@ public class AihrSpeechService {
&& runtime.modelName().toLowerCase(Locale.ROOT).contains("cosyvoice");
}
private static boolean isDashScopeQwenTts(SpeechModel runtime) {
return runtime != null
&& ("dashscope".equalsIgnoreCase(runtime.providerCode()) || "qianwen".equalsIgnoreCase(runtime.providerCode()))
&& QWEN_TTS_MODEL.equals(runtime.modelName());
}
private HttpRequest.Builder authorized(SpeechModel runtime, String path) {
HttpRequest.Builder builder = HttpRequest.newBuilder()
.uri(URI.create(normalizeBaseUrl(runtime.baseUrl()) + path))
@@ -286,6 +441,10 @@ public class AihrSpeechService {
return normalized;
}
private static boolean isBlank(String value) {
return value == null || value.isBlank();
}
private static String truncate(String value) {
if (value == null || value.length() <= 240) {
return value;
backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrSpeechServiceTest.java
+45
View File
@@ -32,6 +32,9 @@ class AihrSpeechServiceTest {
new VoiceProfile("customer", null, 9.0, "intense", "sichuanese"));
var legacy = AihrSpeechService.buildRequestBody(
mapper, runtime, "普通播报", "alex", null);
var qwenVoiceBeforeSwitch = AihrSpeechService.buildRequestBody(
mapper, runtime, "普通播报", null,
new VoiceProfile("neutral", "longanhuan_v3.6", null, null, null));
assertEquals("speech:shifu-warm-v1:cm3hz4wfz02jy106j6z6muix7:ysvbyzypjgmnceokxedr", mentor.path("voice").asText());
assertEquals(0.88, mentor.path("speed").asDouble());
@@ -41,6 +44,7 @@ class AihrSpeechServiceTest {
assertTrue(customer.path("input").asText().contains("情绪强烈"));
assertTrue(customer.path("input").asText().contains("四川话"));
assertEquals("FunAudioLLM/CosyVoice2-0.5B:alex", legacy.path("voice").asText());
assertEquals("FunAudioLLM/CosyVoice2-0.5B:anna", qwenVoiceBeforeSwitch.path("voice").asText());
assertFalse(legacy.has("speed"));
assertFalse(legacy.path("input").asText().contains("<|endofprompt|>"));
}
@@ -60,4 +64,45 @@ class AihrSpeechServiceTest {
assertThrows(IllegalArgumentException.class, () -> AihrSpeechService.buildRequestBody(
mapper, runtime, "测试", null, invalid));
}
@Test
void buildsDashScopeQwenTtsRequestAndNormalizesWorkspaceEndpoint() {
ObjectMapper mapper = new ObjectMapper();
SpeechModel runtime = new SpeechModel(
"dashscope",
"qwen-audio-3.0-tts-flash",
"",
""
);
var body = AihrSpeechService.buildDashScopeRequestBody(
mapper, runtime, "请先确认业主的诉求。",
new VoiceProfile("mentor", null, 0.88, "warm", "shanghai"));
assertEquals("qwen-audio-3.0-tts-flash", body.path("model").asText());
assertEquals("请先确认业主的诉求。", body.path("input").path("text").asText());
assertEquals("longanhuan_v3.6", body.path("input").path("voice").asText());
assertEquals("mp3", body.path("input").path("format").asText());
assertEquals(24000, body.path("input").path("sample_rate").asInt());
assertEquals(0.88, body.path("input").path("rate").asDouble());
assertTrue(body.path("input").path("instruction").asText().contains("上海话"));
assertEquals(
"https://ws-example.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer",
AihrSpeechService.dashScopeSynthesisUri("https://ws-example.cn-beijing.maas.aliyuncs.com/compatible-mode/v1").toString()
);
}
@Test
void extractsDashScopeAudioUrlOnlyWhenPresent() throws Exception {
ObjectMapper mapper = new ObjectMapper();
String response = """
{"output":{"audio":{"url":"http://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/audio.mp3?sig=ok"}}}
""";
assertEquals(
"http://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/audio.mp3?sig=ok",
AihrSpeechService.dashScopeAudioUrl(mapper, response).orElseThrow()
);
assertTrue(AihrSpeechService.dashScopeAudioUrl(mapper, "{\"output\":{}}").isEmpty());
}
}