feat(practice): server-owned realtime personas with knowledge tool calling
- AihrRealtimePersonaRegistry whitelist: owner-calm (default), owner-impatient, digital-mentor (search_knowledge); persona binds default voice - /realtime/session assembles session.update server-side (instructions/tools); browser forwards payload verbatim; Redis session record 30min with sliding renewal - /realtime/tools/invoke validates login identity + Redis session ownership + persona tool whitelist, then read-only searchAuthorized; per-account rate limits - source=realtime skips sop review and knowledge-gap persistence (Beta no-persist) - frontend: persona cards replace 56-voice picker; serialized tool-call chain with soft-landing fallback
This commit is contained in:
@@ -64,6 +64,7 @@
|
||||
- 候选面试复用 `/api/recruit/interview/**`；补充资料走认证的移动端接口，先写 `sys_oss`/MinIO，再写 `aihr_candidate_material`，状态仅为 `待审核/已通过/已驳回`，由管理端 `/recruit/interview` 审核。上传接口不得放入 class-level `@SaIgnore`。
|
||||
- 后端 API 对接先看 `docs/API_INTEGRATION.md`。三角色对练 `/turn`/`/finish` 使用 `AihrPracticeLlmService`，必须保留“未配置/失败回退 seed”的降级链。
|
||||
- 对练语音复用 `/api/ai/asr` 与 `/api/ai/tts`，TTS 保持旧 `voice` 兼容并支持角色音色。浏览器录音不可用或权限失败时，用 `audio/*` file input 继续走同一 ASR，不退回纯文字；新增 multipart 外发同样清洗 filename/contentType 的 CRLF。
|
||||
- 实时对练 Beta（WebRTC）的人设与工具声明只由服务端 `AihrRealtimePersonaRegistry` 白名单组装，浏览器原样转发 session.update，不得恢复前端自组 instructions 或音色 picker；工具执行按登录态 + Redis 会话归属 + 人设白名单三重校验，仅只读知识检索且 `source=realtime` 不落 SOP 评审/知识缺口表；实时 Beta 不计分、不持久化、仅 H5、需 APP 登录态，接口契约见 `docs/API_INTEGRATION.md`。
|
||||
- 后端业务代码不要塞进上游 `ruoyi-demo`；自有 API 放在 `backend/ruoyi-modules/ruoyi-aihr`，再接入 `ruoyi-admin`。
|
||||
- AI 面试、三角色对练、案例、SOP 和三端首页保持真实 API/模型优先与既有降级。知识检索以 MySQL 为事实源，Qdrant/vector/rerank 是可选增强；向量维度不一致必须经状态与重建入口处理，不得静默换成本地向量。
|
||||
- 批量资料和所有视频只走 `/api/knowledge/doc/upload-async` 异步队列；同步 `/api/knowledge/doc/upload` 仅供 SOP 单文件即时预览，禁止把重加工塞回请求线程。格式、大小、超时、OCR/视频解析和重试契约以 `docs/API_INTEGRATION.md` 为准；图片无可用视觉结果时保持「待处理」，视频关键帧不得退回纯 OCR。
|
||||
|
||||
+44
@@ -23,6 +23,11 @@ import org.dromara.aihr.domain.AihrPracticeDto.PracticeAssignmentBatchResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.PracticeAssignmentResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimePersonaOption;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RubricResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RubricAdminResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RubricEnabledRequest;
|
||||
@@ -41,6 +46,8 @@ import org.dromara.aihr.domain.AihrPracticeDto.TurnRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.TurnResponse;
|
||||
import org.dromara.aihr.service.AihrPracticeSeedService;
|
||||
import org.dromara.aihr.service.AihrRealtimePracticeService;
|
||||
import org.dromara.aihr.service.AihrRealtimeSessionService;
|
||||
import org.dromara.aihr.service.AihrRealtimeToolService;
|
||||
import org.dromara.common.core.constant.TenantConstants;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.core.domain.model.LoginUser;
|
||||
@@ -76,6 +83,8 @@ public class AihrPracticeController {
|
||||
|
||||
private final AihrPracticeSeedService practiceSeedService;
|
||||
private final AihrRealtimePracticeService realtimePracticeService;
|
||||
private final AihrRealtimeSessionService realtimeSessionService;
|
||||
private final AihrRealtimeToolService realtimeToolService;
|
||||
|
||||
@GetMapping("/scenarios")
|
||||
public R<List<ScenarioResponse>> scenarios(@RequestParam(required = false) String position,
|
||||
@@ -233,6 +242,41 @@ public class AihrPracticeController {
|
||||
return R.ok(realtimePracticeService.exchangeSdp(request, username));
|
||||
}
|
||||
|
||||
@GetMapping("/realtime/personas")
|
||||
public R<List<RealtimePersonaOption>> realtimePersonas() {
|
||||
String username = currentAppUsername();
|
||||
if (username.isBlank()) {
|
||||
return R.fail("请使用员工端账号开启实时陪练");
|
||||
}
|
||||
return R.ok(realtimeSessionService.listPersonas(username));
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a realtime session: the server assembles the session.update payload
|
||||
* (persona instructions, voice, tool declarations) and records ownership for tool invocation.
|
||||
*/
|
||||
@PostMapping("/realtime/session")
|
||||
public R<RealtimeSessionResponse> realtimeSession(@RequestBody(required = false) RealtimeSessionRequest request) {
|
||||
String username = currentAppUsername();
|
||||
if (username.isBlank()) {
|
||||
return R.fail("请使用员工端账号开启实时陪练");
|
||||
}
|
||||
return R.ok(realtimeSessionService.createSession(request, username, currentAppUserId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side execution bridge for model function calling. Identity comes from the login
|
||||
* token, session ownership and the tool whitelist from the Redis session record.
|
||||
*/
|
||||
@PostMapping("/realtime/tools/invoke")
|
||||
public R<RealtimeToolInvokeResponse> realtimeToolInvoke(@RequestBody RealtimeToolInvokeRequest request) {
|
||||
String username = currentAppUsername();
|
||||
if (username.isBlank()) {
|
||||
return R.fail("请使用员工端账号开启实时陪练");
|
||||
}
|
||||
return R.ok(realtimeToolService.invoke(request, username, currentAppUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/turn")
|
||||
public R<TurnResponse> turn(@RequestBody TurnRequest request) {
|
||||
return R.ok(practiceSeedService.turn(request, requirePracticeActorIdentity("继续训练"), currentAppUserId()));
|
||||
|
||||
+30
-1
@@ -1,6 +1,8 @@
|
||||
package org.dromara.aihr.domain;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public final class AihrPracticeDto {
|
||||
|
||||
@@ -209,12 +211,39 @@ public final class AihrPracticeDto {
|
||||
public record HelpEventResponse(String sessionId, String scenarioId, Integer roundIndex, String recordedAt) {
|
||||
}
|
||||
|
||||
public record RealtimeSdpRequest(String offerSdp) {
|
||||
public record RealtimeSdpRequest(String offerSdp, String sessionId) {
|
||||
public RealtimeSdpRequest(String offerSdp) {
|
||||
this(offerSdp, null);
|
||||
}
|
||||
}
|
||||
|
||||
public record RealtimeSdpResponse(String answerSdp, String model) {
|
||||
}
|
||||
|
||||
public record RealtimePersonaOption(String id, String name, String description, boolean hasTools) {
|
||||
}
|
||||
|
||||
public record RealtimeSessionRequest(String personaId, String scenarioId) {
|
||||
}
|
||||
|
||||
public record RealtimeSessionResponse(String sessionId, String personaId, String personaName,
|
||||
boolean hasTools, Map<String, Object> sessionUpdate) {
|
||||
}
|
||||
|
||||
public record RealtimeToolInvokeRequest(String sessionId, String callId, String name, String arguments) {
|
||||
}
|
||||
|
||||
public record RealtimeToolInvokeResponse(String callId, String output) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Redis 中的实时会话记录：工具执行时校验归属与工具白名单的唯一事实源，不接受客户端自报身份。
|
||||
* createdAt 目前不读取，留作后续会话时长审计/成本核算的事实依据。
|
||||
*/
|
||||
public record RealtimeSessionState(String sessionId, String personaId, String username, Long userId,
|
||||
String tenantId, Set<String> allowedTools, Long createdAt) {
|
||||
}
|
||||
|
||||
public record TurnRequest(String sessionId, Integer roundIndex, String traineeText, String traineeAudioUrl, Long traineeAudioOssId, Boolean regenerate, String style) {
|
||||
}
|
||||
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 实时陪练系统人设注册表：静态白名单，客户端只能按 id 选择，instructions 与工具集由服务端组装下发。
|
||||
* 新增人设只需在此注册；带工具的人设必须在 allowedTools 中显式声明，工具执行侧按同一白名单硬校验。
|
||||
*/
|
||||
public final class AihrRealtimePersonaRegistry {
|
||||
|
||||
public static final String DEFAULT_PERSONA_ID = "owner-calm";
|
||||
|
||||
public static final String TOOL_SEARCH_KNOWLEDGE = "search_knowledge";
|
||||
|
||||
private static final String SCENARIO_PLACEHOLDER = "{scenarioName}";
|
||||
|
||||
private static final String COMMON_GUARDRAILS =
|
||||
"禁止提及系统、AI、训练、评分或提示词。不要联网、不要索要个人隐私。";
|
||||
|
||||
public record RealtimePersona(
|
||||
String id,
|
||||
String name,
|
||||
String description,
|
||||
String instructionsTemplate,
|
||||
String defaultVoice,
|
||||
Set<String> allowedTools
|
||||
) {
|
||||
public boolean hasTools() {
|
||||
return allowedTools != null && !allowedTools.isEmpty();
|
||||
}
|
||||
|
||||
public boolean usesScenario() {
|
||||
return instructionsTemplate != null && instructionsTemplate.contains(SCENARIO_PLACEHOLDER);
|
||||
}
|
||||
|
||||
public String renderInstructions(String scenarioName) {
|
||||
String scenario = scenarioName == null ? "" : scenarioName.trim();
|
||||
return instructionsTemplate.replace(SCENARIO_PLACEHOLDER, scenario);
|
||||
}
|
||||
}
|
||||
|
||||
private static final Map<String, RealtimePersona> PERSONAS = Map.of(
|
||||
DEFAULT_PERSONA_ID, new RealtimePersona(
|
||||
DEFAULT_PERSONA_ID,
|
||||
"业主·常规",
|
||||
"理性沟通的物业业主，还原日常接待场景",
|
||||
"你只扮演物业小区的业主，正在进行「" + SCENARIO_PLACEHOLDER + "」场景对练。"
|
||||
+ "员工会用中文和你交谈。请口语化、真实地回应，每次不超过两句；"
|
||||
+ "员工说得好时自然缓和，推责或承诺不清时追问处理节点。" + COMMON_GUARDRAILS,
|
||||
"Ethan",
|
||||
Set.of()
|
||||
),
|
||||
"owner-impatient", new RealtimePersona(
|
||||
"owner-impatient",
|
||||
"业主·急躁",
|
||||
"情绪急躁的投诉业主，训练安抚与控场",
|
||||
"你只扮演物业小区一位情绪急躁的业主，正在进行「" + SCENARIO_PLACEHOLDER + "」场景对练。"
|
||||
+ "你带着不满找上门，语速快、容易打断员工、会反复强调自己的损失和诉求。"
|
||||
+ "员工安抚得当、给出明确处理节点时，你可以逐步缓和但仍保持较真；"
|
||||
+ "员工推责、敷衍或承诺不清时，你要升级情绪并追问到底。"
|
||||
+ "请口语化、真实地回应，每次不超过两句。" + COMMON_GUARDRAILS,
|
||||
"Ryan",
|
||||
Set.of()
|
||||
),
|
||||
"digital-mentor", new RealtimePersona(
|
||||
"digital-mentor",
|
||||
"数字师傅",
|
||||
"可实时查询知识库的带教师傅，有问必答",
|
||||
"你是物业一线员工的数字师傅，经验丰富、说话接地气。员工会用中文向你请教工作上的问题。"
|
||||
+ "请口语化回答，每次不超过三句，先给结论再给理由。"
|
||||
+ "当员工询问物业规定、操作流程、SOP 类问题时，先调用 search_knowledge 工具查询，"
|
||||
+ "再按查询结果用自己的话转述；查不到时如实说不确定，不要编造。" + COMMON_GUARDRAILS,
|
||||
"Andre",
|
||||
Set.of(TOOL_SEARCH_KNOWLEDGE)
|
||||
)
|
||||
);
|
||||
|
||||
private AihrRealtimePersonaRegistry() {
|
||||
}
|
||||
|
||||
public static RealtimePersona require(String personaId) {
|
||||
RealtimePersona persona = PERSONAS.get(personaId == null ? "" : personaId.trim());
|
||||
if (persona == null) {
|
||||
throw new ServiceException("实时陪练人设不存在或未开放");
|
||||
}
|
||||
return persona;
|
||||
}
|
||||
|
||||
public static RealtimePersona defaultPersona() {
|
||||
return PERSONAS.get(DEFAULT_PERSONA_ID);
|
||||
}
|
||||
|
||||
public static List<RealtimePersona> listPublic() {
|
||||
return List.copyOf(PERSONAS.values());
|
||||
}
|
||||
}
|
||||
+5
@@ -59,6 +59,11 @@ public class AihrRealtimePracticeService {
|
||||
throw new ServiceException("请使用员工端账号开启实时陪练");
|
||||
}
|
||||
enforceRateLimit(account);
|
||||
String sessionId = request.sessionId() == null ? "" : request.sessionId().trim();
|
||||
// sessionId 由客户端回传，仅用于日志关联——限长限字符防日志注入
|
||||
if (sessionId.matches("[A-Za-z0-9_-]{1,64}")) {
|
||||
log.info("realtime sdp exchange bound to session: sessionId={}", sessionId);
|
||||
}
|
||||
|
||||
URI signalingUri = signalingUri(endpoint, model);
|
||||
if (apiKey == null || apiKey.isBlank()) {
|
||||
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimePersonaOption;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionState;
|
||||
import org.dromara.aihr.service.AihrRealtimePersonaRegistry.RealtimePersona;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* 实时陪练会话创建：人设与工具声明的唯一组装点。
|
||||
* 浏览器只转发此处下发的 session.update 载荷，instructions/tools 不接受客户端自组。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AihrRealtimeSessionService {
|
||||
|
||||
private static final String FALLBACK_SCENARIO_NAME = "日常接待";
|
||||
private static final String TRANSCRIPTION_MODEL = "qwen3-asr-flash-realtime";
|
||||
private static final int SESSION_CREATE_RATE_LIMIT = 6;
|
||||
private static final int SESSION_CREATE_RATE_INTERVAL_SECONDS = 60;
|
||||
|
||||
private final AihrPracticeSeedService practiceSeedService;
|
||||
private final AihrRealtimeSessionStore sessionStore;
|
||||
|
||||
public List<RealtimePersonaOption> listPersonas(String username) {
|
||||
String key = "aihr:practice:realtime:personas:" + LoginHelper.getTenantId() + ":" + username;
|
||||
enforceRateLimit(key, 30, "实时陪练人设查询过于频繁，请稍后重试");
|
||||
return AihrRealtimePersonaRegistry.listPublic().stream()
|
||||
.map(persona -> new RealtimePersonaOption(persona.id(), persona.name(), persona.description(), persona.hasTools()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
public RealtimeSessionResponse createSession(RealtimeSessionRequest request, String username, Long userId) {
|
||||
String rateKey = "aihr:practice:realtime:session-create:" + LoginHelper.getTenantId() + ":" + username;
|
||||
enforceRateLimit(rateKey, SESSION_CREATE_RATE_LIMIT, "实时陪练开启过于频繁，请稍后重试");
|
||||
String personaId = request == null || StringUtils.isBlank(request.personaId())
|
||||
? AihrRealtimePersonaRegistry.DEFAULT_PERSONA_ID
|
||||
: request.personaId().trim();
|
||||
RealtimePersona persona = AihrRealtimePersonaRegistry.require(personaId);
|
||||
String scenarioName = resolveScenarioName(persona, request == null ? null : request.scenarioId());
|
||||
|
||||
String sessionId = "rt_" + UUID.randomUUID().toString().replace("-", "");
|
||||
Map<String, Object> sessionUpdate = buildSessionUpdate(persona, scenarioName);
|
||||
|
||||
String tenantId = LoginHelper.getTenantId();
|
||||
RealtimeSessionState state = new RealtimeSessionState(
|
||||
sessionId, persona.id(), username, userId, tenantId, persona.allowedTools(), System.currentTimeMillis());
|
||||
sessionStore.save(state);
|
||||
log.info("realtime session created: persona={}, hasTools={}, usesScenario={}",
|
||||
persona.id(), persona.hasTools(), persona.usesScenario());
|
||||
return new RealtimeSessionResponse(sessionId, persona.id(), persona.name(), persona.hasTools(), sessionUpdate);
|
||||
}
|
||||
|
||||
private void enforceRateLimit(String key, int limit, String exceededMessage) {
|
||||
try {
|
||||
if (sessionStore.rateLimit(key, limit, SESSION_CREATE_RATE_INTERVAL_SECONDS) < 0) {
|
||||
throw new ServiceException(exceededMessage, 429);
|
||||
}
|
||||
} catch (ServiceException known) {
|
||||
throw known;
|
||||
} catch (RuntimeException unavailable) {
|
||||
log.warn("realtime session rate limiter unavailable");
|
||||
throw new ServiceException("实时陪练限流暂时不可用，请稍后重试", 503);
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveScenarioName(RealtimePersona persona, String scenarioId) {
|
||||
if (!persona.usesScenario()) {
|
||||
return "";
|
||||
}
|
||||
if (StringUtils.isBlank(scenarioId)) {
|
||||
return FALLBACK_SCENARIO_NAME;
|
||||
}
|
||||
String name = practiceSeedService.scenario(scenarioId.trim()).name();
|
||||
return StringUtils.isBlank(name) ? FALLBACK_SCENARIO_NAME : name.trim();
|
||||
}
|
||||
|
||||
private Map<String, Object> buildSessionUpdate(RealtimePersona persona, String scenarioName) {
|
||||
Map<String, Object> session = new LinkedHashMap<>();
|
||||
session.put("modalities", List.of("text", "audio"));
|
||||
session.put("voice", persona.defaultVoice());
|
||||
session.put("input_audio_format", "pcm");
|
||||
session.put("output_audio_format", "pcm");
|
||||
session.put("input_audio_transcription", Map.of("model", TRANSCRIPTION_MODEL));
|
||||
session.put("instructions", persona.renderInstructions(scenarioName));
|
||||
Map<String, Object> turnDetection = new LinkedHashMap<>();
|
||||
turnDetection.put("type", "semantic_vad");
|
||||
turnDetection.put("threshold", 0.5);
|
||||
turnDetection.put("prefix_padding_ms", 500);
|
||||
turnDetection.put("silence_duration_ms", 800);
|
||||
session.put("turn_detection", turnDetection);
|
||||
if (persona.hasTools()) {
|
||||
session.put("tools", List.of(searchKnowledgeToolDeclaration()));
|
||||
}
|
||||
Map<String, Object> event = new LinkedHashMap<>();
|
||||
event.put("type", "session.update");
|
||||
event.put("session", session);
|
||||
return event;
|
||||
}
|
||||
|
||||
/**
|
||||
* qwen realtime 工具声明为嵌套格式：{type:'function', function:{name,description,parameters}}。
|
||||
*/
|
||||
private Map<String, Object> searchKnowledgeToolDeclaration() {
|
||||
Map<String, Object> query = new LinkedHashMap<>();
|
||||
query.put("type", "string");
|
||||
query.put("description", "要检索的物业知识库问题");
|
||||
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||
parameters.put("type", "object");
|
||||
parameters.put("properties", Map.of("query", query));
|
||||
parameters.put("required", List.of("query"));
|
||||
Map<String, Object> function = new LinkedHashMap<>();
|
||||
function.put("name", AihrRealtimePersonaRegistry.TOOL_SEARCH_KNOWLEDGE);
|
||||
function.put("description", "查询物业知识库中的规定、操作流程与 SOP，员工请教专业问题时使用");
|
||||
function.put("parameters", parameters);
|
||||
return Map.of("type", "function", "function", function);
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionState;
|
||||
import org.dromara.common.redis.utils.RedisUtils;
|
||||
import org.redisson.api.RateType;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 实时陪练会话的 Redis 存取与限流的唯一出口。
|
||||
* 独立成组件是为了让会话/工具服务在单元测试中可注入替身——RedisUtils 的静态初始化依赖
|
||||
* Spring 上下文，无法在纯单测中加载。
|
||||
*/
|
||||
@Component
|
||||
public class AihrRealtimeSessionStore {
|
||||
|
||||
private static final Duration SESSION_TTL = Duration.ofMinutes(30);
|
||||
private static final String SESSION_KEY_PREFIX = "aihr:practice:realtime:session:";
|
||||
|
||||
public void save(RealtimeSessionState state) {
|
||||
RedisUtils.setCacheObject(key(state.tenantId(), state.sessionId()), state, SESSION_TTL);
|
||||
}
|
||||
|
||||
public RealtimeSessionState find(String tenantId, String sessionId) {
|
||||
return RedisUtils.getCacheObject(key(tenantId, sessionId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 会话仍活跃时滑动续期，避免长会话中途工具失效；不做无限续期，窗口仍以最近一次活动起算 30 分钟。
|
||||
*/
|
||||
public void touch(String tenantId, String sessionId) {
|
||||
RedisUtils.expire(key(tenantId, sessionId), SESSION_TTL);
|
||||
}
|
||||
|
||||
public long rateLimit(String key, int limit, int intervalSeconds) {
|
||||
return RedisUtils.rateLimiter(key, RateType.OVERALL, limit, intervalSeconds);
|
||||
}
|
||||
|
||||
static String key(String tenantId, String sessionId) {
|
||||
return SESSION_KEY_PREFIX + tenantId + ":" + sessionId;
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionState;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeResponse;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SearchRequest;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SearchResponse;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SnippetResponse;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeAppDto.AuthenticatedApp;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgeAccessService;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgeAppService;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 实时陪练工具执行桥：模型 function calling 的服务端落点。
|
||||
* 身份取登录态、会话归属取 Redis 记录、工具白名单取人设注册表，三者不接受客户端自报。
|
||||
* 只走浅层检索（searchAuthorized），不经过 queryInternal，避免 LLM 答案生成/审计/记忆副作用——
|
||||
* 语音应答由实时模型基于 output 自行组织。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AihrRealtimeToolService {
|
||||
|
||||
private static final int MAX_ARGUMENTS_CHARS = 4096;
|
||||
private static final int MAX_QUERY_CHARS = 200;
|
||||
private static final int MAX_OUTPUT_CHARS = 1200;
|
||||
private static final int SNIPPET_LIMIT = 3;
|
||||
private static final int TOOL_RATE_LIMIT = 20;
|
||||
private static final int TOOL_RATE_INTERVAL_SECONDS = 60;
|
||||
private static final String TOOL_RATE_KEY_PREFIX = "aihr:practice:realtime:tool:";
|
||||
|
||||
private static final String SOFT_PARAM_ERROR = "工具参数格式有误，请员工换个问法再试。";
|
||||
private static final String SOFT_NO_SPACE = "暂无可访问的知识库内容，请如实告知员工你不确定。";
|
||||
private static final String SOFT_NO_RESULT = "知识库中未找到相关内容，请如实告知员工你不确定。";
|
||||
private static final String SOFT_SEARCH_FAILED = "知识库查询暂时不可用，请如实告知员工稍后再试。";
|
||||
|
||||
private final AihrSopSeedService sopSeedService;
|
||||
private final AihrKnowledgePrincipalResolver principalResolver;
|
||||
private final AihrKnowledgeAccessService accessService;
|
||||
private final AihrKnowledgeAppService appService;
|
||||
private final AihrRealtimeSessionStore sessionStore;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public RealtimeToolInvokeResponse invoke(RealtimeToolInvokeRequest request, String username, Long userId) {
|
||||
String sessionId = request == null || request.sessionId() == null ? "" : request.sessionId().trim();
|
||||
String callId = request == null || request.callId() == null ? "" : request.callId().trim();
|
||||
String name = request == null || request.name() == null ? "" : request.name().trim();
|
||||
String arguments = request == null ? null : request.arguments();
|
||||
if (sessionId.isEmpty() || callId.isEmpty() || name.isEmpty()) {
|
||||
throw new ServiceException("实时陪练工具调用请求无效", 400);
|
||||
}
|
||||
if (arguments != null && arguments.length() > MAX_ARGUMENTS_CHARS) {
|
||||
throw new ServiceException("实时陪练工具调用请求无效", 400);
|
||||
}
|
||||
|
||||
String tenantId = LoginHelper.getTenantId();
|
||||
RealtimeSessionState state = sessionStore.find(tenantId, sessionId);
|
||||
if (state == null) {
|
||||
throw new ServiceException("实时陪练会话已过期，请重新开始", 410);
|
||||
}
|
||||
if (!java.util.Objects.equals(state.username(), username) || !java.util.Objects.equals(state.userId(), userId)) {
|
||||
throw new ServiceException("无权操作该实时陪练会话", 403);
|
||||
}
|
||||
if (!state.allowedTools().contains(name)) {
|
||||
throw new ServiceException("当前人设不支持该工具", 400);
|
||||
}
|
||||
enforceToolRateLimit(state);
|
||||
|
||||
String output = executeSearchKnowledge(arguments);
|
||||
// 会话仍活跃，滑动续期，避免长会话中途工具失效
|
||||
sessionStore.touch(state.tenantId(), state.sessionId());
|
||||
log.info("realtime tool invoked: sessionId={}, name={}", sessionId, name);
|
||||
return new RealtimeToolInvokeResponse(callId, output);
|
||||
}
|
||||
|
||||
private void enforceToolRateLimit(RealtimeSessionState state) {
|
||||
// 按账号而非会话限流：会话可反复创建，按 sessionId 限流会被会话扇出绕过
|
||||
String key = TOOL_RATE_KEY_PREFIX + state.tenantId() + ":" + state.username();
|
||||
try {
|
||||
if (sessionStore.rateLimit(key, TOOL_RATE_LIMIT, TOOL_RATE_INTERVAL_SECONDS) < 0) {
|
||||
throw new ServiceException("知识库查询过于频繁，请稍后重试", 429);
|
||||
}
|
||||
} catch (ServiceException known) {
|
||||
throw known;
|
||||
} catch (RuntimeException unavailable) {
|
||||
log.warn("realtime tool rate limiter unavailable");
|
||||
throw new ServiceException("实时陪练限流暂时不可用，请稍后重试", 503);
|
||||
}
|
||||
}
|
||||
|
||||
private String executeSearchKnowledge(String argumentsJson) {
|
||||
String query = parseQuery(argumentsJson);
|
||||
if (query == null) {
|
||||
return SOFT_PARAM_ERROR;
|
||||
}
|
||||
AihrKnowledgePrincipal principal = principalResolver.current();
|
||||
AuthenticatedApp app = appService.requireSessionApp(principal.tenantId(), principal.clientKey());
|
||||
Set<Long> spaceIds;
|
||||
try {
|
||||
spaceIds = accessService.resolveInternalSpaceIds(principal, app, List.of(), "READ");
|
||||
} catch (ServiceException forbidden) {
|
||||
// 员工无知识授权时 resolveInternalSpaceIds 抛 403 而非返回空集——软着陆让模型如实回答
|
||||
return SOFT_NO_SPACE;
|
||||
}
|
||||
if (spaceIds == null || spaceIds.isEmpty()) {
|
||||
return SOFT_NO_SPACE;
|
||||
}
|
||||
SearchResponse response;
|
||||
try {
|
||||
response = sopSeedService.searchAuthorized(
|
||||
new SearchRequest(query, null, null, "realtime", SNIPPET_LIMIT),
|
||||
principal.extPartyId(), spaceIds);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("realtime knowledge search failed（处理错误已隐藏）");
|
||||
return SOFT_SEARCH_FAILED;
|
||||
}
|
||||
List<SnippetResponse> snippets = response == null || response.snippets() == null
|
||||
? List.of() : response.snippets();
|
||||
if (snippets.isEmpty()) {
|
||||
return SOFT_NO_RESULT;
|
||||
}
|
||||
StringBuilder output = new StringBuilder();
|
||||
for (SnippetResponse snippet : snippets) {
|
||||
if (snippet == null || StringUtils.isBlank(snippet.text())) {
|
||||
continue;
|
||||
}
|
||||
if (output.length() > 0) {
|
||||
output.append('\n');
|
||||
}
|
||||
if (StringUtils.isNotBlank(snippet.title())) {
|
||||
output.append('「').append(snippet.title().trim()).append('」');
|
||||
}
|
||||
output.append(snippet.text().trim());
|
||||
}
|
||||
if (output.length() == 0) {
|
||||
return SOFT_NO_RESULT;
|
||||
}
|
||||
return output.length() > MAX_OUTPUT_CHARS
|
||||
? output.substring(0, MAX_OUTPUT_CHARS)
|
||||
: output.toString();
|
||||
}
|
||||
|
||||
private String parseQuery(String argumentsJson) {
|
||||
if (StringUtils.isBlank(argumentsJson)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JsonNode node = objectMapper.readTree(argumentsJson);
|
||||
String query = node.path("query").asText("").trim();
|
||||
if (query.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return query.length() > MAX_QUERY_CHARS ? query.substring(0, MAX_QUERY_CHARS) : query;
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -2422,7 +2422,8 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
private void recordKnowledgeGap(String queryText, String category, String position, String source) {
|
||||
if (isBlank(queryText)) {
|
||||
// realtime：实时对练 Beta 不持久化——语音会话衍生的未命中问题不写缺口表
|
||||
if (isBlank(queryText) || "realtime".equals(normalizeSearchSource(source))) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
@@ -2464,7 +2465,8 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
static boolean skipSopReview(String source) {
|
||||
return "verify_demo_questions".equals(source);
|
||||
// realtime：实时对练 Beta 不持久化——语音会话衍生的检索不进评审队列
|
||||
return "verify_demo_questions".equals(source) || "realtime".equals(source);
|
||||
}
|
||||
|
||||
static boolean shouldCreateSopReview(String queryText, String source) {
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import org.dromara.aihr.service.AihrRealtimePersonaRegistry.RealtimePersona;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Tag("dev")
|
||||
class AihrRealtimePersonaRegistryTest {
|
||||
|
||||
@Test
|
||||
void requireReturnsRegisteredPersona() {
|
||||
RealtimePersona persona = AihrRealtimePersonaRegistry.require("digital-mentor");
|
||||
assertNotNull(persona);
|
||||
assertEquals("数字师傅", persona.name());
|
||||
assertEquals(Set.of(AihrRealtimePersonaRegistry.TOOL_SEARCH_KNOWLEDGE), persona.allowedTools());
|
||||
assertTrue(persona.hasTools());
|
||||
}
|
||||
|
||||
@Test
|
||||
void requireRejectsUnknownPersonaId() {
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> AihrRealtimePersonaRegistry.require("no-such-persona"));
|
||||
assertEquals("实时陪练人设不存在或未开放", error.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultPersonaIsCalmOwnerWithoutTools() {
|
||||
RealtimePersona persona = AihrRealtimePersonaRegistry.defaultPersona();
|
||||
assertEquals(AihrRealtimePersonaRegistry.DEFAULT_PERSONA_ID, persona.id());
|
||||
assertFalse(persona.hasTools());
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyPersonaHasVoiceAndOnlyOwnersUseScenario() {
|
||||
for (RealtimePersona persona : AihrRealtimePersonaRegistry.listPublic()) {
|
||||
assertNotNull(persona.defaultVoice());
|
||||
assertFalse(persona.defaultVoice().isBlank(), persona.id() + " must declare a default voice");
|
||||
if (persona.id().startsWith("owner-")) {
|
||||
assertTrue(persona.usesScenario(), persona.id() + " should render scenario context");
|
||||
} else {
|
||||
assertFalse(persona.usesScenario(), persona.id() + " must not depend on scenarios");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void renderInstructionsFillsScenarioPlaceholder() {
|
||||
RealtimePersona owner = AihrRealtimePersonaRegistry.require("owner-calm");
|
||||
String rendered = owner.renderInstructions("投诉接待");
|
||||
assertTrue(rendered.contains("投诉接待"));
|
||||
assertFalse(rendered.contains("{scenarioName}"));
|
||||
}
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionResponse;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionState;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.ScenarioResponse;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@Tag("dev")
|
||||
class AihrRealtimeSessionServiceTest {
|
||||
|
||||
@Test
|
||||
void blankPersonaIdFallsBackToDefaultPersona() {
|
||||
AihrPracticeSeedService seedService = mock(AihrPracticeSeedService.class);
|
||||
AihrRealtimeSessionStore store = mock(AihrRealtimeSessionStore.class);
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(seedService, store);
|
||||
|
||||
try (MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class)) {
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
|
||||
RealtimeSessionResponse response = service.createSession(
|
||||
new RealtimeSessionRequest(null, null), "13900000000", 1L);
|
||||
|
||||
assertEquals(AihrRealtimePersonaRegistry.DEFAULT_PERSONA_ID, response.personaId());
|
||||
assertFalse(response.hasTools());
|
||||
assertTrue(response.sessionId().startsWith("rt_"));
|
||||
assertSessionUpdateShape(response.sessionUpdate());
|
||||
assertFalse(sessionOf(response).containsKey("tools"));
|
||||
|
||||
ArgumentCaptor<RealtimeSessionState> saved = ArgumentCaptor.forClass(RealtimeSessionState.class);
|
||||
verify(store).save(saved.capture());
|
||||
assertEquals("000000", saved.getValue().tenantId());
|
||||
assertEquals("13900000000", saved.getValue().username());
|
||||
assertEquals(1L, saved.getValue().userId());
|
||||
assertEquals(response.sessionId(), saved.getValue().sessionId());
|
||||
assertTrue(saved.getValue().allowedTools().isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownPersonaIdIsRejected() {
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(
|
||||
mock(AihrPracticeSeedService.class), mock(AihrRealtimeSessionStore.class));
|
||||
assertThrows(ServiceException.class,
|
||||
() -> service.createSession(new RealtimeSessionRequest("ghost", null), "13900000000", 1L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mentorPersonaDeclaresNestedSearchKnowledgeTool() {
|
||||
AihrPracticeSeedService seedService = mock(AihrPracticeSeedService.class);
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(
|
||||
seedService, mock(AihrRealtimeSessionStore.class));
|
||||
|
||||
try (MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class)) {
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
|
||||
RealtimeSessionResponse response = service.createSession(
|
||||
new RealtimeSessionRequest("digital-mentor", "complaint-water"), "13900000000", 1L);
|
||||
|
||||
assertTrue(response.hasTools());
|
||||
Map<String, Object> session = sessionOf(response);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Map<String, Object>> tools = (List<Map<String, Object>>) session.get("tools");
|
||||
assertEquals(1, tools.size());
|
||||
assertEquals("function", tools.get(0).get("type"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> function = (Map<String, Object>) tools.get(0).get("function");
|
||||
assertEquals("search_knowledge", function.get("name"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> parameters = (Map<String, Object>) function.get("parameters");
|
||||
assertEquals(List.of("query"), parameters.get("required"));
|
||||
String instructions = String.valueOf(session.get("instructions"));
|
||||
assertFalse(instructions.contains("{scenarioName}"));
|
||||
// 师傅人设不消费场景，不应触发场景查询
|
||||
verify(seedService, never()).scenario(any());
|
||||
verify(seedService, never()).scenario(any(), eq(false));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerPersonaFillsScenarioNameIntoInstructions() {
|
||||
AihrPracticeSeedService seedService = mock(AihrPracticeSeedService.class);
|
||||
ScenarioResponse scenario = mock(ScenarioResponse.class);
|
||||
when(scenario.name()).thenReturn("投诉接待");
|
||||
when(seedService.scenario("complaint-water")).thenReturn(scenario);
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(
|
||||
seedService, mock(AihrRealtimeSessionStore.class));
|
||||
|
||||
try (MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class)) {
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
|
||||
RealtimeSessionResponse response = service.createSession(
|
||||
new RealtimeSessionRequest("owner-impatient", "complaint-water"), "13900000000", 1L);
|
||||
|
||||
String instructions = String.valueOf(sessionOf(response).get("instructions"));
|
||||
assertTrue(instructions.contains("投诉接待"));
|
||||
assertFalse(instructions.contains("{scenarioName}"));
|
||||
assertFalse(sessionOf(response).containsKey("tools"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerPersonaWithoutScenarioIdUsesFallbackName() {
|
||||
AihrPracticeSeedService seedService = mock(AihrPracticeSeedService.class);
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(
|
||||
seedService, mock(AihrRealtimeSessionStore.class));
|
||||
|
||||
try (MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class)) {
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
|
||||
RealtimeSessionResponse response = service.createSession(
|
||||
new RealtimeSessionRequest("owner-calm", " "), "13900000000", 1L);
|
||||
|
||||
String instructions = String.valueOf(sessionOf(response).get("instructions"));
|
||||
assertFalse(instructions.contains("{scenarioName}"));
|
||||
assertFalse(instructions.contains("「」"));
|
||||
verify(seedService, never()).scenario(any());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void sessionCreationIsRateLimitedPerAccount() {
|
||||
AihrRealtimeSessionStore store = mock(AihrRealtimeSessionStore.class);
|
||||
when(store.rateLimit(any(), anyInt(), anyInt())).thenReturn(-1L);
|
||||
AihrRealtimeSessionService service = new AihrRealtimeSessionService(
|
||||
mock(AihrPracticeSeedService.class), store);
|
||||
|
||||
try (MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class)) {
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> service.createSession(new RealtimeSessionRequest(null, null), "13900000000", 1L));
|
||||
assertEquals(429, error.getCode());
|
||||
// 限流拦截在创建之前，不得写入任何会话记录
|
||||
verify(store, never()).save(any());
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertSessionUpdateShape(Map<String, Object> sessionUpdate) {
|
||||
assertEquals("session.update", sessionUpdate.get("type"));
|
||||
Map<String, Object> session = castSession(sessionUpdate);
|
||||
assertEquals(List.of("text", "audio"), session.get("modalities"));
|
||||
assertEquals("pcm", session.get("input_audio_format"));
|
||||
assertEquals("pcm", session.get("output_audio_format"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> transcription = (Map<String, Object>) session.get("input_audio_transcription");
|
||||
assertEquals("qwen3-asr-flash-realtime", transcription.get("model"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> turnDetection = (Map<String, Object>) session.get("turn_detection");
|
||||
assertEquals("semantic_vad", turnDetection.get("type"));
|
||||
assertFalse(session.containsKey("enable_search"));
|
||||
}
|
||||
|
||||
private static Map<String, Object> sessionOf(RealtimeSessionResponse response) {
|
||||
return castSession(response.sessionUpdate());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> castSession(Map<String, Object> sessionUpdate) {
|
||||
return (Map<String, Object>) sessionUpdate.get("session");
|
||||
}
|
||||
}
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSessionState;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeRequest;
|
||||
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeToolInvokeResponse;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SearchResponse;
|
||||
import org.dromara.aihr.domain.AihrSopDto.SnippetResponse;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgeAppDto.AuthenticatedApp;
|
||||
import org.dromara.aihr.knowledge.domain.AihrKnowledgePrincipal;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgeAccessService;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgeAppService;
|
||||
import org.dromara.aihr.knowledge.service.AihrKnowledgePrincipalResolver;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@Tag("dev")
|
||||
class AihrRealtimeToolServiceTest {
|
||||
|
||||
private static final String USERNAME = "13900000000";
|
||||
private static final Long USER_ID = 1L;
|
||||
private static final RealtimeSessionState MENTOR_SESSION = new RealtimeSessionState(
|
||||
"rt_abc", "digital-mentor", USERNAME, USER_ID, "000000",
|
||||
Set.of(AihrRealtimePersonaRegistry.TOOL_SEARCH_KNOWLEDGE), 0L);
|
||||
|
||||
private AihrSopSeedService sopSeedService;
|
||||
private AihrKnowledgePrincipalResolver principalResolver;
|
||||
private AihrKnowledgeAccessService accessService;
|
||||
private AihrKnowledgeAppService appService;
|
||||
private AihrRealtimeSessionStore sessionStore;
|
||||
private AihrRealtimeToolService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
sopSeedService = mock(AihrSopSeedService.class);
|
||||
principalResolver = mock(AihrKnowledgePrincipalResolver.class);
|
||||
accessService = mock(AihrKnowledgeAccessService.class);
|
||||
appService = mock(AihrKnowledgeAppService.class);
|
||||
sessionStore = mock(AihrRealtimeSessionStore.class);
|
||||
service = new AihrRealtimeToolService(
|
||||
sopSeedService, principalResolver, accessService, appService, sessionStore, new ObjectMapper());
|
||||
lenient().when(sessionStore.find("000000", "rt_abc")).thenReturn(MENTOR_SESSION);
|
||||
lenient().when(sessionStore.rateLimit(anyString(), anyInt(), anyInt())).thenReturn(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void expiredSessionReturns410() {
|
||||
when(sessionStore.find("000000", "rt_abc")).thenReturn(null);
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> service.invoke(request("not-json"), USERNAME, USER_ID));
|
||||
assertEquals(410, error.getCode());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void foreignSessionReturns403() {
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
assertThrows(ServiceException.class,
|
||||
() -> service.invoke(request("{}"), "other-user", USER_ID));
|
||||
assertThrows(ServiceException.class,
|
||||
() -> service.invoke(request("{}"), USERNAME, 99L));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void undeclaredToolIsRejectedToContainModelHallucination() {
|
||||
RealtimeSessionState ownerSession = new RealtimeSessionState(
|
||||
"rt_abc", "owner-calm", USERNAME, USER_ID, "000000", Set.of(), 0L);
|
||||
when(sessionStore.find("000000", "rt_abc")).thenReturn(ownerSession);
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
RealtimeToolInvokeRequest hallucinated = new RealtimeToolInvokeRequest(
|
||||
"rt_abc", "call_1", "delete_everything", "{}");
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> service.invoke(hallucinated, USERNAME, USER_ID));
|
||||
assertEquals(400, error.getCode());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidArgumentsJsonFallsBackToSoftOutput() {
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
RealtimeToolInvokeResponse response = service.invoke(request("not-json{{"), USERNAME, USER_ID);
|
||||
assertEquals("call_1", response.callId());
|
||||
assertTrue(response.output().contains("换个问法"));
|
||||
Mockito.verifyNoInteractions(sopSeedService);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void forbiddenKnowledgeAccessFallsBackToSoftOutput() {
|
||||
// 生产真实路径：员工无知识授权时 resolveInternalSpaceIds 抛 403，而不是返回空集
|
||||
AihrKnowledgePrincipal principal = new AihrKnowledgePrincipal(
|
||||
"000000", USER_ID, "app_user", USERNAME, Set.of("employee"), Set.of(), "app");
|
||||
when(principalResolver.current()).thenReturn(principal);
|
||||
when(appService.requireSessionApp("000000", "app")).thenReturn(mock(AuthenticatedApp.class));
|
||||
when(accessService.resolveInternalSpaceIds(eq(principal), any(), eq(List.of()), eq("READ")))
|
||||
.thenThrow(new ServiceException("当前请求没有可访问的知识空间", 403));
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
RealtimeToolInvokeResponse response = service.invoke(
|
||||
request("{\"query\":\"装修垃圾怎么清运\"}"), USERNAME, USER_ID);
|
||||
assertTrue(response.output().contains("暂无可访问"));
|
||||
verify(sopSeedService, never()).searchAuthorized(any(), any(), any());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchResultsAreJoinedAndTruncatedForSpeech() {
|
||||
stubPrincipalWithSpaces(Set.of(7L));
|
||||
SearchResponse search = mock(SearchResponse.class);
|
||||
when(search.snippets()).thenReturn(List.of(
|
||||
new SnippetResponse("装修管理规约", "装修垃圾需袋装化并堆放到指定点位，" + "长".repeat(1300), 1L),
|
||||
new SnippetResponse("清运流程", "联系物业预约清运时间", 2L)
|
||||
));
|
||||
when(sopSeedService.searchAuthorized(any(), eq(USERNAME), eq(Set.of(7L)))).thenReturn(search);
|
||||
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
RealtimeToolInvokeResponse response = service.invoke(
|
||||
request("{\"query\":\"装修垃圾\"}"), USERNAME, USER_ID);
|
||||
assertTrue(response.output().startsWith("「装修管理规约」"));
|
||||
assertEquals(1200, response.output().length());
|
||||
verify(sessionStore).touch("000000", "rt_abc");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptySearchResultAsksModelToAdmitUncertainty() {
|
||||
stubPrincipalWithSpaces(Set.of(7L));
|
||||
SearchResponse search = mock(SearchResponse.class);
|
||||
when(search.snippets()).thenReturn(List.of());
|
||||
when(sopSeedService.searchAuthorized(any(), any(), any())).thenReturn(search);
|
||||
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
RealtimeToolInvokeResponse response = service.invoke(
|
||||
request("{\"query\":\"不存在的规定\"}"), USERNAME, USER_ID);
|
||||
assertTrue(response.output().contains("未找到相关内容"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rateLimitExceededReturns429() {
|
||||
when(sessionStore.rateLimit(anyString(), anyInt(), anyInt())).thenReturn(-1L);
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> service.invoke(request("{\"query\":\"q\"}"), USERNAME, USER_ID));
|
||||
assertEquals(429, error.getCode());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rateLimiterOutageFailsClosed() {
|
||||
when(sessionStore.rateLimit(anyString(), anyInt(), anyInt()))
|
||||
.thenThrow(new RuntimeException("redis down"));
|
||||
try (MockedStatic<LoginHelper> login = mockTenant()) {
|
||||
ServiceException error = assertThrows(ServiceException.class,
|
||||
() -> service.invoke(request("{\"query\":\"q\"}"), USERNAME, USER_ID));
|
||||
assertEquals(503, error.getCode());
|
||||
}
|
||||
}
|
||||
|
||||
private void stubPrincipalWithSpaces(Set<Long> spaceIds) {
|
||||
AihrKnowledgePrincipal principal = new AihrKnowledgePrincipal(
|
||||
"000000", USER_ID, "app_user", USERNAME, Set.of("employee"), Set.of(), "app");
|
||||
when(principalResolver.current()).thenReturn(principal);
|
||||
when(appService.requireSessionApp("000000", "app")).thenReturn(mock(AuthenticatedApp.class));
|
||||
when(accessService.resolveInternalSpaceIds(eq(principal), any(), eq(List.of()), eq("READ")))
|
||||
.thenReturn(spaceIds);
|
||||
}
|
||||
|
||||
private static RealtimeToolInvokeRequest request(String arguments) {
|
||||
return new RealtimeToolInvokeRequest("rt_abc", "call_1",
|
||||
AihrRealtimePersonaRegistry.TOOL_SEARCH_KNOWLEDGE, arguments);
|
||||
}
|
||||
|
||||
private static MockedStatic<LoginHelper> mockTenant() {
|
||||
MockedStatic<LoginHelper> login = Mockito.mockStatic(LoginHelper.class);
|
||||
login.when(LoginHelper::getTenantId).thenReturn("000000");
|
||||
return login;
|
||||
}
|
||||
}
|
||||
+8
@@ -53,6 +53,14 @@ public class AihrSopSeedServiceTest {
|
||||
assertFalse(AihrSopSeedService.skipSopReview("knowledge_search"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void realtimeSourceSkipsPersistenceSideEffects() {
|
||||
// 实时对练 Beta 不持久化：检索不进评审队列
|
||||
assertTrue(AihrSopSeedService.skipSopReview("realtime"));
|
||||
assertFalse(AihrSopSeedService.shouldCreateSopReview("物业费怎么交", "realtime"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void emptySopQuestionCannotCreateReviewSample() {
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
| M2 训练预习与求助 | `GET /api/train/practice/scenarios/{id}/prep-card`、`POST /api/train/practice/sessions/{sessionId}/help` | 均要求已登录；预习卡只使用当前租户已启用的 `prep_card/json_prep` 模板，模型返回必须严格为 3 条要点、3 条红线、2 条话术，否则回退场景卡。求助接口只允许 APP 员工访问其本人、同租户的活动训练会话；客户端可传 `roundIndex` 仅为兼容字段，服务端按会话实际进度落库，返回 `{sessionId,scenarioId,roundIndex,recordedAt}`。迁移表为 `aihr_practice_help_event`，不在请求路径执行 DDL。该表已随 2026-07-24 完整包发布并纳入远端 `62/62` 结构预检；完整训练写入与正式试点仍需另验。 |
|
||||
| 正式试点数据导出 | `GET /api/train/practice/export?startDate=YYYY-MM-DD&endDate=YYYY-MM-DD` | 起止日期必填且包含结束日；只统计窗口内能通过唯一手机号或外部 ID 映射到在职组织快照的正式会话，排除重复手机号和身份碰撞。完训定义为每人至少 10 次，校准必须关联同一窗口内正式会话；CSV 同时给出校准命中数、SOP 可用数、满意度响应数/平均分，以及明细级 AI 分、人工校准分、校准人、校准时间、最终采用分、满意度分和意见，避免用四舍五入后的比率反推门禁状态；汇总和明细均携带正式人员及项目口径，不混入历史 seed/开发身份 |
|
||||
| 对练语音 | `POST /api/ai/asr`（multipart 字段 `file`，≤5MB）、`POST /api/ai/tts`（JSON `{text≤300字, voice?, voiceProfile?:{role,voice?,speed?,emotion?,dialect?}, practiceContext?:{sessionId,turnIndex,role:'customer'}}`，`dialect` 仅接受 `mandarin/cantonese/sichuanese`，非法值返回业务码 `400`；旧 `voice` 兼容；成功返回 `ossId`，客户端播放地址为受控的内联 `data:audio/*`，不返回原始 OSS URL）、`GET /api/aihr/mobile/oss/{ossId}` | 走 OpenAI-compatible audio 接口；生产 SiliconFlow CosyVoice2 按角色映射老师傅/业主/面试官音色，业主对练再按已有情绪分切换平静/严肃/强烈语气和语速；dialect 作为语气提示传给支持表现力提示词的 TTS，不含克隆，也不等同于方言正式验收。`practiceContext` 仅允许已认证 APP 员工把本次服务端生成的业主 TTS 绑定到本人当前或刚完成会话的指定原话术回合；服务端重新校验租户、员工身份、回合和文本，不接受员工端传 OSS ID 或任意角色。设备语音降级按方言选择语言：粤语使用 `zh-HK`，四川话无法由设备语音可靠模拟时明确提示用户转用服务端语音或文字，不伪装成普通话。模型管理需启用 `category=asr/tts` 配置；移动端优先用浏览器录音，`getUserMedia/MediaRecorder` 不可用或麦克风权限失败时，用 `audio/*` file input 选择/录制音频后继续调同一 ASR 接口；ASR/TTS 未配置或失败返回 fail，前端降级设备语音或文本；训练/每日题录音只通过 `audioOssId` 走受保护下载，历史客户端传入的 HTTP `audioUrl` 不再回显；TTS 成功音频仍写入 `sys_oss` 留痕，同时用内联 data URL 保持旧客户端可播放；移动端 OSS 下载只允许经本人或主管项目范围验证的 APP 身份，后台账号若需回放必须使用未来独立、受审计的管理端契约。 |
|
||||
| 实时对练 Beta（WebRTC） | `POST /api/train/practice/realtime/sdp`、`GET /realtime/personas`、`POST /realtime/session`、`POST /realtime/tools/invoke` | 浏览器 WebRTC 直连阿里云（音频不过服务端），后端仅代理 SDP 交换（3 次/分/账号，sessionId 仅作日志关联且限字符）。人设为服务端 `AihrRealtimePersonaRegistry` 白名单：`owner-calm` 业主·常规（默认）、`owner-impatient` 业主·急躁、`digital-mentor` 数字师傅；人设绑定默认音色，客户端只选不编。`session.update`（instructions/音色/turn_detection/tools 声明）由 `/realtime/session` 组装下发，浏览器原样转发；personaId 空白回落默认人设；会话记录写 Redis 30min（成功工具调用滑动续期），创建限流 6 次/分/账号。`digital-mentor` 声明 `search_knowledge` 工具：模型 function calling 命中（`response.function_call_arguments.done`）时浏览器回传 `/realtime/tools/invoke`，服务端按登录态 + Redis 会话归属 + 人设工具白名单三重校验后走 `searchAuthorized` 浅层检索（不生成 LLM 答案、不落 `aihr_agent_run`；`source=realtime` 跳过 SOP 评审与知识缺口写入），output 拼接 ≤3 条片段并截断 1200 字；工具按账号限流 20 次/分，无授权/无结果/超时一律返回软着陆文案让模型如实作答，不中断语音会话。实时 Beta 不计分、不持久化训练数据；仅 H5（App-Plus 不支持），需 APP 员工登录态；不启用模型 `enable_search`。配置 `AIHR_QWEN_REALTIME_ENDPOINT` / `AIHR_QWEN_REALTIME_API_KEY` / `AIHR_QWEN_REALTIME_MODEL`（模型白名单 `qwen3.5-omni-(flash|plus)-realtime`），密钥只放 `.env.local` 或外部环境变量。 |
|
||||
| 案例沉淀 `/knowledge/cases` | `GET /api/knowledge/case/capabilities`、`POST /api/knowledge/case/upload`、`/organize`、`/curate`、`GET /records`、`GET /records/{caseId}`、`POST /records/{caseId}/review`、`GET /records/{caseId}/media` | `/capabilities` 返回服务端判定的案例提交/查看能力，移动端不再向普通员工展示无权提交的素材表单；`/upload` 改为 multipart 真实语音上传并走 ASR，服务端只接受 MP3/WAV/M4A/WebM/OGG/AAC/FLAC，成功后原始音频写入 `sys_oss`，案例记录只保存 `mediaOssId` 供受保护媒体接口读取，不向客户端回传原始 `mediaUrl`；`/organize` 用真实转写调 chat 模型整理案例，未配置模型时按真实 transcript 本地结构化，并从背景外的真实摘要项提取学习点；APP 用户必须先以认证手机号精确校验在职 `person_phone`，再由可信主体解析 `aihr_org_snapshot` 项目范围；案例 SQL 不得把 `person_phone` 与 `ext_party_id` 作为替代查询键。上传、整理、入库、列表和详情均按该项目范围校验，未完成正式组织映射时安全拒绝，不接受前端伪造项目范围；员工列表/详情只返回 `已入库` 案例，管理端系统用户保留全局运营视图；移动端和管理端案例详情通过受保护媒体接口回放原始音频，主管/项目负责人可提交脱敏点评；预渲染视频样片仍待正式媒体资产接入 |
|
||||
| 案例媒体安全 | `GET /api/knowledge/case/records/{caseId}/media` | 案例详情只返回 `mediaOssId`，不返回原始 `sys_oss.url`；媒体下载会重复执行登录、后台角色或 APP 项目范围校验，再由服务端流式读取 OSS。管理端与 `mobile-uni` 通过鉴权 blob/temp 文件播放，关闭详情页时释放本地对象 URL |
|
||||
| 问·数字师傅工作 Agent `/pages/user/sop/index` | `POST /api/aihr/agent/messages`、`POST /api/aihr/agent/messages/media`、`POST /api/aihr/agent/actions/{draftId}/confirm`、`POST /api/aihr/agent/actions/{draftId}/dismiss` | 移动端文字、ASR 转写和图片/视频统一进入 Agent。服务端根据消息规划 `KNOWLEDGE_QA/RESOURCE_DELIVERY/LIVE_MY_WORK/LIVE_TEAM_WORK/PRACTICE_COACHING/CAPTURE_FACT/MEDIA_UNDERSTANDING/WEB_RESEARCH/CLARIFY/SOCIAL`，再由固定策略校验当前登录身份和工具权限；客户端不提交 `toolCode`，模型也不能自由指定身份、SQL、URL 或任意工具。知识、训练概况、当前待办、确认式记忆、媒体分析和全网查询均复用现有领域服务；普通图片问答只做本次视觉分析，不要求先具备知识空间，用户明确问现场制度/流程时才校验授权空间并补充 SOP。全网查询先返回 `NEEDS_INPUT`，只有用户明确同意后才调用外部服务。统一响应使用结构化 `status/sourceSummary/citations/resources/actionDraft/clarification`；按钮不从答案文本推断。写入动作只接受 30 分钟有效的不透明 `draftId`，最终确认继续复用既有版本、幂等与可见性规则。每次运行只审计租户、主体、意图、工具、状态、来源类型、耗时和错误码，不保存原始问题、答案或附件。 |
|
||||
|
||||
@@ -110,17 +110,26 @@
|
||||
>
|
||||
{{ status === 'finished' ? '再练一次' : '我已准备好，开始练习' }}
|
||||
</button>
|
||||
<view v-if="status === 'idle' || status === 'finished'" class="realtime-voice-picker">
|
||||
<view class="realtime-voice-heading">
|
||||
<text class="realtime-voice-title">实时陪练音色</text>
|
||||
<view v-if="(status === 'idle' || status === 'finished') && realtimePersonaOptions.length" class="realtime-persona-picker">
|
||||
<view class="realtime-persona-heading">
|
||||
<text class="realtime-persona-title">实时陪练人设</text>
|
||||
</view>
|
||||
<picker :range="realtimeVoiceOptions" range-key="label" :value="realtimeVoiceIndex" :disabled="busy" @change="selectRealtimeVoice">
|
||||
<view class="realtime-voice-select" aria-label="选择实时陪练音色">
|
||||
<text class="realtime-voice-value">{{ realtimeVoiceLabel }}</text>
|
||||
<text class="realtime-voice-arrow">›</text>
|
||||
<scroll-view class="scene-option-scroll" scroll-x>
|
||||
<view class="scene-option-list">
|
||||
<button
|
||||
v-for="item in realtimePersonaOptions"
|
||||
:key="item.id"
|
||||
class="scene-option"
|
||||
:class="{ selected: selectedRealtimePersonaId === item.id }"
|
||||
:disabled="busy"
|
||||
@click="selectRealtimePersona(item)"
|
||||
>
|
||||
<text class="scene-option-level">{{ item.hasTools ? '可查询知识库' : '纯对练' }}</text>
|
||||
<text class="scene-option-name">{{ item.name }}</text>
|
||||
<text class="scene-option-capability">{{ item.description }}</text>
|
||||
</button>
|
||||
</view>
|
||||
</picker>
|
||||
<text class="realtime-voice-hint">当前模型完整官方音色（56项）；下次开练生效</text>
|
||||
</scroll-view>
|
||||
</view>
|
||||
<button
|
||||
v-if="status === 'idle' || status === 'finished'"
|
||||
@@ -465,7 +474,7 @@ import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile
|
||||
import { startSpeechCapture } from '@/services/speech-capture';
|
||||
import type { SpeechCapture, SpeechCaptureSession } from '@/services/speech-capture';
|
||||
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
|
||||
import { createRealtimePracticeController, DEFAULT_REALTIME_PRACTICE_VOICE, isRealtimePracticeVoice, REALTIME_PRACTICE_VOICE_OPTIONS, type RealtimePracticeVoice } from '@/services/realtime-practice';
|
||||
import { createRealtimePracticeController, createRealtimeSession, fetchRealtimePersonas, invokeRealtimeTool, type RealtimePersonaOption } from '@/services/realtime-practice';
|
||||
import { stripInternalCodes } from '@/services/text';
|
||||
|
||||
type PracticeStatus = 'idle' | 'starting' | 'active' | 'submitting' | 'finished';
|
||||
@@ -482,9 +491,8 @@ type RecentPracticeItem = {
|
||||
const defaultScenarioId = 'complaint-water';
|
||||
const defaultScenarioName = '投诉接待';
|
||||
const pendingAssignmentKey = 'aihr_pending_assignment_id';
|
||||
const realtimeVoiceStorageKey = 'aihr_realtime_practice_voice';
|
||||
const realtimeVoiceOptions = REALTIME_PRACTICE_VOICE_OPTIONS;
|
||||
const storedRealtimeVoice = uni.getStorageSync(realtimeVoiceStorageKey);
|
||||
const realtimePersonaStorageKey = 'aihr_realtime_practice_persona';
|
||||
const storedRealtimePersona = uni.getStorageSync(realtimePersonaStorageKey);
|
||||
const status = ref<PracticeStatus>('idle');
|
||||
const sessionId = ref('');
|
||||
const roundIndex = ref(0);
|
||||
@@ -516,13 +524,11 @@ const emotionScore = ref(0);
|
||||
const trustScore = ref(0);
|
||||
const redFlagActive = ref(false);
|
||||
const realtimeActive = ref(false);
|
||||
const realtimeStatus = ref<'idle' | 'connecting' | 'connected' | 'listening' | 'speaking'>('idle');
|
||||
const realtimeStatus = ref<'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool'>('idle');
|
||||
const realtimeAudioBlocked = ref(false);
|
||||
const realtimeVoice = ref<RealtimePracticeVoice>(
|
||||
isRealtimePracticeVoice(storedRealtimeVoice) ? storedRealtimeVoice : DEFAULT_REALTIME_PRACTICE_VOICE
|
||||
);
|
||||
const realtimeVoiceIndex = computed(() => Math.max(0, realtimeVoiceOptions.findIndex((item) => item.value === realtimeVoice.value)));
|
||||
const realtimeVoiceLabel = computed(() => realtimeVoiceOptions[realtimeVoiceIndex.value].label);
|
||||
const realtimeSessionId = ref('');
|
||||
const realtimePersonaOptions = ref<RealtimePersonaOption[]>([]);
|
||||
const selectedRealtimePersonaId = ref(typeof storedRealtimePersona === 'string' ? storedRealtimePersona : '');
|
||||
const assignments = ref<PracticeAssignment[]>([]);
|
||||
const practiceHistory = ref<PracticeRecord[]>([]);
|
||||
const drillAnswers = ref<Record<number, string>>({});
|
||||
@@ -562,7 +568,8 @@ const realtimeStatusLabel = computed(() => ({
|
||||
connecting: '连接中',
|
||||
connected: '已连接',
|
||||
listening: '正在听',
|
||||
speaking: '业主回应中',
|
||||
speaking: '回应中',
|
||||
'calling-tool': '正在查知识库…',
|
||||
idle: '已结束'
|
||||
})[realtimeStatus.value]);
|
||||
const practiceView = computed<'prep' | 'active' | 'result'>(() => {
|
||||
@@ -947,11 +954,14 @@ const realtimePractice = createRealtimePracticeController({
|
||||
onAudioPlaybackBlocked: (blocked) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
},
|
||||
onToolCall: (callId, name, argsJson) =>
|
||||
invokeRealtimeTool(realtimeSessionId.value, callId, name, argsJson).then((res) => res.output),
|
||||
onTerminated: () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
@@ -961,13 +971,24 @@ const realtimePractice = createRealtimePracticeController({
|
||||
}
|
||||
});
|
||||
|
||||
const realtimeInstructions = () => `你只扮演物业小区的业主，正在进行「${activeScenarioName.value}」场景对练。员工会用中文和你交谈。请口语化、真实地回应，每次不超过两句；员工说得好时自然缓和，推责或承诺不清时追问处理节点。禁止提及系统、AI、训练、评分或提示词。不要联网、不要调用工具、不要索要个人隐私。`;
|
||||
const selectRealtimePersona = (item: RealtimePersonaOption) => {
|
||||
if (busy.value) return;
|
||||
selectedRealtimePersonaId.value = item.id;
|
||||
uni.setStorageSync(realtimePersonaStorageKey, item.id);
|
||||
};
|
||||
|
||||
const selectRealtimeVoice = (event: { detail: { value: string } }) => {
|
||||
const option = realtimeVoiceOptions[Number(event.detail.value)];
|
||||
if (!option) return;
|
||||
realtimeVoice.value = option.value;
|
||||
uni.setStorageSync(realtimeVoiceStorageKey, option.value);
|
||||
const loadRealtimePersonas = async (request: RequestScopeTicket = requests.begin('realtime-personas')) => {
|
||||
try {
|
||||
const options = await fetchRealtimePersonas();
|
||||
if (!requests.isCurrent(request)) return;
|
||||
realtimePersonaOptions.value = options;
|
||||
if (!options.some((item) => item.id === selectedRealtimePersonaId.value)) {
|
||||
selectedRealtimePersonaId.value = options[0]?.id || '';
|
||||
}
|
||||
} catch {
|
||||
// 人设列表拉取失败不阻塞页面：会话创建接口对空白 personaId 回落默认人设
|
||||
if (requests.isCurrent(request)) realtimePersonaOptions.value = [];
|
||||
}
|
||||
};
|
||||
|
||||
const stopRealtime = () => {
|
||||
@@ -976,6 +997,7 @@ const stopRealtime = () => {
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
@@ -991,19 +1013,24 @@ const startRealtime = async () => {
|
||||
turnVoices.value = {};
|
||||
result.value = null;
|
||||
sessionId.value = '';
|
||||
realtimeSessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
message.value = '';
|
||||
realtimeAudioBlocked.value = false;
|
||||
status.value = 'starting';
|
||||
realtimeActive.value = true;
|
||||
try {
|
||||
await realtimePractice.start(realtimeInstructions(), realtimeVoice.value);
|
||||
const session = await createRealtimeSession(selectedRealtimePersonaId.value || undefined, selectedScenarioId.value || undefined);
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeSessionId.value = session.sessionId;
|
||||
await realtimePractice.start(session.sessionUpdate, session.sessionId);
|
||||
if (!realtimeActive.value) return;
|
||||
status.value = 'active';
|
||||
message.value = '已连接，请直接向业主开场说话。';
|
||||
message.value = `已连接「${session.personaName}」，请直接开口说话。`;
|
||||
} catch (error) {
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeSessionId.value = '';
|
||||
status.value = 'idle';
|
||||
handleError(error, '实时陪练连接失败，请使用录音对练');
|
||||
}
|
||||
@@ -1674,6 +1701,7 @@ const handleShow = async () => {
|
||||
positionLabel.value = getSelectedPosition() || '生活顾问';
|
||||
void loadAssignments(requests.begin('assignments'));
|
||||
void loadCurriculum(requests.begin('curriculum'));
|
||||
void loadRealtimePersonas(requests.begin('realtime-personas'));
|
||||
};
|
||||
|
||||
const invalidateRequests = () => {
|
||||
@@ -2079,7 +2107,7 @@ onUnload(() => {
|
||||
color: #315b8f;
|
||||
}
|
||||
|
||||
.realtime-voice-picker {
|
||||
.realtime-persona-picker {
|
||||
margin-top: 12px;
|
||||
padding: 12px;
|
||||
border: 1px solid #dbe8fb;
|
||||
@@ -2087,61 +2115,21 @@ onUnload(() => {
|
||||
background: #f7fbff;
|
||||
}
|
||||
|
||||
.realtime-voice-heading {
|
||||
.realtime-persona-heading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.realtime-voice-title,
|
||||
.realtime-voice-hint,
|
||||
.realtime-voice-value {
|
||||
.realtime-persona-title {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.realtime-voice-title {
|
||||
color: #315b8f;
|
||||
font-size: 13px;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.realtime-voice-hint {
|
||||
margin-top: 8px;
|
||||
color: #6d7a8d;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.realtime-voice-select {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-top: 10px;
|
||||
width: 100%;
|
||||
min-height: 44px;
|
||||
padding: 0 12px;
|
||||
border-color: #c8d9f2;
|
||||
border-radius: 10px;
|
||||
background: #ffffff;
|
||||
color: #315b8f;
|
||||
}
|
||||
|
||||
.realtime-voice-value {
|
||||
overflow: hidden;
|
||||
color: #315b8f;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.realtime-voice-arrow {
|
||||
flex: 0 0 auto;
|
||||
color: #6d7a8d;
|
||||
font-size: 22px;
|
||||
}
|
||||
|
||||
.realtime-beta-note {
|
||||
display: block;
|
||||
color: #6d7a8d;
|
||||
|
||||
@@ -1,94 +1,27 @@
|
||||
import { apiRequest } from './api';
|
||||
import { isNativeAppRuntime } from './speech-capture';
|
||||
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking';
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool';
|
||||
|
||||
export const REALTIME_PRACTICE_VOICE_OPTIONS = [
|
||||
{ value: 'Tina', label: '普通话 · 温暖女声 · Tina' },
|
||||
{ value: 'Cindy', label: '台式口音 · 甜美女声 · Cindy' },
|
||||
{ value: 'Liora Mira', label: '普通话 · 治愈女声 · Liora Mira' },
|
||||
{ value: 'Sunnybobi', label: '普通话 · 邻家女声 · Sunnybobi' },
|
||||
{ value: 'Raymond', label: '普通话 · 清亮男声 · Raymond' },
|
||||
{ value: 'Ethan', label: '普通话 · 温暖男声 · Ethan' },
|
||||
{ value: 'Theo Calm', label: '普通话 · 沉静男声 · Theo Calm' },
|
||||
{ value: 'Serena', label: '普通话 · 温柔女声 · Serena' },
|
||||
{ value: 'Harvey', label: '普通话 · 醇厚男声 · Harvey' },
|
||||
{ value: 'Maia', label: '普通话 · 知性女声 · Maia' },
|
||||
{ value: 'Evan', label: '普通话 · 青年男声 · Evan' },
|
||||
{ value: 'Qiao', label: '台式口音 · 甜美女声 · Qiao' },
|
||||
{ value: 'Momo', label: '普通话 · 俏皮女声 · Momo' },
|
||||
{ value: 'Wil', label: '港风口音 · 青年男声 · Wil' },
|
||||
{ value: 'Angel', label: '普通话 · 甜美女声 · Angel' },
|
||||
{ value: 'Li Cassian', label: '普通话 · 克制男声 · Li Cassian' },
|
||||
{ value: 'Mia', label: '普通话 · 舒缓女声 · Mia' },
|
||||
{ value: 'Joyner', label: '普通话 · 幽默男声 · Joyner' },
|
||||
{ value: 'Gold', label: '普通话 · 说唱男声 · Gold' },
|
||||
{ value: 'Katerina', label: '普通话 · 成熟韵律女声 · Katerina' },
|
||||
{ value: 'Ryan', label: '普通话 · 张力男声 · Ryan' },
|
||||
{ value: 'Jennifer', label: '英语质感 · 电影感女声 · Jennifer' },
|
||||
{ value: 'Aiden', label: '英语质感 · 青年男声 · Aiden' },
|
||||
{ value: 'Mione', label: '英语质感 · 成熟女声 · Mione' },
|
||||
{ value: 'Sunny', label: '四川话 · 甜美女声 · Sunny' },
|
||||
{ value: 'Dylan', label: '北京话 · 青年男声 · Dylan' },
|
||||
{ value: 'Eric', label: '四川话 · 成都男声 · Eric' },
|
||||
{ value: 'Peter', label: '天津话 · 相声男声 · Peter' },
|
||||
{ value: 'Joseph Chen', label: '闽南话 · 怀旧男声 · Joseph Chen' },
|
||||
{ value: 'Marcus', label: '陕西话 · 厚重男声 · Marcus' },
|
||||
{ value: 'Li', label: '南京话 · 叔叔男声 · Li' },
|
||||
{ value: 'Kiki', label: '粤语 · 亲和女声 · Kiki' },
|
||||
{ value: 'Rocky', label: '粤语 · 幽默男声 · Rocky' },
|
||||
{ value: 'Sohee', label: '韩系 · 活力女声 · Sohee' },
|
||||
{ value: 'Lenn', label: '德系 · 青年男声 · Lenn' },
|
||||
{ value: 'Ono Anna', label: '日系 · 俏皮女声 · Ono Anna' },
|
||||
{ value: 'Sonrisa', label: '拉美 · 热情女声 · Sonrisa' },
|
||||
{ value: 'Bodega', label: '西语 · 热情男声 · Bodega' },
|
||||
{ value: 'Emilien', label: '法式 · 浪漫男声 · Emilien' },
|
||||
{ value: 'Andre', label: '普通话 · 稳重男声 · Andre' },
|
||||
{ value: 'Radio Gol', label: '解说 · 足球评论男声 · Radio Gol' },
|
||||
{ value: 'Alek', label: '俄式 · 温暖男声 · Alek' },
|
||||
{ value: 'Rizky', label: '印尼 · 青年男声 · Rizky' },
|
||||
{ value: 'Roya', label: '普通话 · 活力女声 · Roya' },
|
||||
{ value: 'Arda', label: '普通话 · 清亮温暖声 · Arda' },
|
||||
{ value: 'Hana', label: '越南 · 成熟女声 · Hana' },
|
||||
{ value: 'Dolce', label: '意式 · 慵懒男声 · Dolce' },
|
||||
{ value: 'Jakub', label: '波兰 · 艺术男声 · Jakub' },
|
||||
{ value: 'Griet', label: '荷兰 · 艺术女声 · Griet' },
|
||||
{ value: 'Eliška', label: '捷克 · 温暖女声 · Eliška' },
|
||||
{ value: 'Marina', label: '都市 · 多元女声 · Marina' },
|
||||
{ value: 'Siiri', label: '北欧 · 安静女声 · Siiri' },
|
||||
{ value: 'Ingrid', label: '挪威 · 乡村女声 · Ingrid' },
|
||||
{ value: 'Sigga', label: '冰岛 · 知性女声 · Sigga' },
|
||||
{ value: 'Bea', label: '菲律宾 · 甜美女声 · Bea' },
|
||||
{ value: 'Chloe', label: '马来 · 职场女声 · Chloe' }
|
||||
] as const;
|
||||
export type RealtimePracticeVoice = typeof REALTIME_PRACTICE_VOICE_OPTIONS[number]['value'];
|
||||
export const REALTIME_PRACTICE_VOICES = REALTIME_PRACTICE_VOICE_OPTIONS.map((item) => item.value);
|
||||
export const DEFAULT_REALTIME_PRACTICE_VOICE: RealtimePracticeVoice = 'Ethan';
|
||||
|
||||
export const isRealtimePracticeVoice = (value: unknown): value is RealtimePracticeVoice =>
|
||||
typeof value === 'string' && REALTIME_PRACTICE_VOICE_OPTIONS.some((item) => item.value === value);
|
||||
|
||||
export const normalizeRealtimePracticeVoice = (value: unknown): RealtimePracticeVoice =>
|
||||
isRealtimePracticeVoice(value) ? value : DEFAULT_REALTIME_PRACTICE_VOICE;
|
||||
|
||||
export const createRealtimePracticeSessionUpdate = (instructions: string, voice: unknown) => ({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'session.update',
|
||||
session: {
|
||||
modalities: ['text', 'audio'],
|
||||
voice: normalizeRealtimePracticeVoice(voice),
|
||||
input_audio_format: 'pcm',
|
||||
output_audio_format: 'pcm',
|
||||
input_audio_transcription: { model: 'qwen3-asr-flash-realtime' },
|
||||
instructions,
|
||||
turn_detection: {
|
||||
type: 'semantic_vad',
|
||||
threshold: 0.5,
|
||||
prefix_padding_ms: 500,
|
||||
silence_duration_ms: 800
|
||||
export interface RealtimePersonaOption {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
hasTools: boolean;
|
||||
}
|
||||
|
||||
export interface RealtimeSessionResponse {
|
||||
sessionId: string;
|
||||
personaId: string;
|
||||
personaName: string;
|
||||
hasTools: boolean;
|
||||
sessionUpdate: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface RealtimeToolInvokeResponse {
|
||||
callId: string;
|
||||
output: string;
|
||||
}
|
||||
});
|
||||
|
||||
interface RealtimeSdpResponse {
|
||||
answerSdp: string;
|
||||
@@ -98,6 +31,9 @@ interface RealtimeSdpResponse {
|
||||
interface RealtimeEvent {
|
||||
type?: string;
|
||||
transcript?: string;
|
||||
call_id?: string;
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
error?: { message?: string };
|
||||
}
|
||||
|
||||
@@ -112,16 +48,40 @@ export interface RealtimePracticeCallbacks {
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeController {
|
||||
start: (instructions: string, voice?: RealtimePracticeVoice) => Promise<void>;
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export const fetchRealtimePersonas = () =>
|
||||
apiRequest<RealtimePersonaOption[]>({
|
||||
url: '/api/train/practice/realtime/personas',
|
||||
method: 'GET',
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
export const createRealtimeSession = (personaId?: string, scenarioId?: string) =>
|
||||
apiRequest<RealtimeSessionResponse>({
|
||||
url: '/api/train/practice/realtime/session',
|
||||
method: 'POST',
|
||||
data: { personaId: personaId || null, scenarioId: scenarioId || null },
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
export const invokeRealtimeTool = (sessionId: string, callId: string, name: string, argsJson: string) =>
|
||||
apiRequest<RealtimeToolInvokeResponse>({
|
||||
url: '/api/train/practice/realtime/tools/invoke',
|
||||
method: 'POST',
|
||||
data: { sessionId, callId, name, arguments: argsJson },
|
||||
timeout: 20000
|
||||
});
|
||||
|
||||
const isSupported = () =>
|
||||
!isNativeAppRuntime()
|
||||
&& typeof navigator !== 'undefined'
|
||||
@@ -151,6 +111,8 @@ const waitForIceComplete = (connection: RTCPeerConnection) =>
|
||||
const normalizeSdp = (sdp: string) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const trimTranscript = (value?: string) => (value || '').trim();
|
||||
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const createRealtimePracticeController = (callbacks: RealtimePracticeCallbacks): RealtimePracticeController => {
|
||||
let activeAttempt = 0;
|
||||
let connection: RTCPeerConnection | null = null;
|
||||
@@ -159,6 +121,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
let controlChannel: RTCDataChannel | null = null;
|
||||
let cancelSessionWait: (() => void) | null = null;
|
||||
let sessionWaitAttempt: number | null = null;
|
||||
let toolCallChain: Promise<void> = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt: number) => activeAttempt === attempt;
|
||||
|
||||
@@ -176,9 +139,40 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
connection = null;
|
||||
};
|
||||
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, instructions: string, voice: RealtimePracticeVoice) => {
|
||||
// 服务端下发的 session.update 载荷原样转发，仅补 event_id
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, sessionUpdate: Record<string, unknown>) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify(createRealtimePracticeSessionUpdate(instructions, voice)));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt: number, event: RealtimeEvent) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
// 多次工具调用串行化，避免打断竞态下回传交错
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output: string;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw: string, attempt: number, provider: ProviderEventHandlers) => {
|
||||
@@ -202,6 +196,9 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
@@ -224,6 +221,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
@@ -248,9 +246,8 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (instructions: string, voice: RealtimePracticeVoice = DEFAULT_REALTIME_PRACTICE_VOICE) => {
|
||||
const start = async (sessionUpdate: Record<string, unknown>, sessionId?: string) => {
|
||||
if (!isSupported()) throw new Error('当前设备不支持实时语音，请使用录音对练');
|
||||
const selectedVoice = normalizeRealtimePracticeVoice(voice);
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
@@ -292,7 +289,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, instructions, selectedVoice);
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
@@ -358,7 +355,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp: normalizeSdp(offerSdp) },
|
||||
data: { offerSdp: normalizeSdp(offerSdp), sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
if (!ensureCurrent()) return;
|
||||
|
||||
@@ -1,71 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import test from 'node:test';
|
||||
import vm from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
|
||||
const source = (path) => readFile(new URL(path, import.meta.url), 'utf8');
|
||||
|
||||
const realtimeVoiceRuntime = async () => {
|
||||
const service = await source('../src/services/realtime-practice.ts');
|
||||
const compiled = ts.transpileModule(service, {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2020
|
||||
}
|
||||
});
|
||||
const runtimeModule = { exports: {} };
|
||||
vm.runInNewContext(compiled.outputText, {
|
||||
Date,
|
||||
exports: runtimeModule.exports,
|
||||
module: runtimeModule,
|
||||
require: (specifier) => {
|
||||
if (specifier === './api') return { apiRequest: async () => ({}) };
|
||||
if (specifier === './speech-capture') return { isNativeAppRuntime: () => false };
|
||||
throw new Error(`Unexpected module: ${specifier}`);
|
||||
}
|
||||
});
|
||||
return runtimeModule.exports;
|
||||
};
|
||||
|
||||
test('实时陪练通过受认证 SDP 代理接入，前端不携带供应商密钥', async () => {
|
||||
const [service, page, backendService] = await Promise.all([
|
||||
test('实时陪练通过受认证代理接入，前端不携带供应商密钥、不自组会话配置', async () => {
|
||||
const [service, page, backendService, backendSessionService, backendPersonaRegistry] = await Promise.all([
|
||||
source('../src/services/realtime-practice.ts'),
|
||||
source('../src/pages/user/practice/index.vue'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePracticeService.java')
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePracticeService.java'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimeSessionService.java'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePersonaRegistry.java')
|
||||
]);
|
||||
|
||||
// 传输与事件协议
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/sdp/);
|
||||
assert.match(service, /semantic_vad/);
|
||||
assert.match(service, /conversation\.item\.input_audio_transcription\.completed/);
|
||||
assert.match(service, /response\.audio_transcript\.done/);
|
||||
assert.match(service, /session\.created/);
|
||||
assert.match(service, /session\.updated/);
|
||||
assert.match(service, /REALTIME_PRACTICE_VOICES/);
|
||||
assert.match(service, /createRealtimePracticeSessionUpdate/);
|
||||
assert.match(service, /activeAttempt/);
|
||||
assert.match(service, /resumeAudio/);
|
||||
|
||||
// 方案B 架构：session.update 由服务端组装，浏览器只原样转发；instructions/音色不下放前端
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/session/);
|
||||
assert.match(service, /sendSessionUpdate/);
|
||||
assert.doesNotMatch(service, /instructions/);
|
||||
assert.doesNotMatch(service, /REALTIME_PRACTICE_VOICES|createRealtimePracticeSessionUpdate|normalizeRealtimePracticeVoice/);
|
||||
assert.doesNotMatch(service, /semantic_vad/);
|
||||
assert.doesNotMatch(service, /DASHSCOPE_API_KEY|Authorization:\s*Bearer/);
|
||||
|
||||
// function calling 工具桥：事件分支、串行链、软着陆兜底、回传协议
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/tools\/invoke/);
|
||||
assert.match(service, /response\.function_call_arguments\.done/);
|
||||
assert.match(service, /function_call_output/);
|
||||
assert.match(service, /response\.create/);
|
||||
assert.match(service, /toolCallChain/);
|
||||
assert.match(service, /工具暂时不可用，请直接回应员工/);
|
||||
assert.match(service, /calling-tool/);
|
||||
|
||||
// 服务端是会话配置的唯一组装点
|
||||
assert.match(backendSessionService, /session\.update/);
|
||||
assert.match(backendSessionService, /semantic_vad/);
|
||||
assert.match(backendSessionService, /TOOL_SEARCH_KNOWLEDGE/);
|
||||
assert.match(backendPersonaRegistry, /search_knowledge/);
|
||||
assert.match(backendPersonaRegistry, /owner-calm/);
|
||||
assert.match(backendPersonaRegistry, /digital-mentor/);
|
||||
assert.match(backendService, /实时陪练限流暂时不可用/);
|
||||
|
||||
// 页面：人设卡片选择 + 本地持久化；音色 picker 已删除
|
||||
assert.match(page, /实时陪练 Beta/);
|
||||
assert.match(page, /实时陪练音色/);
|
||||
assert.match(page, /<picker :range="realtimeVoiceOptions" range-key="label"/);
|
||||
assert.match(page, /aihr_realtime_practice_voice/);
|
||||
assert.match(page, /selectRealtimeVoice/);
|
||||
assert.match(page, /createRealtimePracticeController/);
|
||||
assert.match(page, /实时陪练人设/);
|
||||
assert.match(page, /aihr_realtime_practice_persona/);
|
||||
assert.match(page, /selectRealtimePersona/);
|
||||
assert.match(page, /fetchRealtimePersonas/);
|
||||
assert.match(page, /createRealtimeSession/);
|
||||
assert.match(page, /正在查知识库/);
|
||||
assert.doesNotMatch(page, /realtimeVoice|selectRealtimeVoice|aihr_realtime_practice_voice/);
|
||||
});
|
||||
|
||||
test('实时陪练音色白名单会写入会话配置，非法值回退默认音色', async () => {
|
||||
const runtime = await realtimeVoiceRuntime();
|
||||
const voices = Array.from(runtime.REALTIME_PRACTICE_VOICES);
|
||||
test('工具调用串行化与竞态防护：链式执行、attempt 双检、stop 重置', async () => {
|
||||
const service = await source('../src/services/realtime-practice.ts');
|
||||
|
||||
assert.deepEqual(voices, [
|
||||
'Tina', 'Cindy', 'Liora Mira', 'Sunnybobi', 'Raymond', 'Ethan', 'Theo Calm', 'Serena', 'Harvey', 'Maia', 'Evan', 'Qiao', 'Momo', 'Wil', 'Angel', 'Li Cassian', 'Mia', 'Joyner', 'Gold', 'Katerina', 'Ryan', 'Jennifer', 'Aiden', 'Mione', 'Sunny', 'Dylan', 'Eric', 'Peter', 'Joseph Chen', 'Marcus', 'Li', 'Kiki', 'Rocky', 'Sohee', 'Lenn', 'Ono Anna', 'Sonrisa', 'Bodega', 'Emilien', 'Andre', 'Radio Gol', 'Alek', 'Rizky', 'Roya', 'Arda', 'Hana', 'Dolce', 'Jakub', 'Griet', 'Eliška', 'Marina', 'Siiri', 'Ingrid', 'Sigga', 'Bea', 'Chloe'
|
||||
]);
|
||||
for (const voice of voices) {
|
||||
assert.equal(runtime.normalizeRealtimePracticeVoice(voice), voice);
|
||||
assert.equal(runtime.createRealtimePracticeSessionUpdate('测试', voice).session.voice, voice);
|
||||
}
|
||||
assert.equal(runtime.normalizeRealtimePracticeVoice('untrusted-voice'), 'Ethan');
|
||||
assert.equal(runtime.createRealtimePracticeSessionUpdate('测试', 'untrusted-voice').session.voice, 'Ethan');
|
||||
// 串行链必须基于上一次调用收尾，且带末端 catch 防毒化
|
||||
assert.match(service, /toolCallChain = toolCallChain\.then/);
|
||||
assert.match(service, /\.catch\(\(\) => \{/);
|
||||
// stop() 递增 attempt 使迟到的工具回调失效，并重置链
|
||||
assert.match(service, /activeAttempt \+= 1/);
|
||||
assert.match(service, /toolCallChain = Promise\.resolve\(\)/);
|
||||
// 回传前双检：attempt 仍当前 + 数据通道仍打开
|
||||
assert.match(service, /!isCurrent\(attempt\) \|\| channel\.readyState !== 'open'/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user