feat(practice): server-owned realtime personas with knowledge tool calling
- AihrRealtimePersonaRegistry whitelist: owner-calm (default), owner-impatient, digital-mentor (search_knowledge); persona binds default voice - /realtime/session assembles session.update server-side (instructions/tools); browser forwards payload verbatim; Redis session record 30min with sliding renewal - /realtime/tools/invoke validates login identity + Redis session ownership + persona tool whitelist, then read-only searchAuthorized; per-account rate limits - source=realtime skips sop review and knowledge-gap persistence (Beta no-persist) - frontend: persona cards replace 56-voice picker; serialized tool-call chain with soft-landing fallback
This commit is contained in:
@@ -110,17 +110,26 @@
|
||||
>
|
||||
{{ status === 'finished' ? '再练一次' : '我已准备好，开始练习' }}
|
||||
</button>
|
||||
<view v-if="status === 'idle' || status === 'finished'" class="realtime-voice-picker">
|
||||
<view class="realtime-voice-heading">
|
||||
<text class="realtime-voice-title">实时陪练音色</text>
|
||||
<view v-if="(status === 'idle' || status === 'finished') && realtimePersonaOptions.length" class="realtime-persona-picker">
|
||||
<view class="realtime-persona-heading">
|
||||
<text class="realtime-persona-title">实时陪练人设</text>
|
||||
</view>
|
||||
<picker :range="realtimeVoiceOptions" range-key="label" :value="realtimeVoiceIndex" :disabled="busy" @change="selectRealtimeVoice">
|
||||
<view class="realtime-voice-select" aria-label="选择实时陪练音色">
|
||||
<text class="realtime-voice-value">{{ realtimeVoiceLabel }}</text>
|
||||
<text class="realtime-voice-arrow">›</text>
|
||||
<scroll-view class="scene-option-scroll" scroll-x>
|
||||
<view class="scene-option-list">
|
||||
<button
|
||||
v-for="item in realtimePersonaOptions"
|
||||
:key="item.id"
|
||||
class="scene-option"
|
||||
:class="{ selected: selectedRealtimePersonaId === item.id }"
|
||||
:disabled="busy"
|
||||
@click="selectRealtimePersona(item)"
|
||||
>
|
||||
<text class="scene-option-level">{{ item.hasTools ? '可查询知识库' : '纯对练' }}</text>
|
||||
<text class="scene-option-name">{{ item.name }}</text>
|
||||
<text class="scene-option-capability">{{ item.description }}</text>
|
||||
</button>
|
||||
</view>
|
||||
</picker>
|
||||
<text class="realtime-voice-hint">当前模型完整官方音色（56项）；下次开练生效</text>
|
||||
</scroll-view>
|
||||
</view>
|
||||
<button
|
||||
v-if="status === 'idle' || status === 'finished'"
|
||||
@@ -465,7 +474,7 @@ import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile
|
||||
import { startSpeechCapture } from '@/services/speech-capture';
|
||||
import type { SpeechCapture, SpeechCaptureSession } from '@/services/speech-capture';
|
||||
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
|
||||
import { createRealtimePracticeController, DEFAULT_REALTIME_PRACTICE_VOICE, isRealtimePracticeVoice, REALTIME_PRACTICE_VOICE_OPTIONS, type RealtimePracticeVoice } from '@/services/realtime-practice';
|
||||
import { createRealtimePracticeController, createRealtimeSession, fetchRealtimePersonas, invokeRealtimeTool, type RealtimePersonaOption } from '@/services/realtime-practice';
|
||||
import { stripInternalCodes } from '@/services/text';
|
||||
|
||||
type PracticeStatus = 'idle' | 'starting' | 'active' | 'submitting' | 'finished';
|
||||
@@ -482,9 +491,8 @@ type RecentPracticeItem = {
|
||||
const defaultScenarioId = 'complaint-water';
|
||||
const defaultScenarioName = '投诉接待';
|
||||
const pendingAssignmentKey = 'aihr_pending_assignment_id';
|
||||
const realtimeVoiceStorageKey = 'aihr_realtime_practice_voice';
|
||||
const realtimeVoiceOptions = REALTIME_PRACTICE_VOICE_OPTIONS;
|
||||
const storedRealtimeVoice = uni.getStorageSync(realtimeVoiceStorageKey);
|
||||
const realtimePersonaStorageKey = 'aihr_realtime_practice_persona';
|
||||
const storedRealtimePersona = uni.getStorageSync(realtimePersonaStorageKey);
|
||||
const status = ref<PracticeStatus>('idle');
|
||||
const sessionId = ref('');
|
||||
const roundIndex = ref(0);
|
||||
@@ -516,13 +524,11 @@ const emotionScore = ref(0);
|
||||
const trustScore = ref(0);
|
||||
const redFlagActive = ref(false);
|
||||
const realtimeActive = ref(false);
|
||||
const realtimeStatus = ref<'idle' | 'connecting' | 'connected' | 'listening' | 'speaking'>('idle');
|
||||
const realtimeStatus = ref<'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool'>('idle');
|
||||
const realtimeAudioBlocked = ref(false);
|
||||
const realtimeVoice = ref<RealtimePracticeVoice>(
|
||||
isRealtimePracticeVoice(storedRealtimeVoice) ? storedRealtimeVoice : DEFAULT_REALTIME_PRACTICE_VOICE
|
||||
);
|
||||
const realtimeVoiceIndex = computed(() => Math.max(0, realtimeVoiceOptions.findIndex((item) => item.value === realtimeVoice.value)));
|
||||
const realtimeVoiceLabel = computed(() => realtimeVoiceOptions[realtimeVoiceIndex.value].label);
|
||||
const realtimeSessionId = ref('');
|
||||
const realtimePersonaOptions = ref<RealtimePersonaOption[]>([]);
|
||||
const selectedRealtimePersonaId = ref(typeof storedRealtimePersona === 'string' ? storedRealtimePersona : '');
|
||||
const assignments = ref<PracticeAssignment[]>([]);
|
||||
const practiceHistory = ref<PracticeRecord[]>([]);
|
||||
const drillAnswers = ref<Record<number, string>>({});
|
||||
@@ -562,7 +568,8 @@ const realtimeStatusLabel = computed(() => ({
|
||||
connecting: '连接中',
|
||||
connected: '已连接',
|
||||
listening: '正在听',
|
||||
speaking: '业主回应中',
|
||||
speaking: '回应中',
|
||||
'calling-tool': '正在查知识库…',
|
||||
idle: '已结束'
|
||||
})[realtimeStatus.value]);
|
||||
const practiceView = computed<'prep' | 'active' | 'result'>(() => {
|
||||
@@ -947,11 +954,14 @@ const realtimePractice = createRealtimePracticeController({
|
||||
onAudioPlaybackBlocked: (blocked) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
},
|
||||
onToolCall: (callId, name, argsJson) =>
|
||||
invokeRealtimeTool(realtimeSessionId.value, callId, name, argsJson).then((res) => res.output),
|
||||
onTerminated: () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
@@ -961,13 +971,24 @@ const realtimePractice = createRealtimePracticeController({
|
||||
}
|
||||
});
|
||||
|
||||
const realtimeInstructions = () => `你只扮演物业小区的业主，正在进行「${activeScenarioName.value}」场景对练。员工会用中文和你交谈。请口语化、真实地回应，每次不超过两句；员工说得好时自然缓和，推责或承诺不清时追问处理节点。禁止提及系统、AI、训练、评分或提示词。不要联网、不要调用工具、不要索要个人隐私。`;
|
||||
const selectRealtimePersona = (item: RealtimePersonaOption) => {
|
||||
if (busy.value) return;
|
||||
selectedRealtimePersonaId.value = item.id;
|
||||
uni.setStorageSync(realtimePersonaStorageKey, item.id);
|
||||
};
|
||||
|
||||
const selectRealtimeVoice = (event: { detail: { value: string } }) => {
|
||||
const option = realtimeVoiceOptions[Number(event.detail.value)];
|
||||
if (!option) return;
|
||||
realtimeVoice.value = option.value;
|
||||
uni.setStorageSync(realtimeVoiceStorageKey, option.value);
|
||||
const loadRealtimePersonas = async (request: RequestScopeTicket = requests.begin('realtime-personas')) => {
|
||||
try {
|
||||
const options = await fetchRealtimePersonas();
|
||||
if (!requests.isCurrent(request)) return;
|
||||
realtimePersonaOptions.value = options;
|
||||
if (!options.some((item) => item.id === selectedRealtimePersonaId.value)) {
|
||||
selectedRealtimePersonaId.value = options[0]?.id || '';
|
||||
}
|
||||
} catch {
|
||||
// 人设列表拉取失败不阻塞页面：会话创建接口对空白 personaId 回落默认人设
|
||||
if (requests.isCurrent(request)) realtimePersonaOptions.value = [];
|
||||
}
|
||||
};
|
||||
|
||||
const stopRealtime = () => {
|
||||
@@ -976,6 +997,7 @@ const stopRealtime = () => {
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
@@ -991,19 +1013,24 @@ const startRealtime = async () => {
|
||||
turnVoices.value = {};
|
||||
result.value = null;
|
||||
sessionId.value = '';
|
||||
realtimeSessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
message.value = '';
|
||||
realtimeAudioBlocked.value = false;
|
||||
status.value = 'starting';
|
||||
realtimeActive.value = true;
|
||||
try {
|
||||
await realtimePractice.start(realtimeInstructions(), realtimeVoice.value);
|
||||
const session = await createRealtimeSession(selectedRealtimePersonaId.value || undefined, selectedScenarioId.value || undefined);
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeSessionId.value = session.sessionId;
|
||||
await realtimePractice.start(session.sessionUpdate, session.sessionId);
|
||||
if (!realtimeActive.value) return;
|
||||
status.value = 'active';
|
||||
message.value = '已连接，请直接向业主开场说话。';
|
||||
message.value = `已连接「${session.personaName}」，请直接开口说话。`;
|
||||
} catch (error) {
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeSessionId.value = '';
|
||||
status.value = 'idle';
|
||||
handleError(error, '实时陪练连接失败，请使用录音对练');
|
||||
}
|
||||
@@ -1674,6 +1701,7 @@ const handleShow = async () => {
|
||||
positionLabel.value = getSelectedPosition() || '生活顾问';
|
||||
void loadAssignments(requests.begin('assignments'));
|
||||
void loadCurriculum(requests.begin('curriculum'));
|
||||
void loadRealtimePersonas(requests.begin('realtime-personas'));
|
||||
};
|
||||
|
||||
const invalidateRequests = () => {
|
||||
@@ -2079,7 +2107,7 @@ onUnload(() => {
|
||||
color: #315b8f;
|
||||
}
|
||||
|
||||
.realtime-voice-picker {
|
||||
.realtime-persona-picker {
|
||||
margin-top: 12px;
|
||||
padding: 12px;
|
||||
border: 1px solid #dbe8fb;
|
||||
@@ -2087,61 +2115,21 @@ onUnload(() => {
|
||||
background: #f7fbff;
|
||||
}
|
||||
|
||||
.realtime-voice-heading {
|
||||
.realtime-persona-heading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.realtime-voice-title,
|
||||
.realtime-voice-hint,
|
||||
.realtime-voice-value {
|
||||
.realtime-persona-title {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.realtime-voice-title {
|
||||
color: #315b8f;
|
||||
font-size: 13px;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.realtime-voice-hint {
|
||||
margin-top: 8px;
|
||||
color: #6d7a8d;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.realtime-voice-select {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-top: 10px;
|
||||
width: 100%;
|
||||
min-height: 44px;
|
||||
padding: 0 12px;
|
||||
border-color: #c8d9f2;
|
||||
border-radius: 10px;
|
||||
background: #ffffff;
|
||||
color: #315b8f;
|
||||
}
|
||||
|
||||
.realtime-voice-value {
|
||||
overflow: hidden;
|
||||
color: #315b8f;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.realtime-voice-arrow {
|
||||
flex: 0 0 auto;
|
||||
color: #6d7a8d;
|
||||
font-size: 22px;
|
||||
}
|
||||
|
||||
.realtime-beta-note {
|
||||
display: block;
|
||||
color: #6d7a8d;
|
||||
|
||||
@@ -1,94 +1,27 @@
|
||||
import { apiRequest } from './api';
|
||||
import { isNativeAppRuntime } from './speech-capture';
|
||||
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking';
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool';
|
||||
|
||||
export const REALTIME_PRACTICE_VOICE_OPTIONS = [
|
||||
{ value: 'Tina', label: '普通话 · 温暖女声 · Tina' },
|
||||
{ value: 'Cindy', label: '台式口音 · 甜美女声 · Cindy' },
|
||||
{ value: 'Liora Mira', label: '普通话 · 治愈女声 · Liora Mira' },
|
||||
{ value: 'Sunnybobi', label: '普通话 · 邻家女声 · Sunnybobi' },
|
||||
{ value: 'Raymond', label: '普通话 · 清亮男声 · Raymond' },
|
||||
{ value: 'Ethan', label: '普通话 · 温暖男声 · Ethan' },
|
||||
{ value: 'Theo Calm', label: '普通话 · 沉静男声 · Theo Calm' },
|
||||
{ value: 'Serena', label: '普通话 · 温柔女声 · Serena' },
|
||||
{ value: 'Harvey', label: '普通话 · 醇厚男声 · Harvey' },
|
||||
{ value: 'Maia', label: '普通话 · 知性女声 · Maia' },
|
||||
{ value: 'Evan', label: '普通话 · 青年男声 · Evan' },
|
||||
{ value: 'Qiao', label: '台式口音 · 甜美女声 · Qiao' },
|
||||
{ value: 'Momo', label: '普通话 · 俏皮女声 · Momo' },
|
||||
{ value: 'Wil', label: '港风口音 · 青年男声 · Wil' },
|
||||
{ value: 'Angel', label: '普通话 · 甜美女声 · Angel' },
|
||||
{ value: 'Li Cassian', label: '普通话 · 克制男声 · Li Cassian' },
|
||||
{ value: 'Mia', label: '普通话 · 舒缓女声 · Mia' },
|
||||
{ value: 'Joyner', label: '普通话 · 幽默男声 · Joyner' },
|
||||
{ value: 'Gold', label: '普通话 · 说唱男声 · Gold' },
|
||||
{ value: 'Katerina', label: '普通话 · 成熟韵律女声 · Katerina' },
|
||||
{ value: 'Ryan', label: '普通话 · 张力男声 · Ryan' },
|
||||
{ value: 'Jennifer', label: '英语质感 · 电影感女声 · Jennifer' },
|
||||
{ value: 'Aiden', label: '英语质感 · 青年男声 · Aiden' },
|
||||
{ value: 'Mione', label: '英语质感 · 成熟女声 · Mione' },
|
||||
{ value: 'Sunny', label: '四川话 · 甜美女声 · Sunny' },
|
||||
{ value: 'Dylan', label: '北京话 · 青年男声 · Dylan' },
|
||||
{ value: 'Eric', label: '四川话 · 成都男声 · Eric' },
|
||||
{ value: 'Peter', label: '天津话 · 相声男声 · Peter' },
|
||||
{ value: 'Joseph Chen', label: '闽南话 · 怀旧男声 · Joseph Chen' },
|
||||
{ value: 'Marcus', label: '陕西话 · 厚重男声 · Marcus' },
|
||||
{ value: 'Li', label: '南京话 · 叔叔男声 · Li' },
|
||||
{ value: 'Kiki', label: '粤语 · 亲和女声 · Kiki' },
|
||||
{ value: 'Rocky', label: '粤语 · 幽默男声 · Rocky' },
|
||||
{ value: 'Sohee', label: '韩系 · 活力女声 · Sohee' },
|
||||
{ value: 'Lenn', label: '德系 · 青年男声 · Lenn' },
|
||||
{ value: 'Ono Anna', label: '日系 · 俏皮女声 · Ono Anna' },
|
||||
{ value: 'Sonrisa', label: '拉美 · 热情女声 · Sonrisa' },
|
||||
{ value: 'Bodega', label: '西语 · 热情男声 · Bodega' },
|
||||
{ value: 'Emilien', label: '法式 · 浪漫男声 · Emilien' },
|
||||
{ value: 'Andre', label: '普通话 · 稳重男声 · Andre' },
|
||||
{ value: 'Radio Gol', label: '解说 · 足球评论男声 · Radio Gol' },
|
||||
{ value: 'Alek', label: '俄式 · 温暖男声 · Alek' },
|
||||
{ value: 'Rizky', label: '印尼 · 青年男声 · Rizky' },
|
||||
{ value: 'Roya', label: '普通话 · 活力女声 · Roya' },
|
||||
{ value: 'Arda', label: '普通话 · 清亮温暖声 · Arda' },
|
||||
{ value: 'Hana', label: '越南 · 成熟女声 · Hana' },
|
||||
{ value: 'Dolce', label: '意式 · 慵懒男声 · Dolce' },
|
||||
{ value: 'Jakub', label: '波兰 · 艺术男声 · Jakub' },
|
||||
{ value: 'Griet', label: '荷兰 · 艺术女声 · Griet' },
|
||||
{ value: 'Eliška', label: '捷克 · 温暖女声 · Eliška' },
|
||||
{ value: 'Marina', label: '都市 · 多元女声 · Marina' },
|
||||
{ value: 'Siiri', label: '北欧 · 安静女声 · Siiri' },
|
||||
{ value: 'Ingrid', label: '挪威 · 乡村女声 · Ingrid' },
|
||||
{ value: 'Sigga', label: '冰岛 · 知性女声 · Sigga' },
|
||||
{ value: 'Bea', label: '菲律宾 · 甜美女声 · Bea' },
|
||||
{ value: 'Chloe', label: '马来 · 职场女声 · Chloe' }
|
||||
] as const;
|
||||
export type RealtimePracticeVoice = typeof REALTIME_PRACTICE_VOICE_OPTIONS[number]['value'];
|
||||
export const REALTIME_PRACTICE_VOICES = REALTIME_PRACTICE_VOICE_OPTIONS.map((item) => item.value);
|
||||
export const DEFAULT_REALTIME_PRACTICE_VOICE: RealtimePracticeVoice = 'Ethan';
|
||||
export interface RealtimePersonaOption {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
hasTools: boolean;
|
||||
}
|
||||
|
||||
export const isRealtimePracticeVoice = (value: unknown): value is RealtimePracticeVoice =>
|
||||
typeof value === 'string' && REALTIME_PRACTICE_VOICE_OPTIONS.some((item) => item.value === value);
|
||||
export interface RealtimeSessionResponse {
|
||||
sessionId: string;
|
||||
personaId: string;
|
||||
personaName: string;
|
||||
hasTools: boolean;
|
||||
sessionUpdate: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export const normalizeRealtimePracticeVoice = (value: unknown): RealtimePracticeVoice =>
|
||||
isRealtimePracticeVoice(value) ? value : DEFAULT_REALTIME_PRACTICE_VOICE;
|
||||
|
||||
export const createRealtimePracticeSessionUpdate = (instructions: string, voice: unknown) => ({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'session.update',
|
||||
session: {
|
||||
modalities: ['text', 'audio'],
|
||||
voice: normalizeRealtimePracticeVoice(voice),
|
||||
input_audio_format: 'pcm',
|
||||
output_audio_format: 'pcm',
|
||||
input_audio_transcription: { model: 'qwen3-asr-flash-realtime' },
|
||||
instructions,
|
||||
turn_detection: {
|
||||
type: 'semantic_vad',
|
||||
threshold: 0.5,
|
||||
prefix_padding_ms: 500,
|
||||
silence_duration_ms: 800
|
||||
}
|
||||
}
|
||||
});
|
||||
interface RealtimeToolInvokeResponse {
|
||||
callId: string;
|
||||
output: string;
|
||||
}
|
||||
|
||||
interface RealtimeSdpResponse {
|
||||
answerSdp: string;
|
||||
@@ -98,6 +31,9 @@ interface RealtimeSdpResponse {
|
||||
interface RealtimeEvent {
|
||||
type?: string;
|
||||
transcript?: string;
|
||||
call_id?: string;
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
error?: { message?: string };
|
||||
}
|
||||
|
||||
@@ -112,16 +48,40 @@ export interface RealtimePracticeCallbacks {
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeController {
|
||||
start: (instructions: string, voice?: RealtimePracticeVoice) => Promise<void>;
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export const fetchRealtimePersonas = () =>
|
||||
apiRequest<RealtimePersonaOption[]>({
|
||||
url: '/api/train/practice/realtime/personas',
|
||||
method: 'GET',
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
export const createRealtimeSession = (personaId?: string, scenarioId?: string) =>
|
||||
apiRequest<RealtimeSessionResponse>({
|
||||
url: '/api/train/practice/realtime/session',
|
||||
method: 'POST',
|
||||
data: { personaId: personaId || null, scenarioId: scenarioId || null },
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
export const invokeRealtimeTool = (sessionId: string, callId: string, name: string, argsJson: string) =>
|
||||
apiRequest<RealtimeToolInvokeResponse>({
|
||||
url: '/api/train/practice/realtime/tools/invoke',
|
||||
method: 'POST',
|
||||
data: { sessionId, callId, name, arguments: argsJson },
|
||||
timeout: 20000
|
||||
});
|
||||
|
||||
const isSupported = () =>
|
||||
!isNativeAppRuntime()
|
||||
&& typeof navigator !== 'undefined'
|
||||
@@ -151,6 +111,8 @@ const waitForIceComplete = (connection: RTCPeerConnection) =>
|
||||
const normalizeSdp = (sdp: string) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const trimTranscript = (value?: string) => (value || '').trim();
|
||||
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const createRealtimePracticeController = (callbacks: RealtimePracticeCallbacks): RealtimePracticeController => {
|
||||
let activeAttempt = 0;
|
||||
let connection: RTCPeerConnection | null = null;
|
||||
@@ -159,6 +121,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
let controlChannel: RTCDataChannel | null = null;
|
||||
let cancelSessionWait: (() => void) | null = null;
|
||||
let sessionWaitAttempt: number | null = null;
|
||||
let toolCallChain: Promise<void> = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt: number) => activeAttempt === attempt;
|
||||
|
||||
@@ -176,9 +139,40 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
connection = null;
|
||||
};
|
||||
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, instructions: string, voice: RealtimePracticeVoice) => {
|
||||
// 服务端下发的 session.update 载荷原样转发，仅补 event_id
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, sessionUpdate: Record<string, unknown>) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify(createRealtimePracticeSessionUpdate(instructions, voice)));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt: number, event: RealtimeEvent) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
// 多次工具调用串行化，避免打断竞态下回传交错
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output: string;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw: string, attempt: number, provider: ProviderEventHandlers) => {
|
||||
@@ -202,6 +196,9 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
@@ -224,6 +221,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
@@ -248,9 +246,8 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (instructions: string, voice: RealtimePracticeVoice = DEFAULT_REALTIME_PRACTICE_VOICE) => {
|
||||
const start = async (sessionUpdate: Record<string, unknown>, sessionId?: string) => {
|
||||
if (!isSupported()) throw new Error('当前设备不支持实时语音，请使用录音对练');
|
||||
const selectedVoice = normalizeRealtimePracticeVoice(voice);
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
@@ -292,7 +289,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, instructions, selectedVoice);
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
@@ -358,7 +355,7 @@ export const createRealtimePracticeController = (callbacks: RealtimePracticeCall
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp: normalizeSdp(offerSdp) },
|
||||
data: { offerSdp: normalizeSdp(offerSdp), sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
if (!ensureCurrent()) return;
|
||||
|
||||
Reference in New Issue
Block a user